wassname/Castor
1
0
Fork 0
mirror of https://github.com/wassname/Castor.git synced 2026-09-09 11:13:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

e2e castorini/Castor castorini/data castorini/models (#7)

Browse Source 
Initial integration of Castor components for e2e QA demo.
This commit is contained in:
gauravbaruah
2017-04-02 15:24:33 -04:00
committed by Jimmy Lin
parent 31928de93c
commit d4ac752cf2
26 changed files with 272 additions and 660 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3 -3
View File
@@ -2,14 +2,14 @@
Pytorch deep learning models.
1. [SM model](./sm-model/README.md): Similarity between question and candidate answers.
1. [SM model](./sm_model/): Similarity between question and candidate answers.
## Setting up Pytorch
You need Python 3.6 to use the models in this repository.
As per [pytorch.org](pytorch.org)
As per [pytorch.org](pytorch.org),
> [Anaconda](https://www.continuum.io/downloads) is our recommended package manager
```conda install pytorch torchvision -c soumith```
@@ -27,6 +27,6 @@ CUDA installation guide for linux can be found [here](http://docs.nvidia.com/cud
**NOTE**: Install CUDA libraries **before** installing conda and pytorch.
## Castor-data
## data for models
Sourcing and pre-processing of input data for each model is described in respective ```model/README.md```'s
sm-model/SMCastorini.py → castorini_smmodel_bridge.py
+42 -39
View File
@@ -1,4 +1,5 @@
import os
import sys
import pickle
import string
from collections import defaultdict
@@ -8,21 +9,23 @@ import torch
from nltk.tokenize import TreebankWordTokenizer
from torch.autograd import Variable
from model import QAModel
from sm_model import model
sys.modules['model'] = model
class SMModelCastorini(object):
class SMModelBridge(object):
def __init__(self, model_file, word_embeddings_cache_file, stopwords_file, word2dfs_file):
# init torch random seeds
torch.manual_seed(1234)
np.random.seed(1234)
np.random.seed(1234)
# load model
self.model = QAModel.load('', model_file)
self.model = model.QAModel.load(model_file)
# load vectors
self.vec_dim = self._preload_cached_embeddings(word_embeddings_cache_file)
self.unk_term_vec = np.random.uniform(-0.25, 0.25, self.vec_dim)
self.unk_term_vec = np.random.uniform(-0.25, 0.25, self.vec_dim)
# stopwords
self.stoplist = set([line.strip() for line in open(stopwords_file)])
@@ -34,16 +37,16 @@ class SMModelCastorini(object):
def _preload_cached_embeddings(self, cache_file):
with open(cache_file + '.dimensions') as d:
vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
self.W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim))
with open(cache_file + '.vocab') as f:
with open(cache_file + '.vocab') as f:
w2v_vocab_list = map(str.strip, f.readlines())
self.vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)}
self.vocab_dict = {w:k for k, w in enumerate(w2v_vocab_list)}
return vec_dim
@@ -64,7 +67,7 @@ class SMModelCastorini(object):
feats_overlap = []
for a in a_list:
question = q_str.split()
answer = a.split()
answer = a.split()
# q_set = set(question)
# a_set = set(answer)
q_set = set([q for q in question if q not in stoplist])
@@ -82,43 +85,40 @@ class SMModelCastorini(object):
df_overlap = 0.0
for w in word_overlap:
df_overlap += word2df[w]
if len(q_set) == 0 and len(a_set) == 0:
df_overlap = 0
else:
df_overlap /= (len(q_set) + len(a_set))
feats_overlap.append(np.array([
overlap,
df_overlap,
]))
feats_overlap.append(np.array([overlap, df_overlap]))
return np.array(feats_overlap)
def make_input_matrix(self, sentence):
terms = sentence.strip().split()
def make_input_matrix(self, sentence):
terms = sentence.strip().split()
# word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor)
word_embeddings = torch.zeros(len(terms), self.vec_dim).type(torch.DoubleTensor)
for i in range(len(terms)):
word = terms[i]
word = terms[i]
if word not in self.vocab_dict:
emb = torch.from_numpy(self.unk_term_vec)
else:
emb = torch.from_numpy(self.W[self.vocab_dict[word]])
word_embeddings[i] = emb
emb = torch.from_numpy(self.W[self.vocab_dict[word]])
word_embeddings[i] = emb
input_tensor = torch.zeros(1, self.vec_dim, len(terms))
input_tensor[0] = torch.transpose(word_embeddings, 0, 1)
return input_tensor
def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats):
assert(1 == len(batch_ques))
def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats):
assert(1 == len(batch_ques))
tensorized_inputs = []
for i in range(len(batch_ques)):
xq = Variable(self.make_input_matrix(batch_ques[i]))
xq = Variable(self.make_input_matrix(batch_ques[i]))
xs = Variable(self.make_input_matrix(batch_sents[i]))
ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
ext_feats =torch.unsqueeze(ext_feats, 0)
ext_feats = torch.unsqueeze(ext_feats, 0)
tensorized_inputs.append((xq, xs, ext_feats))
return tensorized_inputs
@@ -128,30 +128,33 @@ class SMModelCastorini(object):
q_str, a_list = self.parser(question, answers)
# calculate overlap features
overlap_feats = self.compute_overlap_features(q_str,a_list, stoplist=None, word2df=self.word2dfs)
overlap_feats_stoplist = self.compute_overlap_features(q_str, a_list, stoplist=self.stoplist, word2df=self.word2dfs)
overlap_feats = self.compute_overlap_features(q_str, a_list, \
stoplist=None, word2df=self.word2dfs)
overlap_feats_stoplist = self.compute_overlap_features(q_str, a_list, \
stoplist=self.stoplist, word2df=self.word2dfs)
overlap_feats_vec = np.hstack([overlap_feats, overlap_feats_stoplist])
# run through the model
scores_sentences = []
for i in range(len(a_list)):
xq, xa, x_ext_feats = self.get_tensorized_inputs([q], [a_list[i]], [overlap_feats_vec[i]])[0]
pred = self.model(xq, xa, x_ext_feats)
xq, xa, x_ext_feats = self.get_tensorized_inputs([q_str], [a_list[i]], \
[overlap_feats_vec[i]])[0]
pred = self.model(xq, xa, x_ext_feats)
pred = torch.exp(pred)
scores_sentences.append((pred.data.squeeze()[1], a_list[i]))
return scores_sentences
if __name__ == "__main__":
smmodel = SMModelCastorini('sm.model.py3',
'../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache',
'stopwords.txt',
'word2dfs.p')
q = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?"
a = [
smmodel = SMModelBridge('../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor',
'../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache',
'../data/TrecQA/stopwords.txt',
'../data/TrecQA/word2dfs.p')
question = "who is the author of the book , `` the iron lady : a biography of margaret thatcher '' ?"
answers = [
"the iron lady ; a biography of margaret thatcher by hugo young -lrb- farrar , straus & giroux -rrb-",
"in this same revisionist mold , hugo young , the distinguished british journalist , has performed a brilliant \
dissection of the notion of thatcher as a conservative icon .",
@@ -163,8 +166,8 @@ if __name__ == "__main__":
"this is not the answer",
"asdfawe asdf sertse dgfsgsfg"
]
ss = smmodel.rerank_candidate_answers(q, a)
print('Question:', q)
ss = smmodel.rerank_candidate_answers(question, answers)
print('Question:', question)
for score, sentence in ss:
print(score, '\t', sentence)
sm-model/README.md
-33
View File
@@ -1,33 +0,0 @@
## Similarity Measure model (SM model)
#### References:
1. Aliaksei Severyn and Alessandro Moschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738
#### TODOs:
1. figure out if the L2 regularization is correct
2. Batch size of 50 (current batch_size = 1)
#### Getting the data
TODO:
#### Running it
``1.`` Make TrecEval:
```
$ cd trec_eval-8.0
$ make clean
$ make
```
``2.`` Get the Overlapping features for Q and A:
```
$ python overlap_features.py ../../data/TrecQA
```
``3.`` To run the S&M model on TrecQA, please follow the same parameter setting:
```
$ python main.py ../../data/aquaint+wiki.txt.gz.ndim\=50.bin ../../data/TrecQA/ sm --batch_size 1
```
sm-model/overlap_features.py
-190
View File
@@ -1,190 +0,0 @@
import sys
import re
import os
import numpy as np
import argparse
#from nltk.corpus import stopwords
from nltk.stem.porter import *
from collections import defaultdict
def load_data(dname):
stemmer = PorterStemmer()
qids, questions, answers, labels = [], [], [], []
print(dname)
with open(dname+'a.toks') as f:
for line in f:
line = unicode(line, errors='ignore')
question = line.strip().split()
question = [stemmer.stem(word) for word in question]
questions.append(question)
with open(dname+'b.toks') as f:
for line in f:
line = unicode(line, errors='ignore')
answer = line.decode('utf-8').strip().split()
answer = [stemmer.stem(word) for word in answer]
answers.append(answer)
with open(dname+'id.txt') as f:
for line in f:
qids.append(line.strip())
with open(dname+'sim.txt') as f:
for line in f:
labels.append(int(line.strip()))
return qids, questions, answers, labels
def compute_overlap_features(questions, answers, word2df=None, stoplist=None):
word2df = word2df if word2df else {}
stoplist = stoplist if stoplist else set()
feats_overlap = []
for question, answer in zip(questions, answers):
# q_set = set(question)
# a_set = set(answer)
q_set = set([q for q in question if q not in stoplist])
a_set = set([a for a in answer if a not in stoplist])
word_overlap = q_set.intersection(a_set)
# overlap = float(len(word_overlap)) / (len(q_set) * len(a_set) + 1e-8)
if len(q_set) == 0 and len(a_set) == 0:
overlap = 0
else:
overlap = float(len(word_overlap)) / (len(q_set) + len(a_set))
# q_set = set([q for q in question if q not in stoplist])
# a_set = set([a for a in answer if a not in stoplist])
word_overlap = q_set.intersection(a_set)
df_overlap = 0.0
for w in word_overlap:
df_overlap += word2df[w]
if len(q_set) == 0 and len(a_set) == 0:
df_overlap = 0
else:
df_overlap /= (len(q_set) + len(a_set))
feats_overlap.append(np.array([
overlap,
df_overlap,
]))
return np.array(feats_overlap)
def compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length):
stoplist = stoplist if stoplist else []
feats_overlap = []
q_indices, a_indices = [], []
for question, answer in zip(questions, answers):
q_set = set([q for q in question if q not in stoplist])
a_set = set([a for a in answer if a not in stoplist])
word_overlap = q_set.intersection(a_set)
q_idx = np.ones(q_max_sent_length) * 2
for i, q in enumerate(question):
value = 0
if q in word_overlap:
value = 1
q_idx[i] = value
q_indices.append(q_idx)
#### ERROR
# a_idx = np.ones(a_max_sent_length) * 2
# for i, q in enumerate(question):
# value = 0
# if q in word_overlap:
a_idx = np.ones(a_max_sent_length) * 2
for i, a in enumerate(answer):
value = 0
if a in word_overlap:
value = 1
a_idx[i] = value
a_indices.append(a_idx)
q_indices = np.vstack(q_indices).astype('int32')
a_indices = np.vstack(a_indices).astype('int32')
return q_indices, a_indices
def compute_dfs(docs):
word2df = defaultdict(float)
for doc in docs:
for w in set(doc):
word2df[w] += 1.0
num_docs = len(docs)
for w, value in word2df.iteritems():
word2df[w] /= np.math.log(num_docs / value)
return word2df
if __name__ == '__main__':
ap = argparse.ArgumentParser(description="compute overlap features for SM model")
ap.add_argument("dataset", help="path/to/dataset-directory", default="../../data/TrecQA")
ap.add_argument("--train_all", help="will generate overlap features for the train-all dataset", action="store_true")
args = ap.parse_args()
stoplist = set([line.strip() for line in open('stopwords.txt')])
import string
punct = set(string.punctuation)
stoplist.update(punct)
#stoplist = None
all_questions, all_answers, all_qids = [], [], []
base_dir = args.dataset
# base_dir = '../../data/' + sys.argv[1] + '/'
# sub_dirs = ['train/', 'raw-dev/', 'test.minimal/','test.complete/']
# sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/']
sub_dirs = ['train/', 'clean-dev/', 'clean-test/']
if args.train_all:
sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/']
for sub in sub_dirs:
qids, questions, answers, labels = load_data(base_dir+sub)
all_questions.extend(questions)
all_answers.extend(answers)
all_qids.extend(qids)
seen = set()
unique_questions = []
for q, qid in zip(all_questions, all_qids):
if qid not in seen:
seen.add(qid)
unique_questions.append(q)
docs = all_answers + unique_questions
word2dfs = compute_dfs(docs)
print(word2dfs.items()[:10])
q_max_sent_length = max(map(lambda x: len(x), all_questions))
a_max_sent_length = max(map(lambda x: len(x), all_answers))
print('q_max_sent_length', q_max_sent_length)
print('a_max_sent_length', a_max_sent_length)
for sub in sub_dirs:
print(sub)
qids, questions, answers, labels = load_data(base_dir+sub)
overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs)
overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs)
overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist])
print(overlap_feats[:3])
print('overlap_feats', overlap_feats.shape)
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
print("Scaling overlap features")
overlap_feats = scaler.fit_transform(overlap_feats)
print(overlap_feats[:3])
'''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length)
print(q_overlap_indices[:3], a_overlap_indices[:3])
print('q_overlap_indices', q_overlap_indices.shape)
print('a_overlap_indices', a_overlap_indices.shape)'''
with open(base_dir+sub+'overlap_feats.txt', 'w') as f:
for i in range(overlap_feats.shape[0]):
for j in range(4):
f.write(str(overlap_feats[i][j]) + ' ')
f.write('\n')
'''with open(base_dir+sub+'overlap_indices.txt', 'w') as f:
for i in range(q_overlap_indices.shape[0]):
for j in range(q_max_sent_length):
f.write(str(q_overlap_indices[i][j]) + ' ')
for j in range(a_max_sent_length):
f.write(str(a_overlap_indices[i][j]) + ' ')
f.write('\n')'''
sm-model/stopwords.txt
-238
View File
@@ -1,238 +0,0 @@
's
I
a
aboard
about
above
across
after
afterwards
against
agin
ago
agreed-upon
ah
alas
albeit
all
all-over
almost
along
alongside
altho
although
amid
amidst
among
amongst
an
and
another
any
anyone
anything
around
as
aside
astride
at
atop
avec
away
back
be
because
before
beforehand
behind
behynde
below
beneath
beside
besides
between
bewteen
beyond
bi
both
but
by
ca.
de
des
despite
do
down
due
durin
during
each
eh
either
en
every
ever
everyone
everything
except
far
fer
for
from
go
goddamn
goody
gosh
half
have
he
hell
her
herself
hey
him
himself
his
ho
how
however
i
if
in
inside
insofar
instead
into
it
its
itself
la
le
les
lest
lieu
like
me
minus
moreover
my
myself
near
near-by
nearer
nearest
neither
nevertheless
next
no
nor
not
nothing
notwithstanding
o
o'er
of
off
on
once
one
oneself
only
onto
or
other
others
otherwise
our
ours
ourselves
out
outside
outta
over
per
rather
regardless
round
se
she
should
since
so
some
someone
something
than
that
the
their
them
themselves
then
there
therefore
these
they
thine
this
those
thou
though
through
throughout
thru
till
to
together
toward
towardes
towards
uh
under
underneath
unless
unlike
until
unto
up
upon
uppon
us
via
vis-a-vis
vis-�-vis
we
well
what
whatever
whatsoever
when
whenever
where
whereas
wherefore
whereupon
whether
which
whichever
while
who
whoever
whom
whose
why
with
withal
within
without
ye
yea
yeah
yes
yet
yonder
you
your
yours
yourself
yourselves
sm-model/.gitignore → sm_model/.gitignore
View File
sm_model/README.md
+48
View File
@@ -0,0 +1,48 @@
## SM model
#### References:
1. Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738
#### TODOs:
1. figure out if the L2 regularization is correct
2. Batch size of 50 (current batch_size = 1)
#### Requirements
gensim==1.0.1
nltk==3.2.2
numpy==1.11.3
pandas==0.19.2
torch==0.1.11+b13b701
#### Getting the data
git clone [castorini/data](https://github.com/castorini/data)
castorini/data contains:
```word2vec/aquaint+wiki.txt.gz.ndim=50.bin```: word embeddings.
Note that a memory mapped cache will be created on first use on your disk, when you run ```main.py``` (below).
```TrecQA/```: the directory with the input data for training the model.
Follow instructions in castorini/data/TrecQA/README.md to preprocess data for it to be ingestable by the model.
#### Running the model
``1.`` Make TrecEval:
```
$ cd trec_eval-8.0
$ make clean
$ make
```
``2.`` To run the S&M model on TrecQA, please follow the same parameter setting:
```
$ python main.py ../../model/sm.model.aquaint.train-all --train_all
```
The final model will be saved to ```../../model/sm.model.aquaint.train-all```
Run ```python main.py -h``` for more default options.
sm_model/__init__.py
View File
sm-model/main.py → sm_model/main.py
+56 -44
View File
@@ -8,13 +8,10 @@ import sys
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import utils
from model import QAModel
from train import Trainer
from model import QAModel
# logging setup
import logging
@@ -41,14 +38,16 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder)
# Call TrecEval code to calc MAP and MRR
df_submission = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rank', 'sim', 'run_id'])
df_submission = pd.DataFrame(index=np.arange(N), \
columns=['qid', 'iter', 'docno', 'rank', 'sim', 'run_id'])
df_submission['qid'] = qids_test
df_submission['iter'] = 0
df_submission['docno'] = np.arange(N)
df_submission['rank'] = 0
df_submission['sim'] = test_scores
df_submission['run_id'] = 'smmodel'
df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), header=False, index=False, sep=' ')
df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), \
header=False, index=False, sep=' ')
df_gold = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rel'])
df_gold['qid'] = qids_test
@@ -60,8 +59,8 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
# subprocess.call("/bin/sh run_eval.sh '{}'".format(args.dataset_folder), shell=True)
pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(args.dataset_folder))
p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
pout, perr = p.communicate()
pout, perr = p.communicate()
lines = pout.split(b'\n')
map = float(lines[0].strip().split()[-1])
mrr = float(lines[1].strip().split()[-1])
@@ -70,33 +69,46 @@ def compute_map_mrr(dataset_folder, set_folder, test_scores):
if __name__ == "__main__":
ap = argparse.ArgumentParser(description='pytorch port of the SM model')
ap.add_argument('word_vectors_file', help='NOTE: a cache will be created for faster loading for word vectors')
ap.add_argument('dataset_folder', help='directory containing train, dev, test sets')
ap.add_argument('model_fname', help='model will be saved in args.dataset_folder/<model_fname>')
ap = argparse.ArgumentParser(description='pytorch port of the SM model', \
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
ap.add_argument('model_outfile', help='file to save final model')
ap.add_argument('--word_vectors_file', \
help='NOTE: a cache will be created for faster loading for word vectors',
default="../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin")
ap.add_argument('--dataset_folder', help='directory containing train, dev, test sets', \
default="../../data/TrecQA")
ap.add_argument('--classes', type=int, default=2)
# system arguments
# TODO: add arguments for CUDA
ap.add_argument('--num_threads', help="the number of simultaneous processes to run", type=int, default=4)
ap.add_argument('--num_threads', help="the number of simultaneous processes to run", \
type=int, default=4)
# training arguments
ap.add_argument('--batch_size', type=int, default=1)
ap.add_argument('--filter_width', type=int, default=5)
ap.add_argument('--batch_size', type=int, default=1, help="training mini-batch size")
ap.add_argument('--filter_width', type=int, default=5, help="number of convolution channels")
ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float)
ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float)
ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float)
ap.add_argument('--train_all', help='switches to train-all set', action="store_true")
# epoch related arguments
ap.add_argument('--epochs', type=int, default=25)
ap.add_argument('--patience', type=int, default=5, help="if there is no appreciable change in model after <patience> epochs, then stop")
ap.add_argument('--epochs', type=int, default=25, help="number of trainin epochs")
ap.add_argument('--patience', type=int, default=5, \
help="if there is no appreciable change in model after <patience> epochs, then stop")
# debugging arguments
ap.add_argument('--debugSingleBatch', action="store_true", help="will stop program after training 1 input batch")
ap.add_argument('--num_conv_filters', help="the number of convolution channels (lesser is faster)", default=100, type=int)
ap.add_argument('--no_ext_feats', action="store_true", help="will not include external features in the model")
ap.add_argument('--debug_single_batch', action="store_true", \
help="will stop program after training 1 input batch")
ap.add_argument('--num_conv_filters', default=100, type=int, \
help="the number of convolution channels (lesser is faster)")
ap.add_argument('--no_ext_feats', action="store_true", \
help="will not include external features in the model")
ap.add_argument('--no_loss_reg', help="no loss regularization", action="store_true")
ap.add_argument('--test_on_each_epoch', help='runs test on each epoch to track final performance', action="store_true")
ap.add_argument('--test_on_each_epoch', action="store_true", \
help='runs test on each epoch to track final performance')
args = ap.parse_args()
@@ -108,17 +120,17 @@ if __name__ == "__main__":
train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test'
# cache word embeddings
cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache'
cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache'
utils.cache_word_embeddings(args.word_vectors_file, cache_file)
vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file)
# instantiate model
net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats) #filter width is 5
QAModel.save(net, args.dataset_folder, args.model_fname)
net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats)
QAModel.save(net, args.model_outfile)
torch.set_num_threads(args.num_threads)
trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim)
logger.info("Loading input data...")
trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set)
@@ -127,38 +139,38 @@ if __name__ == "__main__":
best_model = 0
for i in range(args.epochs):
logger.info('------------- Training epoch {} --------------'.format(i+1))
train_accuracy = trainer.train(train_set, args.batch_size, args.debugSingleBatch)
if args.debugSingleBatch: sys.exit(0)
logger.info('------------- Training epoch {} --------------'.format(i+1))
train_accuracy = trainer.train(train_set, args.batch_size, args.debug_single_batch)
if args.debug_single_batch: sys.exit(0)
dev_scores = trainer.test(dev_set, args.batch_size)
dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores)
logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr))
if dev_map - best_map > 1e-3: # new map is better than best map
if dev_map - best_map > 1e-3: # new map is better than best map
best_model = i
best_map = dev_map
QAModel.save(net, args.dataset_folder, args.model_fname)
QAModel.save(net, args.model_outfile)
logger.info('Achieved better dev_map ... saved model')
if args.test_on_each_epoch:
test_scores = trainer.test(test_set, args.batch_size)
if args.test_on_each_epoch:
test_scores = trainer.test(test_set, args.batch_size)
map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
logger.info("------- MAP {}, MRR {}".format(map, mrr))
if (i - best_model) >= args.patience:
logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model))
break
logger.info(' ------------ Training epochs completed! ------------')
logger.info(' ------------ Training epochs completed! ------------')
logger.info('Best MAP in training phase = {:.4f}'.format(best_map))
trained_model = QAModel.load(args.dataset_folder, args.model_fname)
evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim)
trained_model = QAModel.load(args.model_outfile)
evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim)
evaluator.load_input_data(args.dataset_folder, cache_file, None, None, test_set)
test_scores = evaluator.test(test_set, args.batch_size)
map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores)
logger.info("------- MAP {}, MRR {}".format(map, mrr))
sm-model/model.py → sm_model/model.py
+24 -26
View File
@@ -1,10 +1,6 @@
import os
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
# logging setup
import logging
@@ -19,18 +15,19 @@ logger.addHandler(ch)
class QAModel(nn.Module):
@staticmethod
def save(model, out_folder, model_fname):
torch.save(model, os.path.join(out_folder, model_fname))
@staticmethod
def load(in_folder, model_fname):
return torch.load(os.path.join(in_folder, model_fname))
def __init__(self, input_n_dim, filter_width, conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2):
def save(model, model_fname):
torch.save(model, model_fname)
@staticmethod
def load(model_fname):
return torch.load(model_fname)
def __init__(self, input_n_dim, filter_width, \
conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2):
super(QAModel, self).__init__()
self.no_ext_feats = no_ext_feats
@@ -40,25 +37,26 @@ class QAModel(nn.Module):
self.conv_q = nn.Sequential(
nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),
nn.Tanh()
nn.Tanh()
)
self.conv_a = nn.Sequential(
nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),
nn.Tanh()
nn.Tanh()
)
self.combined_feature_vector = nn.Linear(2*self.conv_channels + (0 if no_ext_feats else ext_feats_size), n_hidden)
#TODO: add +1 to Linear layer^. Will need change in forward function
self.combined_feature_vector = nn.Linear(2*self.conv_channels + \
(0 if no_ext_feats else ext_feats_size), n_hidden)
# TODO: add +1 to Linear layer^. Will need change in forward function
self.combined_features_activation = nn.Tanh()
self.dropout = nn.Dropout(0.5)
self.hidden = nn.Linear(n_hidden, n_classes)
self.logsoftmax = nn.LogSoftmax()
def forward(self, question, answer, ext_feats):
q = self.conv_q.forward(question)
def forward(self, question, answer, ext_feats):
q = self.conv_q.forward(question)
q = F.max_pool1d(q, q.size()[2])
q = q.view(-1, self.conv_channels)
# logger.debug('forward q: {}'.format(q))
@@ -69,12 +67,12 @@ class QAModel(nn.Module):
x = None
if self.no_ext_feats:
x = torch.cat([q, a], 1)
x = torch.cat([q, a], 1)
# logger.debug('no_ext_feats')
else:
x = torch.cat([q, a, ext_feats], 1)
x = torch.cat([q, a, ext_feats], 1)
# logger.debug('with ext_feats')
# logger.debug('featvec x: {}'.format(x))
# logger.debug(x.creator)
sm-model/run_eval.sh → sm_model/run_eval.sh
View File
sm-model/train.py → sm_model/train.py
+78 -70
View File
@@ -1,5 +1,3 @@
import argparse
import time
import numpy as np
@@ -23,71 +21,75 @@ logger.addHandler(ch)
class Trainer(object):
def __init__(self, model, eta, mom, no_loss_reg, vec_dim):
# set the random seeds for every instance of trainer.
# set the random seeds for every instance of trainer.
# needed to ensure reproduction of random word vectors for out of vocab terms
torch.manual_seed(1234)
np.random.seed(1234)
self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim)
self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim)
self.reg = 1e-5
self.no_loss_reg = no_loss_reg
self.model = model
self.criterion = nn.CrossEntropyLoss()
#self.criterion = nn.NLLLoss()
self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, weight_decay=(0 if no_loss_reg else self.reg))
self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, \
weight_decay=(0 if no_loss_reg else self.reg))
self.datasets = {}
self.embeddings = {}
self.vec_dim = vec_dim
def load_input_data(self, dataset_root_folder, word_vectors_cache_file, train_set_folder, dev_set_folder, test_set_folder):
def load_input_data(self, dataset_root_folder, word_vectors_cache_file, \
train_set_folder, dev_set_folder, test_set_folder):
for set_folder in [test_set_folder, dev_set_folder, train_set_folder]:
if set_folder:
self.datasets[set_folder] = utils.read_in_dataset(dataset_root_folder, set_folder)
# NOTE: self.datasets[set_folder] = questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats
self.embeddings[set_folder] = utils.load_cached_embeddings(word_vectors_cache_file,
self.datasets[set_folder][3], [] if "train" in set_folder else self.unk_term)
def regularize_loss(self, loss):
# NOTE: self.datasets[set_folder] = questions, sentences, labels,
# vocab, maxlen_q, maxlen_s, ext_feats
self.embeddings[set_folder] = utils.load_cached_embeddings( \
word_vectors_cache_file, self.datasets[set_folder][3], \
[] if "train" in set_folder else self.unk_term)
def regularize_loss(self, loss):
flattened_params = []
for p in self.model.parameters():
f = p.data.clone()
flattened_params.append(f.view(-1))
fp = torch.cat(flattened_params)
loss = loss + 0.5 * self.reg * fp.norm() * fp.norm()
# for p in self.model.parameters():
# loss = loss + 0.5 * self.reg * p.norm() * p.norm()
return loss
return loss
def _train(self, xq, xa, ext_feats, ys):
self.optimizer.zero_grad()
output = self.model(xq, xa, ext_feats)
loss = self.criterion(output, ys)
self.optimizer.zero_grad()
output = self.model(xq, xa, ext_feats)
loss = self.criterion(output, ys)
# logger.debug('loss after criterion {}'.format(loss))
# NOTE: regularizing location 1
if not self.no_loss_reg:
loss = self.regularize_loss(loss)
loss = self.regularize_loss(loss)
# logger.debug('loss after regularizing {}'.format(loss))
loss.backward()
# logger.debug('AFTER backward')
#logger.debug('params {}'.format([p for p in self.model.parameters()]))
# logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()]))
# NOTE: regularizing location 2. It would seem that location 1 is correct?
#if not self.no_loss_reg:
# loss = self.regularize_loss(loss)
@@ -102,18 +104,19 @@ class Trainer(object):
return loss.data[0], self.pred_equals_y(output, ys)
def pred_equals_y(self, pred, y):
_, best = pred.max(1)
best = best.data.long().squeeze()
def pred_equals_y(self, pred, y):
_, best = pred.max(1)
best = best.data.long().squeeze()
return torch.sum(y.data.long() == best)
def test(self, set_folder, batch_size):
logger.info('----- Predictions on {} '.format(set_folder))
questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder]
questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \
self.datasets[set_folder]
word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim
self.model.eval()
batch_size = 1
@@ -123,31 +126,32 @@ class Trainer(object):
num_batches = np.ceil(len(questions)/batch_size)
y_pred = np.zeros(len(questions))
ypc = 0
for k in range(int(num_batches)):
batch_start = k * batch_size
batch_end = (k+1) * batch_size
# convert raw questions and sentences to tensors
batch_inputs, batch_labels = self.get_tensorized_inputs(
questions[batch_start:batch_end],
sentences[batch_start:batch_end],
labels[batch_start:batch_end],
ext_feats[batch_start:batch_end],
word_vectors, vocab, vec_dim
questions[batch_start:batch_end],
sentences[batch_start:batch_end],
labels[batch_start:batch_end],
ext_feats[batch_start:batch_end],
word_vectors, vocab, vec_dim
)
xq, xa, x_ext_feats = batch_inputs[0]
y = batch_labels[0]
pred = self.model(xq, xa, x_ext_feats)
loss = self.criterion(pred, y)
pred = self.model(xq, xa, x_ext_feats)
loss = self.criterion(pred, y)
pred = torch.exp(pred)
total_loss += loss
# total_correct += self.pred_equals_y(pred, y)
y_pred[ypc] = pred.data.squeeze()[1] # we want to score for relevance, NOT the predicted class
ypc += 1
y_pred[ypc] = pred.data.squeeze()[1]
# ^ we want to score for relevance, NOT the predicted class
ypc += 1
# logger.info('{}_correct {}'.format(set_folder, total_correct))
# logger.info('{}_loss {}'.format(set_folder, total_loss.data[0]))
logger.info('{} total {}'.format(set_folder, len(labels)))
@@ -157,12 +161,13 @@ class Trainer(object):
return y_pred
def train(self, set_folder, batch_size, debugSingleBatch):
def train(self, set_folder, batch_size, debug_single_batch):
train_start_time = time.time()
questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = self.datasets[set_folder]
questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \
self.datasets[set_folder]
word_vectors, vec_dim = self.embeddings[set_folder], self.vec_dim
# set model for training modep
self.model.train()
@@ -175,23 +180,24 @@ class Trainer(object):
# convert raw questions and sentences to tensors
batch_inputs, batch_labels = self.get_tensorized_inputs(
questions[batch_start:batch_end],
sentences[batch_start:batch_end],
labels[batch_start:batch_end],
ext_feats[batch_start:batch_end],
word_vectors, vocab, vec_dim
questions[batch_start:batch_end],
sentences[batch_start:batch_end],
labels[batch_start:batch_end],
ext_feats[batch_start:batch_end],
word_vectors, vocab, vec_dim
)
xq, xa, x_ext_feats = batch_inputs[0]
ys = batch_labels[0]
batch_loss, batch_correct = self._train(xq, xa, x_ext_feats, ys)
batch_loss, batch_correct = self._train(xq, xa, x_ext_feats, ys)
# logger.debug('batch_loss {}, batch_correct {}'.format(batch_loss, batch_correct))
train_loss += batch_loss
# train_correct += batch_correct
if debugSingleBatch: break
if debug_single_batch:
break
# logger.info('train_correct {}'.format(train_correct))
logger.info('train_loss {}'.format(train_loss))
@@ -201,23 +207,25 @@ class Trainer(object):
))
logger.info('training time = {:.3f} seconds'.format(time.time() - train_start_time))
return train_correct/num_batches
def make_input_matrix(self, sentence, word_vectors, vec_dim):
terms = sentence.strip().split()
# word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor)
def make_input_matrix(self, sentence, word_vectors, vec_dim):
terms = sentence.strip().split()[:60]
# NOTE: we are truncating the inputs to 60 words.
word_embeddings = torch.zeros(len(terms), vec_dim).type(torch.DoubleTensor)
for i in range(len(terms)):
word = terms[i]
emb = torch.from_numpy(word_vectors[word])
word_embeddings[i] = emb
word = terms[i]
emb = torch.from_numpy(word_vectors[word])
word_embeddings[i] = emb
input_tensor = torch.zeros(1, vec_dim, len(terms))
input_tensor[0] = torch.transpose(word_embeddings, 0 , 1)
input_tensor[0] = torch.transpose(word_embeddings, 0, 1)
return input_tensor
def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, word_vectors, vocab, vec_dim):
def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, \
word_vectors, vocab, vec_dim):
batch_size = len(batch_ques)
# NOTE: ideal batch size is one, because sentences are all of different length.
# In other words, we have no option but to feed in sentences one by one into the model
@@ -231,10 +239,10 @@ class Trainer(object):
tensorized_inputs = []
for i in range(len(batch_ques)):
xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim)) #, requires_grad=False)
xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim)) #, requires_grad=False)
xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim))
xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim))
ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
ext_feats =torch.unsqueeze(ext_feats, 0)
ext_feats = torch.unsqueeze(ext_feats, 0)
y[i] = batch_labels[i]
tensorized_inputs.append((xq, xs, ext_feats))
sm-model/trec_eval-8.0/Makefile → sm_model/trec_eval-8.0/Makefile
View File
sm-model/trec_eval-8.0/README → sm_model/trec_eval-8.0/README
View File
sm-model/trec_eval-8.0/buf.h → sm_model/trec_eval-8.0/buf.h
View File
sm-model/trec_eval-8.0/buf_util.c → sm_model/trec_eval-8.0/buf_util.c
View File
sm-model/trec_eval-8.0/common.h → sm_model/trec_eval-8.0/common.h
View File
sm-model/trec_eval-8.0/error_msgs.c → sm_model/trec_eval-8.0/error_msgs.c
View File
sm-model/trec_eval-8.0/form_trvec.c → sm_model/trec_eval-8.0/form_trvec.c
View File
sm-model/trec_eval-8.0/get_qrels.c → sm_model/trec_eval-8.0/get_qrels.c
View File
sm-model/trec_eval-8.0/get_top.c → sm_model/trec_eval-8.0/get_top.c
View File
sm-model/trec_eval-8.0/smart_error.h → sm_model/trec_eval-8.0/smart_error.h
View File
sm-model/trec_eval-8.0/sysfunc.h → sm_model/trec_eval-8.0/sysfunc.h
View File
sm-model/trec_eval-8.0/tr_vec.h → sm_model/trec_eval-8.0/tr_vec.h
View File
sm-model/trec_eval-8.0/trec_eval.c → sm_model/trec_eval-8.0/trec_eval.c
View File
sm-model/utils.py → sm_model/utils.py
+21 -17
View File
@@ -1,4 +1,4 @@
# file input output
# file input output
import os
import sys
@@ -27,7 +27,8 @@ def logargs(func):
def cache_word_embeddings(word_embeddings_file, cache_file):
if not word_embeddings_file.endswith('.gz'):
logger.warning('WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file))
logger.warning('WARNING: expecting a .gz file. Is the {} in the correct \
format?'.format(word_embeddings_file))
vocab_size, vec_dim = 0, 0
@@ -36,8 +37,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file):
if not os.path.exists(os.path.dirname(cache_file)):
# make cache folder if needed
os.mkdir(os.path.dirname(cache_file))
logger.info('caching the word embeddings in np.memmap format')
logger.info('caching the word embeddings in np.memmap format')
wv = KeyedVectors.load_word2vec_format(word_embeddings_file, binary=True)
# print len(wv.syn0), wv.syn0.shape
# print len(wv.syn0norm) if wv.syn0norm else None
@@ -50,7 +50,7 @@ def cache_word_embeddings(word_embeddings_file, cache_file):
with open(cache_file + '.dimensions', 'w') as f:
logger.info('writing out dimensions for {}'.format(word_embeddings_file))
print(wv.syn0.shape[0], wv.syn0.shape[1], file=f)
vocab_size, vec_dim = wv.syn0.shape
vocab_size, vec_dim = wv.syn0.shape
del fp, wv
print('cached {} into {}'.format(word_embeddings_file, cache_file))
@@ -63,9 +63,8 @@ def load_embedding_dimensions(cache_file):
vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
return vocab_size, vec_dim
def load_cached_embeddings(cache_file, vocab_list, oov_vec = []):
logger.debug('loading cached embeddings ')
def load_cached_embeddings(cache_file, vocab_list, oov_vec=[]):
logger.debug('loading cached embeddings ')
with open(cache_file + '.dimensions') as d:
vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
@@ -76,8 +75,8 @@ def load_cached_embeddings(cache_file, vocab_list, oov_vec = []):
logger.debug('loading vocab')
w2v_vocab_list = map(str.strip, f.readlines())
vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)}
vocab_dict = {w:k for k, w in enumerate(w2v_vocab_list)}
# Read w2v for vocab appears in Q and A
w2v_dict = {}
for word in vocab_list:
@@ -86,7 +85,8 @@ def load_cached_embeddings(cache_file, vocab_list, oov_vec = []):
if word in vocab_dict:
w2v_dict[word] = W[vocab_dict[word]]
else:
w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) if len(oov_vec) == 0 else oov_vec
w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) \
if len(oov_vec) == 0 else oov_vec
#w2v_dict[word] = W[vocab_dict["unk"]]
return w2v_dict
@@ -99,12 +99,16 @@ def read_in_dataset(dataset_folder, set_folder):
max_q = 0
max_s = 0
set_path = os.path.join(dataset_folder, set_folder)
len_q_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines()]
questions = [line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines()]
len_s_list =[len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines()]
len_q_list = [len(q.split()) for q in questions]
sentences = [line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines()]
len_s_list = [len(s.split()) for s in sentences]
labels = [int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()]
ext_feats = np.array([list(map(float, line.strip().split(' '))) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()])
ext_feats = np.array([list(map(float, line.strip().split(' '))) \
for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines()])
#y = torch.from_numpy(labels)
#return questions, sentences, y
@@ -121,11 +125,11 @@ def get_test_qids_labels(dataset_folder, set_folder):
if __name__ == "__main__":
vocab = ["unk", "idontreallythinkthiswordexists", "hello"]
w2v_dict, vec_dim = load_cached_embeddings("../../data/word2vec-models/aquaint+wiki.txt.gz.ndim=50.cache", vocab)
w2v_dict, vec_dim = load_cached_embeddings("../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache", vocab)
for w, v in w2v_dict.iteritems():
print(w)
print(v)