From d877f176a4b78511200233315aef3af8d5f556ca Mon Sep 17 00:00:00 2001 From: Gaurav Baruah Date: Mon, 27 Mar 2017 12:10:10 -0400 Subject: [PATCH] Castor sm-model first commit --- sm-model/README.md | 30 +++ sm-model/main.py | 129 +++++++++ sm-model/model.py | 81 ++++++ sm-model/overlap_features.py | 186 +++++++++++++ sm-model/run_eval.sh | 9 + sm-model/stopwords.txt | 238 +++++++++++++++++ sm-model/train.py | 240 +++++++++++++++++ sm-model/trec_eval-8.0/Makefile | 141 ++++++++++ sm-model/trec_eval-8.0/README | 385 +++++++++++++++++++++++++++ sm-model/trec_eval-8.0/buf.h | 13 + sm-model/trec_eval-8.0/buf_util.c | 86 ++++++ sm-model/trec_eval-8.0/common.h | 31 +++ sm-model/trec_eval-8.0/error_msgs.c | 93 +++++++ sm-model/trec_eval-8.0/form_trvec.c | 201 ++++++++++++++ sm-model/trec_eval-8.0/get_qrels.c | 169 ++++++++++++ sm-model/trec_eval-8.0/get_top.c | 192 +++++++++++++ sm-model/trec_eval-8.0/smart_error.h | 24 ++ sm-model/trec_eval-8.0/sysfunc.h | 39 +++ sm-model/trec_eval-8.0/tr_vec.h | 21 ++ sm-model/trec_eval-8.0/trec_eval.c | 256 ++++++++++++++++++ sm-model/utils.py | 156 +++++++++++ 21 files changed, 2720 insertions(+) create mode 100644 sm-model/README.md create mode 100644 sm-model/main.py create mode 100644 sm-model/model.py create mode 100644 sm-model/overlap_features.py create mode 100755 sm-model/run_eval.sh create mode 100644 sm-model/stopwords.txt create mode 100644 sm-model/train.py create mode 100755 sm-model/trec_eval-8.0/Makefile create mode 100755 sm-model/trec_eval-8.0/README create mode 100755 sm-model/trec_eval-8.0/buf.h create mode 100755 sm-model/trec_eval-8.0/buf_util.c create mode 100755 sm-model/trec_eval-8.0/common.h create mode 100755 sm-model/trec_eval-8.0/error_msgs.c create mode 100755 sm-model/trec_eval-8.0/form_trvec.c create mode 100755 sm-model/trec_eval-8.0/get_qrels.c create mode 100755 sm-model/trec_eval-8.0/get_top.c create mode 100755 sm-model/trec_eval-8.0/smart_error.h create mode 100755 sm-model/trec_eval-8.0/sysfunc.h create mode 100755 sm-model/trec_eval-8.0/tr_vec.h create mode 100755 sm-model/trec_eval-8.0/trec_eval.c create mode 100644 sm-model/utils.py diff --git a/sm-model/README.md b/sm-model/README.md new file mode 100644 index 0000000..78df4aa --- /dev/null +++ b/sm-model/README.md @@ -0,0 +1,30 @@ +## Similarity Measure model (SM model) + +#### References: +1. Aliaksei Severyn and Alessandro Moschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 + + +#### TODOs: +1. figure out if the L2 regularization is correct +2. Batch size of 50 (current batch_size = 1) + + +#### Running it + +``1.`` Make TrecEval: +``` +$ cd trec_eval-8.0 +$ make clean +$ make +``` + +``2.`` Get the Overlapping features for Q and A: +``` +$ python overlap_features.py TrecQA +``` + +``3.`` To run the S&M model on TrecQA, please follow the same parameter setting: +``` +$ python main.py ../../data/aquaint+wiki.txt.gz.ndim\=50.bin ../../data/TrecQA/ sm --batch_size 1 +``` + diff --git a/sm-model/main.py b/sm-model/main.py new file mode 100644 index 0000000..758cb47 --- /dev/null +++ b/sm-model/main.py @@ -0,0 +1,129 @@ +import os +import sys +import time +import glob +import argparse +import numpy as np + +import pandas as pd +import subprocess + +import torch +import torch.optim as optim +import torch.nn as nn +from torch.autograd import Variable + + +from model import QAModel +import utils +from train import Trainer + +# logging setup +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + +def logargs(func): + def inner(*args, **kwargs): + logger.info('%s : %s %s' % (func.__name__, args, kwargs)) + return func(*args, **kwargs) + return inner + + +def compute_map_mrr(dataset_folder, set_folder, test_scores): + logger.info( "Running trec_eval script..." ) + N = len(test_scores) + + qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder) + + # Call TrecEval code to calc MAP and MRR + df_submission = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rank', 'sim', 'run_id']) + df_submission['qid'] = qids_test + df_submission['iter'] = 0 + df_submission['docno'] = np.arange(N) + df_submission['rank'] = 0 + df_submission['sim'] = test_scores + df_submission['run_id'] = 'smmodel' + df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), header=False, index=False, sep=' ') + + df_gold = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rel']) + df_gold['qid'] = qids_test + df_gold['iter'] = 0 + df_gold['docno'] = np.arange(N) + df_gold['rel'] = y_test + df_gold.to_csv(os.path.join(args.dataset_folder, 'gold.txt'), header=False, index=False, sep=' ') + + subprocess.call("/bin/sh run_eval.sh '{}'".format(args.dataset_folder), shell=True) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description='pytorch port of the SM model') + ap.add_argument('word_vectors_file', help='NOTE: a cache will be created for faster loading for word vectors') + ap.add_argument('dataset_folder', help='directory containing train, dev, test sets') + ap.add_argument('model_fname', help='model will be saved in args.dataset_folder/') + ap.add_argument('--batch_size', type=int, default=1) + ap.add_argument('--filter_width', type=int, default=5) + ap.add_argument('--epochs', type=int, default=25) + ap.add_argument('--eta', help='Initial learning rate', default=0.01, type=float) + ap.add_argument('--mom', help='SGD Momentum', default=0.9, type=float) + ap.add_argument('--classes', type=int, default=2) + ap.add_argument('--patience', type=int, default=5, help="if there is no appreciable change in model after epochs, then stop") + + args = ap.parse_args() + + torch.manual_seed(1234) + np.random.seed(1234) + + # cache word embeddings + cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache' + utils.cache_word_embeddings(args.word_vectors_file, cache_file) + + vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file) + + # instantiate model + net = QAModel(vec_dim, args.filter_width) #filter width is 5 + QAModel.save(net, args.dataset_folder, args.model_fname) + + trainer = Trainer(net) + + best_accuracy = 0.0 + best_model = 0 + + for i in range(args.epochs): + logger.info('Training epoch {} -------------'.format(i+1)) + train_accuracy = trainer.train(args.dataset_folder, 'train', args.batch_size, cache_file) + # sys.exit(0) + dev_accuracy, dev_scores = trainer.test(args.dataset_folder, 'clean-dev', args.batch_size, cache_file) + if dev_accuracy > best_accuracy: + best_model = i + best_accuracy = dev_accuracy + QAModel.save(net, args.dataset_folder, args.model_fname) + logger.info('Achieved better dev_accuracy ... saved model') + + compute_map_mrr(args.dataset_folder, 'clean-dev', dev_scores) + + if (i - best_model) >= args.patience: + logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model)) + break + + logger.info('Training epochs completed ------------') + logger.info('Best accuracy in training phase = {:.4f}'.format(best_accuracy)) + + logger.info('Evaluating over test set...') + model = QAModel.load(args.dataset_folder, args.model_fname) + + evaluator = Trainer(model) + test_accuracy, test_scores = evaluator.test(args.dataset_folder, 'clean-test', args.batch_size, cache_file) + + logger.info('Test set accuracy = {:.4f}'.format(test_accuracy)) + + compute_map_mrr(args.dataset_folder, 'clean-test', test_scores) + + + diff --git a/sm-model/model.py b/sm-model/model.py new file mode 100644 index 0000000..4bd7f29 --- /dev/null +++ b/sm-model/model.py @@ -0,0 +1,81 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable +import numpy as np + +import os + +# logging setup +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + + +class QAModel(nn.Module): + + @staticmethod + def save(model, out_folder, model_fname): + torch.save(model, os.path.join(out_folder, model_fname)) + + @staticmethod + def load(in_folder, model_fname): + return torch.load(os.path.join(in_folder, model_fname)) + + def __init__(self, input_n_dim, filter_width, ext_feats_size=4, n_classes=2): + super(QAModel, self).__init__() + + self.conv_channels = 100 + n_hidden = 2*self.conv_channels + 1 + + self.conv_q = nn.Sequential( + nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1), + nn.Tanh() + ) + + self.conv_a = nn.Sequential( + nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1), + nn.Tanh() + ) + + self.combined_feature_vector = nn.Linear(2*self.conv_channels+ext_feats_size, n_hidden) + #TODO: add +1 to Linear layer^. Will need change in forward function + self.combined_features_activation = nn.Tanh() + self.dropout = nn.Dropout(0.5) + self.hidden = nn.Linear(n_hidden, n_classes) + self.logsoftmax = nn.LogSoftmax() + + + def forward(self, question, answer, ext_feats): + + q = self.conv_q.forward(question) + q = F.max_pool1d(q, q.size()[2]) + q = q.view(-1, self.conv_channels) + logger.debug('forward q: {}'.format(q)) + + a = self.conv_a.forward(answer) + a = F.max_pool1d(a, a.size()[2]) + a = a.view(-1, self.conv_channels) + + x = torch.cat([q, a, ext_feats], 1) + # logger.debug('featvec x: {}'.format(x)) + # logger.debug(x.creator) + + x = self.combined_feature_vector.forward(x) + x = self.combined_features_activation.forward(x) + x = self.dropout(x) + x = self.hidden(x) + x = self.logsoftmax(x) + + logger.debug('x data {}'.format(x.data)) + logger.debug('x grad {}'.format(x.grad)) + + return x + + diff --git a/sm-model/overlap_features.py b/sm-model/overlap_features.py new file mode 100644 index 0000000..6374d7b --- /dev/null +++ b/sm-model/overlap_features.py @@ -0,0 +1,186 @@ +import sys +import re +import os +import numpy as np + +import argparse + +#from nltk.corpus import stopwords +from nltk.stem.porter import * +from collections import defaultdict + +def load_data(dname): + stemmer = PorterStemmer() + qids, questions, answers, labels = [], [], [], [] + print dname + with open(dname+'a.toks') as f: + for line in f: + line = unicode(line, errors='ignore') + question = line.strip().split() + question = [stemmer.stem(word) for word in question] + questions.append(question) + with open(dname+'b.toks') as f: + for line in f: + line = unicode(line, errors='ignore') + answer = line.decode('utf-8').strip().split() + answer = [stemmer.stem(word) for word in answer] + answers.append(answer) + with open(dname+'id.txt') as f: + for line in f: + qids.append(line.strip()) + with open(dname+'sim.txt') as f: + for line in f: + labels.append(int(line.strip())) + return qids, questions, answers, labels + +def compute_overlap_features(questions, answers, word2df=None, stoplist=None): + word2df = word2df if word2df else {} + stoplist = stoplist if stoplist else set() + feats_overlap = [] + for question, answer in zip(questions, answers): + # q_set = set(question) + # a_set = set(answer) + q_set = set([q for q in question if q not in stoplist]) + a_set = set([a for a in answer if a not in stoplist]) + word_overlap = q_set.intersection(a_set) + # overlap = float(len(word_overlap)) / (len(q_set) * len(a_set) + 1e-8) + if len(q_set) == 0 and len(a_set) == 0: + overlap = 0 + else: + overlap = float(len(word_overlap)) / (len(q_set) + len(a_set)) + + # q_set = set([q for q in question if q not in stoplist]) + # a_set = set([a for a in answer if a not in stoplist]) + word_overlap = q_set.intersection(a_set) + df_overlap = 0.0 + for w in word_overlap: + df_overlap += word2df[w] + if len(q_set) == 0 and len(a_set) == 0: + df_overlap = 0 + else: + df_overlap /= (len(q_set) + len(a_set)) + + feats_overlap.append(np.array([ + overlap, + df_overlap, + ])) + return np.array(feats_overlap) + +def compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length): + stoplist = stoplist if stoplist else [] + feats_overlap = [] + q_indices, a_indices = [], [] + for question, answer in zip(questions, answers): + q_set = set([q for q in question if q not in stoplist]) + a_set = set([a for a in answer if a not in stoplist]) + word_overlap = q_set.intersection(a_set) + + q_idx = np.ones(q_max_sent_length) * 2 + for i, q in enumerate(question): + value = 0 + if q in word_overlap: + value = 1 + q_idx[i] = value + q_indices.append(q_idx) + + #### ERROR + # a_idx = np.ones(a_max_sent_length) * 2 + # for i, q in enumerate(question): + # value = 0 + # if q in word_overlap: + + a_idx = np.ones(a_max_sent_length) * 2 + for i, a in enumerate(answer): + value = 0 + if a in word_overlap: + value = 1 + a_idx[i] = value + a_indices.append(a_idx) + + q_indices = np.vstack(q_indices).astype('int32') + a_indices = np.vstack(a_indices).astype('int32') + + return q_indices, a_indices + +def compute_dfs(docs): + word2df = defaultdict(float) + for doc in docs: + for w in set(doc): + word2df[w] += 1.0 + num_docs = len(docs) + for w, value in word2df.iteritems(): + word2df[w] /= np.math.log(num_docs / value) + return word2df + +if __name__ == '__main__': + ap = argparse.ArgumentParser(description="compute overlap features for SM model") + ap.add_argument("dataset", help="path/to/dataset-directory", default="../../data/TrecQA") + args = ap.parse_args() + + stoplist = set([line.strip() for line in open('stopwords.txt')]) + import string + punct = set(string.punctuation) + stoplist.update(punct) + #stoplist = None + + all_questions, all_answers, all_qids = [], [], [] + base_dir = args.dataset + # base_dir = '../../data/' + sys.argv[1] + '/' + # sub_dirs = ['train/', 'raw-dev/', 'test.minimal/','test.complete/'] + # sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/'] + sub_dirs = ['train/', 'clean-dev/', 'clean-test/'] + for sub in sub_dirs: + qids, questions, answers, labels = load_data(base_dir+sub) + all_questions.extend(questions) + all_answers.extend(answers) + all_qids.extend(qids) + + seen = set() + unique_questions = [] + for q, qid in zip(all_questions, all_qids): + if qid not in seen: + seen.add(qid) + unique_questions.append(q) + + docs = all_answers + unique_questions + word2dfs = compute_dfs(docs) + print word2dfs.items()[:10] + + q_max_sent_length = max(map(lambda x: len(x), all_questions)) + a_max_sent_length = max(map(lambda x: len(x), all_answers)) + print 'q_max_sent_length', q_max_sent_length + print 'a_max_sent_length', a_max_sent_length + + for sub in sub_dirs: + print sub + qids, questions, answers, labels = load_data(base_dir+sub) + + overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs) + overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs) + overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist]) + print overlap_feats[:3] + print 'overlap_feats', overlap_feats.shape + + from sklearn.preprocessing import StandardScaler + scaler = StandardScaler() + print "Scaling overlap features" + overlap_feats = scaler.fit_transform(overlap_feats) + print overlap_feats[:3] + + '''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length) + print q_overlap_indices[:3], a_overlap_indices[:3] + print 'q_overlap_indices', q_overlap_indices.shape + print 'a_overlap_indices', a_overlap_indices.shape''' + + with open(base_dir+sub+'overlap_feats.txt', 'w') as f: + for i in range(overlap_feats.shape[0]): + for j in range(4): + f.write(str(overlap_feats[i][j]) + ' ') + f.write('\n') + '''with open(base_dir+sub+'overlap_indices.txt', 'w') as f: + for i in range(q_overlap_indices.shape[0]): + for j in range(q_max_sent_length): + f.write(str(q_overlap_indices[i][j]) + ' ') + for j in range(a_max_sent_length): + f.write(str(a_overlap_indices[i][j]) + ' ') + f.write('\n')''' diff --git a/sm-model/run_eval.sh b/sm-model/run_eval.sh new file mode 100755 index 0000000..21202d6 --- /dev/null +++ b/sm-model/run_eval.sh @@ -0,0 +1,9 @@ +#!/bin/csh -f + +exp_dir=$1 +judgement=${exp_dir}/gold.txt +output=${exp_dir}/submission.txt + +./trec_eval-8.0/trec_eval -q -c ${judgement} ${output} > ${output}.treceval +tail -29 ${output}.treceval | grep -e 'map' -e 'recip_rank' +exit 0 diff --git a/sm-model/stopwords.txt b/sm-model/stopwords.txt new file mode 100644 index 0000000..5c78865 --- /dev/null +++ b/sm-model/stopwords.txt @@ -0,0 +1,238 @@ +'s +I +a +aboard +about +above +across +after +afterwards +against +agin +ago +agreed-upon +ah +alas +albeit +all +all-over +almost +along +alongside +altho +although +amid +amidst +among +amongst +an +and +another +any +anyone +anything +around +as +aside +astride +at +atop +avec +away +back +be +because +before +beforehand +behind +behynde +below +beneath +beside +besides +between +bewteen +beyond +bi +both +but +by +ca. +de +des +despite +do +down +due +durin +during +each +eh +either +en +every +ever +everyone +everything +except +far +fer +for +from +go +goddamn +goody +gosh +half +have +he +hell +her +herself +hey +him +himself +his +ho +how +however +i +if +in +inside +insofar +instead +into +it +its +itself +la +le +les +lest +lieu +like +me +minus +moreover +my +myself +near +near-by +nearer +nearest +neither +nevertheless +next +no +nor +not +nothing +notwithstanding +o +o'er +of +off +on +once +one +oneself +only +onto +or +other +others +otherwise +our +ours +ourselves +out +outside +outta +over +per +rather +regardless +round +se +she +should +since +so +some +someone +something +than +that +the +their +them +themselves +then +there +therefore +these +they +thine +this +those +thou +though +through +throughout +thru +till +to +together +toward +towardes +towards +uh +under +underneath +unless +unlike +until +unto +up +upon +uppon +us +via +vis-a-vis +vis-�-vis +we +well +what +whatever +whatsoever +when +whenever +where +whereas +wherefore +whereupon +whether +which +whichever +while +who +whoever +whom +whose +why +with +withal +within +without +ye +yea +yeah +yes +yet +yonder +you +your +yours +yourself +yourselves diff --git a/sm-model/train.py b/sm-model/train.py new file mode 100644 index 0000000..7f4edad --- /dev/null +++ b/sm-model/train.py @@ -0,0 +1,240 @@ +import os +import sys + +import time +import glob +import argparse +import numpy as np + +import torch +import torch.optim as optim +import torch.nn as nn +from torch.autograd import Variable + +import utils + +# logging setup +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + + +class Trainer(object): + + def __init__(self, model): + self.reg = 1e-5 + self.model = model + self.criterion = nn.CrossEntropyLoss() + self.optimizer = optim.SGD(self.model.parameters(), lr=0.001, weight_decay=self.reg) + + + def regularize_loss(self, loss): + + flattened_params = [] + + for p in self.model.parameters(): + f = p.data.clone() + flattened_params.append(f.view(-1)) + + fp = torch.cat(flattened_params) + + loss = loss + 0.5 * self.reg * fp.norm() * fp.norm() + + # for p in self.model.parameters(): + # loss = loss + 0.5 * self.reg * p.norm() * p.norm() + + return loss + + + def _train(self, xq, xa, ext_feats, ys): + + self.optimizer.zero_grad() + output = self.model(xq, xa, ext_feats) + loss = self.criterion(output, ys) + logger.debug('loss after criterion {}'.format(loss)) + + # NOTE: regularizing location 1 + # loss = self.regularize_loss(loss) + # logger.debug('loss after regularizing {}'.format(loss)) + + loss.backward() + + logger.debug('AFTER backward') + #logger.debug('params {}'.format([p for p in self.model.parameters()])) + logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()])) + + + # NOTE: regularizing location 2. It would seem that location 1 is correct? + loss = self.regularize_loss(loss) + logger.debug('loss after regularizing {}'.format(loss)) + + self.optimizer.step() + + logger.debug('AFTER step') + #logger.debug('params {}'.format([p for p in self.model.parameters()])) + logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()])) + + return loss.data[0], self.pred_equals_y(output, ys) + + + def pred_equals_y(self, pred, y): + # logger.debug('pred_equals_y:') + # logger.debug(pred) + # logger.debug(y) + _, best = pred.max(1) + # logger.debug('{} {}'.format(_, best)) + best = best.data.long().squeeze() + # logger.debug(best) + return torch.sum(y.data.long() == best) + + + def test(self, dataset_folder, set_folder, batch_size, word_vectors_cache_file): + logger.info('Predictions on {} -----'.format(set_folder)) + + questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \ + utils.read_in_dataset(dataset_folder, set_folder) + + # load word embeddings for training set vocab + word_vectors, vec_dim = utils.load_cached_embeddings(word_vectors_cache_file, vocab) + + self.model.eval() + + batch_size = 1 + + total_loss = 0.0 + total_correct = 0.0 + num_batches = np.ceil(len(questions)/batch_size ) + y_pred = np.zeros(len(questions)) + ypc = 0 + + for k in xrange(int(num_batches)): + batch_start = k * batch_size + batch_end = (k+1) * batch_size + # convert raw questions and sentences to tensors + batch_inputs, batch_labels = self.get_tensorized_inputs( + questions[batch_start:batch_end], + sentences[batch_start:batch_end], + labels[batch_start:batch_end], + ext_feats[batch_start:batch_end], + word_vectors, vocab, vec_dim + ) + + xq, xa, x_ext_feats = batch_inputs[0] + y = batch_labels[0] + + pred = self.model(xq, xa, x_ext_feats) + loss = self.criterion(pred, y) + total_loss += loss + total_correct += self.pred_equals_y(pred, y) + + p_score, p_class = pred.max(1) + logger.debug('pred {}'.format(pred)) + logger.debug('pred.max {} {}'.format(p_score, p_class)) + logger.debug('score {}'.format(p_score.squeeze())) + logger.debug('score {}'.format(p_score.squeeze()[0])) + y_pred[ypc] = p_score.data.squeeze()[0] + ypc += 1 + + + logger.info('{}_correct {}'.format(set_folder, total_correct)) + logger.info('{}_loss {}'.format(set_folder, total_loss.data[0])) + logger.info('{} total {}'.format(set_folder, len(labels))) + logger.info('{}_loss = {:.4f}, acc = {:.4f}'.format( set_folder, total_loss.data[0]/len(labels), float(total_correct)/len(labels) )) + + return float(total_correct)/len(labels), y_pred + + + def train(self, dataset_folder, set_folder, batch_size, word_vectors_cache_file): + # read in training data + questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \ + utils.read_in_dataset(dataset_folder, set_folder) + + # load word embeddings for training set vocab + word_vectors, vec_dim = utils.load_cached_embeddings(word_vectors_cache_file, vocab) + + # set model for training modep + self.model.train() + + train_loss, train_correct = 0., 0. + num_batches = np.ceil(len(questions)/float(batch_size) ) + + for k in xrange(int(num_batches)): + batch_start = k * batch_size + batch_end = (k+1) * batch_size + + # convert raw questions and sentences to tensors + batch_inputs, batch_labels = self.get_tensorized_inputs( + questions[batch_start:batch_end], + sentences[batch_start:batch_end], + labels[batch_start:batch_end], + ext_feats[batch_start:batch_end], + word_vectors, vocab, vec_dim + ) + + xq, xa, x_ext_feats = batch_inputs[0] + + ys = batch_labels[0] + + batch_loss, batch_correct = self._train(xq, xa, x_ext_feats, ys) + + # logger.debug('batch_loss {}, batch_correct {}'.format(batch_loss, batch_correct)) + train_loss += batch_loss + train_correct += batch_correct + # break + + logger.info('train_correct {}'.format(train_correct)) + logger.info('train_loss {}'.format(train_loss)) + logger.info('total training batches = {}'.format(num_batches)) + logger.info('train_loss = {:.4f}, acc = {:.4f}'.format( + train_loss/num_batches, train_correct/num_batches + )) + return train_correct/num_batches + + + def make_input_matrix(self, sentence, word_vectors, vec_dim): + terms = sentence.strip().split() + # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor) + word_embeddings = torch.zeros(len(terms), vec_dim).type(torch.DoubleTensor) + for i in xrange(len(terms)): + word = terms[i] + emb = torch.from_numpy(word_vectors[word]) + word_embeddings[i] = emb + + input_tensor = torch.zeros(1, vec_dim, len(terms)) + input_tensor[0] = torch.transpose(word_embeddings, 0 , 1) + return input_tensor + + + def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, word_vectors, vocab, vec_dim): + batch_size = len(batch_ques) + # NOTE: ideal batch size is one, because sentences are all of different length. + # In other words, we have no option but to feed in sentences one by one into the model + # and compute loss at the end. + + # TODO: what if the sentences in a batch are all of different lengths? + # - should be have the longest sentence as 2nd dim? + # - would zero endings work for other smaller sentences? + + y = torch.LongTensor(batch_size).type(torch.LongTensor) + + tensorized_inputs = [] + for i in xrange(len(batch_ques)): + xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim) ) #, requires_grad=False) + xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False) + # ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) + ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) + ext_feats =torch.unsqueeze(ext_feats, 0) + y[i] = batch_labels[i] + tensorized_inputs.append((xq, xs, ext_feats)) + + return tensorized_inputs, Variable(y) + + + + diff --git a/sm-model/trec_eval-8.0/Makefile b/sm-model/trec_eval-8.0/Makefile new file mode 100755 index 0000000..f58087b --- /dev/null +++ b/sm-model/trec_eval-8.0/Makefile @@ -0,0 +1,141 @@ +BIN = /home/smart/bin +H = . + +VERSIONID = 8.0 + +# gcc +CC = gcc +CFLAGS = -g -I$H -O3 -Wall -DVERSIONID=\"$(VERSIONID)\" +CFLAGS = -g -I$H -Wall -DVERSIONID=\"$(VERSIONID)\" + +# cc +###CC = cc +###CFLAGS = -I$H -g -DVERSIONID=\"$(VERSIONID)\" + +# Other macros used in some or all makefiles +INSTALL = /bin/mv + + +OBJS = trec_eval.o get_qrels.o get_top.o form_trvec.o measures.o print_meas.o\ + trvec_teval.o buf_util.o error_msgs.o \ + trec_eval_help.o + + +SRCS = trec_eval.c get_qrels.c get_top.c form_trvec.c measures.c print_meas.c\ + trvec_teval.c buf_util.c error_msgs.c \ + trec_eval_help.c + +SRCH = common.h trec_eval.h smart_error.h sysfunc.h tr_vec.h buf.h + +SRCOTHER = README Makefile test bpref_bug + +trec_eval: $(SRCS) Makefile $(SRCH) + $(CC) $(CFLAGS) -o trec_eval $(SRCS) -lm + +install: $(BIN)/trec_eval + +quicktest: trec_eval + ./trec_eval test/qrels.test test/results.test | diff - test/out.test + ./trec_eval -a test/qrels.test test/results.test | diff - test/out.test.a + ./trec_eval -a -q test/qrels.test test/results.test | diff - test/out.test.aq + ./trec_eval -a -q -c test/qrels.test test/results.trunc | diff - test/out.test.aqc + ./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc | diff - test/out.test.aqcM + /bin/echo "Test succeeeded" + +longtest: trec_eval + /bin/rm -rf test.long; mkdir test.long + ./trec_eval test/qrels.test test/results.test > test.long/out.test + ./trec_eval -a test/qrels.test test/results.test > test.long/out.test.a + ./trec_eval -a -q test/qrels.test test/results.test > test.long/out.test.aq + ./trec_eval -a -q -c test/qrels.test test/results.trunc > test.long/out.test.aqc + ./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc > test.long/out.test.aqcM + diff test.long test + +$(BIN)/trec_eval: trec_eval + if [ -f $@ ]; then $(INSTALL) $@ $@.old; fi; + $(INSTALL) trec_eval $@ + +##4########################################################################## +##5########################################################################## +# All code below this line (except for automatically created dependencies) +# is independent of this particular makefile, and should not be changed! +############################################################################# + +######################################################################### +# Odds and ends # +######################################################################### +clean semiclean: + /bin/rm -f *.o *.BAK *~ trec_eval trec_eval.*.shar out.trec_eval Makefile.bak + +shar: + shar -X $(SRCOTHER) $(SRCS) $(SRCH) > trec_eval.$(VERSIONID).shar + +lint: + lint $(SRCS) + +######################################################################### +# Determining program dependencies # +######################################################################### +depend: + grep '^#[ ]*include' *.c \ + | sed -e 's?:[^"]*"\([^"]*\)".*?: \$H/\1?' \ + -e '/ makedep + echo '/^# DO NOT DELETE THIS LINE/+2,$$d' >eddep + echo '$$r makedep' >>eddep + echo 'w' >>eddep + cp Makefile Makefile.bak + ed - Makefile < eddep + /bin/rm eddep makedep + echo '# DEPENDENCIES MUST END AT END OF FILE' >> Makefile + echo '# IF YOU PUT STUFF HERE IT WILL GO AWAY' >> Makefile + echo '# see make depend above' >> Makefile + +# DO NOT DELETE THIS LINE -- make depend uses it + +buf_util.o: ./common.h +buf_util.o: ./sysfunc.h +buf_util.o: ./buf.h +error_msgs.o: ./smart_error.h +error_msgs.o: ./sysfunc.h +form_trvec.o: ./common.h +form_trvec.o: ./sysfunc.h +form_trvec.o: ./smart_error.h +form_trvec.o: ./tr_vec.h +form_trvec.o: ./trec_eval.h +form_trvec.o: ./buf.h +get_qrels.o: ./common.h +get_qrels.o: ./sysfunc.h +get_qrels.o: ./smart_error.h +get_qrels.o: ./trec_eval.h +get_top.o: ./common.h +get_top.o: ./sysfunc.h +get_top.o: ./smart_error.h +get_top.o: ./trec_eval.h +measures.o: ./common.h +measures.o: ./sysfunc.h +measures.o: ./buf.h +measures.o: ./trec_eval.h +print_meas.o: ./common.h +print_meas.o: ./sysfunc.h +print_meas.o: ./buf.h +print_meas.o: ./trec_eval.h +trec_eval.o: ./common.h +trec_eval.o: ./sysfunc.h +trec_eval.o: ./smart_error.h +trec_eval.o: ./tr_vec.h +trec_eval.o: ./trec_eval.h +trec_eval.o: ./buf.h +trec_eval_help.o: ./common.h +trvec_teval.o: ./common.h +trvec_teval.o: ./sysfunc.h +trvec_teval.o: ./smart_error.h +trvec_teval.o: ./tr_vec.h +trvec_teval.o: ./trec_eval.h +# DEPENDENCIES MUST END AT END OF FILE +# IF YOU PUT STUFF HERE IT WILL GO AWAY +# see make depend above diff --git a/sm-model/trec_eval-8.0/README b/sm-model/trec_eval-8.0/README new file mode 100755 index 0000000..6bc8722 --- /dev/null +++ b/sm-model/trec_eval-8.0/README @@ -0,0 +1,385 @@ +trec_eval is the standard tool used by the TREC community for +evaluating an ad hoc retrieval run, given the results file and a +standard set of judged results. + +------------------------------------------------------------------------------ +Installation: Should be as easy as typing "make" in the source directory, +if gcc is available. Otherwise, comment out the gcc lines (lines 5-6) and +uncomment out the cc lines (lines 9-10) +If you wish the trec_eval binary to be placed in a standard location, alter +the first line of Makefile appropriately. + +------------------------------------------------------------------------------ +Testing: sample input and output files are included in the directory test. +"make quicktest" will perform some sample simple evaluations and compare +the results. + +------------------------------------------------------------------------------ +Usage: Most options can be ignored. The only one most folks will need +is the "-q" flag, to indicate whether to output results for individual +queries as well as the averages over all queries. Official TREC usage +might be something like + trec_eval -q -c -M1000 official_qrels submitted_results +to ensure correct evaluation if submitted_results doesn't have results +for all queries, or returns more than 1000 documents per query. + + +------------------------------------------------------------------------------ +Change Log +------------------------------------------------------------------------------ +Version 8.0, full bpref bug fix, see file bpref_bug. I decided to up the + version number since bpref results are incompatible with previous + results (though the changes are small). + +11/8/05: Bpref_bug: New file explaining bug and impact (conclusions after + rerunning all of SIGIR 2004 bpref paper experiments). +11/5/05: Added new measures: micro_prec, micro_recall, micro_bpref. I thought + I had an application for micro_bpref averaging (summing components of + measure over all docs (ignoring topics) and then computing measure), + but micro_bpref still proved a rotten measure. Left code in case + someone ever actually finds an application for valid micro averaging. +11/5/05: Added new measures: old_bpref, old_bpref_top10pRnonrel. These are + the old buggy measures included only for backward comparisons. +11/5/05: trvec_teval.c: Broke apart old trvec_trec_eval to calculate + different types of measures separately. Very hard to decipher + old code (though still difficult with new code) since parts of + the calculations for a measure were so far apart. + +------------------------------------------------------------------------------ +Version 7.4, minor changes from 7.3 + +11/4/05: trvec_teval.c: fixed bpref bug if very low (< R) numbers of non-rel + judgements available (divided by num_nonrel_ret instead of + num_nonrel). (pointed out by Ian Soboroff). +11/3/05: trvec_teval.c: bpref_10, bpref_5 had zero division problems if + no rel docs were retrieved. (pointed out by Ian Soboroff). +10/23/05: form_trvec.c: Added check for duplicate docno's in results and qrels. + (pointed out by Shlomo Geva. Default behavior used to be that + duplicate result docno's were always non-rel, but that changed in + later versions, so had better test explicitly for it and complain). +10/23/05: README: sample invocation of trec_eval had arguments reversed. + (pointed out by Carol Peters). +10/23/05: moved gm_ap to be a major measure (always printed). changed + measures.c, test/out*, README + +------------------------------------------------------------------------------ + +Version 7.3, a reasonably major rewrite from earlier versions in terms +of internal structure and default output format (now relational), but +the input format and measures calculated remain the same (or at least +upward compatible). + +------------------------------------------------------------------------------ +end of ChangeLog +------------------------------------------------------------------------------ + +Adding measures: To add a new measure: + 1. Add space for the measure in TREC_EVAL structure of "trec_eval.h" + 2. Add description of measure in "measures.c". See "trec_eval.h" for + definition of the fields. This description is used for + printing, accumulating, and averaging the measure values. + 3. Calculate the measure in "trvec_teval.c". Unfortunately, this has + gotten very long over the years as more measures are added, but + most of it can be ignored. I should write a simple version of + just the "short" measures so the structure can be seen better. + +------------------------------------------------------------------------------ + +Files: +Makefile Compile and test trec_eval +README This file +test Collection of sample input and output for trec_eval +trec_eval.c Main procedure +get_qrels.c Called by main to read the standard judged documents (qrels) +get_top.c Called by main to read the results file to be evaluated +form_trvec.c Called by main to put the results and qrels for an individual + query in the proper format to be evaluated. +trvec_teval.c Called by main to evaluate an individual query +print_meas.c Called by main to print an evaluated query, and to accumulate + the results for later averaging over the queries. +measures.c Description of the measures used by printing. +trec_eval_help.c Descriptions of trec_eval, the output, and the measures. + +trec_eval.h Basic evaluation structures. +bpref_bug: Description of bug in bpref that existed in trec_eval versions 6 + through 7.3. + +The rest of the files are small utility portions from SMART. +tr_vec.h +smart_error.h +sysfunc.h +buf.h +common.h +buf_util.c +error_msgs.c + +------------------------------------------------------------------------------ + +The rest of this file consists of information printed by "trec_eval -h": +(If you REALLY want a complete list of measures calculated, you can add the +time based measures and run "trec_eval -T -h".) + +trec_eval [-h] [-q] [-a] [-o] [-c] [-l [-N] [-M] [-Ua] [-Ub] [-Uc] [-Ud] [-T] trec_rel_file trec_top_file + +Calculate and print various evaluation measures, evaluating the results +in trec_top_file against the relevance judgements in trec_rel_file. + +There are a fair number of options, of which only the lower case options are +normally ever used. + -h: Print full help message and exit + -q: In addition to summary evaluation, give evaluation for each query + -a: Print all evaluation measures calculated, instead of just the + main official measures for TREC. + -o: Print everything out in old, nonrelational format (default is relational) + -c: Average over the complete set of queries in the relevance judgements + instead of the queries in the intersection of relevance judgements + and results. Missing queries will contribute a value of 0 to all + evaluation measures (which may or may not be reasonable for a + particular evaluation measure, but is reasonable for standard TREC + measures.) + -l: Num indicates the minimum relevance judgement value needed for + a document to be called relevant. (All measures used by TREC eval are + based on binary relevance). Used if trec_rel_file contains relevance + judged on a multi-relevance scale. Default is 1. + -N: Number of docs in collection + -M: Max number of docs per topic to use in evaluation (discard rest). + -Ua: Value to use for 'a' coefficient of utility computation. + relevant nonrelevant + retrieved a b + nonretrieved c d + -Ub: Value to use for 'b' coefficient of utility computation. + -Uc: Value to use for 'c' coefficient of utility computation. + -Ud: Value to use for 'd' coefficient of utility computation. + -J: Calculate all values only over the judged (either relevant or + nonrelevant) documents. All unjudged documents are removed from the + retrieved set before any calculations (possibly leaving an empty set). + DO NOT USE, unless you really know what you're doing - very easy to get + reasonable looking, but invalid, numbers. + -T: Treat similarity as time that document retrieved. Compute + several time-based measures after ranking docs by time retrieved + (first doc (lowest sim) retrieved ranked highest). + Only done if -a selected. + + +Read text tuples from trec_top_file of the form + 030 Q0 ZF08-175-870 0 4238 prise1 + qid iter docno rank sim run_id +giving TREC document numbers (a string) retrieved by query qid +(a string) with similarity sim (a float). The other fields are ignored, +with the exception that the run_id field of the last line is kept and +output. In particular, note that the rank field is ignored here; +internally ranks are assigned by sorting by the sim field with ties +broken deterministicly (using docno). +Sim is assumed to be higher for the docs to be retrieved first. +File may contain no NULL characters. +Lines may contain fields after the run_id; they are ignored. + +Relevance for each docno to qid is determined from text_qrels_file, which +consists of text tuples of the form + qid iter docno rel +giving TREC document numbers (docno, a string) and their relevance (rel, +an integer) to query qid (a string). iter string field is ignored. +Fields are separated by whitespace, string fields can contain no whitespace. +File may contain no NULL characters. + +The text tuples with relevance judgements are converted to TR_VEC form +and then submitted to the SMART evaluation routines. +The qid,did,rank,sim,rel fields of TR_VEC are filled in; +action,iter fields are set to 0. +The rel field is set to -1 if the document was not judged (not in +text_qrels_file). Most measures, but not all, will treat -1 the same as 0, +namely nonrelevant. Note that relevance_level is used to determine if the +document is relevant during score calculations. +Queries for which there are no relevant docs are ignored. +Warning: queries for which there are relevant docs but no retrieved docs +are also ignored by default. This allows systems to evaluate over subsets +of the relevant docs, but means if a system improperly retrieves no docs, +it will not be detected. Use the -c flag to avoid this behavior. + +EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. +Relational Format prints the same values, but all lines are of the form + measure_name query value + +1. Total number of documents over all queries + Retrieved: + Relevant: + Rel_ret: (relevant and retrieved) + These should be self-explanatory. All values are totals over all + queries being evaluated. +2. Interpolated Recall - Precision Averages: + at 0.00 + at 0.10 + ... + at 1.00 + See any standard IR text (especially by Salton) for more details of + recall-precision evaluation. Measures precision (percent of retrieved + docs that are relevant) at various recall levels (after a certain + percentage of all the relevant docs for that query have been retrieved). + 'Interpolated' means that, for example, precision at recall + 0.10 (ie, after 10% of rel docs for a query have been retrieved) is + taken to be MAXIMUM of precision at all recall points >= 0.10. + Values are averaged over all queries (for each of the 11 recall levels). + These values are used for Recall-Precision graphs. +3. Average precision (non-interpolated) over all rel docs + The precision is calculated after each relevant doc is retrieved. + If a relevant doc is not retrieved, its precision is 0.0. + All precision values are then averaged together to get a single number + for the performance of a query. Conceptually this is the area + underneath the recall-precision graph for the query. + The values are then averaged over all queries. +4. Precision: + at 5 docs + at 10 docs + ... + at 1000 docs + The precision (percent of retrieved docs that are relevant) after X + documents (whether relevant or nonrelevant) have been retrieved. + Values averaged over all queries. If X docs were not retrieved + for a query, then all missing docs are assumed to be non-relevant. +5. R-Precision (precision after R (= num_rel for a query) docs retrieved): + Measures precision (or recall, they're the same) after R docs + have been retrieved, where R is the total number of relevant docs + for a query. Thus if a query has 40 relevant docs, then precision + is measured after 40 docs, while if it has 600 relevant docs, precision + is measured after 600 docs. This avoids some of the averaging + problems of the 'precision at X docs' values in (4) above. + If R is greater than the number of docs retrieved for a query, then + the nonretrieved docs are all assumed to be nonrelevant. + +Major measures (again) with their relational names: +num_ret Total number of documents retrieved over all queries +num_rel Total number of relevant documents over all queries +num_rel_ret Total number of relevant documents retrieved over all queries +map Mean Average Precision (MAP) +gm_ap Average Precision. Geometric Mean, q_score=log(MAX(map,.00001)) +R-prec R-Precision (Precision after R (= num-rel for topic) documents retrieved) +bpref Binary Preference, top R judged nonrel +recip_rank Reciprical rank of top relevant document +ircl_prn.0.00 Interpolated Recall - Precision Averages at 0.00 recall +ircl_prn.0.10 Interpolated Recall - Precision Averages at 0.10 recall +ircl_prn.0.20 Interpolated Recall - Precision Averages at 0.20 recall +ircl_prn.0.30 Interpolated Recall - Precision Averages at 0.30 recall +ircl_prn.0.40 Interpolated Recall - Precision Averages at 0.40 recall +ircl_prn.0.50 Interpolated Recall - Precision Averages at 0.50 recall +ircl_prn.0.60 Interpolated Recall - Precision Averages at 0.60 recall +ircl_prn.0.70 Interpolated Recall - Precision Averages at 0.70 recall +ircl_prn.0.80 Interpolated Recall - Precision Averages at 0.80 recall +ircl_prn.0.90 Interpolated Recall - Precision Averages at 0.90 recall +ircl_prn.1.00 Interpolated Recall - Precision Averages at 1.00 recall +P5 Precision after 5 docs retrieved +P10 Precision after 10 docs retrieved +P15 Precision after 15 docs retrieved +P20 Precision after 20 docs retrieved +P30 Precision after 30 docs retrieved +P100 Precision after 100 docs retrieved +P200 Precision after 200 docs retrieved +P500 Precision after 500 docs retrieved +P1000 Precision after 1000 docs retrieved + + +Minor measures with their relational names: +exact_prec Exact Precision over retrieved set +exact_recall Exact Recall over retrieved set +11-pt_avg Average over all 11 points of recall-precision graph +3-pt_avg Average over 3 points of recall-precision graph +avg_doc_prec Rel doc precision averaged over all relevant docs (NOT over topics) +exact_relative_prec Exact relative precision +avg_relative_prec Average relative precision +exact_unranked_avg_prec Exact Unranked Average Precision +exact_relative_unranked_avg_prec Exact Relative Unranked Average Precision +map_at_R Average Precision over first R docs retrieved +int_map Interpolated Mean Average Precision +exact_int_R_rcl_prec Exact R-based-interpolated-Precision +int_map_at_R Average Interpolated Precision for first R docs retrieved +bpref_allnonrel Binary Preference, all judged nonrel +bpref_retnonrel Binary Preference, all retrieved judged nonrel +bpref_topnonrel Binary Preference, top 100 judged nonrel +bpref_top5Rnonrel Binary Preference, top 5R judged nonrel +bpref_top10Rnonrel Binary Preference, top 10R judged nonrel +bpref_top10pRnonrel Binary Preference, top 10 + R judged nonrel +bpref_top25pRnonrel Binary Preference, top 25 + R judged nonrel +bpref_top50pRnonrel Binary Preference, top 50 + R judged nonrel +bpref_top25p2Rnonrel Binary Preference, top 25 + 2*R judged nonrel +bpref_retall Binary Preference, Only retrieved judged rel and nonrel +bpref_5 Binary Preference, top 5 rel, top 5 nonrel +bpref_10 Binary Preference, top 10 rel, top 10 nonrel +bpref_num_all Binary Preference, Number not retrieved before (all judged) +bpref_num_ret Binary Preference, Number retrieved after +bpref_num_correct Binary Preference, Number correct preferences +bpref_num_possible Binary Preference, Number possible correct_preferences +old_bpref Buggy Version 7.3. Binary Preference, top R judged nonrel +old_bpref_top10pRnonrel Buggy Version 7.3. Binary Preference,top 10+R judged nonrel +gm_bpref Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001)) +rank_first_rel Rank of top relevant document (0 if none) +recall5 Recall after 5 docs retrieved +recall10 Recall after 10 docs retrieved +recall15 Recall after 15 docs retrieved +recall20 Recall after 20 docs retrieved +recall30 Recall after 30 docs retrieved +recall100 Recall after 100 docs retrieved +recall200 Recall after 200 docs retrieved +recall500 Recall after 500 docs retrieved +recall1000 Recall after 1000 docs retrieved +0.20R-prec R-based precision- precision after 0.20 * R docs retrieved +0.40R-prec R-based precision- precision after 0.40 * R docs retrieved +0.60R-prec R-based precision- precision after 0.60 * R docs retrieved +0.80R-prec R-based precision- precision after 0.80 * R docs retrieved +1.00R-prec R-based precision- precision after 1.00 * R docs retrieved +1.20R-prec R-based precision- precision after 1.20 * R docs retrieved +1.40R-prec R-based precision- precision after 1.40 * R docs retrieved +1.60R-prec R-based precision- precision after 1.60 * R docs retrieved +1.80R-prec R-based precision- precision after 1.80 * R docs retrieved +2.00R-prec R-based precision- precision after 2.00 * R docs retrieved +relative_prec5 Relative precision after 5 docs retrieved +relative_prec10 Relative precision after 10 docs retrieved +relative_prec15 Relative precision after 15 docs retrieved +relative_prec20 Relative precision after 20 docs retrieved +relative_prec30 Relative precision after 30 docs retrieved +relative_prec100 Relative precision after 100 docs retrieved +relative_prec200 Relative precision after 200 docs retrieved +relative_prec500 Relative precision after 500 docs retrieved +relative_prec1000 Relative precision after 1000 docs retrieved +unranked_avg_prec5 Unranked Average Precision after 5 docs retrieved +unranked_avg_prec10 Unranked Average Precision after 10 docs retrieved +unranked_avg_prec15 Unranked Average Precision after 15 docs retrieved +unranked_avg_prec20 Unranked Average Precision after 20 docs retrieved +unranked_avg_prec30 Unranked Average Precision after 30 docs retrieved +unranked_avg_prec100 Unranked Average Precision after 100 docs retrieved +unranked_avg_prec200 Unranked Average Precision after 200 docs retrieved +unranked_avg_prec500 Unranked Average Precision after 500 docs retrieved +unranked_avg_prec1000 Unranked Average Precision after 1000 docs retrieved +relative_unranked_avg_prec5 Relative Unranked Average Precision after 5 docs retrieved +relative_unranked_avg_prec10 Relative Unranked Average Precision after 10 docs retrieved +relative_unranked_avg_prec15 Relative Unranked Average Precision after 15 docs retrieved +relative_unranked_avg_prec20 Relative Unranked Average Precision after 20 docs retrieved +relative_unranked_avg_prec30 Relative Unranked Average Precision after 30 docs retrieved +relative_unranked_avg_prec100 Relative Unranked Average Precision after 100 docs retrieved +relative_unranked_avg_prec200 Relative Unranked Average Precision after 200 docs retrieved +relative_unranked_avg_prec500 Relative Unranked Average Precision after 500 docs retrieved +relative_unranked_avg_prec1000 Relative Unranked Average Precision after 1000 docs retrieved +utility_1.0_-1.0_0.0_0.0 Utility (a,b,c,d) Coefficients 1.0_-1.0_0.0_0.0 +rcl_at_142_nonrel Recall averaged at X nonrel docs X= 142 +fallout_recall_0 Fallout - Recall Averages- recall after 0 nonrel docs retrieved +fallout_recall_14 Fallout - Recall Averages- recall after 14 nonrel docs retrieved +fallout_recall_28 Fallout - Recall Averages- recall after 28 nonrel docs retrieved +fallout_recall_42 Fallout - Recall Averages- recall after 42 nonrel docs retrieved +fallout_recall_56 Fallout - Recall Averages- recall after 56 nonrel docs retrieved +fallout_recall_71 Fallout - Recall Averages- recall after 71 nonrel docs retrieved +fallout_recall_85 Fallout - Recall Averages- recall after 85 nonrel docs retrieved +fallout_recall_99 Fallout - Recall Averages- recall after 99 nonrel docs retrieved +fallout_recall_113 Fallout - Recall Averages- recall after 113 nonrel docs retrieved +fallout_recall_127 Fallout - Recall Averages- recall after 127 nonrel docs retrieved +fallout_recall_142 Fallout - Recall Averages- recall after 142 nonrel docs retrieved +int_0.20R-prec Interpolated R-based precision, after 0.20 * R docs retrieved +int_0.40R-prec Interpolated R-based precision, after 0.40 * R docs retrieved +int_0.60R-prec Interpolated R-based precision, after 0.60 * R docs retrieved +int_0.80R-prec Interpolated R-based precision, after 0.80 * R docs retrieved +int_1.00R-prec Interpolated R-based precision, after 1.00 * R docs retrieved +int_1.20R-prec Interpolated R-based precision, after 1.20 * R docs retrieved +int_1.40R-prec Interpolated R-based precision, after 1.40 * R docs retrieved +int_1.60R-prec Interpolated R-based precision, after 1.60 * R docs retrieved +int_1.80R-prec Interpolated R-based precision, after 1.80 * R docs retrieved +int_2.00R-prec Interpolated R-based precision, after 2.00 * R docs retrieved +micro_prec Total relevant retrieved documents / Total retrieved documents +micro_recall Total relevant retrieved documents / Total relevant documents +micro_bpref Total correct preferences / Total possible preferences diff --git a/sm-model/trec_eval-8.0/buf.h b/sm-model/trec_eval-8.0/buf.h new file mode 100755 index 0000000..5f2f84c --- /dev/null +++ b/sm-model/trec_eval-8.0/buf.h @@ -0,0 +1,13 @@ +#ifndef BUFH +#define BUFH +/* $Header: /home/smart/release/src/h/buf.h,v 11.0 1992/07/21 18:18:32 chrisb Exp $*/ + +/* structure used for passing around text (buf) which possibly includes + NULLs. see buf_util.c for add_buf(). */ +typedef struct { + int size; + int end; + char *buf; +} SM_BUF; + +#endif /* BUFH */ diff --git a/sm-model/trec_eval-8.0/buf_util.c b/sm-model/trec_eval-8.0/buf_util.c new file mode 100755 index 0000000..754eb07 --- /dev/null +++ b/sm-model/trec_eval-8.0/buf_util.c @@ -0,0 +1,86 @@ +#ifdef RCSID +static char rcsid[] = "$Header: /home/smart/release/src/libgeneral/buf_util.c,v 11.0 1992/07/21 18:21:04 chrisb Exp $"; +#endif + +/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. + + Permission is granted for use of this file in unmodified form for + research purposes. Please contact the SMART project to obtain + permission for other uses. +*/ + +/******************** PROCEDURE DESCRIPTION ************************ + *0 Utility procedure to add the memory contents of new.buf to result.buf + *2 add_buf (new, result) + *3 SM_BUF *new; + *3 SM_BUF *result; + *7 Both new and result are of type + *7 typedef struct { + *7 int size; * allocated space for buf * + *7 int end; * end of valid data in buf * + *7 char *buf; * buffer of arbitrary data * + *7 } SM_BUF; + *7 + *7 Append the data in new to the end of the data in result. The data can + *7 be arbitrary data, eg, include '\0's. + *7 Return UNDEF if can't allocate enough space for the result, 0 otherwise. +***********************************************************************/ + +#include "common.h" +#include "sysfunc.h" +#include "buf.h" + +int +add_buf (new, result) +SM_BUF *new, *result; +{ + if (result->size == 0) { + if (NULL == (result->buf = malloc ((unsigned) new->end * 2 + 1))) + return (UNDEF); + result->size = 2 * new->end + 1; + result->end = 0; + } + else if (new->end >= result->size - result->end) { + if (NULL == (result->buf = + realloc (result->buf, + (unsigned) result->size * 2 + new->end))) + return (UNDEF); + result->size += result->size + new->end; + } + + bcopy (new->buf, &result->buf[result->end], new->end); + result->end += new->end; + + return (0); +} + +/******************** PROCEDURE DESCRIPTION ************************ + *0 Utility procedure to add the string new to result.buf + *2 add_buf_string (new, result) + *3 char *new; + *3 SM_BUF *result; + + *7 Result is of type + *7 typedef struct { + *7 int size; * allocated space for buf * + *7 int end; * end of valid data in buf * + *7 char *buf; * buffer of arbitrary data * + *7 } SM_BUF; + *7 + *7 Append the data in new to the end of the data in result. + *7 Return UNDEF if can't allocate enough space for the result, 0 otherwise. +***********************************************************************/ + +int +add_buf_string (new, result) +char *new; +SM_BUF *result; +{ + SM_BUF temp_buf; + temp_buf.end = strlen (new); + temp_buf.buf = new; + + return (add_buf (&temp_buf, result)); + +} + diff --git a/sm-model/trec_eval-8.0/common.h b/sm-model/trec_eval-8.0/common.h new file mode 100755 index 0000000..80007df --- /dev/null +++ b/sm-model/trec_eval-8.0/common.h @@ -0,0 +1,31 @@ +#ifndef COMMONH +#define COMMONH + +#include + +#ifndef FALSE +#define FALSE 0 +#endif +#ifndef TRUE +#define TRUE 1 +#endif + +#define UNDEF -1 + +#define MAX(A,B) ((A) > (B) ? (A) : (B)) +#define MIN(A,B) ((A) > (B) ? (B) : (A)) + +#ifndef MAXLONG +#define MAXLONG 2147483647L /* largest long int. no. */ +#endif + +/* + * Some useful macros for making malloc et al easier to use. + * Macros handle the casting and the like that's needed. + */ +#define Malloc(n,type) (type *) malloc( (unsigned) ((n)*sizeof(type))) +#define Realloc(loc,n,type) (type *) realloc( (char *)(loc), \ + (unsigned) ((n)*sizeof(type))) +#define Free(loc) (void) free( (char *)(loc) ) + +#endif /* COMMONH */ diff --git a/sm-model/trec_eval-8.0/error_msgs.c b/sm-model/trec_eval-8.0/error_msgs.c new file mode 100755 index 0000000..0eb23d0 --- /dev/null +++ b/sm-model/trec_eval-8.0/error_msgs.c @@ -0,0 +1,93 @@ +#ifdef RCSID +static char rcsid[] = "$Header: /home/smart/release/./src/libgeneral/error_msgs.c,v 10.1 91/11/05 23:49:06 smart Exp Locker: smart $"; +#endif + +/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. + + Permission is granted for use of this file in unmodified form for + research purposes. Please contact the SMART project to obtain + permission for other uses. +*/ + +/******************** PROCEDURE DESCRIPTION ************************ + *0 print a SMART error message + *2 print_error (new_routine, new_message) + *3 char *new_routine; + *3 char *new_message; + *6 Global UNIX variables errno, sys_nerr, sys_errlist are used, as well + *6 as SMART global variables smart_errlist and smart_errno; + + *7 Print an error message to stderr. At point of error determination, + *7 either smart_errno should be set, or (if UNIX library error) errno will + *7 be set. If smart_errno is set, then the routine name that detected the + *7 error and a message are printed. In addition, the routine name that prints + *7 the error and it's message (eg action to be taken) are printed. + *9 smart_errno should be more widely used, in particular to locate the + *9 procedure the error occurs in. Many errors can only get "located" + *9 by setting trace. +***********************************************************************/ + +#include +#include "smart_error.h" +#include "sysfunc.h" + +/* Declarations of external variables defined in "smart_error.h" */ + +int smart_errno; /* If > 0 and <= sys_nerr then refers to */ + /* sys_errlist, else if >= smart_errmin */ + /* and <= smart_errmax, then smart_errlist */ +char *smart_message; /* Message to be printed (often filename) */ +char *smart_routine; /* Major routine issuing error message */ + + +extern int errno; + +char *smart_errlist[] = { + "Inconsistency check", + "Illegal value for seek", + "Illegal mode for object", + "Illegal parameter value" +}; + +void +print_error (new_routine, new_message) +char *new_routine; +char *new_message; +{ + if (smart_errno > 0 && smart_errno < SMART_MINERR) { + (void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n", + new_routine, + smart_routine, + smart_message, + strerror(smart_errno), + new_message); + } + else if (smart_errno >= SMART_MINERR && + smart_errno < SMART_MINERR + SMART_NUMERR) { + (void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n", + new_routine, + smart_routine, + smart_message, + smart_errlist[smart_errno - SMART_MINERR], + new_message); + } + else if (smart_errno == 0 && errno != 0) { + /* Presumably error detected directly by new_routine */ + /* after system call */ + (void) fprintf (stderr, "%s: '%s' - %s\n", + new_routine, + strerror(errno), + new_message); + } + else { + (void) fprintf (stderr, "%s: Undetermined error detected - %s\n", + new_routine, + new_message); + } + + /* Reset the global error indicators */ + errno = 0; + smart_errno = 0; + smart_message = NULL; + smart_routine = NULL; +} diff --git a/sm-model/trec_eval-8.0/form_trvec.c b/sm-model/trec_eval-8.0/form_trvec.c new file mode 100755 index 0000000..432fdab --- /dev/null +++ b/sm-model/trec_eval-8.0/form_trvec.c @@ -0,0 +1,201 @@ +#include "common.h" +#include "sysfunc.h" +#include "smart_error.h" +#include "tr_vec.h" +#include "trec_eval.h" +#include "buf.h" + +static int comp_tr_tup_rank(), comp_tr_tup_did(), comp_tr_docno(), + comp_qrels_docno(), comp_sim_docno(), comp_negsim_docno(); + +/* Space reserved for output TR_TUP tuples */ +static TR_TUP *start_tr_tup; +static long max_tr_tup = 0; + +int +form_trvec (epi, trec_top, trec_qrels, tr_vec, num_rel) +EVAL_PARAM_INFO *epi; +TREC_TOP *trec_top; +TREC_QRELS *trec_qrels; +TR_VEC *tr_vec; +long *num_rel; +{ + TR_TUP *tr_tup; + TEXT_QRELS *qrels_ptr, *end_qrels; + long i; + + /* Reserve space for output tr_tups, if needed */ + if (trec_top->num_text_tr > max_tr_tup) { + if (max_tr_tup > 0) + (void) free ((char *) start_tr_tup); + max_tr_tup += trec_top->num_text_tr; + if (NULL == (start_tr_tup = Malloc (max_tr_tup, TR_TUP))) + return (UNDEF); + } + + /* Sort trec_top by sim, breaking ties lexicographically using docno */ + if (epi->time_flag) { + qsort ((char *) trec_top->text_tr, + (int) trec_top->num_text_tr, + sizeof (TEXT_TR), + comp_negsim_docno); + } + else { + qsort ((char *) trec_top->text_tr, + (int) trec_top->num_text_tr, + sizeof (TEXT_TR), + comp_sim_docno); + } + + /* Add ranks to trec_top (starting at 1) */ + for (i = 0; i < trec_top->num_text_tr; i++) { + trec_top->text_tr[i].rank = i+1; + } + + /* Sort trec_top lexicographically */ + qsort ((char *) trec_top->text_tr, + (int) trec_top->num_text_tr, + sizeof (TEXT_TR), + comp_tr_docno); + + for (i = 1; i < trec_top->num_text_tr; i++) { + if (0 == strcmp (trec_top->text_tr[i].docno, + trec_top->text_tr[i-1].docno)) { + set_error (SM_ILLPA_ERR, "Duplicate top docs docno", trec_top->text_tr[i].docno); + return (UNDEF); + } + } + + /* Sort trec_qrels lexicographically */ + qsort ((char *) trec_qrels->text_qrels, + (int) trec_qrels->num_text_qrels, + sizeof (TEXT_QRELS), + comp_qrels_docno); + + /* Find number of relevant docs, and check for duplicates */ + *num_rel = 0; + for (i = 0; i < trec_qrels->num_text_qrels; i++) { + //if (i > 0 && (0 == strcmp (trec_qrels->text_qrels[i].docno, + // trec_qrels->text_qrels[i-1].docno))) { + // set_error (SM_ILLPA_ERR, "Duplicate qrels docno", trec_qrels->text_qrels[i].docno); + // return (UNDEF); + //} + if (trec_qrels->text_qrels[i].rel >= epi->relevance_level) + (*num_rel)++; + } + + /* Go through trec_top, trec_qrels in parallel to determine which + docno's are in both (ie, which trec_top are relevant). Once relevance + is known, convert trec_top tuple into TR_TUP. */ + tr_tup = start_tr_tup; + qrels_ptr = trec_qrels->text_qrels; + end_qrels = &trec_qrels->text_qrels[trec_qrels->num_text_qrels]; + for (i = 0; i < trec_top->num_text_tr; i++) { + if (trec_top->text_tr[i].rank > epi->max_num_docs_per_topic) + /* Skip if evaluation desired over fewer docs than this rank */ + continue; + while (qrels_ptr < end_qrels && + strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) < 0) + qrels_ptr++; + if (qrels_ptr >= end_qrels || + strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) > 0) { + /* Doc is non-judged */ + tr_tup->rel = -1; + /* Skip unjudged docs if desired */ + if (epi->judged_docs_only_flag) + continue; + } + else { + /* Doc is judged; assign relevance */ + tr_tup->rel = qrels_ptr->rel; + qrels_ptr++; + } + tr_tup->did = i; + tr_tup->rank = trec_top->text_tr[i].rank; + tr_tup->sim = trec_top->text_tr[i].sim; + tr_tup->action = 0; + tr_tup->iter = 0; + tr_tup++; + } + + /* Form the full TR_VEC object for this qid */ + tr_vec->qid = trec_top->qid; + tr_vec->num_tr = tr_tup - start_tr_tup; + tr_vec->tr = start_tr_tup; + + /* If judged_docs_only_flag, then must fix up ranks to reflect unjudged + docs being thrown out. Note: done this way to preserve original + tie-breaking based on text docno */ + if (epi->judged_docs_only_flag) { + /* Sort tuples by increasing rank */ + qsort ((char *) tr_vec->tr, + (int) tr_vec->num_tr, + sizeof (TR_TUP), + comp_tr_tup_rank); + for (i = 0; i < tr_vec->num_tr; i++) { + tr_vec->tr[i].rank = i+1; + } + qsort ((char *) tr_vec->tr, + (int) tr_vec->num_tr, + sizeof (TR_TUP), + comp_tr_tup_did); + } + + return (1); +} + +static int +comp_sim_docno (ptr1, ptr2) +TEXT_TR *ptr1; +TEXT_TR *ptr2; +{ + if (ptr1->sim > ptr2->sim) + return (-1); + if (ptr1->sim < ptr2->sim) + return (1); + return (strcmp (ptr2->docno, ptr1->docno)); +} + +static int +comp_negsim_docno (ptr1, ptr2) +TEXT_TR *ptr1; +TEXT_TR *ptr2; +{ + if (ptr1->sim < ptr2->sim) + return (-1); + if (ptr1->sim > ptr2->sim) + return (1); + return (strcmp (ptr2->docno, ptr1->docno)); +} + +static int +comp_tr_docno (ptr1, ptr2) +TEXT_TR *ptr1; +TEXT_TR *ptr2; +{ + return (strcmp (ptr1->docno, ptr2->docno)); +} + +static int +comp_qrels_docno (ptr1, ptr2) +TEXT_QRELS *ptr1; +TEXT_QRELS *ptr2; +{ + return (strcmp (ptr1->docno, ptr2->docno)); +} + +static int +comp_tr_tup_rank (ptr1, ptr2) +TR_TUP *ptr1; +TR_TUP *ptr2; +{ + return (ptr1->rank - ptr2->rank); +} + +static int +comp_tr_tup_did (ptr1, ptr2) +TR_TUP *ptr1; +TR_TUP *ptr2; +{ + return (ptr1->did - ptr2->did); +} diff --git a/sm-model/trec_eval-8.0/get_qrels.c b/sm-model/trec_eval-8.0/get_qrels.c new file mode 100755 index 0000000..84d51de --- /dev/null +++ b/sm-model/trec_eval-8.0/get_qrels.c @@ -0,0 +1,169 @@ +/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley. */ + + +#include "common.h" +#include "sysfunc.h" +#include "smart_error.h" +#include "trec_eval.h" +#include + + +/* Read all relevance information from text_qrels_file. +Relevance for each docno to qid is determined from text_qrels_file, which +consists of text tuples of the form + qid iter docno rel +giving TREC document numbers (docno, a string) and their relevance (rel, +an integer) to query qid (a string). iter string field is ignored. +Fields are separated by whitespace, string fields can contain no whitespace. +File may contain no NULL characters. +*/ + +int +get_qrels (text_qrels_file, all_trec_qrels) +char *text_qrels_file; +ALL_TREC_QRELS *all_trec_qrels; +{ + int fd; + int size = 0; + char *trec_qrels_buf; + char *ptr; + char *current_qid; + char *qid_ptr, *docno_ptr, *rel_ptr; + long i; + long rel; + TREC_QRELS *current_qrels = NULL; + + /* Read entire file into memory */ + if (-1 == (fd = open (text_qrels_file, 0)) || + -1 == (size = lseek (fd, 0L, 2)) || + NULL == (trec_qrels_buf = malloc ((unsigned) size+2)) || + -1 == lseek (fd, 0L, 0) || + size != read (fd, trec_qrels_buf, size) || + -1 == close (fd)) { + + set_error (SM_ILLPA_ERR, "Cannot read qrels file", "trec_eval"); + return (UNDEF); + } + + current_qid = ""; + + /* Initialize all_trec_qrels */ + all_trec_qrels->num_q_qrels = 0; + all_trec_qrels->max_num_q_qrels = INIT_NUM_QUERIES; + if (NULL == (all_trec_qrels->trec_qrels = Malloc (INIT_NUM_QUERIES, + TREC_QRELS))) + return (UNDEF); + + if (size == 0) + return (0); + + /* Append ending newline if not present, Append NULL terminator */ + if (trec_qrels_buf[size-1] != '\n') { + trec_qrels_buf[size] = '\n'; + size++; + } + trec_qrels_buf[size] = '\0'; + + ptr = trec_qrels_buf; + + while (*ptr) { + /* Get current line */ + /* Get qid */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + qid_ptr = ptr; + while (! isspace (*ptr)) ptr++; + if (*ptr == '\n') { + set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); + return (UNDEF); + } + *ptr++ = '\0'; + /* Skip iter */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + while (! isspace (*ptr)) ptr++; + if (*ptr++ == '\n') { + set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); + return (UNDEF); + } + /* Get docno */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + docno_ptr = ptr; + while (! isspace (*ptr)) ptr++; + if (*ptr == '\n') { + set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); + return (UNDEF); + } + *ptr++ = '\0'; + /* Get relevance */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + if (*ptr == '\n') { + set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); + return (UNDEF); + } + rel_ptr = ptr; + while (! isspace (*ptr)) ptr++; + if (*ptr != '\n') { + *ptr++ = '\0'; + while (*ptr != '\n' && isspace (*ptr)) ptr++; + if (*ptr != '\n') { + set_error (SM_ILLPA_ERR, "malformed qrels line", + "trec_eval"); + return (UNDEF); + } + } + *ptr++ = '\0'; + + if (0 != strcmp (qid_ptr, current_qid)) { + /* Query has changed. Must check if new query or this is more + judgements for an old query */ + for (i = 0; i < all_trec_qrels->num_q_qrels; i++) { + if (0 == strcmp (qid_ptr, all_trec_qrels->trec_qrels[i].qid)) + break; + } + if (i >= all_trec_qrels->num_q_qrels) { + /* New unseen query, add and initialize it */ + if (all_trec_qrels->num_q_qrels >= + all_trec_qrels->max_num_q_qrels) { + all_trec_qrels->max_num_q_qrels *= 10; + if (NULL == (all_trec_qrels->trec_qrels = + Realloc (all_trec_qrels->trec_qrels, + all_trec_qrels->max_num_q_qrels, + TREC_QRELS))) + return (UNDEF); + } + current_qrels = &all_trec_qrels->trec_qrels[i]; + current_qrels->qid = qid_ptr; + current_qrels->num_text_qrels = 0; + current_qrels->max_num_text_qrels = INIT_NUM_RELS; + if (NULL == (current_qrels->text_qrels = + Malloc (INIT_NUM_RELS, TEXT_QRELS))) + return (UNDEF); + all_trec_qrels->num_q_qrels++; + } + else { + /* Old query, just switch current_q_index */ + current_qrels = &all_trec_qrels->trec_qrels[i]; + } + current_qid = current_qrels->qid; + } + + /* Add judgement to current query's list */ + if (current_qrels->num_text_qrels >= + current_qrels->max_num_text_qrels) { + /* Need more space */ + current_qrels->max_num_text_qrels *= 10; + if (NULL == (current_qrels->text_qrels = + Realloc (current_qrels->text_qrels, + current_qrels->max_num_text_qrels, + TEXT_QRELS))) + return (UNDEF); + } + current_qrels->text_qrels[current_qrels->num_text_qrels].docno = + docno_ptr; + rel = atol (rel_ptr); + current_qrels->text_qrels[current_qrels->num_text_qrels++].rel = + rel; + } + + return (1); +} + diff --git a/sm-model/trec_eval-8.0/get_top.c b/sm-model/trec_eval-8.0/get_top.c new file mode 100755 index 0000000..efc6c76 --- /dev/null +++ b/sm-model/trec_eval-8.0/get_top.c @@ -0,0 +1,192 @@ +/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley. */ + + +#include "common.h" +#include "sysfunc.h" +#include "smart_error.h" +#include "trec_eval.h" +#include + +/* Read all retrieved results information from trec_top_file. +Read text tuples from trec_top_file of the form + 030 Q0 ZF08-175-870 0 4238 prise1 + qid iter docno rank sim run_id +giving TREC document numbers (a string) retrieved by query qid +(a string) with similarity sim (a float). The other fields are ignored, +with the exception that the run_id field of the last line is kept and +output. In particular, note that the rank field is ignored here; +internally ranks are assigned by sorting by the sim field with ties +broken determinstically (using docno). +Sim is assumed to be higher for the docs to be retrieved first. +File may contain no NULL characters. +Any field following run_id is ignored. +*/ + +int +get_top (trec_top_file, all_trec_top) +char *trec_top_file; +ALL_TREC_TOP *all_trec_top; +{ + int fd; + int size = 0; + char *trec_top_buf; + char *ptr; + char *current_qid; + char *qid_ptr, *docno_ptr, *sim_ptr; + char *run_id_ptr = ""; + long i; + TREC_TOP *current_top = NULL; + float sim; + + /* Read entire file into memory */ + if (-1 == (fd = open (trec_top_file, 0)) || + -1 == (size = lseek (fd, 0L, 2)) || + NULL == (trec_top_buf = malloc ((unsigned) size+2)) || + -1 == lseek (fd, 0L, 0) || + size != read (fd, trec_top_buf, size) || + -1 == close (fd)) { + + set_error (SM_ILLPA_ERR, "Cannot read qrels file", "trec_eval"); + return (UNDEF); + } + + current_qid = ""; + + /* Initialize all_trec_top */ + all_trec_top->num_q_tr = 0; + all_trec_top->max_num_q_tr = INIT_NUM_QUERIES; + if (NULL == (all_trec_top->trec_top = Malloc (INIT_NUM_QUERIES, + TREC_TOP))) + return (UNDEF); + + if (size == 0) + return (0); + + /* Append ending newline if not present, Append NULL terminator */ + if (trec_top_buf[size-1] != '\n') { + trec_top_buf[size] = '\n'; + size++; + } + trec_top_buf[size] = '\0'; + + ptr = trec_top_buf; + + while (*ptr) { + /* Get current line */ + /* Get qid */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + if (*ptr == '\n') { + /* Ignore blank lines (people seem to insist on them!) */ + ptr++; + continue; + } + qid_ptr = ptr; + while (! isspace (*ptr)) ptr++; + if (*ptr == '\n') { + set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); + return (UNDEF); + } + *ptr++ = '\0'; + /* Skip iter */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + while (! isspace (*ptr)) ptr++; + if (*ptr++ == '\n') { + set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); + return (UNDEF); + } + /* Get docno */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + docno_ptr = ptr; + while (! isspace (*ptr)) ptr++; + if (*ptr == '\n') { + set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); + return (UNDEF); + } + *ptr++ = '\0'; + /* Skip rank */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + while (! isspace (*ptr)) ptr++; + if (*ptr++ == '\n') { + set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); + return (UNDEF); + } + /* Get sim */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + sim_ptr = ptr; + while (! isspace (*ptr)) ptr++; + if (*ptr == '\n') { + set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); + return (UNDEF); + } + *ptr++ = '\0'; + /* Get run_id */ + while (*ptr != '\n' && isspace (*ptr)) ptr++; + if (*ptr == '\n') { + set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); + return (UNDEF); + } + run_id_ptr = ptr; + while (! isspace (*ptr)) ptr++; + if (*ptr != '\n') { + /* Skip over rest of line */ + *ptr++ = '\0'; + while (*ptr != '\n') ptr++; + } + *ptr++ = '\0'; + + if (0 != strcmp (qid_ptr, current_qid)) { + /* Query has changed. Must check if new query or this is more + judgements for an old query */ + for (i = 0; i < all_trec_top->num_q_tr; i++) { + if (0 == strcmp (qid_ptr, all_trec_top->trec_top[i].qid)) + break; + } + if (i >= all_trec_top->num_q_tr) { + /* New unseen query, add and initialize it */ + if (all_trec_top->num_q_tr >= + all_trec_top->max_num_q_tr) { + all_trec_top->max_num_q_tr *= 10; + if (NULL == (all_trec_top->trec_top = + Realloc (all_trec_top->trec_top, + all_trec_top->max_num_q_tr, + TREC_TOP))) + return (UNDEF); + } + current_top = &all_trec_top->trec_top[i]; + current_top->qid = qid_ptr; + current_top->num_text_tr = 0; + current_top->max_num_text_tr = INIT_NUM_RESULTS; + if (NULL == (current_top->text_tr = + Malloc (INIT_NUM_RESULTS, TEXT_TR))) + return (UNDEF); + all_trec_top->num_q_tr++; + } + else { + /* Old query, just switch current_q_index */ + current_top = &all_trec_top->trec_top[i]; + } + current_qid = current_top->qid; + } + + /* Add retrieval docno/sim to current query's list */ + if (current_top->num_text_tr >= + current_top->max_num_text_tr) { + /* Need more space */ + current_top->max_num_text_tr *= 10; + if (NULL == (current_top->text_tr = + Realloc (current_top->text_tr, + current_top->max_num_text_tr, + TEXT_TR))) + return (UNDEF); + } + current_top->text_tr[current_top->num_text_tr].docno = docno_ptr; + sim = atof (sim_ptr); + current_top->text_tr[current_top->num_text_tr].sim = sim; + current_top->text_tr[current_top->num_text_tr++].rank = 0; + } + + all_trec_top->run_id = run_id_ptr; + + return (1); +} + diff --git a/sm-model/trec_eval-8.0/smart_error.h b/sm-model/trec_eval-8.0/smart_error.h new file mode 100755 index 0000000..78d7523 --- /dev/null +++ b/sm-model/trec_eval-8.0/smart_error.h @@ -0,0 +1,24 @@ +#ifndef SMART_ERRORH +#define SMART_ERRORH +#include +#define SMART_MINERR 1000 +#define SM_INCON_ERR 1000 +#define SM_ILLSK_ERR 1001 +#define SM_ILLMD_ERR 1002 +#define SM_ILLPA_ERR 1003 +#define SMART_NUMERR 4 + + +extern int errno; +extern int smart_errno; /* If > 0 and <= sys_nerr then refers to */ + /* sys_errlist, else if >= smart_errmin */ + /* and <= smart_errmax, then smart_errlist */ +extern char *smart_message; /* Message to be printed (often filename) */ +extern char *smart_routine; /* Major routine issuing error message */ + + +#define set_error(n,m,r) { if (n > 0) smart_errno = n;\ + smart_message = m;\ + smart_routine = r; } +#define clr_err() smart_errno = errno = 0 +#endif /* SMART_ERRORH */ diff --git a/sm-model/trec_eval-8.0/sysfunc.h b/sm-model/trec_eval-8.0/sysfunc.h new file mode 100755 index 0000000..64021da --- /dev/null +++ b/sm-model/trec_eval-8.0/sysfunc.h @@ -0,0 +1,39 @@ +#ifndef SYSFUNCH +#define SYSFUNCH +/* Declarations of major functions within standard C libraries */ +/* Once all of the major systems get their act together (and I follow + suit!), this file should just include system header files from + /usr/include. Until then... */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* For time being, define Berkeley constructs in terms of SVR4 constructs*/ +#define bzero(dest,len) memset(dest,'\0',len) +#define bcopy(source,dest,len) memcpy(dest,source,len) +#define srandom(seed) srand(seed) +#define random() rand() + +/* ANSI should give us an offsetof suitable for the implementation; + * otherwise, try a non-portable but commonly supported definition + */ +#ifdef __STDC__ +#include +#endif +#ifndef offsetof +#define offsetof(type, member) ((size_t) \ + ((char *)&((type*)0)->member - (char *)(type *)0)) +#endif + +#endif /* SYSFUNCH */ diff --git a/sm-model/trec_eval-8.0/tr_vec.h b/sm-model/trec_eval-8.0/tr_vec.h new file mode 100755 index 0000000..4b5b868 --- /dev/null +++ b/sm-model/trec_eval-8.0/tr_vec.h @@ -0,0 +1,21 @@ +#ifndef TR_VECH +#define TR_VECH +/* $Header: /home/smart/release/./src/h/tr_vec.h,v 10.1 91/11/05 23:47:35 smart Exp Locker: smart $*/ + +typedef struct { + long did; /* document id */ + long rank; /* Rank of this document */ + char action; /* what action a user has taken with doc */ + char rel; /* whether doc judged relevant(1) or not(0) */ + char iter; /* Number of feedback runs for this query */ + char trtup_unused; /* Presently unused field */ + float sim; /* similarity of did to qid */ +} TR_TUP; + +typedef struct { + char *qid; /* query id */ + long num_tr; /* Number of tuples for tr_vec */ + TR_TUP *tr; /* tuples. Invariant: tr sorted increasing did */ +} TR_VEC; + +#endif /* TR_VECH */ diff --git a/sm-model/trec_eval-8.0/trec_eval.c b/sm-model/trec_eval-8.0/trec_eval.c new file mode 100755 index 0000000..b85b8a7 --- /dev/null +++ b/sm-model/trec_eval-8.0/trec_eval.c @@ -0,0 +1,256 @@ +static char *VersionID = VERSIONID; +/* "Version 7.3 trec_eval Dec 15, 2004"; */ + +/* Copyright (c) 2004, 2003, 1991, 1990, 1984 - Chris Buckley. */ + +/******************** PROCEDURE DESCRIPTION ************************ + *0 Take TREC results text file, TREC qrels file, and evaluate + *1 local.convert.obj.trec_eval + *2 trec_eval [-q] [-a] [-t] [-o] [-v] [-n num] trec_rel_file trec_top_file + + *7 Read text tuples from trec_top_file of the form + *7 030 Q0 ZF08-175-870 0 4238 prise1 + *7 qid iter docno rank sim run_id + *7 giving TREC document numbers (a string) retrieved by query qid + *7 (an integer) with similarity sim (a float). The other fields are ignored. + *7 Input is asssumed to be sorted numerically by qid. + *7 Sim is assumed to be higher for the docs to be retrieved first. + *7 Relevance for each docno to qid is determined from text_qrels_file, which + *7 consists of text tuples of the form + *7 qid iter docno rel + *7 giving TREC document numbers (a string) and their relevance to query qid + *7 (an integer). Tuples are asssumed to be sorted numerically by qid. + *7 The text tuples with relevence judgements are converted to TR_VEC form + *7 and then submitted to the evaluation routines. + *7 + *7 -q: In addition to summary evaluation, give evaluation for each query + *7 -a: Print all evaluation measures calculated, instead of just the + *7 official measures for TREC 2. + *7 -o: Print everything out in old, non-relational format + *7 -v: Print version number and exit + *7 -h: Print full help message and exit + *7 -t: Treat similarity as time that document retrieved. Compute + *7 several time-based measures after ranking docs by time retrieved + *7 (first doc (lowest sim) retrieved ranked highest). + *7 Only done if -a selected. + *7 -J: Calculate all measures only over judged documents that appear + *7 in qrels. (DO NOT USE) + *7 -n: following integer is the number of queries to average over. + *7 -ua: Value to use for 'a' coefficient of utility computation. + *7 -ub: Value to use for 'b' coefficient of utility computation. + *7 -uc: Value to use for 'c' coefficient of utility computation. + *7 -ud: Value to use for 'd' coefficient of utility computation. + *7 -N: Number of docs in collection + *7 -M:Max number of results to evaluate per topic + + *8 Procedure is to read all the docs retrieved for a query, and all the + *8 relevant docs for that query, + *8 sort and rank the retrieved docs by sim/docno, + *8 and look up docno in the relevant docs to determine relevance. + *8 The qid,did,rank,sim,rel fields of of TR_VEC are filled in; + *8 action,iter fields are set to 0. + *8 Queries for which there are no relevant docs are ignored completely. + +***********************************************************************/ + +#include "common.h" +#include "sysfunc.h" +#include "smart_error.h" +#include "tr_vec.h" +#include "trec_eval.h" +#include "buf.h" + +void print_error(); +void old_print_trec_eval_list(); +void print_rel_trec_eval_list(); + +int trec_eval_help(EVAL_PARAM_INFO *epi); +int accumulate_results (TREC_EVAL *query_eval, TREC_EVAL *accum_eval); +int get_top (char *trec_top_file, ALL_TREC_TOP *all_trec_top); +int get_qrels (char *text_qrels_file, ALL_TREC_QRELS *all_trec_qrels); +int form_trvec (EVAL_PARAM_INFO *ep, TREC_TOP *trec_top, + TREC_QRELS *trec_qrels, TR_VEC *tr_vec, long *num_rel); +int trvec_trec_eval (EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, + TREC_EVAL *eval, long num_rel, long num_nonrel); + +static char *usage = "Usage: trec_eval [-h] [-q] [-a] [-o] [-v] trec_rel_file trec_top_file\n\ + -h: Give full help information, including other options\n\ + -q: In addition to summary evaluation, give evaluation for each query\n\ + -a: Print all evaluation measures, instead of just official measures\n\ + -o: Print requested measures in old non-relational format\n"; + + +int +main (argc, argv) +int argc; +char *argv[]; +{ + char *trec_rel_file, *trec_top_file; + ALL_TREC_TOP all_trec_top; + ALL_TREC_QRELS all_trec_qrels; + TREC_EVAL accum_eval, query_eval; + TR_VEC tr_vec; + long num_rel; + long num_eval_q; + long i,j; + EVAL_PARAM_INFO epi; + + /* Initialize static info before getting program optional args */ + epi.query_flag = epi.all_flag = epi.time_flag = epi.average_complete_flag = 0; + epi.judged_docs_only_flag = 0; + epi.relation_flag = 1; + epi.utility_a = UTILITY_A; epi.utility_b = UTILITY_B; + epi.utility_c = UTILITY_C; epi.utility_d = UTILITY_D; + epi.num_docs_in_coll = 0; + epi.relevance_level = 1; + epi.max_num_docs_per_topic = MAXLONG; + + /* Should use getopts, but some people may not have it. */ + /* This keeps growing over the years. Should redo */ + while (argc > 1 && argv[1][0] == '-') { + if (argv[1][1] == 'q') + epi.query_flag++; + else if (argv[1][1] == 'v') { + fprintf (stderr, "trec_eval version %s\n", VersionID); + exit (0); + } + else if (argv[1][1] == 'h') { + (void) trec_eval_help(&epi); + exit (0); + } + else if (argv[1][1] == 'a') + epi.all_flag++; + else if (argv[1][1] == 'o') + epi.relation_flag = 0; + else if (argv[1][1] == 'c') { + epi.average_complete_flag++; + } + else if (argv[1][1] == 'l') { + epi.relevance_level = atol (&argv[1][2]); + } + else if (argv[1][1] == 'J') { + epi.judged_docs_only_flag++; + } + else if (argv[1][1] == 'N') + epi.num_docs_in_coll = atol (&argv[1][2]); + else if (argv[1][1] == 'M') + epi.max_num_docs_per_topic = atol (&argv[1][2]); + else if (argv[1][1] == 'U') { + if (argv[1][2] == 'a') + epi.utility_a = atof (&argv[1][3]); + else if (argv[1][2] == 'b') + epi.utility_b = atof (&argv[1][3]); + else if (argv[1][2] == 'c') + epi.utility_c = atof (&argv[1][3]); + else if (argv[1][2] == 'd') + epi.utility_d = atof (&argv[1][3]); + else { + (void) fputs (usage,stderr); + exit (1); + } + } + else if (argv[1][1] == 'T') + epi.time_flag++; + else { + (void) fputs (usage,stderr); + exit (1); + } + argc--; argv++; + } + + if (argc != 3) { + (void) fputs (usage,stderr); + exit (1); + } + + trec_rel_file = argv[1]; + trec_top_file = argv[2]; + + /* Get qrels and top results information for all queries from the + input text files */ + if (UNDEF == get_qrels (trec_rel_file, &all_trec_qrels) || + UNDEF == get_top (trec_top_file, &all_trec_top)) { + print_error ("trec_eval: input error", "Quit"); + exit (2); + } + + /* For each topic which has both qrels and top results information, + calculate, possibly print (if query_flag), and accumulate + evaluation measures. */ + num_eval_q = 0; + (void) memset ((void *) &accum_eval, 0, sizeof (TREC_EVAL)); + accum_eval.qid = "All"; + + for (i = 0; i < all_trec_top.num_q_tr; i++) { + /* Find rel info for this query (skip if no rel info) */ + for (j = 0; j < all_trec_qrels.num_q_qrels; j++) { + if (0 == strcmp (all_trec_top.trec_top[i].qid, + all_trec_qrels.trec_qrels[j].qid)) + break; + } + if (j >= all_trec_qrels.num_q_qrels) + continue; + + /* Form results/rel into SMART TR_VEC form */ + if (UNDEF == form_trvec (&epi, + &all_trec_top.trec_top[i], + &all_trec_qrels.trec_qrels[j], + &tr_vec, + &num_rel)) { + print_error ("trec_eval: form_tr_vec error", "Quit"); + exit (3); + } + + /* Evaluate results/rel for this query */ + if (UNDEF == trvec_trec_eval (&epi, + &tr_vec, + &query_eval, + num_rel, + all_trec_qrels.trec_qrels[j].num_text_qrels - num_rel)) { + print_error ("trec_eval: evaluation error", "Quit"); + exit (4); + } + + /* Print results for this query, if desired */ + if (epi.query_flag) { + if (epi.relation_flag) + print_rel_trec_eval_list (1, &epi, &query_eval, (SM_BUF *) NULL); + else + old_print_trec_eval_list (&epi, &query_eval, 1, (SM_BUF *) NULL); + } + + /* Accumulate results for later averaging */ + if (UNDEF == accumulate_results (&query_eval, &accum_eval)) { + print_error ("trec_eval: accumulation error", "Quit"); + exit (5); + } + + num_eval_q++; + } + + /******** REMOVE THIS ONCE WARNING FLAG ADDED */ + /* Warn if numq_flag_num < num_eval_q */ + if (num_eval_q == 0) { + set_error (SM_INCON_ERR, + "No queries with both results and relevance info", + "trec_eval"); + return (UNDEF); + print_error ("trec_eval", "Quit"); + exit (6); + } + + if (epi.average_complete_flag) { + /* Want to average over possibly missing queries. Pass in actual + * number of queries in num_orig_queries */ + accum_eval.num_orig_queries = accum_eval.num_queries; + accum_eval.num_queries = all_trec_qrels.num_q_qrels; + } + + /* Print final evaluation results */ + if (epi.relation_flag) + print_rel_trec_eval_list (0, &epi, &accum_eval, (SM_BUF *) NULL); + else + old_print_trec_eval_list (&epi, &accum_eval, 1, (SM_BUF *) NULL); + + exit (0); +} diff --git a/sm-model/utils.py b/sm-model/utils.py new file mode 100644 index 0000000..246cd01 --- /dev/null +++ b/sm-model/utils.py @@ -0,0 +1,156 @@ +# file input output +# LATER model input output as well + +import os +import sys +import numpy as np +from gensim.models.keyedvectors import KeyedVectors + +import torch + +import cPickle + + + +# logging setup +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + + + +def load_bin_vec(fname, words): + """ + Loads 300x1 word vecs from Google (Mikolov) word2vec + """ + print fname + vocab = set(words) + word_vecs = {} + with open(fname, "rb") as f: + header = f.readline() + vocab_size, layer1_size = map(int, header.split()) + binary_len = numpy.dtype('float32').itemsize * layer1_size + print 'vocab_size, layer1_size', vocab_size, layer1_size + count = 0 + for i, line in enumerate(xrange(vocab_size)): + if i % 100000 == 0: + print '.', + word = [] + while True: + ch = f.read(1) + if ch == ' ': + word = ''.join(word) + break + if ch != '\n': + word.append(ch) + if word in vocab: + count += 1 + word_vecs[word] = numpy.fromstring(f.read(binary_len), dtype='float32') + else: + f.read(binary_len) + print "done" + print "Words found in wor2vec embeddings", count + return word_vecs + + +def logargs(func): + def inner(*args, **kwargs): + logger.info('%s : %s %s' % (func.__name__, args, kwargs)) + return func(*args, **kwargs) + return inner + + +def cache_word_embeddings(word_embeddings_file, cache_file): + if not word_embeddings_file.endswith('.gz'): + logger.warning( 'WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file)) + + vocab_size, vec_dim = 0, 0 + + if not os.path.exists(cache_file): + # cache does not exist + if not os.path.exists(os.path.dirname(cache_file)): + # make cache folder if needed + os.mkdir(os.path.dirname(cache_file)) + + logger.info( 'caching the word embeddings in np.memmap format' ) + wv = KeyedVectors.load_word2vec_format(word_embeddings_file, binary=True) + # print len(wv.syn0), wv.syn0.shape + # print len(wv.syn0norm) if wv.syn0norm else None + fp = np.memmap(cache_file, dtype=np.double, mode='w+', shape=wv.syn0.shape) + fp[:] = wv.syn0[:] + with open(cache_file + '.vocab', 'w') as f: + logger.info( 'writing out vocab for {}'.format(word_embeddings_file)) + for _, w in sorted( (voc.index, word) for word, voc in wv.vocab.items()): + print >> f, w.encode('utf-8') + with open(cache_file + '.dimensions', 'w') as f: + logger.info( 'writing out dimensions for {}'.format(word_embeddings_file)) + print >> f, wv.syn0.shape[0], wv.syn0.shape[1] + vocab_size, vec_dim = wv.syn0.shape + del fp, wv + print 'cached {} into {}'.format(word_embeddings_file, cache_file) + + return vocab_size, vec_dim + + +def load_embedding_dimensions(cache_file): + vocab_size, vec_dim = 0, 0 + with open(cache_file + '.dimensions') as d: + vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] + return vocab_size, vec_dim + + +def load_cached_embeddings(cache_file, vocab_list): + logger.debug( 'loading cached embeddings ') + w2v_dict = {} + with open(cache_file + '.dimensions') as d: + vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] + + W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim)) + + with open(cache_file + '.vocab') as f: + logger.debug( 'loading vocab') + w2v_vocab_list = map(str.strip, f.readlines()) + + vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)} + + # Read w2v for vocab appears in Q and A + for word in vocab_list: + if word in vocab_dict: + w2v_dict[word] = W[vocab_dict[word]] + else: + w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) + return w2v_dict, vec_dim + + +def read_in_dataset(dataset_folder, set_folder): + """ + read in the data to return (question, sentence, label) + set_folder = {train|dev|test} + """ + max_q = 0 + max_s = 0 + set_path = os.path.join(dataset_folder, set_folder) + len_q_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines() ] + questions = [ line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines() ] + len_s_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines() ] + sentences = [ line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines() ] + labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ]) + ext_feats = np.array([ map(float, line.strip().split(' ')) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ]) + + #y = torch.from_numpy(labels) + #return questions, sentences, y + + vocab = [ line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines() ] + return questions, sentences, labels, vocab, max(len_q_list), max(len_s_list), ext_feats + +def get_test_qids_labels(dataset_folder, set_folder): + set_path = os.path.join(dataset_folder, set_folder) + qids = [ line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines() ] + labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ]) + return qids, labels \ No newline at end of file