diff --git a/sm_cnn/.gitignore b/sm_cnn/.gitignore index 85d6be8..edd6884 100644 --- a/sm_cnn/.gitignore +++ b/sm_cnn/.gitignore @@ -1,5 +1,5 @@ *pyc -trec_eval-8.0/trec_eval -data/ +*.pt +text/ trained_models/ -trec_eval-8.0/trec_eval.dSYM +data/ diff --git a/sm_cnn/README.md b/sm_cnn/README.md index 49b69e0..c416852 100644 --- a/sm_cnn/README.md +++ b/sm_cnn/README.md @@ -1,80 +1,127 @@ ## SM model #### References: -1. Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 +1. Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural +Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information +Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 -### Requirements -gensim==1.0.1 -nltk==3.2.2 -numpy==1.11.3 -pandas==0.19.2 -torch==0.1.11+b13b701 - -Please install the requirements. See [Castor/README.md](../README.md) for pytorch installation details. - -### Training the model - -1. Setup repository layout. - -``` -mkdir castorini -cd castorini +### Setup +Clone and create the dataset: +```bash git clone https://github.com/castorini/data.git -git clone https://github.com/castorini/models.git git clone https://github.com/castorini/Castor.git ``` -This should generate: +You should you see the following tree: ``` +. ├── Castor │   ├── README.md +│   ├── baseline_results.tsv │   ├── idf_baseline │   ├── kim_cnn -│   ├── simple_qa_rnn -│   └── sm_cnn/ -├── data -│   ├── README.md -│   ├── TrecQA/ -│   └── word2vec/ -└── models +│   ├── mp_cnn +│   ├── setup.py +│   ├── sm_cnn +└── data + ├── GloVe + ├── ParagramEmbeddings ├── README.md - └── sm_cnn/ + ├── SimpleQuestions_v2 + ├── TrecQA + ├── WikiQA + ├── msrvid + ├── requirements.txt + ├── sick + ├── twitterPPDB + ├── utils + └── word2vec ``` -2. Preprocess data - -``` -cd data/TrecQA -python3 parse.py -python3 overlap_features.py -python3 build_vocab.py -``` - -3. Download word embeddings from [here](https://drive.google.com/folderview?id=0B-yipfgecoSBfkZlY2FFWEpDR3M4Qkw5U055MWJrenE5MTBFVXlpRnd0QjZaMDQxejh1cWs&usp=sharing) and save the ``aquaint+wiki.txt.gz.ndim=50.bin`` into ``data/word2vec/``. - - -4. Train the model - -Make trec_eval - -``` +To create the dataset: +```bash cd Castor/sm_cnn/ -cd trec_eval-8.0 -make clean && make -cd .. +./create_dataset.sh ``` -To train the S&M model on TrecQA + +We use `trec_eval` for evaluation: + +```bash +cd ../utils/ +./get_trec_eval.sh ``` -python main.py ../../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor --paper-ext-feats + +### Training +Download the word2vec model from [here] (https://drive.google.com/file/d/0B2u_nClt6NbzUmhOZU55eEo4QWM/view?usp=sharing) +and copy it to the `data/` folder. + +You can train the SM model for the 4 following configurations: +1. __random__ - the word embedddings are initialized randomly and are tuned during training +2. __static__ - the word embeddings are static (Severyn and Moschitti, SIGIR'15) +3. __non-static__ - the word embeddings are tuned during training +4. __multichannel__ - contains static and non-static channels for question and answer conv layers + +To train on GPU 0 with static configuration: + +```bash +python train.py --mode static --gpu 0 ``` -**To use the GPU, add `--cuda`.** -The final model will be saved to ```../../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor``` +NB: pass `--no_cuda` to use CPU -_NOTE:_ On first run, the program will create a memory-mapped cache for word e mbeddings (943MB) in ``data/word2vec``. -The cache allows for faster loading of data in future runs. +The trained model will be save to: +``` +saves/static_best_model.pt +``` -Run ```python main.py -h``` for more default options. +### Testing the model +``` +python main.py --trained_model saves/TREC/multichannel_best_model.pt +``` + +### Evaluation + +The performance on TrecQA dataset: + +### TrecQA: + +#### Best dev +Metric |rand |static|non-static|multichannel +-------|------|------|----------|------------ +MAP |0.8096|0.8162|0.8387 | 0.8274 +MRR |0.8560|0.8918|0.9058 | 0.8818 + +#### Test +Metric |rand |static|non-static|multichannel +-------|-------|------|----------|------------ +MAP |0.7441 |0.7524|0.7688 |0.7641 +MRR |0.8172 |0.8012|0.8144 |0.8174 + +### WikiQA: + +#### Best dev +Metric |rand |static|non-static|multichannel +-------|------|------|----------|------------ +MAP |0.7109|0.7204|0.7049 | 0.7245 +MRR |0.7169|0.7234|0.7075 | 0.7259 + +#### Test +Metric |rand |static|non-static|multichannel +-------|-------|------|----------|------------ +MAP |0.6313 |0.6378|0.6455 |0.6476 +MRR |0.6522 |0.6542|0.6689 |0.6646 + +NB: The results on WikiQA are based on the SM model hyperparameters. + + +### To create your own word2vec.pt file + ++ Download word2vec from [here](https://drive.google.com/drive/u/0/folders/0B-yipfgecoSBfkZlY2FFWEpDR3M4Qkw5U055MWJrenE5MTBFVXlpRnd0QjZaMDQxejh1cWs) +to the `data/` folder + +```bash +python utils.py --input data/aquaint+wiki.txt.gz.ndim=50.bin +``` \ No newline at end of file diff --git a/sm_cnn/__init__.py b/sm_cnn/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sm_modified_cnn/args.py b/sm_cnn/args.py similarity index 100% rename from sm_modified_cnn/args.py rename to sm_cnn/args.py diff --git a/sm_cnn/bridge.py b/sm_cnn/bridge.py index 0c836ab..c6e48de 100644 --- a/sm_cnn/bridge.py +++ b/sm_cnn/bridge.py @@ -3,117 +3,75 @@ import os import sys from collections import Counter import argparse +import random -import re -import string import numpy as np import torch from nltk.tokenize import TreebankWordTokenizer -from torch.autograd import Variable -from py4j.java_gateway import JavaGateway +from torchtext import data from sm_cnn import model from sm_cnn.external_features import compute_overlap, compute_idf_weighted_overlap, stopped +from sm_cnn.trec_dataset import TrecDataset +from sm_cnn.wiki_dataset import WikiDataset +from anserini_dependency.RetrieveSentences import RetrieveSentences sys.modules['model'] = model - class SMModelBridge(object): - def __init__(self, model_file, word_embeddings_cache_file, index_path): - # init torch random seeds - torch.manual_seed(1234) - np.random.seed(1234) + def __init__(self, args): + if not args.cuda: + args.gpu = -1 + if torch.cuda.is_available() and args.cuda: + print("Note: You are using GPU for training") + torch.cuda.set_device(args.gpu) + torch.cuda.manual_seed(args.seed) + if torch.cuda.is_available() and not args.cuda: + print("Warning: You have Cuda but do not use it. You are using CPU for training") - # load model - self.model = model.QAModel.load(model_file) - self.model_file = model_file + torch.manual_seed(args.seed) + np.random.seed(args.seed) + random.seed(args.seed) - # load vectors - self.vec_dim = self._preload_cached_embeddings(word_embeddings_cache_file) - self.unk_term_vec = np.random.uniform(-0.25, 0.25, self.vec_dim) - - self.index = index_path + self.QID = data.Field(sequential=False) + self.QUESTION = data.Field(batch_first=True) + self.ANSWER = data.Field(batch_first=True) + self.LABEL = data.Field(sequential=False) + self.EXTERNAL = data.Field(sequential=False, tensor_type=torch.FloatTensor, batch_first=True, use_vocab=False, + preprocessing=data.Pipeline(lambda arr, _, train: [float(y) for y in arr])) - def _preload_cached_embeddings(self, cache_file): + if 'TrecQA' in args.dataset: + train, dev, test = TrecDataset.splits(self.QID, self.QUESTION, self.ANSWER, self.EXTERNAL, self.LABEL) + elif 'WikiQA' in args.dataset: + train, dev, test = WikiDataset.splits(self.QID, self.QUESTION, self.ANSWER, self.EXTERNAL, self.LABEL) + else: + print("Unsupported dataset") + exit() - with open(cache_file + '.dimensions') as d: - vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] + self.QID.build_vocab(train, dev, test) + self.QUESTION.build_vocab(train, dev, test) + self.ANSWER.build_vocab(train, dev, test) + self.LABEL.build_vocab(train, dev, test) + self.retrieveSentencesObj = RetrieveSentences(args) + self.idf_json = self.retrieveSentencesObj.getTermIdfJSON() - self.W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim)) + if args.cuda: + self.model = torch.load(args.model, map_location=lambda storage, location: storage.cuda(args.gpu)) + else: + self.model = torch.load(args.model, map_location=lambda storage, location: storage) - with open(cache_file + '.vocab') as f: - w2v_vocab_list = map(str.strip, f.readlines()) - - self.vocab_dict = {w:k for k, w in enumerate(w2v_vocab_list)} - return vec_dim - - - def parse(self, sentence, flags): + def parse(self, sentence): s_toks = TreebankWordTokenizer().tokenize(sentence) sentence = ' '.join(s_toks).lower() - - # model_input_args = self.model_file.split('.') - # punctuation = model_input_args[-3].split('-')[1] - # dash_words = model_input_args[-2].split('_')[1] - - if flags["dash_words"] == "split": - def split_hyphenated_words(sentence): - rtokens = [] - for term in sentence.split(): - for t in term.split('-'): - if t: - rtokens.append(t) - return ' '.join(rtokens) - sentence = split_hyphenated_words(sentence) - - if flags["punctuation"] == "remove": - regex = re.compile('[{}]'.format(re.escape(string.punctuation))) - def remove_punctuation(sentence): - rtokens = [] - for term in sentence.split(): - for t in regex.sub(' ', term).strip().split(): - if t: - rtokens.append(t) - return ' '.join(rtokens) - sentence = remove_punctuation(sentence) - return sentence - - def make_input_matrix(self, sentence): - terms = sentence.strip().split()[:60] - # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor) - word_embeddings = torch.zeros(len(terms), self.vec_dim).type(torch.DoubleTensor) - for i in range(len(terms)): - word = terms[i] - if word not in self.vocab_dict: - emb = torch.from_numpy(self.unk_term_vec) - else: - emb = torch.from_numpy(self.W[self.vocab_dict[word]]) - word_embeddings[i] = emb - input_tensor = torch.zeros(1, self.vec_dim, len(terms)) - input_tensor[0] = torch.transpose(word_embeddings, 0, 1) - return input_tensor - - - def get_tensorized_inputs(self, batch_ques, batch_sents, batch_ext_feats): - assert(1 == len(batch_ques)) - tensorized_inputs = [] - for i in range(len(batch_ques)): - xq = Variable(self.make_input_matrix(batch_ques[i])) - xs = Variable(self.make_input_matrix(batch_sents[i])) - ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) - ext_feats = torch.unsqueeze(ext_feats, 0) - tensorized_inputs.append((xq, xs, ext_feats)) - return tensorized_inputs - - def rerank_candidate_answers(self, question, answers, idf_json, flags): + def rerank_candidate_answers(self, question, answers): # run through the model scores_sentences = [] - question = self.parse(question, flags) - term_idfs = json.loads(idf_json) + question = self.parse(question) + term_idfs = json.loads(self.idf_json) term_idfs = dict((k, float(v)) for k, v in term_idfs.items()) for term in question.split(): @@ -121,7 +79,7 @@ class SMModelBridge(object): term_idfs[term] = 0.0 for answer in answers: - answer = self.parse(answer, flags) + answer = self.parse(answer) for term in answer.split(): if term not in term_idfs: term_idfs[term] = 0.0 @@ -132,73 +90,56 @@ class SMModelBridge(object): compute_overlap(stopped([question]), stopped([answer])) idf_weighted_overlap_no_stopwords =\ compute_idf_weighted_overlap(stopped([question]), stopped([answer]), term_idfs) - ext_feats = [np.array(feats) for feats in zip(overlap, idf_weighted_overlap,\ - overlap_no_stopwords, idf_weighted_overlap_no_stopwords)] + ext_feats = str(overlap[0]) + " " + str(idf_weighted_overlap[0]) + " " + \ + str(overlap_no_stopwords[0]) + " " + str(idf_weighted_overlap_no_stopwords[0]) - xq, xa, x_ext_feats = self.get_tensorized_inputs([question], [answer], \ - ext_feats)[0] - pred = self.model(xq, xa, x_ext_feats) - pred = torch.exp(pred) - scores_sentences.append((pred.data.squeeze()[1], answer)) + + fields = [('question', self.QUESTION), ('answer', self.ANSWER), ('ext_feat', self.EXTERNAL)] + example = data.Example.fromlist([question, answer, ext_feats], fields) + this_question = self.QUESTION.numericalize(self.QUESTION.pad([example.question]), args.gpu) + this_answer = self.ANSWER.numericalize(self.ANSWER.pad([example.answer]), args.gpu) + this_external = self.EXTERNAL.numericalize(self.EXTERNAL.pad([example.ext_feat]), args.gpu) + self.model.eval() + scores = self.model(this_question, this_answer, this_external) + scores_sentences.append((scores[:, 2].cpu().data.numpy(), answer)) return scores_sentences -def get_term_idf_json_list(index_path, sent_list): - gateway = JavaGateway() - index = gateway.jvm.java.lang.String(index_path) - pyserini = gateway.jvm.io.anserini.py4j.PyseriniEntryPoint() - pyserini.initializeWithIndex(index_path) - java_list = gateway.jvm.java.util.ArrayList() - - for l in sent_list: - java_list.add(l) - - json_object = pyserini.getTermIdfJSONs(java_list) - return json_object - if __name__ == "__main__": - ap = argparse.ArgumentParser(description="Bridge Demo. Produces scores in trec_eval format", + parser = argparse.ArgumentParser(description="Bridge Demo. Produces scores in trec_eval format", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - ap.add_argument('model', help="the path to the saved model file") - ap.add_argument('--word-embeddings-cache', help="the embeddings 'cache' file",\ - default='../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache') - ap.add_argument('index_path', help="the path to the source corpus index") - # ap.add_argument('--paper-ext-feats', action="store_true", \ - # help="external features as per the paper") - ap.add_argument('--dataset-folder', help="the QA dataset folder {TrecQA|WikiQA}", - default='../../data/TrecQA/') - ap.add_argument("--punctuation", choices=["keep", "remove"], default="keep") - ap.add_argument("--dash-words", choices=["keep", "split"], default="keep") + parser.add_argument('--model', help="the path to the saved model file") + parser.add_argument('--dataset', help="the QA dataset folder {TrecQA|WikiQA}", default='../../data/TrecQA/') + parser.add_argument("--index", help="Lucene index", required=True) + parser.add_argument("--embeddings", help="Path of the word2vec index", default="") + parser.add_argument("--topics", help="topics file", default="") + parser.add_argument("--query", help="a single query", default="where was newton born ?") + parser.add_argument("--hits", help="max number of hits to return", default=100) + parser.add_argument("--scorer", help="passage scores", default="Idf") + parser.add_argument("--k", help="top-k passages to be retrieved", default=1) + parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') + parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU + parser.add_argument('--seed', type=int, default=3435) - args = ap.parse_args() + args = parser.parse_args() - smmodel = SMModelBridge( - args.model, - args.word_embeddings_cache, - args.index_path - ) + if not args.cuda: + args.gpu = -1 + + smmodel = SMModelBridge(args) train_set, dev_set, test_set = 'train', 'dev', 'test' - if 'TrecQA' in args.dataset_folder: + if 'TrecQA' in args.dataset: train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test' - flags = { - "punctuation": args.punctuation, - "dash_words": args.dash_words - } - for split in [dev_set, test_set]: outfile = open('bridge.{}.scores'.format(split), 'w') - questions = [q.strip() for q in \ - open(os.path.join(args.dataset_folder, split, 'a.toks')).readlines()] - answers = [q.strip() for q in \ - open(os.path.join(args.dataset_folder, split, 'b.toks')).readlines()] - labels = [q.strip() for q in \ - open(os.path.join(args.dataset_folder, split, 'sim.txt')).readlines()] - qids = [q.strip() for q in \ - open(os.path.join(args.dataset_folder, split, 'id.txt')).readlines()] + questions = [q.strip() for q in open(os.path.join(args.dataset, split, 'a.toks')).readlines()] + answers = [q.strip() for q in open(os.path.join(args.dataset, split, 'b.toks')).readlines()] + labels = [q.strip() for q in open(os.path.join(args.dataset, split, 'sim.txt')).readlines()] + qids = [q.strip() for q in open(os.path.join(args.dataset, split, 'id.txt')).readlines()] qid_question = dict(zip(qids, questions)) q_counts = Counter(questions) @@ -207,23 +148,21 @@ if __name__ == "__main__": docid_counter = 0 all_questions_answers = questions + answers - idf_json = get_term_idf_json_list(args.index_path, all_questions_answers) - for qid, question in sorted(qid_question.items(), key=lambda x: float(x[0])): num_answers = q_counts[question] q_answers = answers[answers_offset: answers_offset + num_answers] answers_offset += num_answers - sentence_scores = smmodel.rerank_candidate_answers(question, q_answers, idf_json, flags) + sentence_scores = smmodel.rerank_candidate_answers(question, q_answers) for score, sentence in sentence_scores: print('{} Q0 {} 0 {} sm_cnn_bridge.{}.run'.format( qid, docid_counter, score, - os.path.basename(args.dataset_folder) + os.path.basename(args.dataset) ), file=outfile) docid_counter += 1 - if 'WikiQA' in args.dataset_folder: + if 'WikiQA' in args.dataset: docid_counter = 0 outfile.close() diff --git a/sm_modified_cnn/create_dataset.sh b/sm_cnn/create_dataset.sh similarity index 100% rename from sm_modified_cnn/create_dataset.sh rename to sm_cnn/create_dataset.sh diff --git a/sm_cnn/experimental_settings.py b/sm_cnn/experimental_settings.py deleted file mode 100644 index 77a4b8a..0000000 --- a/sm_cnn/experimental_settings.py +++ /dev/null @@ -1,160 +0,0 @@ -# The following 5 conditions vary -# idf_source, stopwords_and_stemming, punctuation, words-with-hyphens -import argparse -import itertools -import shlex -import subprocess - -class Setting(object): - def __init__(self, label, value_flag_map): - self.label = label - self.choice_flags = value_flag_map - - def get_settings(self): - return self.choice_flags.keys() - - def get_choice(self, setting): - return self.choice_flags[setting] - - def get_options(self): - options = [] - for key in self.choice_flags.keys(): - options.append("{}:{}".format(self.label, key)) - return options - -class Experiments(object): - def __init__(self, qa_dataset, word_embeddings_file): - self.settings = {} - self.combinations = [] - self.qa_data = qa_dataset - self.w2v_file = word_embeddings_file - #self.cmd_root = "python main.py --dataset_folder {} --word_vectors_file {} --run-name-prefix run --epochs 1 --num_conv_filters 5".format(self.qa_data, self.w2v_file) - self.cmd_root = "python main.py --dataset_folder {} --word_vectors_file {} --run-name-prefix run --paper-ext-feats".format(self.qa_data, self.w2v_file) - self.eval_cmd_root = "../../Anserini/eval/trec_eval.9.0/trec_eval -m map -m recip_rank -m bpref" - self.rbp_cmd_root = "rbp_eval" - - def add_setting(self, setting): - self.settings[setting.label] = setting - self._setup_combinations() - - - def _setup_combinations(self): - all_settings = [] - for setting in self.settings.values(): - all_settings.append(setting.get_options()) - - self.combinations = [] - for c in itertools.product(*all_settings): - self.combinations.append(c) - - def _run_cmd(self, cmd): - pargs = shlex.split(cmd) - p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, \ - bufsize=1, universal_newlines=True) - pout, perr = p.communicate() - return pout, perr - - def _run_eval(self): - for split in ['raw-dev', 'raw-test']: - cmd = '{} {}/{}.qrel run.{}.smrun'.format(self.eval_cmd_root, - self.qa_data, split, split) - out, err = self._run_cmd(cmd) - print(split, '------' ) - # trec_eval scores - metrics = [] - scores = [] - for line in str(out).split('\n'): - if not line.strip().split(): continue - fields = line.strip().split() - metrics.append(fields[0]) - scores.append(fields[-1]) - - # rbp_eval scores - cmd = '{} {}/{}.qrel run.{}.smrun'.format(self.rbp_cmd_root, - self.qa_data, split, split) - out, err = self._run_cmd(cmd) - for line in str(out).split('\n'): - if not line.startswith('p= 0.50'): continue - metrics.append('rbp_p0.5') - scores.append(' '.join(line.strip().split()[-2:])) - - print('\t'.join(metrics)) - print('\t'.join(scores)) - - - def run(self, indices): - """ - runs a particular combination of settings - """ - for ci in indices: - combo = self.combinations[ci] - print(combo) - cmd_args = [] - # set model name - model_name = 'sm_cnn.' - for setting_choice in combo: - setting, choice = setting_choice.split(':') - model_name += '{}-{}.'.format(setting, choice) - cmd_args.append(self.settings[setting].choice_flags[choice]) - model_name += 'model' - cmd = '{} {} {}'.format(self.cmd_root, ' '.join(cmd_args), model_name) - print(cmd) - out, err = self._run_cmd(cmd) - with open(model_name + '.log', 'w') as lf: - print('---------- OUT ------------', file=lf) - print(out, file=lf) - print('---------- ERR ------------', file=lf) - print(err, file=lf) - self._run_eval() - - def run_all(self): - """ - runs all experiments - """ - pass - - def list_settings(self): - """ - lists all settings - """ - for c in enumerate(self.combinations): - print(c) - print("--run X Y Z to run combinations number X Y Z") - - -if __name__ == "__main__": - ap = argparse.ArgumentParser(description="Lists exerimental settings and runs experiments") - ap.add_argument("--list", help="lists all available experimental settings combinations", - action="store_true") - ap.add_argument("--run", help='runs experimenal setting combination NUMBER(s)', - nargs="+", type=int) - ap.add_argument("indexPath", help="required for some combination of experiments") - ap.add_argument('qa_data', help="path to the QA dataset", - choices=['../../data/TrecQA', '../../data/WikiQA']) - ap.add_argument('word_embeddings_file', help="the word embeddings file") - - args = ap.parse_args() - - experiments = Experiments(args.qa_data, args.word_embeddings_file) - - experiments.add_setting(Setting('idf_source', { - 'qa-data':'', - 'corpus-index': '--index-for-corpusIDF {}'.format(args.indexPath) - })) - - experiments.add_setting(Setting('punctuation', { - 'keep': '', - 'remove': '--stop-punct' - })) - - experiments.add_setting(Setting('dash_words', { - 'keep': '', - 'split': '--dash-split' - })) - - experiments.list_settings() - - if args.run: - experiments.run(args.run) - - diff --git a/sm_cnn/main.py b/sm_cnn/main.py index a36a921..f4bbf8f 100644 --- a/sm_cnn/main.py +++ b/sm_cnn/main.py @@ -1,23 +1,15 @@ -import argparse - -import os -import shlex -import subprocess -import sys - import numpy as np -import pandas as pd -import torch - -import utils -from external_features import stopped, stemmed, compute_idf_weighted_overlap, compute_overlap,\ - get_qadata_only_idf, set_external_features_as_per_paper,\ - set_external_features_as_per_paper_and_stem -from train import Trainer -from model import QAModel - -# logging setup +import random import logging + +import torch +from torchtext import data + +from args import get_args +from utils.relevancy_metrics import get_map_mrr +from trec_dataset import TrecDataset +from wiki_dataset import WikiDataset + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @@ -27,185 +19,86 @@ formatter = logging.Formatter('%(levelname)s - %(message)s') ch.setFormatter(formatter) logger.addHandler(ch) -def logargs(func): - def inner(*args, **kwargs): - logger.info('%s : %s %s' % (func.__name__, args, kwargs)) - return func(*args, **kwargs) - return inner +args = get_args() +config = args + +torch.manual_seed(args.seed) + +if not args.cuda: + args.gpu = -1 +if torch.cuda.is_available() and args.cuda: + logger.info("Note: You are using GPU for training") + torch.cuda.set_device(args.gpu) + torch.cuda.manual_seed(args.seed) +if torch.cuda.is_available() and not args.cuda: + logger.info("Warning: You have Cuda but do not use it. You are using CPU for training") +np.random.seed(args.seed) +random.seed(args.seed) + +QID = data.Field(sequential=False) +QUESTION = data.Field(batch_first=True) +ANSWER = data.Field(batch_first=True) +LABEL = data.Field(sequential=False) +EXTERNAL = data.Field(sequential=True, tensor_type=torch.FloatTensor, batch_first=True, use_vocab=False, + postprocessing=data.Pipeline(lambda arr, _, train: [float(y) for y in arr])) + +if config.dataset == 'TREC': + train, dev, test = TrecDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) +elif config.dataset == 'wiki': + train, dev, test = WikiDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) +else: + print("Unsupported dataset") + exit() + +QID.build_vocab(train, dev, test) +QUESTION.build_vocab(train, dev, test) +ANSWER.build_vocab(train, dev, test) +LABEL.build_vocab(train, dev, test) + +train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False, + sort=False, shuffle=True) +dev_iter = data.Iterator(dev, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, + sort=False, shuffle=False) +test_iter = data.Iterator(test, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, + sort=False, shuffle=False) + +config.target_class = len(LABEL.vocab) +config.questions_num = len(QUESTION.vocab) +config.answers_num = len(ANSWER.vocab) +print("Label dict:", LABEL.vocab.itos) + +if args.cuda: + model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu)) +else: + model = torch.load(args.trained_model, map_location=lambda storage,location: storage) + +index2label = np.array(LABEL.vocab.itos) +index2qid = np.array(QID.vocab.itos) -def compute_map_mrr(dataset_folder, set_folder, test_scores, run_name_prefix=None): - # logger.info("Running trec_eval script...") - N = len(test_scores) +def predict(dataset, test_mode, dataset_iter): + model.eval() + dataset_iter.init_epoch() - qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder) + qids = [] + predictions = [] + labels = [] + for dev_batch_idx, dev_batch in enumerate(dataset_iter): + qid_array = index2qid[np.transpose(dev_batch.qid.cpu().data.numpy())] + true_label_array = index2label[np.transpose(dev_batch.label.cpu().data.numpy())] - # Call TrecEval code to calc MAP and MRR - df_submission = pd.DataFrame(index=np.arange(N), \ - columns=['qid', 'iter', 'docno', 'rank', 'sim', 'run_id']) - df_submission['qid'] = qids_test - df_submission['iter'] = 0 - df_submission['docno'] = np.arange(N) - df_submission['rank'] = 0 - df_submission['sim'] = test_scores - df_submission['run_id'] = 'smmodel' - df_submission.to_csv(os.path.join(dataset_folder, 'submission.txt'), \ - header=False, index=False, sep=' ') - if run_name_prefix: - df_submission.to_csv('{}.{}.smrun'.format(run_name_prefix, set_folder),\ - header=False, index=False, sep=" ") + scores = model(dev_batch.question, dev_batch.answer, dev_batch.ext_feat) + score_array = scores[:, 2].cpu().data.numpy() - df_gold = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rel']) - df_gold['qid'] = qids_test - df_gold['iter'] = 0 - df_gold['docno'] = np.arange(N) - df_gold['rel'] = y_test - df_gold.to_csv(os.path.join(args.dataset_folder, 'gold.txt'), header=False, index=False, sep=' ') + qids.extend(qid_array.tolist()) + predictions.extend(score_array.tolist()) + labels.extend(true_label_array.tolist()) - # subprocess.call("/bin/sh run_eval.sh '{}'".format(args.dataset_folder), shell=True) - pargs = shlex.split("/bin/sh run_eval.sh '{}'".format(dataset_folder)) - p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - pout, perr = p.communicate() + dev_map, dev_mrr = get_map_mrr(qids, predictions, labels) + logger.info("{} {}".format(dev_map, dev_mrr)) - lines = pout.split(b'\n') - map = float(lines[0].strip().split()[-1]) - mrr = float(lines[1].strip().split()[-1]) - return map, mrr +# Run the model on the dev set +predict(config.dataset, 'dev', dataset_iter=dev_iter) - -if __name__ == "__main__": - ap = argparse.ArgumentParser(description='pytorch port of the SM model', \ - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - ap.add_argument('model_outfile', help='file to save final model') - ap.add_argument('--word_vectors_file', \ - help='NOTE: a cache will be created for faster loading for word vectors',\ - default="../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin") - ap.add_argument('--dataset_folder', help='directory containing train, dev, test sets', \ - default="../../data/TrecQA") - ap.add_argument('--classes', type=int, default=2) - # external features related arguments - ap.add_argument('--no-ext-feats', action="store_true", \ - help="will not include external features in the model") - ap.add_argument('--paper-ext-feats', action="store_true", default=True, \ - help="external features as per the paper") - ap.add_argument('--paper-ext-feats-stem', action="store_true", \ - help="external features as per the paper") - # system arguments - ap.add_argument('--cuda', action='store_true', help='use CUDA if available') - ap.add_argument('--num_threads', help="the number of simultaneous processes to run", \ - type=int, default=4) - # training arguments - ap.add_argument('--batch_size', type=int, default=1, help="training mini-batch size") - ap.add_argument('--filter_width', type=int, default=5, help="number of convolution channels") - ap.add_argument('--eta', help='Initial learning rate', default=0.001, type=float) - ap.add_argument('--mom', help='SGD Momentum', default=0.0, type=float) - ap.add_argument('--train', help='switches to train set', action="store_true") - # epoch related arguments - ap.add_argument('--epochs', type=int, default=25, help="number of training epochs") - ap.add_argument('--patience', type=int, default=5, \ - help="if there is no appreciable change in model after epochs, then stop") - # debugging arguments - ap.add_argument('--debug_single_batch', action="store_true", \ - help="will stop program after training 1 input batch") - ap.add_argument('--num_conv_filters', default=100, type=int, \ - help="the number of convolution channels (lesser is faster)") - ap.add_argument('--no_loss_reg', help="no loss regularization", action="store_true") - ap.add_argument('--test_on_each_epoch', action="store_true", \ - help='runs test on each epoch to track final performance') - ap.add_argument("--skip-training", help="will load pre-trained model", action="store_true") - ap.add_argument("--run-name-prefix", help="will output train|dev|test runs with provided prefix") - ap.add_argument("--stop-punct", help='removes punctuation', action="store_true") - ap.add_argument("--dash-split", help="split words containing hyphens", action="store_true") - ap.add_argument("--index-for-corpusIDF", help="fetches idf from Index. provide index path. will\ - generate a vocabFile") - ap.add_argument('--seed', help='Random seed', type=int, default=1234) - ap.add_argument('--nocudnn', help='Disable the CuDNN backend', action="store_true") - - args = ap.parse_args() - - torch.manual_seed(args.seed) - np.random.seed(args.seed) - if args.cuda and torch.cuda.is_available(): - torch.cuda.manual_seed(args.seed) - if args.nocudnn: - torch.backends.cudnn.enabled = False - - torch.set_num_threads(args.num_threads) - - train_set, dev_set, test_set = 'train-all', 'raw-dev', 'raw-test' - if args.train: - train_set, dev_set, test_set = 'train', 'clean-dev', 'clean-test' - - # cache word embeddings - cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache' - utils.cache_word_embeddings(args.word_vectors_file, cache_file) - - vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file) - - # instantiate model - net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats, cuda=args.cuda) - - # initialize the trainer - trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim, args.cuda) - logger.info("Loading input data...") - # load input data - trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set) - logger.info("Setting up external features...") - # setup external features - # TODO: remember to update args.* in testing loop below - if args.paper_ext_feats: - logger.info("--paper-ext-feats") - ext_feats_for_splits = \ - set_external_features_as_per_paper(trainer, args.index_for_corpusIDF) - # ^^ we are saving the features to be used while testing at the end of training - elif args.paper_ext_feats_stem: - logger.info("--paper-ext-feats-stem") - ext_feats_for_splits = \ - set_external_features_as_per_paper_and_stem(trainer, args.index_for_corpusIDF) - - - if not args.skip_training: - best_map = 0.0 - best_model = 0 - - for i in range(args.epochs): - logger.info('------------- Training epoch {} --------------'.format(i+1)) - train_accuracy = trainer.train(train_set, args.batch_size, args.debug_single_batch) - if args.debug_single_batch: sys.exit(0) - - dev_scores = trainer.test(dev_set, args.batch_size) - - dev_map, dev_mrr = compute_map_mrr(args.dataset_folder, dev_set, dev_scores) - logger.info("------- MAP {}, MRR {}".format(dev_map, dev_mrr)) - - if dev_map - best_map > 1e-3: # new map is better than best map - best_model = i - best_map = dev_map - - QAModel.save(net, args.model_outfile) - logger.info('Achieved better dev_map ... saved model') - - if args.test_on_each_epoch: - test_scores = trainer.test(test_set, args.batch_size) - map, mrr = compute_map_mrr(args.dataset_folder, test_set, test_scores) - logger.info("------- MAP {}, MRR {}".format(map, mrr)) - - if (i - best_model) >= args.patience: - logger.warning('No improvement since the last {} epochs. Stopping training'\ - .format(i - best_model)) - break - - logger.info(' ------------ Training epochs completed! ------------') - logger.info('Best dev MAP in training phase = {:.4f}'.format(best_map)) - - trained_model = QAModel.load(args.model_outfile) - evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim, args.cuda) - - for split in [test_set, dev_set]: - evaluator.load_input_data(args.dataset_folder, cache_file, None, None, split) - if args.paper_ext_feats or args.paper_ext_feats_stem: - evaluator.data_splits[split][-1] = ext_feats_for_splits[split] - #set_external_features_as_per_paper(evaluator) - split_scores = evaluator.test(split, args.batch_size) - map, mrr = compute_map_mrr(args.dataset_folder, split, split_scores, args.run_name_prefix) - logger.info("-------{} MAP {}, MRR {}".format(split, map, mrr)) +# Run the model on the test set +predict(config.dataset, 'test', dataset_iter=test_iter) diff --git a/sm_cnn/model.py b/sm_cnn/model.py index 3fe3755..42085ea 100644 --- a/sm_cnn/model.py +++ b/sm_cnn/model.py @@ -1,92 +1,79 @@ import torch import torch.nn as nn + import torch.nn.functional as F -# logging setup -import logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +class SmPlusPlus(nn.Module): + def __init__(self, config): + super(SmPlusPlus, self).__init__() + output_channel = config.output_channel + questions_num = config.questions_num + answers_num = config.answers_num + words_dim = config.words_dim + filter_width = config.filter_width + self.mode = config.mode -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) + n_classes = config.target_class + ext_feats_size = 4 - -class QAModel(nn.Module): - - @staticmethod - def save(model, model_fname): - torch.save(model, model_fname) - - - @staticmethod - def load(model_fname): - return torch.load(model_fname) - - - def __init__(self, input_n_dim, filter_width, \ - conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2, cuda=False): - super(QAModel, self).__init__() - - self.no_ext_feats = no_ext_feats - - self.conv_channels = conv_filters - n_hidden = 2*self.conv_channels + (0 if no_ext_feats else ext_feats_size) - - self.conv_q = nn.Sequential( - nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1), - nn.Tanh() - ) - - self.conv_a = nn.Sequential( - nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1), - nn.Tanh() - ) - - self.combined_feature_vector = nn.Linear(2*self.conv_channels + \ - (0 if no_ext_feats else ext_feats_size), n_hidden) - - self.combined_features_activation = nn.Tanh() - self.dropout = nn.Dropout(0.5) - self.hidden = nn.Linear(n_hidden, n_classes) - self.logsoftmax = nn.LogSoftmax() - - if cuda and torch.cuda.is_available(): - self.conv_q, self.conv_a = self.conv_q.cuda(), self.conv_a.cuda() - self.combined_feature_vector = self.combined_feature_vector.cuda() - self.combined_features_activation = self.combined_features_activation.cuda() - self.dropout, self.hidden, self.logsoftmax = self.dropout.cuda(), self.hidden.cuda(), self.logsoftmax.cuda() - - def forward(self, question, answer, ext_feats): - - q = self.conv_q.forward(question) - q = F.max_pool1d(q, q.size()[2]) - q = q.view(-1, self.conv_channels) - # logger.debug('forward q: {}'.format(q)) - - a = self.conv_a.forward(answer) - a = F.max_pool1d(a, a.size()[2]) - a = a.view(-1, self.conv_channels) - - x = None - if self.no_ext_feats: - x = torch.cat([q, a], 1) - # logger.debug('no_ext_feats') + if self.mode == 'multichannel': + input_channel = 2 else: - x = torch.cat([q, a, ext_feats], 1) - # logger.debug('with ext_feats') + input_channel = 1 - # logger.debug('featvec x: {}'.format(x)) - # logger.debug(x.creator) + self.question_embed = nn.Embedding(questions_num, words_dim) + self.answer_embed = nn.Embedding(answers_num, words_dim) + self.static_question_embed = nn.Embedding(questions_num, words_dim) + self.nonstatic_question_embed = nn.Embedding(questions_num, words_dim) + self.static_answer_embed = nn.Embedding(answers_num, words_dim) + self.nonstatic_answer_embed = nn.Embedding(answers_num, words_dim) + self.static_question_embed.weight.requires_grad = False + self.static_answer_embed.weight.requires_grad = False - x = self.combined_feature_vector.forward(x) - x = self.combined_features_activation.forward(x) + self.conv_q = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0)) + self.conv_a = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0)) + + self.dropout = nn.Dropout(config.dropout) + n_hidden = 2 * output_channel + ext_feats_size + + self.combined_feature_vector = nn.Linear(n_hidden, n_hidden) + self.hidden = nn.Linear(n_hidden, n_classes) + + def forward(self, x_question, x_answer, x_ext): + if self.mode == 'rand': + question = self.question_embed(x_question).unsqueeze(1) + answer = self.answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + # actual SM model mode (Severyn & Moschitti, 2015) + elif self.mode == 'static': + question = self.static_question_embed(x_question).unsqueeze(1) + answer = self.static_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + elif self.mode == 'non-static': + question = self.nonstatic_question_embed(x_question).unsqueeze(1) + answer = self.nonstatic_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + elif self.mode == 'multichannel': + question_static = self.static_question_embed(x_question) + answer_static = self.static_answer_embed(x_answer) + question_nonstatic = self.nonstatic_question_embed(x_question) + answer_nonstatic = self.nonstatic_answer_embed(x_answer) + question = torch.stack([question_static, question_nonstatic], dim=1) + answer = torch.stack([answer_static, answer_nonstatic], dim=1) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + else: + print("Unsupported Mode") + exit() + + # append external features and feed to fc + x.append(x_ext) + x = torch.cat(x, 1) + + x = F.tanh(self.combined_feature_vector(x)) x = self.dropout(x) x = self.hidden(x) - x = self.logsoftmax(x) - - return x - - + return x \ No newline at end of file diff --git a/sm_cnn/nn-architecture.png b/sm_cnn/nn-architecture.png deleted file mode 100644 index 3734c85..0000000 Binary files a/sm_cnn/nn-architecture.png and /dev/null differ diff --git a/sm_modified_cnn/overlap_features.py b/sm_cnn/overlap_features.py similarity index 100% rename from sm_modified_cnn/overlap_features.py rename to sm_cnn/overlap_features.py diff --git a/sm_cnn/requirements.txt b/sm_cnn/requirements.txt index 20759b0..0c28e6b 100644 --- a/sm_cnn/requirements.txt +++ b/sm_cnn/requirements.txt @@ -1,5 +1,4 @@ -gensim==1.0.1 -nltk==3.2.1 -numpy==1.11.3 -pandas==0.19.2 -pytorch==0.1.12 +nltk==3.2.4 +numpy==1.13.1 +pytorch==0.2.0 +torchtext==0.2.0 diff --git a/sm_cnn/run_eval.sh b/sm_cnn/run_eval.sh deleted file mode 100755 index 21202d6..0000000 --- a/sm_cnn/run_eval.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/csh -f - -exp_dir=$1 -judgement=${exp_dir}/gold.txt -output=${exp_dir}/submission.txt - -./trec_eval-8.0/trec_eval -q -c ${judgement} ${output} > ${output}.treceval -tail -29 ${output}.treceval | grep -e 'map' -e 'recip_rank' -exit 0 diff --git a/sm_cnn/train.py b/sm_cnn/train.py index 4af588e..768cedc 100644 --- a/sm_cnn/train.py +++ b/sm_cnn/train.py @@ -1,261 +1,210 @@ import time - +import os import numpy as np +import random + import torch import torch.nn as nn -import torch.optim as optim -from torch.autograd import Variable +from torchtext import data -import utils +from args import get_args +from model import SmPlusPlus +from utils.relevancy_metrics import get_map_mrr +from trec_dataset import TrecDataset +from wiki_dataset import WikiDataset +from evaluate import evaluate -# logging setup -import logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +args = get_args() +config = args -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) +torch.manual_seed(args.seed) + +def set_vectors(field, vector_path): + if os.path.isfile(vector_path): + stoi, vectors, dim = torch.load(vector_path) + field.vocab.vectors = torch.Tensor(len(field.vocab), dim) + + for i, token in enumerate(field.vocab.itos): + wv_index = stoi.get(token, None) + if wv_index is not None: + field.vocab.vectors[i] = vectors[wv_index] + else: + # initialize with U(-0.25, 0.25) vectors + field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.25, 0.25) + else: + print("Error: Need word embedding pt file") + exit(1) + return field + +# Set default configuration in : args.py +args = get_args() +config = args + +# Set random seed for reproducibility +torch.manual_seed(args.seed) +torch.backends.cudnn.deterministic = True +if not args.cuda: + args.gpu = -1 +if torch.cuda.is_available() and args.cuda: + print("Note: You are using GPU for training") + torch.cuda.set_device(args.gpu) + torch.cuda.manual_seed(args.seed) +if torch.cuda.is_available() and not args.cuda: + print("You have Cuda but you're using CPU for training.") +np.random.seed(args.seed) +random.seed(args.seed) + +QID = data.Field(sequential=False) +QUESTION = data.Field(batch_first=True) +ANSWER = data.Field(batch_first=True) +LABEL = data.Field(sequential=False) +EXTERNAL = data.Field(sequential=True, tensor_type=torch.FloatTensor, batch_first=True, use_vocab=False, + postprocessing=data.Pipeline(lambda arr, _, train: [float(y) for y in arr])) + +if config.dataset == 'TREC': + train, dev, test = TrecDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) +elif config.dataset == 'wiki': + train, dev, test = WikiDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) +else: + print("Unsupported dataset") + exit() + +QID.build_vocab(train, dev, test) +QUESTION.build_vocab(train, dev, test) +ANSWER.build_vocab(train, dev, test) +LABEL.build_vocab(train, dev, test) -class Trainer(object): +QUESTION = set_vectors(QUESTION, args.vector_cache) +ANSWER = set_vectors(ANSWER, args.vector_cache) - def __init__(self, model, eta, mom, no_loss_reg, vec_dim, cuda=False): - # set the random seeds for every instance of trainer. - # needed to ensure reproduction of random word vectors for out of vocab terms - torch.manual_seed(1234) - np.random.seed(1234) - self.cuda = cuda - self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim) +train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False, + sort=False, shuffle=True) +dev_iter = data.Iterator(dev, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, + sort=False, shuffle=False) +test_iter = data.Iterator(test, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, + sort=False, shuffle=False) - self.reg = 1e-5 - self.no_loss_reg = no_loss_reg - self.model = model - self.criterion = nn.CrossEntropyLoss() - #self.criterion = nn.NLLLoss() - self.optimizer = optim.SGD(self.model.parameters(), lr=eta, momentum=mom, \ - weight_decay=(0 if no_loss_reg else self.reg)) +config.target_class = len(LABEL.vocab) +config.questions_num = len(QUESTION.vocab) +config.answers_num = len(ANSWER.vocab) - self.data_splits = {} - self.embeddings = {} - self.vec_dim = vec_dim +print("Dataset {} Mode {}".format(args.dataset, args.mode)) +print("VOCAB num", len(QUESTION.vocab)) +print("LABEL.target_class:", len(LABEL.vocab)) +print("LABELS:", LABEL.vocab.itos) +print("Train instance", len(train)) +print("Dev instance", len(dev)) +print("Test instance", len(test)) + +if args.resume_snapshot: + if args.cuda: + model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage.cuda(args.gpu)) + else: + model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage) +else: + model = SmPlusPlus(config) + model.static_question_embed.weight.data.copy_(QUESTION.vocab.vectors) + model.nonstatic_question_embed.weight.data.copy_(QUESTION.vocab.vectors) + model.static_answer_embed.weight.data.copy_(ANSWER.vocab.vectors) + model.nonstatic_answer_embed.weight.data.copy_(ANSWER.vocab.vectors) + + if args.cuda: + model.cuda() + print("Shift model to GPU") - def load_input_data(self, dataset_root_folder, word_vectors_cache_file, \ - train_set_folder, dev_set_folder, test_set_folder, load_ext_feats=True): - for set_folder in [test_set_folder, dev_set_folder, train_set_folder]: - if set_folder: - questions, sentences, labels, maxlen_q, maxlen_s, vocab = \ - utils.read_in_dataset(dataset_root_folder, set_folder) +parameter = filter(lambda p: p.requires_grad, model.parameters()) - self.data_splits[set_folder] = [questions, sentences, labels, maxlen_q, maxlen_s] +# the SM model originally follows SGD but Adadelta is used here +optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay) +criterion = nn.CrossEntropyLoss() +early_stop = False +best_dev_map = 0 +iterations = 0 +iters_not_improved = 0 +epoch = 0 +start = time.time() +header = ' Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss Accuracy Dev/Accuracy' +dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(',')) +log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(',')) +os.makedirs(args.save_path, exist_ok=True) +os.makedirs(os.path.join(args.save_path, args.dataset), exist_ok=True) +print(header) - default_ext_feats = [np.zeros(4)] * len(self.data_splits[set_folder][0]) - self.data_splits[set_folder].append(default_ext_feats) +index2label = np.array(LABEL.vocab.itos) +index2qid = np.array(QID.vocab.itos) +index2question = np.array(ANSWER.vocab.itos) - utils.load_cached_embeddings(word_vectors_cache_file, vocab, self.embeddings, - [] if "train" in set_folder else self.unk_term) +while True: + if early_stop: + print("Early Stopping. Epoch: {}, Best Dev Acc: {}".format(epoch, best_dev_map)) + break + epoch += 1 + train_iter.init_epoch() + n_correct, n_total = 0, 0 + for batch_idx, batch in enumerate(train_iter): + iterations += 1 + model.train(); optimizer.zero_grad() + scores = model(batch.question, batch.answer, batch.ext_feat) + n_correct += (torch.max(scores, 1)[1].view(batch.label.size()).data == batch.label.data).sum() + n_total += batch.batch_size + train_acc = 100. * n_correct / n_total - def regularize_loss(self, loss): - - flattened_params = [] - - for p in self.model.parameters(): - f = p.data.clone() - flattened_params.append(f.view(-1)) - - fp = torch.cat(flattened_params) - - loss = loss + 0.5 * self.reg * fp.norm() * fp.norm() - - # for p in self.model.parameters(): - # loss = loss + 0.5 * self.reg * p.norm() * p.norm() - - return loss - - - def _train(self, xq, xa, ext_feats, ys): - - self.optimizer.zero_grad() - output = self.model(xq, xa, ext_feats) - loss = self.criterion(output, ys) - # logger.debug('loss after criterion {}'.format(loss)) - - # NOTE: regularizing location 1 - if not self.no_loss_reg: - loss = self.regularize_loss(loss) - # logger.debug('loss after regularizing {}'.format(loss)) - + loss = criterion(scores, batch.label) loss.backward() + optimizer.step() - # logger.debug('AFTER backward') - #logger.debug('params {}'.format([p for p in self.model.parameters()])) - # logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()])) + # Evaluate performance on validation set + if iterations % args.dev_every == 1: + # switch model into evaluation mode + model.eval() + dev_iter.init_epoch() + n_dev_correct = 0 + dev_losses = [] - # NOTE: regularizing location 2. It would seem that location 1 is correct? - #if not self.no_loss_reg: - # loss = self.regularize_loss(loss) - # logger.debug('loss after regularizing {}'.format(loss)) + qids = [] + predictions = [] + labels = [] + for dev_batch_idx, dev_batch in enumerate(dev_iter): + qid_array = index2qid[np.transpose(dev_batch.qid.cpu().data.numpy())] + true_label_array = index2label[np.transpose(dev_batch.label.cpu().data.numpy())] - self.optimizer.step() + scores = model(dev_batch.question, dev_batch.answer, dev_batch.ext_feat) + n_dev_correct += (torch.max(scores, 1)[1].view(dev_batch.label.size()).data == dev_batch.label.data).sum() + dev_loss = criterion(scores, dev_batch.label) + dev_losses.append(dev_loss.data[0]) + index_label = np.transpose(torch.max(scores, 1)[1].view(dev_batch.label.size()).cpu().data.numpy()) + label_array = index2label[index_label] + # get the relevance scores + score_array = scores[:, 2].cpu().data.numpy() - # logger.debug('AFTER step') - #logger.debug('params {}'.format([p for p in self.model.parameters()])) - # logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()])) + qids.extend(qid_array.tolist()) + predictions.extend(score_array.tolist()) + labels.extend(true_label_array.tolist()) - return loss.data[0], self.pred_equals_y(output, ys) + dev_map, dev_mrr = get_map_mrr(qids, predictions, labels) + print(dev_log_template.format(time.time() - start, + epoch, iterations, 1 + batch_idx, len(train_iter), + 100. * (1 + batch_idx) / len(train_iter), loss.data[0], + sum(dev_losses) / len(dev_losses), train_acc, dev_map)) + # Update validation results + if dev_map > best_dev_map: + iters_not_improved = 0 + best_dev_map = dev_map + snapshot_path = os.path.join(args.save_path, args.dataset, args.mode+'_best_model.pt') + torch.save(model, snapshot_path) + else: + iters_not_improved += 1 + if iters_not_improved >= args.patience: + early_stop = True + break - def pred_equals_y(self, pred, y): - _, best = pred.max(1) - best = best.data.long().squeeze() - return torch.sum(y.data.long() == best) - - - def test(self, set_folder, batch_size): - logger.info('----- Predictions on {} '.format(set_folder)) - - questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = \ - self.data_splits[set_folder] - word_vectors, vec_dim = self.embeddings, self.vec_dim - - self.model.eval() - - batch_size = 1 - - total_loss = 0.0 - total_correct = 0.0 - num_batches = np.ceil(len(questions)/batch_size) - y_pred = np.zeros(len(questions)) - ypc = 0 - - for k in range(int(num_batches)): - batch_start = k * batch_size - batch_end = (k+1) * batch_size - # convert raw questions and sentences to tensors - batch_inputs, batch_labels = self.get_tensorized_inputs( - questions[batch_start:batch_end], - sentences[batch_start:batch_end], - labels[batch_start:batch_end], - ext_feats[batch_start:batch_end], - word_vectors, vec_dim - ) - - xq, xa, x_ext_feats = batch_inputs[0] - y = batch_labels[0] - - pred = self.model(xq, xa, x_ext_feats) - loss = self.criterion(pred, y) - pred = torch.exp(pred) - total_loss += loss - # total_correct += self.pred_equals_y(pred, y) - - y_pred[ypc] = pred.data.squeeze()[1] - # ^ we want to score for relevance, NOT the predicted class - ypc += 1 - - # logger.info('{}_correct {}'.format(set_folder, total_correct)) - # logger.info('{}_loss {}'.format(set_folder, total_loss.data[0])) - logger.info('{} total {}'.format(set_folder, len(labels))) - # logger.info('{}_loss = {:.4f}, acc = {:.4f}'.format(set_folder, total_loss.data[0]/len(labels), float(total_correct)/len(labels)) - #logger.info('{}_loss = {:.4f}'.format(set_folder, total_loss.data[0]/len(labels))) - - return y_pred - - - def train(self, set_folder, batch_size, debug_single_batch): - train_start_time = time.time() - - questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = \ - self.data_splits[set_folder] - word_vectors, vec_dim = self.embeddings, self.vec_dim - - # set model for training modep - self.model.train() - - train_loss, train_correct = 0., 0. - num_batches = np.ceil(len(questions)/float(batch_size)) - - for k in range(int(num_batches)): - batch_start = k * batch_size - batch_end = (k+1) * batch_size - - # convert raw questions and sentences to tensors - batch_inputs, batch_labels = self.get_tensorized_inputs( - questions[batch_start:batch_end], - sentences[batch_start:batch_end], - labels[batch_start:batch_end], - ext_feats[batch_start:batch_end], - word_vectors, vec_dim - ) - - xq, xa, x_ext_feats = batch_inputs[0] - - ys = batch_labels[0] - - batch_loss, batch_correct = self._train(xq, xa, x_ext_feats, ys) - - # logger.debug('batch_loss {}, batch_correct {}'.format(batch_loss, batch_correct)) - train_loss += batch_loss - # train_correct += batch_correct - if debug_single_batch: - break - - # logger.info('train_correct {}'.format(train_correct)) - logger.info('train_loss {}'.format(train_loss)) - logger.info('total training batches = {}'.format(num_batches)) - logger.info('train_loss = {:.4f}'.format( - train_loss/num_batches - )) - logger.info('training time = {:.3f} seconds'.format(time.time() - train_start_time)) - return train_correct/num_batches - - - def make_input_matrix(self, sentence, word_vectors, vec_dim): - terms = sentence.strip().split()[:60] - # NOTE: we are truncating the inputs to 60 words. - - word_embeddings = torch.zeros(len(terms), vec_dim).type(torch.DoubleTensor) - for i in range(len(terms)): - word = terms[i] - emb = torch.from_numpy(word_vectors[word]) - word_embeddings[i] = emb - - input_tensor = torch.zeros(1, vec_dim, len(terms)) - input_tensor[0] = torch.transpose(word_embeddings, 0, 1) - if self.cuda and torch.cuda.is_available(): - input_tensor = input_tensor.cuda() - return input_tensor - - - def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, \ - word_vectors, vec_dim): - batch_size = len(batch_ques) - # NOTE: ideal batch size is one, because sentences are all of different length. - # In other words, we have no option but to feed in sentences one by one into the model - # and compute loss at the end. - - # TODO: what if the sentences in a batch are all of different lengths? - # - should be have the longest sentence as 2nd dim? - # - would zero endings work for other smaller sentences? - - y = torch.LongTensor(batch_size).type(torch.LongTensor) - if self.cuda and torch.cuda.is_available(): - y = y.cuda() - - tensorized_inputs = [] - for i in range(len(batch_ques)): - xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim)) - xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim)) - ext_feats = torch.FloatTensor(batch_ext_feats[i]) - if self.cuda and torch.cuda.is_available(): - ext_feats = ext_feats.cuda() - ext_feats = Variable(ext_feats) - ext_feats = torch.unsqueeze(ext_feats, 0) - y[i] = batch_labels[i] - tensorized_inputs.append((xq, xs, ext_feats)) - - return tensorized_inputs, Variable(y) + if iterations % args.log_every == 1: + # print progress message + print(log_template.format(time.time() - start, + epoch, iterations, 1 + batch_idx, len(train_iter), + 100. * (1 + batch_idx) / len(train_iter), loss.data[0], ' ' * 8, + n_correct / n_total * 100, ' ' * 12)) diff --git a/sm_modified_cnn/trec_dataset.py b/sm_cnn/trec_dataset.py similarity index 67% rename from sm_modified_cnn/trec_dataset.py rename to sm_cnn/trec_dataset.py index 3621c49..9298475 100644 --- a/sm_modified_cnn/trec_dataset.py +++ b/sm_cnn/trec_dataset.py @@ -1,16 +1,14 @@ from torchtext import data -import os class TrecDataset(data.TabularDataset): dirname = 'data' @classmethod - def splits(cls, question_id, question_field, answer_field, external_field, label_field, - train='train.tsv', validation='dev.tsv', test='test.tsv'): + def splits(cls, question_id, question_field, answer_field, external_field, label_field, root='.data', + train='trecqa.train.tsv', validation='trecqa.dev.tsv', test='trecqa.test.tsv'): path = './data' - prefix_name = 'trecqa.' return super(TrecDataset, cls).splits( - os.path.join(path, prefix_name), train, validation, test, + path, root, train, validation, test, format='TSV', fields=[('qid', question_id), ('label', label_field), ('question', question_field), ('answer', answer_field), ('ext_feat', external_field)] ) diff --git a/sm_cnn/trec_eval-8.0/Makefile b/sm_cnn/trec_eval-8.0/Makefile deleted file mode 100755 index f58087b..0000000 --- a/sm_cnn/trec_eval-8.0/Makefile +++ /dev/null @@ -1,141 +0,0 @@ -BIN = /home/smart/bin -H = . - -VERSIONID = 8.0 - -# gcc -CC = gcc -CFLAGS = -g -I$H -O3 -Wall -DVERSIONID=\"$(VERSIONID)\" -CFLAGS = -g -I$H -Wall -DVERSIONID=\"$(VERSIONID)\" - -# cc -###CC = cc -###CFLAGS = -I$H -g -DVERSIONID=\"$(VERSIONID)\" - -# Other macros used in some or all makefiles -INSTALL = /bin/mv - - -OBJS = trec_eval.o get_qrels.o get_top.o form_trvec.o measures.o print_meas.o\ - trvec_teval.o buf_util.o error_msgs.o \ - trec_eval_help.o - - -SRCS = trec_eval.c get_qrels.c get_top.c form_trvec.c measures.c print_meas.c\ - trvec_teval.c buf_util.c error_msgs.c \ - trec_eval_help.c - -SRCH = common.h trec_eval.h smart_error.h sysfunc.h tr_vec.h buf.h - -SRCOTHER = README Makefile test bpref_bug - -trec_eval: $(SRCS) Makefile $(SRCH) - $(CC) $(CFLAGS) -o trec_eval $(SRCS) -lm - -install: $(BIN)/trec_eval - -quicktest: trec_eval - ./trec_eval test/qrels.test test/results.test | diff - test/out.test - ./trec_eval -a test/qrels.test test/results.test | diff - test/out.test.a - ./trec_eval -a -q test/qrels.test test/results.test | diff - test/out.test.aq - ./trec_eval -a -q -c test/qrels.test test/results.trunc | diff - test/out.test.aqc - ./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc | diff - test/out.test.aqcM - /bin/echo "Test succeeeded" - -longtest: trec_eval - /bin/rm -rf test.long; mkdir test.long - ./trec_eval test/qrels.test test/results.test > test.long/out.test - ./trec_eval -a test/qrels.test test/results.test > test.long/out.test.a - ./trec_eval -a -q test/qrels.test test/results.test > test.long/out.test.aq - ./trec_eval -a -q -c test/qrels.test test/results.trunc > test.long/out.test.aqc - ./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc > test.long/out.test.aqcM - diff test.long test - -$(BIN)/trec_eval: trec_eval - if [ -f $@ ]; then $(INSTALL) $@ $@.old; fi; - $(INSTALL) trec_eval $@ - -##4########################################################################## -##5########################################################################## -# All code below this line (except for automatically created dependencies) -# is independent of this particular makefile, and should not be changed! -############################################################################# - -######################################################################### -# Odds and ends # -######################################################################### -clean semiclean: - /bin/rm -f *.o *.BAK *~ trec_eval trec_eval.*.shar out.trec_eval Makefile.bak - -shar: - shar -X $(SRCOTHER) $(SRCS) $(SRCH) > trec_eval.$(VERSIONID).shar - -lint: - lint $(SRCS) - -######################################################################### -# Determining program dependencies # -######################################################################### -depend: - grep '^#[ ]*include' *.c \ - | sed -e 's?:[^"]*"\([^"]*\)".*?: \$H/\1?' \ - -e '/ makedep - echo '/^# DO NOT DELETE THIS LINE/+2,$$d' >eddep - echo '$$r makedep' >>eddep - echo 'w' >>eddep - cp Makefile Makefile.bak - ed - Makefile < eddep - /bin/rm eddep makedep - echo '# DEPENDENCIES MUST END AT END OF FILE' >> Makefile - echo '# IF YOU PUT STUFF HERE IT WILL GO AWAY' >> Makefile - echo '# see make depend above' >> Makefile - -# DO NOT DELETE THIS LINE -- make depend uses it - -buf_util.o: ./common.h -buf_util.o: ./sysfunc.h -buf_util.o: ./buf.h -error_msgs.o: ./smart_error.h -error_msgs.o: ./sysfunc.h -form_trvec.o: ./common.h -form_trvec.o: ./sysfunc.h -form_trvec.o: ./smart_error.h -form_trvec.o: ./tr_vec.h -form_trvec.o: ./trec_eval.h -form_trvec.o: ./buf.h -get_qrels.o: ./common.h -get_qrels.o: ./sysfunc.h -get_qrels.o: ./smart_error.h -get_qrels.o: ./trec_eval.h -get_top.o: ./common.h -get_top.o: ./sysfunc.h -get_top.o: ./smart_error.h -get_top.o: ./trec_eval.h -measures.o: ./common.h -measures.o: ./sysfunc.h -measures.o: ./buf.h -measures.o: ./trec_eval.h -print_meas.o: ./common.h -print_meas.o: ./sysfunc.h -print_meas.o: ./buf.h -print_meas.o: ./trec_eval.h -trec_eval.o: ./common.h -trec_eval.o: ./sysfunc.h -trec_eval.o: ./smart_error.h -trec_eval.o: ./tr_vec.h -trec_eval.o: ./trec_eval.h -trec_eval.o: ./buf.h -trec_eval_help.o: ./common.h -trvec_teval.o: ./common.h -trvec_teval.o: ./sysfunc.h -trvec_teval.o: ./smart_error.h -trvec_teval.o: ./tr_vec.h -trvec_teval.o: ./trec_eval.h -# DEPENDENCIES MUST END AT END OF FILE -# IF YOU PUT STUFF HERE IT WILL GO AWAY -# see make depend above diff --git a/sm_cnn/trec_eval-8.0/README b/sm_cnn/trec_eval-8.0/README deleted file mode 100755 index 6bc8722..0000000 --- a/sm_cnn/trec_eval-8.0/README +++ /dev/null @@ -1,385 +0,0 @@ -trec_eval is the standard tool used by the TREC community for -evaluating an ad hoc retrieval run, given the results file and a -standard set of judged results. - ------------------------------------------------------------------------------- -Installation: Should be as easy as typing "make" in the source directory, -if gcc is available. Otherwise, comment out the gcc lines (lines 5-6) and -uncomment out the cc lines (lines 9-10) -If you wish the trec_eval binary to be placed in a standard location, alter -the first line of Makefile appropriately. - ------------------------------------------------------------------------------- -Testing: sample input and output files are included in the directory test. -"make quicktest" will perform some sample simple evaluations and compare -the results. - ------------------------------------------------------------------------------- -Usage: Most options can be ignored. The only one most folks will need -is the "-q" flag, to indicate whether to output results for individual -queries as well as the averages over all queries. Official TREC usage -might be something like - trec_eval -q -c -M1000 official_qrels submitted_results -to ensure correct evaluation if submitted_results doesn't have results -for all queries, or returns more than 1000 documents per query. - - ------------------------------------------------------------------------------- -Change Log ------------------------------------------------------------------------------- -Version 8.0, full bpref bug fix, see file bpref_bug. I decided to up the - version number since bpref results are incompatible with previous - results (though the changes are small). - -11/8/05: Bpref_bug: New file explaining bug and impact (conclusions after - rerunning all of SIGIR 2004 bpref paper experiments). -11/5/05: Added new measures: micro_prec, micro_recall, micro_bpref. I thought - I had an application for micro_bpref averaging (summing components of - measure over all docs (ignoring topics) and then computing measure), - but micro_bpref still proved a rotten measure. Left code in case - someone ever actually finds an application for valid micro averaging. -11/5/05: Added new measures: old_bpref, old_bpref_top10pRnonrel. These are - the old buggy measures included only for backward comparisons. -11/5/05: trvec_teval.c: Broke apart old trvec_trec_eval to calculate - different types of measures separately. Very hard to decipher - old code (though still difficult with new code) since parts of - the calculations for a measure were so far apart. - ------------------------------------------------------------------------------- -Version 7.4, minor changes from 7.3 - -11/4/05: trvec_teval.c: fixed bpref bug if very low (< R) numbers of non-rel - judgements available (divided by num_nonrel_ret instead of - num_nonrel). (pointed out by Ian Soboroff). -11/3/05: trvec_teval.c: bpref_10, bpref_5 had zero division problems if - no rel docs were retrieved. (pointed out by Ian Soboroff). -10/23/05: form_trvec.c: Added check for duplicate docno's in results and qrels. - (pointed out by Shlomo Geva. Default behavior used to be that - duplicate result docno's were always non-rel, but that changed in - later versions, so had better test explicitly for it and complain). -10/23/05: README: sample invocation of trec_eval had arguments reversed. - (pointed out by Carol Peters). -10/23/05: moved gm_ap to be a major measure (always printed). changed - measures.c, test/out*, README - ------------------------------------------------------------------------------- - -Version 7.3, a reasonably major rewrite from earlier versions in terms -of internal structure and default output format (now relational), but -the input format and measures calculated remain the same (or at least -upward compatible). - ------------------------------------------------------------------------------- -end of ChangeLog ------------------------------------------------------------------------------- - -Adding measures: To add a new measure: - 1. Add space for the measure in TREC_EVAL structure of "trec_eval.h" - 2. Add description of measure in "measures.c". See "trec_eval.h" for - definition of the fields. This description is used for - printing, accumulating, and averaging the measure values. - 3. Calculate the measure in "trvec_teval.c". Unfortunately, this has - gotten very long over the years as more measures are added, but - most of it can be ignored. I should write a simple version of - just the "short" measures so the structure can be seen better. - ------------------------------------------------------------------------------- - -Files: -Makefile Compile and test trec_eval -README This file -test Collection of sample input and output for trec_eval -trec_eval.c Main procedure -get_qrels.c Called by main to read the standard judged documents (qrels) -get_top.c Called by main to read the results file to be evaluated -form_trvec.c Called by main to put the results and qrels for an individual - query in the proper format to be evaluated. -trvec_teval.c Called by main to evaluate an individual query -print_meas.c Called by main to print an evaluated query, and to accumulate - the results for later averaging over the queries. -measures.c Description of the measures used by printing. -trec_eval_help.c Descriptions of trec_eval, the output, and the measures. - -trec_eval.h Basic evaluation structures. -bpref_bug: Description of bug in bpref that existed in trec_eval versions 6 - through 7.3. - -The rest of the files are small utility portions from SMART. -tr_vec.h -smart_error.h -sysfunc.h -buf.h -common.h -buf_util.c -error_msgs.c - ------------------------------------------------------------------------------- - -The rest of this file consists of information printed by "trec_eval -h": -(If you REALLY want a complete list of measures calculated, you can add the -time based measures and run "trec_eval -T -h".) - -trec_eval [-h] [-q] [-a] [-o] [-c] [-l [-N] [-M] [-Ua] [-Ub] [-Uc] [-Ud] [-T] trec_rel_file trec_top_file - -Calculate and print various evaluation measures, evaluating the results -in trec_top_file against the relevance judgements in trec_rel_file. - -There are a fair number of options, of which only the lower case options are -normally ever used. - -h: Print full help message and exit - -q: In addition to summary evaluation, give evaluation for each query - -a: Print all evaluation measures calculated, instead of just the - main official measures for TREC. - -o: Print everything out in old, nonrelational format (default is relational) - -c: Average over the complete set of queries in the relevance judgements - instead of the queries in the intersection of relevance judgements - and results. Missing queries will contribute a value of 0 to all - evaluation measures (which may or may not be reasonable for a - particular evaluation measure, but is reasonable for standard TREC - measures.) - -l: Num indicates the minimum relevance judgement value needed for - a document to be called relevant. (All measures used by TREC eval are - based on binary relevance). Used if trec_rel_file contains relevance - judged on a multi-relevance scale. Default is 1. - -N: Number of docs in collection - -M: Max number of docs per topic to use in evaluation (discard rest). - -Ua: Value to use for 'a' coefficient of utility computation. - relevant nonrelevant - retrieved a b - nonretrieved c d - -Ub: Value to use for 'b' coefficient of utility computation. - -Uc: Value to use for 'c' coefficient of utility computation. - -Ud: Value to use for 'd' coefficient of utility computation. - -J: Calculate all values only over the judged (either relevant or - nonrelevant) documents. All unjudged documents are removed from the - retrieved set before any calculations (possibly leaving an empty set). - DO NOT USE, unless you really know what you're doing - very easy to get - reasonable looking, but invalid, numbers. - -T: Treat similarity as time that document retrieved. Compute - several time-based measures after ranking docs by time retrieved - (first doc (lowest sim) retrieved ranked highest). - Only done if -a selected. - - -Read text tuples from trec_top_file of the form - 030 Q0 ZF08-175-870 0 4238 prise1 - qid iter docno rank sim run_id -giving TREC document numbers (a string) retrieved by query qid -(a string) with similarity sim (a float). The other fields are ignored, -with the exception that the run_id field of the last line is kept and -output. In particular, note that the rank field is ignored here; -internally ranks are assigned by sorting by the sim field with ties -broken deterministicly (using docno). -Sim is assumed to be higher for the docs to be retrieved first. -File may contain no NULL characters. -Lines may contain fields after the run_id; they are ignored. - -Relevance for each docno to qid is determined from text_qrels_file, which -consists of text tuples of the form - qid iter docno rel -giving TREC document numbers (docno, a string) and their relevance (rel, -an integer) to query qid (a string). iter string field is ignored. -Fields are separated by whitespace, string fields can contain no whitespace. -File may contain no NULL characters. - -The text tuples with relevance judgements are converted to TR_VEC form -and then submitted to the SMART evaluation routines. -The qid,did,rank,sim,rel fields of TR_VEC are filled in; -action,iter fields are set to 0. -The rel field is set to -1 if the document was not judged (not in -text_qrels_file). Most measures, but not all, will treat -1 the same as 0, -namely nonrelevant. Note that relevance_level is used to determine if the -document is relevant during score calculations. -Queries for which there are no relevant docs are ignored. -Warning: queries for which there are relevant docs but no retrieved docs -are also ignored by default. This allows systems to evaluate over subsets -of the relevant docs, but means if a system improperly retrieves no docs, -it will not be detected. Use the -c flag to avoid this behavior. - -EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. -Relational Format prints the same values, but all lines are of the form - measure_name query value - -1. Total number of documents over all queries - Retrieved: - Relevant: - Rel_ret: (relevant and retrieved) - These should be self-explanatory. All values are totals over all - queries being evaluated. -2. Interpolated Recall - Precision Averages: - at 0.00 - at 0.10 - ... - at 1.00 - See any standard IR text (especially by Salton) for more details of - recall-precision evaluation. Measures precision (percent of retrieved - docs that are relevant) at various recall levels (after a certain - percentage of all the relevant docs for that query have been retrieved). - 'Interpolated' means that, for example, precision at recall - 0.10 (ie, after 10% of rel docs for a query have been retrieved) is - taken to be MAXIMUM of precision at all recall points >= 0.10. - Values are averaged over all queries (for each of the 11 recall levels). - These values are used for Recall-Precision graphs. -3. Average precision (non-interpolated) over all rel docs - The precision is calculated after each relevant doc is retrieved. - If a relevant doc is not retrieved, its precision is 0.0. - All precision values are then averaged together to get a single number - for the performance of a query. Conceptually this is the area - underneath the recall-precision graph for the query. - The values are then averaged over all queries. -4. Precision: - at 5 docs - at 10 docs - ... - at 1000 docs - The precision (percent of retrieved docs that are relevant) after X - documents (whether relevant or nonrelevant) have been retrieved. - Values averaged over all queries. If X docs were not retrieved - for a query, then all missing docs are assumed to be non-relevant. -5. R-Precision (precision after R (= num_rel for a query) docs retrieved): - Measures precision (or recall, they're the same) after R docs - have been retrieved, where R is the total number of relevant docs - for a query. Thus if a query has 40 relevant docs, then precision - is measured after 40 docs, while if it has 600 relevant docs, precision - is measured after 600 docs. This avoids some of the averaging - problems of the 'precision at X docs' values in (4) above. - If R is greater than the number of docs retrieved for a query, then - the nonretrieved docs are all assumed to be nonrelevant. - -Major measures (again) with their relational names: -num_ret Total number of documents retrieved over all queries -num_rel Total number of relevant documents over all queries -num_rel_ret Total number of relevant documents retrieved over all queries -map Mean Average Precision (MAP) -gm_ap Average Precision. Geometric Mean, q_score=log(MAX(map,.00001)) -R-prec R-Precision (Precision after R (= num-rel for topic) documents retrieved) -bpref Binary Preference, top R judged nonrel -recip_rank Reciprical rank of top relevant document -ircl_prn.0.00 Interpolated Recall - Precision Averages at 0.00 recall -ircl_prn.0.10 Interpolated Recall - Precision Averages at 0.10 recall -ircl_prn.0.20 Interpolated Recall - Precision Averages at 0.20 recall -ircl_prn.0.30 Interpolated Recall - Precision Averages at 0.30 recall -ircl_prn.0.40 Interpolated Recall - Precision Averages at 0.40 recall -ircl_prn.0.50 Interpolated Recall - Precision Averages at 0.50 recall -ircl_prn.0.60 Interpolated Recall - Precision Averages at 0.60 recall -ircl_prn.0.70 Interpolated Recall - Precision Averages at 0.70 recall -ircl_prn.0.80 Interpolated Recall - Precision Averages at 0.80 recall -ircl_prn.0.90 Interpolated Recall - Precision Averages at 0.90 recall -ircl_prn.1.00 Interpolated Recall - Precision Averages at 1.00 recall -P5 Precision after 5 docs retrieved -P10 Precision after 10 docs retrieved -P15 Precision after 15 docs retrieved -P20 Precision after 20 docs retrieved -P30 Precision after 30 docs retrieved -P100 Precision after 100 docs retrieved -P200 Precision after 200 docs retrieved -P500 Precision after 500 docs retrieved -P1000 Precision after 1000 docs retrieved - - -Minor measures with their relational names: -exact_prec Exact Precision over retrieved set -exact_recall Exact Recall over retrieved set -11-pt_avg Average over all 11 points of recall-precision graph -3-pt_avg Average over 3 points of recall-precision graph -avg_doc_prec Rel doc precision averaged over all relevant docs (NOT over topics) -exact_relative_prec Exact relative precision -avg_relative_prec Average relative precision -exact_unranked_avg_prec Exact Unranked Average Precision -exact_relative_unranked_avg_prec Exact Relative Unranked Average Precision -map_at_R Average Precision over first R docs retrieved -int_map Interpolated Mean Average Precision -exact_int_R_rcl_prec Exact R-based-interpolated-Precision -int_map_at_R Average Interpolated Precision for first R docs retrieved -bpref_allnonrel Binary Preference, all judged nonrel -bpref_retnonrel Binary Preference, all retrieved judged nonrel -bpref_topnonrel Binary Preference, top 100 judged nonrel -bpref_top5Rnonrel Binary Preference, top 5R judged nonrel -bpref_top10Rnonrel Binary Preference, top 10R judged nonrel -bpref_top10pRnonrel Binary Preference, top 10 + R judged nonrel -bpref_top25pRnonrel Binary Preference, top 25 + R judged nonrel -bpref_top50pRnonrel Binary Preference, top 50 + R judged nonrel -bpref_top25p2Rnonrel Binary Preference, top 25 + 2*R judged nonrel -bpref_retall Binary Preference, Only retrieved judged rel and nonrel -bpref_5 Binary Preference, top 5 rel, top 5 nonrel -bpref_10 Binary Preference, top 10 rel, top 10 nonrel -bpref_num_all Binary Preference, Number not retrieved before (all judged) -bpref_num_ret Binary Preference, Number retrieved after -bpref_num_correct Binary Preference, Number correct preferences -bpref_num_possible Binary Preference, Number possible correct_preferences -old_bpref Buggy Version 7.3. Binary Preference, top R judged nonrel -old_bpref_top10pRnonrel Buggy Version 7.3. Binary Preference,top 10+R judged nonrel -gm_bpref Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001)) -rank_first_rel Rank of top relevant document (0 if none) -recall5 Recall after 5 docs retrieved -recall10 Recall after 10 docs retrieved -recall15 Recall after 15 docs retrieved -recall20 Recall after 20 docs retrieved -recall30 Recall after 30 docs retrieved -recall100 Recall after 100 docs retrieved -recall200 Recall after 200 docs retrieved -recall500 Recall after 500 docs retrieved -recall1000 Recall after 1000 docs retrieved -0.20R-prec R-based precision- precision after 0.20 * R docs retrieved -0.40R-prec R-based precision- precision after 0.40 * R docs retrieved -0.60R-prec R-based precision- precision after 0.60 * R docs retrieved -0.80R-prec R-based precision- precision after 0.80 * R docs retrieved -1.00R-prec R-based precision- precision after 1.00 * R docs retrieved -1.20R-prec R-based precision- precision after 1.20 * R docs retrieved -1.40R-prec R-based precision- precision after 1.40 * R docs retrieved -1.60R-prec R-based precision- precision after 1.60 * R docs retrieved -1.80R-prec R-based precision- precision after 1.80 * R docs retrieved -2.00R-prec R-based precision- precision after 2.00 * R docs retrieved -relative_prec5 Relative precision after 5 docs retrieved -relative_prec10 Relative precision after 10 docs retrieved -relative_prec15 Relative precision after 15 docs retrieved -relative_prec20 Relative precision after 20 docs retrieved -relative_prec30 Relative precision after 30 docs retrieved -relative_prec100 Relative precision after 100 docs retrieved -relative_prec200 Relative precision after 200 docs retrieved -relative_prec500 Relative precision after 500 docs retrieved -relative_prec1000 Relative precision after 1000 docs retrieved -unranked_avg_prec5 Unranked Average Precision after 5 docs retrieved -unranked_avg_prec10 Unranked Average Precision after 10 docs retrieved -unranked_avg_prec15 Unranked Average Precision after 15 docs retrieved -unranked_avg_prec20 Unranked Average Precision after 20 docs retrieved -unranked_avg_prec30 Unranked Average Precision after 30 docs retrieved -unranked_avg_prec100 Unranked Average Precision after 100 docs retrieved -unranked_avg_prec200 Unranked Average Precision after 200 docs retrieved -unranked_avg_prec500 Unranked Average Precision after 500 docs retrieved -unranked_avg_prec1000 Unranked Average Precision after 1000 docs retrieved -relative_unranked_avg_prec5 Relative Unranked Average Precision after 5 docs retrieved -relative_unranked_avg_prec10 Relative Unranked Average Precision after 10 docs retrieved -relative_unranked_avg_prec15 Relative Unranked Average Precision after 15 docs retrieved -relative_unranked_avg_prec20 Relative Unranked Average Precision after 20 docs retrieved -relative_unranked_avg_prec30 Relative Unranked Average Precision after 30 docs retrieved -relative_unranked_avg_prec100 Relative Unranked Average Precision after 100 docs retrieved -relative_unranked_avg_prec200 Relative Unranked Average Precision after 200 docs retrieved -relative_unranked_avg_prec500 Relative Unranked Average Precision after 500 docs retrieved -relative_unranked_avg_prec1000 Relative Unranked Average Precision after 1000 docs retrieved -utility_1.0_-1.0_0.0_0.0 Utility (a,b,c,d) Coefficients 1.0_-1.0_0.0_0.0 -rcl_at_142_nonrel Recall averaged at X nonrel docs X= 142 -fallout_recall_0 Fallout - Recall Averages- recall after 0 nonrel docs retrieved -fallout_recall_14 Fallout - Recall Averages- recall after 14 nonrel docs retrieved -fallout_recall_28 Fallout - Recall Averages- recall after 28 nonrel docs retrieved -fallout_recall_42 Fallout - Recall Averages- recall after 42 nonrel docs retrieved -fallout_recall_56 Fallout - Recall Averages- recall after 56 nonrel docs retrieved -fallout_recall_71 Fallout - Recall Averages- recall after 71 nonrel docs retrieved -fallout_recall_85 Fallout - Recall Averages- recall after 85 nonrel docs retrieved -fallout_recall_99 Fallout - Recall Averages- recall after 99 nonrel docs retrieved -fallout_recall_113 Fallout - Recall Averages- recall after 113 nonrel docs retrieved -fallout_recall_127 Fallout - Recall Averages- recall after 127 nonrel docs retrieved -fallout_recall_142 Fallout - Recall Averages- recall after 142 nonrel docs retrieved -int_0.20R-prec Interpolated R-based precision, after 0.20 * R docs retrieved -int_0.40R-prec Interpolated R-based precision, after 0.40 * R docs retrieved -int_0.60R-prec Interpolated R-based precision, after 0.60 * R docs retrieved -int_0.80R-prec Interpolated R-based precision, after 0.80 * R docs retrieved -int_1.00R-prec Interpolated R-based precision, after 1.00 * R docs retrieved -int_1.20R-prec Interpolated R-based precision, after 1.20 * R docs retrieved -int_1.40R-prec Interpolated R-based precision, after 1.40 * R docs retrieved -int_1.60R-prec Interpolated R-based precision, after 1.60 * R docs retrieved -int_1.80R-prec Interpolated R-based precision, after 1.80 * R docs retrieved -int_2.00R-prec Interpolated R-based precision, after 2.00 * R docs retrieved -micro_prec Total relevant retrieved documents / Total retrieved documents -micro_recall Total relevant retrieved documents / Total relevant documents -micro_bpref Total correct preferences / Total possible preferences diff --git a/sm_cnn/trec_eval-8.0/buf.h b/sm_cnn/trec_eval-8.0/buf.h deleted file mode 100755 index 5f2f84c..0000000 --- a/sm_cnn/trec_eval-8.0/buf.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef BUFH -#define BUFH -/* $Header: /home/smart/release/src/h/buf.h,v 11.0 1992/07/21 18:18:32 chrisb Exp $*/ - -/* structure used for passing around text (buf) which possibly includes - NULLs. see buf_util.c for add_buf(). */ -typedef struct { - int size; - int end; - char *buf; -} SM_BUF; - -#endif /* BUFH */ diff --git a/sm_cnn/trec_eval-8.0/buf_util.c b/sm_cnn/trec_eval-8.0/buf_util.c deleted file mode 100755 index 754eb07..0000000 --- a/sm_cnn/trec_eval-8.0/buf_util.c +++ /dev/null @@ -1,86 +0,0 @@ -#ifdef RCSID -static char rcsid[] = "$Header: /home/smart/release/src/libgeneral/buf_util.c,v 11.0 1992/07/21 18:21:04 chrisb Exp $"; -#endif - -/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. - - Permission is granted for use of this file in unmodified form for - research purposes. Please contact the SMART project to obtain - permission for other uses. -*/ - -/******************** PROCEDURE DESCRIPTION ************************ - *0 Utility procedure to add the memory contents of new.buf to result.buf - *2 add_buf (new, result) - *3 SM_BUF *new; - *3 SM_BUF *result; - *7 Both new and result are of type - *7 typedef struct { - *7 int size; * allocated space for buf * - *7 int end; * end of valid data in buf * - *7 char *buf; * buffer of arbitrary data * - *7 } SM_BUF; - *7 - *7 Append the data in new to the end of the data in result. The data can - *7 be arbitrary data, eg, include '\0's. - *7 Return UNDEF if can't allocate enough space for the result, 0 otherwise. -***********************************************************************/ - -#include "common.h" -#include "sysfunc.h" -#include "buf.h" - -int -add_buf (new, result) -SM_BUF *new, *result; -{ - if (result->size == 0) { - if (NULL == (result->buf = malloc ((unsigned) new->end * 2 + 1))) - return (UNDEF); - result->size = 2 * new->end + 1; - result->end = 0; - } - else if (new->end >= result->size - result->end) { - if (NULL == (result->buf = - realloc (result->buf, - (unsigned) result->size * 2 + new->end))) - return (UNDEF); - result->size += result->size + new->end; - } - - bcopy (new->buf, &result->buf[result->end], new->end); - result->end += new->end; - - return (0); -} - -/******************** PROCEDURE DESCRIPTION ************************ - *0 Utility procedure to add the string new to result.buf - *2 add_buf_string (new, result) - *3 char *new; - *3 SM_BUF *result; - - *7 Result is of type - *7 typedef struct { - *7 int size; * allocated space for buf * - *7 int end; * end of valid data in buf * - *7 char *buf; * buffer of arbitrary data * - *7 } SM_BUF; - *7 - *7 Append the data in new to the end of the data in result. - *7 Return UNDEF if can't allocate enough space for the result, 0 otherwise. -***********************************************************************/ - -int -add_buf_string (new, result) -char *new; -SM_BUF *result; -{ - SM_BUF temp_buf; - temp_buf.end = strlen (new); - temp_buf.buf = new; - - return (add_buf (&temp_buf, result)); - -} - diff --git a/sm_cnn/trec_eval-8.0/common.h b/sm_cnn/trec_eval-8.0/common.h deleted file mode 100755 index 80007df..0000000 --- a/sm_cnn/trec_eval-8.0/common.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef COMMONH -#define COMMONH - -#include - -#ifndef FALSE -#define FALSE 0 -#endif -#ifndef TRUE -#define TRUE 1 -#endif - -#define UNDEF -1 - -#define MAX(A,B) ((A) > (B) ? (A) : (B)) -#define MIN(A,B) ((A) > (B) ? (B) : (A)) - -#ifndef MAXLONG -#define MAXLONG 2147483647L /* largest long int. no. */ -#endif - -/* - * Some useful macros for making malloc et al easier to use. - * Macros handle the casting and the like that's needed. - */ -#define Malloc(n,type) (type *) malloc( (unsigned) ((n)*sizeof(type))) -#define Realloc(loc,n,type) (type *) realloc( (char *)(loc), \ - (unsigned) ((n)*sizeof(type))) -#define Free(loc) (void) free( (char *)(loc) ) - -#endif /* COMMONH */ diff --git a/sm_cnn/trec_eval-8.0/error_msgs.c b/sm_cnn/trec_eval-8.0/error_msgs.c deleted file mode 100755 index 0eb23d0..0000000 --- a/sm_cnn/trec_eval-8.0/error_msgs.c +++ /dev/null @@ -1,93 +0,0 @@ -#ifdef RCSID -static char rcsid[] = "$Header: /home/smart/release/./src/libgeneral/error_msgs.c,v 10.1 91/11/05 23:49:06 smart Exp Locker: smart $"; -#endif - -/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. - - Permission is granted for use of this file in unmodified form for - research purposes. Please contact the SMART project to obtain - permission for other uses. -*/ - -/******************** PROCEDURE DESCRIPTION ************************ - *0 print a SMART error message - *2 print_error (new_routine, new_message) - *3 char *new_routine; - *3 char *new_message; - *6 Global UNIX variables errno, sys_nerr, sys_errlist are used, as well - *6 as SMART global variables smart_errlist and smart_errno; - - *7 Print an error message to stderr. At point of error determination, - *7 either smart_errno should be set, or (if UNIX library error) errno will - *7 be set. If smart_errno is set, then the routine name that detected the - *7 error and a message are printed. In addition, the routine name that prints - *7 the error and it's message (eg action to be taken) are printed. - *9 smart_errno should be more widely used, in particular to locate the - *9 procedure the error occurs in. Many errors can only get "located" - *9 by setting trace. -***********************************************************************/ - -#include -#include "smart_error.h" -#include "sysfunc.h" - -/* Declarations of external variables defined in "smart_error.h" */ - -int smart_errno; /* If > 0 and <= sys_nerr then refers to */ - /* sys_errlist, else if >= smart_errmin */ - /* and <= smart_errmax, then smart_errlist */ -char *smart_message; /* Message to be printed (often filename) */ -char *smart_routine; /* Major routine issuing error message */ - - -extern int errno; - -char *smart_errlist[] = { - "Inconsistency check", - "Illegal value for seek", - "Illegal mode for object", - "Illegal parameter value" -}; - -void -print_error (new_routine, new_message) -char *new_routine; -char *new_message; -{ - if (smart_errno > 0 && smart_errno < SMART_MINERR) { - (void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n", - new_routine, - smart_routine, - smart_message, - strerror(smart_errno), - new_message); - } - else if (smart_errno >= SMART_MINERR && - smart_errno < SMART_MINERR + SMART_NUMERR) { - (void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n", - new_routine, - smart_routine, - smart_message, - smart_errlist[smart_errno - SMART_MINERR], - new_message); - } - else if (smart_errno == 0 && errno != 0) { - /* Presumably error detected directly by new_routine */ - /* after system call */ - (void) fprintf (stderr, "%s: '%s' - %s\n", - new_routine, - strerror(errno), - new_message); - } - else { - (void) fprintf (stderr, "%s: Undetermined error detected - %s\n", - new_routine, - new_message); - } - - /* Reset the global error indicators */ - errno = 0; - smart_errno = 0; - smart_message = NULL; - smart_routine = NULL; -} diff --git a/sm_cnn/trec_eval-8.0/form_trvec.c b/sm_cnn/trec_eval-8.0/form_trvec.c deleted file mode 100755 index 432fdab..0000000 --- a/sm_cnn/trec_eval-8.0/form_trvec.c +++ /dev/null @@ -1,201 +0,0 @@ -#include "common.h" -#include "sysfunc.h" -#include "smart_error.h" -#include "tr_vec.h" -#include "trec_eval.h" -#include "buf.h" - -static int comp_tr_tup_rank(), comp_tr_tup_did(), comp_tr_docno(), - comp_qrels_docno(), comp_sim_docno(), comp_negsim_docno(); - -/* Space reserved for output TR_TUP tuples */ -static TR_TUP *start_tr_tup; -static long max_tr_tup = 0; - -int -form_trvec (epi, trec_top, trec_qrels, tr_vec, num_rel) -EVAL_PARAM_INFO *epi; -TREC_TOP *trec_top; -TREC_QRELS *trec_qrels; -TR_VEC *tr_vec; -long *num_rel; -{ - TR_TUP *tr_tup; - TEXT_QRELS *qrels_ptr, *end_qrels; - long i; - - /* Reserve space for output tr_tups, if needed */ - if (trec_top->num_text_tr > max_tr_tup) { - if (max_tr_tup > 0) - (void) free ((char *) start_tr_tup); - max_tr_tup += trec_top->num_text_tr; - if (NULL == (start_tr_tup = Malloc (max_tr_tup, TR_TUP))) - return (UNDEF); - } - - /* Sort trec_top by sim, breaking ties lexicographically using docno */ - if (epi->time_flag) { - qsort ((char *) trec_top->text_tr, - (int) trec_top->num_text_tr, - sizeof (TEXT_TR), - comp_negsim_docno); - } - else { - qsort ((char *) trec_top->text_tr, - (int) trec_top->num_text_tr, - sizeof (TEXT_TR), - comp_sim_docno); - } - - /* Add ranks to trec_top (starting at 1) */ - for (i = 0; i < trec_top->num_text_tr; i++) { - trec_top->text_tr[i].rank = i+1; - } - - /* Sort trec_top lexicographically */ - qsort ((char *) trec_top->text_tr, - (int) trec_top->num_text_tr, - sizeof (TEXT_TR), - comp_tr_docno); - - for (i = 1; i < trec_top->num_text_tr; i++) { - if (0 == strcmp (trec_top->text_tr[i].docno, - trec_top->text_tr[i-1].docno)) { - set_error (SM_ILLPA_ERR, "Duplicate top docs docno", trec_top->text_tr[i].docno); - return (UNDEF); - } - } - - /* Sort trec_qrels lexicographically */ - qsort ((char *) trec_qrels->text_qrels, - (int) trec_qrels->num_text_qrels, - sizeof (TEXT_QRELS), - comp_qrels_docno); - - /* Find number of relevant docs, and check for duplicates */ - *num_rel = 0; - for (i = 0; i < trec_qrels->num_text_qrels; i++) { - //if (i > 0 && (0 == strcmp (trec_qrels->text_qrels[i].docno, - // trec_qrels->text_qrels[i-1].docno))) { - // set_error (SM_ILLPA_ERR, "Duplicate qrels docno", trec_qrels->text_qrels[i].docno); - // return (UNDEF); - //} - if (trec_qrels->text_qrels[i].rel >= epi->relevance_level) - (*num_rel)++; - } - - /* Go through trec_top, trec_qrels in parallel to determine which - docno's are in both (ie, which trec_top are relevant). Once relevance - is known, convert trec_top tuple into TR_TUP. */ - tr_tup = start_tr_tup; - qrels_ptr = trec_qrels->text_qrels; - end_qrels = &trec_qrels->text_qrels[trec_qrels->num_text_qrels]; - for (i = 0; i < trec_top->num_text_tr; i++) { - if (trec_top->text_tr[i].rank > epi->max_num_docs_per_topic) - /* Skip if evaluation desired over fewer docs than this rank */ - continue; - while (qrels_ptr < end_qrels && - strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) < 0) - qrels_ptr++; - if (qrels_ptr >= end_qrels || - strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) > 0) { - /* Doc is non-judged */ - tr_tup->rel = -1; - /* Skip unjudged docs if desired */ - if (epi->judged_docs_only_flag) - continue; - } - else { - /* Doc is judged; assign relevance */ - tr_tup->rel = qrels_ptr->rel; - qrels_ptr++; - } - tr_tup->did = i; - tr_tup->rank = trec_top->text_tr[i].rank; - tr_tup->sim = trec_top->text_tr[i].sim; - tr_tup->action = 0; - tr_tup->iter = 0; - tr_tup++; - } - - /* Form the full TR_VEC object for this qid */ - tr_vec->qid = trec_top->qid; - tr_vec->num_tr = tr_tup - start_tr_tup; - tr_vec->tr = start_tr_tup; - - /* If judged_docs_only_flag, then must fix up ranks to reflect unjudged - docs being thrown out. Note: done this way to preserve original - tie-breaking based on text docno */ - if (epi->judged_docs_only_flag) { - /* Sort tuples by increasing rank */ - qsort ((char *) tr_vec->tr, - (int) tr_vec->num_tr, - sizeof (TR_TUP), - comp_tr_tup_rank); - for (i = 0; i < tr_vec->num_tr; i++) { - tr_vec->tr[i].rank = i+1; - } - qsort ((char *) tr_vec->tr, - (int) tr_vec->num_tr, - sizeof (TR_TUP), - comp_tr_tup_did); - } - - return (1); -} - -static int -comp_sim_docno (ptr1, ptr2) -TEXT_TR *ptr1; -TEXT_TR *ptr2; -{ - if (ptr1->sim > ptr2->sim) - return (-1); - if (ptr1->sim < ptr2->sim) - return (1); - return (strcmp (ptr2->docno, ptr1->docno)); -} - -static int -comp_negsim_docno (ptr1, ptr2) -TEXT_TR *ptr1; -TEXT_TR *ptr2; -{ - if (ptr1->sim < ptr2->sim) - return (-1); - if (ptr1->sim > ptr2->sim) - return (1); - return (strcmp (ptr2->docno, ptr1->docno)); -} - -static int -comp_tr_docno (ptr1, ptr2) -TEXT_TR *ptr1; -TEXT_TR *ptr2; -{ - return (strcmp (ptr1->docno, ptr2->docno)); -} - -static int -comp_qrels_docno (ptr1, ptr2) -TEXT_QRELS *ptr1; -TEXT_QRELS *ptr2; -{ - return (strcmp (ptr1->docno, ptr2->docno)); -} - -static int -comp_tr_tup_rank (ptr1, ptr2) -TR_TUP *ptr1; -TR_TUP *ptr2; -{ - return (ptr1->rank - ptr2->rank); -} - -static int -comp_tr_tup_did (ptr1, ptr2) -TR_TUP *ptr1; -TR_TUP *ptr2; -{ - return (ptr1->did - ptr2->did); -} diff --git a/sm_cnn/trec_eval-8.0/get_qrels.c b/sm_cnn/trec_eval-8.0/get_qrels.c deleted file mode 100755 index 84d51de..0000000 --- a/sm_cnn/trec_eval-8.0/get_qrels.c +++ /dev/null @@ -1,169 +0,0 @@ -/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley. */ - - -#include "common.h" -#include "sysfunc.h" -#include "smart_error.h" -#include "trec_eval.h" -#include - - -/* Read all relevance information from text_qrels_file. -Relevance for each docno to qid is determined from text_qrels_file, which -consists of text tuples of the form - qid iter docno rel -giving TREC document numbers (docno, a string) and their relevance (rel, -an integer) to query qid (a string). iter string field is ignored. -Fields are separated by whitespace, string fields can contain no whitespace. -File may contain no NULL characters. -*/ - -int -get_qrels (text_qrels_file, all_trec_qrels) -char *text_qrels_file; -ALL_TREC_QRELS *all_trec_qrels; -{ - int fd; - int size = 0; - char *trec_qrels_buf; - char *ptr; - char *current_qid; - char *qid_ptr, *docno_ptr, *rel_ptr; - long i; - long rel; - TREC_QRELS *current_qrels = NULL; - - /* Read entire file into memory */ - if (-1 == (fd = open (text_qrels_file, 0)) || - -1 == (size = lseek (fd, 0L, 2)) || - NULL == (trec_qrels_buf = malloc ((unsigned) size+2)) || - -1 == lseek (fd, 0L, 0) || - size != read (fd, trec_qrels_buf, size) || - -1 == close (fd)) { - - set_error (SM_ILLPA_ERR, "Cannot read qrels file", "trec_eval"); - return (UNDEF); - } - - current_qid = ""; - - /* Initialize all_trec_qrels */ - all_trec_qrels->num_q_qrels = 0; - all_trec_qrels->max_num_q_qrels = INIT_NUM_QUERIES; - if (NULL == (all_trec_qrels->trec_qrels = Malloc (INIT_NUM_QUERIES, - TREC_QRELS))) - return (UNDEF); - - if (size == 0) - return (0); - - /* Append ending newline if not present, Append NULL terminator */ - if (trec_qrels_buf[size-1] != '\n') { - trec_qrels_buf[size] = '\n'; - size++; - } - trec_qrels_buf[size] = '\0'; - - ptr = trec_qrels_buf; - - while (*ptr) { - /* Get current line */ - /* Get qid */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - qid_ptr = ptr; - while (! isspace (*ptr)) ptr++; - if (*ptr == '\n') { - set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); - return (UNDEF); - } - *ptr++ = '\0'; - /* Skip iter */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - while (! isspace (*ptr)) ptr++; - if (*ptr++ == '\n') { - set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); - return (UNDEF); - } - /* Get docno */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - docno_ptr = ptr; - while (! isspace (*ptr)) ptr++; - if (*ptr == '\n') { - set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); - return (UNDEF); - } - *ptr++ = '\0'; - /* Get relevance */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - if (*ptr == '\n') { - set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval"); - return (UNDEF); - } - rel_ptr = ptr; - while (! isspace (*ptr)) ptr++; - if (*ptr != '\n') { - *ptr++ = '\0'; - while (*ptr != '\n' && isspace (*ptr)) ptr++; - if (*ptr != '\n') { - set_error (SM_ILLPA_ERR, "malformed qrels line", - "trec_eval"); - return (UNDEF); - } - } - *ptr++ = '\0'; - - if (0 != strcmp (qid_ptr, current_qid)) { - /* Query has changed. Must check if new query or this is more - judgements for an old query */ - for (i = 0; i < all_trec_qrels->num_q_qrels; i++) { - if (0 == strcmp (qid_ptr, all_trec_qrels->trec_qrels[i].qid)) - break; - } - if (i >= all_trec_qrels->num_q_qrels) { - /* New unseen query, add and initialize it */ - if (all_trec_qrels->num_q_qrels >= - all_trec_qrels->max_num_q_qrels) { - all_trec_qrels->max_num_q_qrels *= 10; - if (NULL == (all_trec_qrels->trec_qrels = - Realloc (all_trec_qrels->trec_qrels, - all_trec_qrels->max_num_q_qrels, - TREC_QRELS))) - return (UNDEF); - } - current_qrels = &all_trec_qrels->trec_qrels[i]; - current_qrels->qid = qid_ptr; - current_qrels->num_text_qrels = 0; - current_qrels->max_num_text_qrels = INIT_NUM_RELS; - if (NULL == (current_qrels->text_qrels = - Malloc (INIT_NUM_RELS, TEXT_QRELS))) - return (UNDEF); - all_trec_qrels->num_q_qrels++; - } - else { - /* Old query, just switch current_q_index */ - current_qrels = &all_trec_qrels->trec_qrels[i]; - } - current_qid = current_qrels->qid; - } - - /* Add judgement to current query's list */ - if (current_qrels->num_text_qrels >= - current_qrels->max_num_text_qrels) { - /* Need more space */ - current_qrels->max_num_text_qrels *= 10; - if (NULL == (current_qrels->text_qrels = - Realloc (current_qrels->text_qrels, - current_qrels->max_num_text_qrels, - TEXT_QRELS))) - return (UNDEF); - } - current_qrels->text_qrels[current_qrels->num_text_qrels].docno = - docno_ptr; - rel = atol (rel_ptr); - current_qrels->text_qrels[current_qrels->num_text_qrels++].rel = - rel; - } - - return (1); -} - diff --git a/sm_cnn/trec_eval-8.0/get_top.c b/sm_cnn/trec_eval-8.0/get_top.c deleted file mode 100755 index efc6c76..0000000 --- a/sm_cnn/trec_eval-8.0/get_top.c +++ /dev/null @@ -1,192 +0,0 @@ -/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley. */ - - -#include "common.h" -#include "sysfunc.h" -#include "smart_error.h" -#include "trec_eval.h" -#include - -/* Read all retrieved results information from trec_top_file. -Read text tuples from trec_top_file of the form - 030 Q0 ZF08-175-870 0 4238 prise1 - qid iter docno rank sim run_id -giving TREC document numbers (a string) retrieved by query qid -(a string) with similarity sim (a float). The other fields are ignored, -with the exception that the run_id field of the last line is kept and -output. In particular, note that the rank field is ignored here; -internally ranks are assigned by sorting by the sim field with ties -broken determinstically (using docno). -Sim is assumed to be higher for the docs to be retrieved first. -File may contain no NULL characters. -Any field following run_id is ignored. -*/ - -int -get_top (trec_top_file, all_trec_top) -char *trec_top_file; -ALL_TREC_TOP *all_trec_top; -{ - int fd; - int size = 0; - char *trec_top_buf; - char *ptr; - char *current_qid; - char *qid_ptr, *docno_ptr, *sim_ptr; - char *run_id_ptr = ""; - long i; - TREC_TOP *current_top = NULL; - float sim; - - /* Read entire file into memory */ - if (-1 == (fd = open (trec_top_file, 0)) || - -1 == (size = lseek (fd, 0L, 2)) || - NULL == (trec_top_buf = malloc ((unsigned) size+2)) || - -1 == lseek (fd, 0L, 0) || - size != read (fd, trec_top_buf, size) || - -1 == close (fd)) { - - set_error (SM_ILLPA_ERR, "Cannot read qrels file", "trec_eval"); - return (UNDEF); - } - - current_qid = ""; - - /* Initialize all_trec_top */ - all_trec_top->num_q_tr = 0; - all_trec_top->max_num_q_tr = INIT_NUM_QUERIES; - if (NULL == (all_trec_top->trec_top = Malloc (INIT_NUM_QUERIES, - TREC_TOP))) - return (UNDEF); - - if (size == 0) - return (0); - - /* Append ending newline if not present, Append NULL terminator */ - if (trec_top_buf[size-1] != '\n') { - trec_top_buf[size] = '\n'; - size++; - } - trec_top_buf[size] = '\0'; - - ptr = trec_top_buf; - - while (*ptr) { - /* Get current line */ - /* Get qid */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - if (*ptr == '\n') { - /* Ignore blank lines (people seem to insist on them!) */ - ptr++; - continue; - } - qid_ptr = ptr; - while (! isspace (*ptr)) ptr++; - if (*ptr == '\n') { - set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); - return (UNDEF); - } - *ptr++ = '\0'; - /* Skip iter */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - while (! isspace (*ptr)) ptr++; - if (*ptr++ == '\n') { - set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); - return (UNDEF); - } - /* Get docno */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - docno_ptr = ptr; - while (! isspace (*ptr)) ptr++; - if (*ptr == '\n') { - set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); - return (UNDEF); - } - *ptr++ = '\0'; - /* Skip rank */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - while (! isspace (*ptr)) ptr++; - if (*ptr++ == '\n') { - set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); - return (UNDEF); - } - /* Get sim */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - sim_ptr = ptr; - while (! isspace (*ptr)) ptr++; - if (*ptr == '\n') { - set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); - return (UNDEF); - } - *ptr++ = '\0'; - /* Get run_id */ - while (*ptr != '\n' && isspace (*ptr)) ptr++; - if (*ptr == '\n') { - set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval"); - return (UNDEF); - } - run_id_ptr = ptr; - while (! isspace (*ptr)) ptr++; - if (*ptr != '\n') { - /* Skip over rest of line */ - *ptr++ = '\0'; - while (*ptr != '\n') ptr++; - } - *ptr++ = '\0'; - - if (0 != strcmp (qid_ptr, current_qid)) { - /* Query has changed. Must check if new query or this is more - judgements for an old query */ - for (i = 0; i < all_trec_top->num_q_tr; i++) { - if (0 == strcmp (qid_ptr, all_trec_top->trec_top[i].qid)) - break; - } - if (i >= all_trec_top->num_q_tr) { - /* New unseen query, add and initialize it */ - if (all_trec_top->num_q_tr >= - all_trec_top->max_num_q_tr) { - all_trec_top->max_num_q_tr *= 10; - if (NULL == (all_trec_top->trec_top = - Realloc (all_trec_top->trec_top, - all_trec_top->max_num_q_tr, - TREC_TOP))) - return (UNDEF); - } - current_top = &all_trec_top->trec_top[i]; - current_top->qid = qid_ptr; - current_top->num_text_tr = 0; - current_top->max_num_text_tr = INIT_NUM_RESULTS; - if (NULL == (current_top->text_tr = - Malloc (INIT_NUM_RESULTS, TEXT_TR))) - return (UNDEF); - all_trec_top->num_q_tr++; - } - else { - /* Old query, just switch current_q_index */ - current_top = &all_trec_top->trec_top[i]; - } - current_qid = current_top->qid; - } - - /* Add retrieval docno/sim to current query's list */ - if (current_top->num_text_tr >= - current_top->max_num_text_tr) { - /* Need more space */ - current_top->max_num_text_tr *= 10; - if (NULL == (current_top->text_tr = - Realloc (current_top->text_tr, - current_top->max_num_text_tr, - TEXT_TR))) - return (UNDEF); - } - current_top->text_tr[current_top->num_text_tr].docno = docno_ptr; - sim = atof (sim_ptr); - current_top->text_tr[current_top->num_text_tr].sim = sim; - current_top->text_tr[current_top->num_text_tr++].rank = 0; - } - - all_trec_top->run_id = run_id_ptr; - - return (1); -} - diff --git a/sm_cnn/trec_eval-8.0/help.info b/sm_cnn/trec_eval-8.0/help.info deleted file mode 100755 index eb243c7..0000000 --- a/sm_cnn/trec_eval-8.0/help.info +++ /dev/null @@ -1,264 +0,0 @@ -trec_eval [-h] [-q] [-a] [-o] [-c] [-l [-N] [-M] [-Ua] [-Ub] [-Uc] [-Ud] [-T] trec_rel_file trec_top_file - -Calculate and print various evaluation measures, evaluating the results -in trec_top_file against the relevance judgements in trec_rel_file. - -There are a fair number of options, of which only the lower case options are -normally ever used. - -h: Print full help message and exit - -q: In addition to summary evaluation, give evaluation for each query - -a: Print all evaluation measures calculated, instead of just the - main official measures for TREC. - -o: Print everything out in old, nonrelational format (default is relational) - -c: Average over the complete set of queries in the relevance judgements - instead of the queries in the intersection of relevance judgements - and results. Missing queries will contribute a value of 0 to all - evaluation measures (which may or may not be reasonable for a - particular evaluation measure, but is reasonable for standard TREC - measures.) - -l: Num indicates the minimum relevance judgement value needed for - a document to be called relevant. (All measures used by TREC eval are - based on binary relevance). Used if trec_rel_file contains relevance - judged on a multi-relevance scale. Default is 1. - -N: Number of docs in collection - -M: Max number of docs per topic to use in evaluation (discard rest). - -Ua: Value to use for 'a' coefficient of utility computation. - relevant nonrelevant - retrieved a b - nonretrieved c d - -Ub: Value to use for 'b' coefficient of utility computation. - -Uc: Value to use for 'c' coefficient of utility computation. - -Ud: Value to use for 'd' coefficient of utility computation. - -J: Calculate all values only over the judged (either relevant or - nonrelevant) documents. All unjudged documents are removed from the - retrieved set before any calculations (possibly leaving an empty set). - DO NOT USE, unless you really know what you're doing - very easy to get - reasonable looking, but invalid, numbers. - -T: Treat similarity as time that document retrieved. Compute - several time-based measures after ranking docs by time retrieved - (first doc (lowest sim) retrieved ranked highest). - Only done if -a selected. - - -Read text tuples from trec_top_file of the form - 030 Q0 ZF08-175-870 0 4238 prise1 - qid iter docno rank sim run_id -giving TREC document numbers (a string) retrieved by query qid -(a string) with similarity sim (a float). The other fields are ignored, -with the exception that the run_id field of the last line is kept and -output. In particular, note that the rank field is ignored here; -internally ranks are assigned by sorting by the sim field with ties -broken deterministicly (using docno). -Sim is assumed to be higher for the docs to be retrieved first. -File may contain no NULL characters. -Lines may contain fields after the run_id; they are ignored. - -Relevance for each docno to qid is determined from text_qrels_file, which -consists of text tuples of the form - qid iter docno rel -giving TREC document numbers (docno, a string) and their relevance (rel, -an integer) to query qid (a string). iter string field is ignored. -Fields are separated by whitespace, string fields can contain no whitespace. -File may contain no NULL characters. - -The text tuples with relevance judgements are converted to TR_VEC form -and then submitted to the SMART evaluation routines. -The qid,did,rank,sim,rel fields of TR_VEC are filled in; -action,iter fields are set to 0. -The rel field is set to -1 if the document was not judged (not in -text_qrels_file). Most measures, but not all, will treat -1 the same as 0, -namely nonrelevant. Note that relevance_level is used to determine if the -document is relevant during score calculations. -Queries for which there are no relevant docs are ignored. -Warning: queries for which there are relevant docs but no retrieved docs -are also ignored by default. This allows systems to evaluate over subsets -of the relevant docs, but means if a system improperly retrieves no docs, -it will not be detected. Use the -c flag to avoid this behavior. - -EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. -Relational Format prints the same values, but all lines are of the form - measure_name query value - -1. Total number of documents over all queries - Retrieved: - Relevant: - Rel_ret: (relevant and retrieved) - These should be self-explanatory. All values are totals over all - queries being evaluated. -2. Interpolated Recall - Precision Averages: - at 0.00 - at 0.10 - ... - at 1.00 - See any standard IR text (especially by Salton) for more details of - recall-precision evaluation. Measures precision (percent of retrieved - docs that are relevant) at various recall levels (after a certain - percentage of all the relevant docs for that query have been retrieved). - 'Interpolated' means that, for example, precision at recall - 0.10 (ie, after 10% of rel docs for a query have been retrieved) is - taken to be MAXIMUM of precision at all recall points >= 0.10. - Values are averaged over all queries (for each of the 11 recall levels). - These values are used for Recall-Precision graphs. -3. Average precision (non-interpolated) over all rel docs - The precision is calculated after each relevant doc is retrieved. - If a relevant doc is not retrieved, its precision is 0.0. - All precision values are then averaged together to get a single number - for the performance of a query. Conceptually this is the area - underneath the recall-precision graph for the query. - The values are then averaged over all queries. -4. Precision: - at 5 docs - at 10 docs - ... - at 1000 docs - The precision (percent of retrieved docs that are relevant) after X - documents (whether relevant or nonrelevant) have been retrieved. - Values averaged over all queries. If X docs were not retrieved - for a query, then all missing docs are assumed to be non-relevant. -5. R-Precision (precision after R (= num_rel for a query) docs retrieved): - Measures precision (or recall, they're the same) after R docs - have been retrieved, where R is the total number of relevant docs - for a query. Thus if a query has 40 relevant docs, then precision - is measured after 40 docs, while if it has 600 relevant docs, precision - is measured after 600 docs. This avoids some of the averaging - problems of the 'precision at X docs' values in (4) above. - If R is greater than the number of docs retrieved for a query, then - the nonretrieved docs are all assumed to be nonrelevant. - -Major measures (again) with their relational names: -num_ret Total number of documents retrieved over all queries -num_rel Total number of relevant documents over all queries -num_rel_ret Total number of relevant documents retrieved over all queries -map Mean Average Precision (MAP) -gm_ap Average Precision. Geometric Mean, q_score=log(MAX(map,.00001)) -R-prec R-Precision (Precision after R (= num-rel for topic) documents retrieved) -bpref Binary Preference, top R judged nonrel -recip_rank Reciprical rank of top relevant document -ircl_prn.0.00 Interpolated Recall - Precision Averages at 0.00 recall -ircl_prn.0.10 Interpolated Recall - Precision Averages at 0.10 recall -ircl_prn.0.20 Interpolated Recall - Precision Averages at 0.20 recall -ircl_prn.0.30 Interpolated Recall - Precision Averages at 0.30 recall -ircl_prn.0.40 Interpolated Recall - Precision Averages at 0.40 recall -ircl_prn.0.50 Interpolated Recall - Precision Averages at 0.50 recall -ircl_prn.0.60 Interpolated Recall - Precision Averages at 0.60 recall -ircl_prn.0.70 Interpolated Recall - Precision Averages at 0.70 recall -ircl_prn.0.80 Interpolated Recall - Precision Averages at 0.80 recall -ircl_prn.0.90 Interpolated Recall - Precision Averages at 0.90 recall -ircl_prn.1.00 Interpolated Recall - Precision Averages at 1.00 recall -P5 Precision after 5 docs retrieved -P10 Precision after 10 docs retrieved -P15 Precision after 15 docs retrieved -P20 Precision after 20 docs retrieved -P30 Precision after 30 docs retrieved -P100 Precision after 100 docs retrieved -P200 Precision after 200 docs retrieved -P500 Precision after 500 docs retrieved -P1000 Precision after 1000 docs retrieved - - -Minor measures with their relational names: -exact_prec Exact Precision over retrieved set -exact_recall Exact Recall over retrieved set -11-pt_avg Average over all 11 points of recall-precision graph -3-pt_avg Average over 3 points of recall-precision graph -avg_doc_prec Rel doc precision averaged over all relevant docs (NOT over topics) -exact_relative_prec Exact relative precision -avg_relative_prec Average relative precision -exact_unranked_avg_prec Exact Unranked Average Precision -exact_relative_unranked_avg_prec Exact Relative Unranked Average Precision -map_at_R Average Precision over first R docs retrieved -int_map Interpolated Mean Average Precision -exact_int_R_rcl_prec Exact R-based-interpolated-Precision -int_map_at_R Average Interpolated Precision for first R docs retrieved -bpref_allnonrel Binary Preference, all judged nonrel -bpref_retnonrel Binary Preference, all retrieved judged nonrel -bpref_topnonrel Binary Preference, top 100 judged nonrel -bpref_top5Rnonrel Binary Preference, top 5R judged nonrel -bpref_top10Rnonrel Binary Preference, top 10R judged nonrel -bpref_top10pRnonrel Binary Preference, top 10 + R judged nonrel -bpref_top25pRnonrel Binary Preference, top 25 + R judged nonrel -bpref_top50pRnonrel Binary Preference, top 50 + R judged nonrel -bpref_top25p2Rnonrel Binary Preference, top 25 + 2*R judged nonrel -bpref_retall Binary Preference, Only retrieved judged rel and nonrel -bpref_5 Binary Preference, top 5 rel, top 5 nonrel -bpref_10 Binary Preference, top 10 rel, top 10 nonrel -bpref_num_all Binary Preference, Number not retrieved before (all judged) -bpref_num_ret Binary Preference, Number retrieved after -bpref_num_correct Binary Preference, Number correct preferences -bpref_num_possible Binary Preference, Number possible correct_preferences -old_bpref Buggy Version 7.3. Binary Preference, top R judged nonrel -old_bpref_top10pRnonrel Buggy Version 7.3. Binary Preference,top 10+R judged nonrel -gm_bpref Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001)) -rank_first_rel Rank of top relevant document (0 if none) -recall5 Recall after 5 docs retrieved -recall10 Recall after 10 docs retrieved -recall15 Recall after 15 docs retrieved -recall20 Recall after 20 docs retrieved -recall30 Recall after 30 docs retrieved -recall100 Recall after 100 docs retrieved -recall200 Recall after 200 docs retrieved -recall500 Recall after 500 docs retrieved -recall1000 Recall after 1000 docs retrieved -0.20R-prec R-based precision- precision after 0.20 * R docs retrieved -0.40R-prec R-based precision- precision after 0.40 * R docs retrieved -0.60R-prec R-based precision- precision after 0.60 * R docs retrieved -0.80R-prec R-based precision- precision after 0.80 * R docs retrieved -1.00R-prec R-based precision- precision after 1.00 * R docs retrieved -1.20R-prec R-based precision- precision after 1.20 * R docs retrieved -1.40R-prec R-based precision- precision after 1.40 * R docs retrieved -1.60R-prec R-based precision- precision after 1.60 * R docs retrieved -1.80R-prec R-based precision- precision after 1.80 * R docs retrieved -2.00R-prec R-based precision- precision after 2.00 * R docs retrieved -relative_prec5 Relative precision after 5 docs retrieved -relative_prec10 Relative precision after 10 docs retrieved -relative_prec15 Relative precision after 15 docs retrieved -relative_prec20 Relative precision after 20 docs retrieved -relative_prec30 Relative precision after 30 docs retrieved -relative_prec100 Relative precision after 100 docs retrieved -relative_prec200 Relative precision after 200 docs retrieved -relative_prec500 Relative precision after 500 docs retrieved -relative_prec1000 Relative precision after 1000 docs retrieved -unranked_avg_prec5 Unranked Average Precision after 5 docs retrieved -unranked_avg_prec10 Unranked Average Precision after 10 docs retrieved -unranked_avg_prec15 Unranked Average Precision after 15 docs retrieved -unranked_avg_prec20 Unranked Average Precision after 20 docs retrieved -unranked_avg_prec30 Unranked Average Precision after 30 docs retrieved -unranked_avg_prec100 Unranked Average Precision after 100 docs retrieved -unranked_avg_prec200 Unranked Average Precision after 200 docs retrieved -unranked_avg_prec500 Unranked Average Precision after 500 docs retrieved -unranked_avg_prec1000 Unranked Average Precision after 1000 docs retrieved -relative_unranked_avg_prec5 Relative Unranked Average Precision after 5 docs retrieved -relative_unranked_avg_prec10 Relative Unranked Average Precision after 10 docs retrieved -relative_unranked_avg_prec15 Relative Unranked Average Precision after 15 docs retrieved -relative_unranked_avg_prec20 Relative Unranked Average Precision after 20 docs retrieved -relative_unranked_avg_prec30 Relative Unranked Average Precision after 30 docs retrieved -relative_unranked_avg_prec100 Relative Unranked Average Precision after 100 docs retrieved -relative_unranked_avg_prec200 Relative Unranked Average Precision after 200 docs retrieved -relative_unranked_avg_prec500 Relative Unranked Average Precision after 500 docs retrieved -relative_unranked_avg_prec1000 Relative Unranked Average Precision after 1000 docs retrieved -utility_1.0_-1.0_0.0_0.0 Utility (a,b,c,d) Coefficients 1.0_-1.0_0.0_0.0 -rcl_at_142_nonrel Recall averaged at X nonrel docs X= 142 -fallout_recall_0 Fallout - Recall Averages- recall after 0 nonrel docs retrieved -fallout_recall_14 Fallout - Recall Averages- recall after 14 nonrel docs retrieved -fallout_recall_28 Fallout - Recall Averages- recall after 28 nonrel docs retrieved -fallout_recall_42 Fallout - Recall Averages- recall after 42 nonrel docs retrieved -fallout_recall_56 Fallout - Recall Averages- recall after 56 nonrel docs retrieved -fallout_recall_71 Fallout - Recall Averages- recall after 71 nonrel docs retrieved -fallout_recall_85 Fallout - Recall Averages- recall after 85 nonrel docs retrieved -fallout_recall_99 Fallout - Recall Averages- recall after 99 nonrel docs retrieved -fallout_recall_113 Fallout - Recall Averages- recall after 113 nonrel docs retrieved -fallout_recall_127 Fallout - Recall Averages- recall after 127 nonrel docs retrieved -fallout_recall_142 Fallout - Recall Averages- recall after 142 nonrel docs retrieved -int_0.20R-prec Interpolated R-based precision, after 0.20 * R docs retrieved -int_0.40R-prec Interpolated R-based precision, after 0.40 * R docs retrieved -int_0.60R-prec Interpolated R-based precision, after 0.60 * R docs retrieved -int_0.80R-prec Interpolated R-based precision, after 0.80 * R docs retrieved -int_1.00R-prec Interpolated R-based precision, after 1.00 * R docs retrieved -int_1.20R-prec Interpolated R-based precision, after 1.20 * R docs retrieved -int_1.40R-prec Interpolated R-based precision, after 1.40 * R docs retrieved -int_1.60R-prec Interpolated R-based precision, after 1.60 * R docs retrieved -int_1.80R-prec Interpolated R-based precision, after 1.80 * R docs retrieved -int_2.00R-prec Interpolated R-based precision, after 2.00 * R docs retrieved -micro_prec Total relevant retrieved documents / Total retrieved documents -micro_recall Total relevant retrieved documents / Total relevant documents -micro_bpref Total correct preferences / Total possible preferences diff --git a/sm_cnn/trec_eval-8.0/measures.c b/sm_cnn/trec_eval-8.0/measures.c deleted file mode 100755 index 8bf6263..0000000 --- a/sm_cnn/trec_eval-8.0/measures.c +++ /dev/null @@ -1,282 +0,0 @@ -#ifdef RCSID -static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $"; -#endif - -/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. - - Permission is granted for use of this file in unmodified form for - research purposes. Please contact the SMART project to obtain - permission for other uses. -*/ - -#include "common.h" -#include "sysfunc.h" -#include "buf.h" -#include "trec_eval.h" - -static long cutoff[] = CUTOFF_VALUES; -static char param_val[20]; -char *get_param_str_ircl_prn(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%4.2f", (float) index / (NUM_RP_PTS -1)); - return (param_val); -} -char *get_param_str_cutoff(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%ld", cutoff[index]); - return (param_val); -} -char *get_param_str_Rcutoff(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%4.2f", - (float) MAX_RPREC * (index+1) /(float) (NUM_PREC_PTS - 1)); - return (param_val); -} -char *get_param_str_utility(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f", - epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d); - return (param_val); -} -char *get_param_str_maxfallout(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%ld", (long) MAX_FALL_RET); - return (param_val); -} -char *get_param_str_fall_recall(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%ld", - (long) (MAX_FALL_RET * index) / (NUM_FR_PTS - 1)); - return (param_val); -} -char *get_param_str_time_cutoff(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%ld", - (long) (index * MAX_TIME / NUM_TIME_PTS)); - return (param_val); -} -char *get_param_str_time_utility_cutoff(epi, index) -EVAL_PARAM_INFO *epi; -long index; -{ - sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f-%ld", - epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d, - (long) (index * MAX_TIME / NUM_TIME_PTS)); - return (param_val); -} - -SINGLE_MEASURE sing_meas[] = { - {"num_ret", "Total number of documents retrieved over all queries", - 1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_ret)}, - {"num_rel", "Total number of relevant documents over all queries", - 1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel)}, - {"num_rel_ret", "Total number of relevant documents retrieved over all queries", - 1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel_ret)}, - {"map", "Mean Average Precision (MAP)", - 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_recall_precis)}, - {"gm_ap","Average Precision. Geometric Mean, q_score=log(MAX(map,.00001))", - 0, 1, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_ap)}, - {"R-prec", "R-Precision (Precision after R (= num-rel for topic) documents retrieved)", - 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, R_recall_precis)}, - {"bpref", "Binary Preference, top R judged nonrel", - 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref)}, - {"recip_rank", "Reciprical rank of top relevant document", - 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, recip_rank)}, - /* end of short output measures (the major ones) */ - - {"exact_prec", "Exact Precision over retrieved set", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_precis)}, - {"exact_recall", "Exact Recall over retrieved set", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_recall)}, - {"11-pt_avg", "Average over all 11 points of recall-precision graph", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av11_recall_precis)}, - {"3-pt_avg", "Average over 3 points of recall-precision graph", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av3_recall_precis)}, - {"avg_doc_prec", "Rel doc precision averaged over all relevant docs (NOT over topics)", - 0, 0, 0, 0, 0, 0, 1, 0, offsetof(TREC_EVAL, avg_doc_prec)}, - {"exact_relative_prec", "Exact relative precision", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_precis)}, - {"avg_relative_prec", "Average relative precision", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_rel_precis)}, - {"exact_unranked_avg_prec", "Exact Unranked Average Precision", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_uap)}, - {"exact_relative_unranked_avg_prec", "Exact Relative Unranked Average Precision", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_uap)}, - {"map_at_R", "Average Precision over first R docs retrieved", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_R_precis)}, - {"int_map", "Interpolated Mean Average Precision", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_recall_precis)}, - {"exact_int_R_rcl_prec", "Exact R-based-interpolated-Precision", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_R_recall_precis)}, - {"int_map_at_R", "Average Interpolated Precision for first R docs retrieved", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_R_precis)}, - {"time_integral_prec", "Time: Average Integral Precision", - 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_precis)}, - {"time_integral_relative_prec", "Time: Average Integral Relative Precision", - 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_relprecis)}, - {"time_integral_uap", "Time: Average Integral Unranked Precision", - 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_uap)}, - {"time_integral_relative_uap", "Time: Average Integral Unranked Relative Precision", - 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_reluap)}, - {"time_integral_cum_rel", "Time: Average (Integral) cumulative number relevant", - 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_cum_rel)}, - {"bpref_allnonrel", "Binary Preference, all judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_allnonrel)}, - {"bpref_retnonrel", "Binary Preference, all retrieved judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retnonrel)}, - {"bpref_topnonrel", "Binary Preference, top 100 judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_topnonrel)}, - {"bpref_top5Rnonrel", "Binary Preference, top 5R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top5Rnonrel)}, - {"bpref_top10Rnonrel", "Binary Preference, top 10R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10Rnonrel)}, - {"bpref_top10pRnonrel", "Binary Preference, top 10 + R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10pRnonrel)}, - {"bpref_top25pRnonrel", "Binary Preference, top 25 + R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25pRnonrel)}, - {"bpref_top50pRnonrel", "Binary Preference, top 50 + R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top50pRnonrel)}, - {"bpref_top25p2Rnonrel", "Binary Preference, top 25 + 2*R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25p2Rnonrel)}, - {"bpref_retall", "Binary Preference, Only retrieved judged rel and nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retall)}, - {"bpref_5", "Binary Preference, top 5 rel, top 5 nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_5)}, - {"bpref_10", "Binary Preference, top 10 rel, top 10 nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_10)}, - {"bpref_num_all", "Binary Preference, Number not retrieved before (all judged)", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_all)}, - {"bpref_num_ret", "Binary Preference, Number retrieved after", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_ret)}, - {"bpref_num_correct", "Binary Preference, Number correct preferences", - 1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_correct)}, - {"bpref_num_possible", "Binary Preference, Number possible correct_preferences", - 1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_possible)}, - {"old_bpref", "Buggy Version 7.3. Binary Preference, top R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref)}, - {"old_bpref_top10pRnonrel", "Buggy Version 7.3. Binary Preference,top 10+R judged nonrel", - 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref_top10pRnonrel)}, - {"gm_bpref", "Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001))", - 0, 0, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_bpref)}, - {"rank_first_rel", "Rank of top relevant document (0 if none)", - 1, 0, 0, 1, 0, 0, 0, 0, offsetof(TREC_EVAL, rank_first_rel)}, -}; - -int num_sing_meas = sizeof (sing_meas) / sizeof (sing_meas[0]); - -PARAMETERIZED_MEASURE param_meas[] = { - {"Interpolated Recall - Precision Averages", - 0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, int_recall_precis[0]), NUM_RP_PTS, - "ircl_prn.%s", " at %s recall", - get_param_str_ircl_prn}, - {"Precision", - 0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, precis_cut[0]), NUM_CUTOFF, - "P%s", " after %s docs retrieved", - get_param_str_cutoff}, - /* end of short output measures (the major ones) */ - - {"Recall", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, recall_cut[0]), NUM_CUTOFF, - "recall%s", " after %s docs retrieved", - get_param_str_cutoff}, - {"R-based precision", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, R_prec_cut[0]), NUM_PREC_PTS-1, - "%sR-prec", "- precision after %s * R docs retrieved", - get_param_str_Rcutoff}, - {"Relative precision", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_precis_cut[0]), NUM_CUTOFF, - "relative_prec%s", " after %s docs retrieved", - get_param_str_cutoff}, - {"Unranked Average Precision", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, uap_cut[0]), NUM_CUTOFF, - "unranked_avg_prec%s", " after %s docs retrieved", - get_param_str_cutoff}, - {"Relative Unranked Average Precision", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_uap_cut[0]), NUM_CUTOFF, - "relative_unranked_avg_prec%s", " after %s docs retrieved", - get_param_str_cutoff}, - {"Utility (a,b,c,d)", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, exact_utility), 1, - "utility_%s", " Coefficients %s ", - get_param_str_utility}, - {"Recall averaged at X nonrel docs", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, av_fall_recall), 1, - "rcl_at_%s_nonrel", " X= %s ", - get_param_str_maxfallout}, - {"Fallout - Recall Averages", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, fall_recall[0]), NUM_FR_PTS, - "fallout_recall_%s", "- recall after %s nonrel docs retrieved", - get_param_str_fall_recall}, - {"Interpolated R-based precision,", - 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, int_R_prec_cut[0]), NUM_PREC_PTS-1, - "int_%sR-prec", " after %s * R docs retrieved", - get_param_str_Rcutoff}, - {"Time: Utility (a,b,c,d):", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, av_time_utility), 1, - "time_integral_utility_%s", " Coefficients %s ", - get_param_str_utility}, - {"Time: num_rel at cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_rel[0]), NUM_TIME_PTS, - "time_num_rel_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: num_nonrel at cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_nrel[0]), NUM_TIME_PTS, - "time_num_nonrel_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: cumulative rel at cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_cum_rel[0]), NUM_TIME_PTS, - "time_cum_rel_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: precision at time cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS, - "time_precis_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: precision at time cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS, - "time_precis_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: relative precision at time cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_relprecis[0]), NUM_TIME_PTS, - "time_relative_precis_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: unranked precision at time cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_uap[0]), NUM_TIME_PTS, - "time_uap_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: relative unranked precision at time cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_reluap[0]), NUM_TIME_PTS, - "time_relative_uap_%s", " after %s seconds", - get_param_str_time_cutoff}, - {"Time: utility at time cutoff:", - 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_utility[0]), NUM_TIME_PTS, - "time_utility_%s", " after %s seconds", - get_param_str_time_utility_cutoff}, -}; - -int num_param_meas = sizeof (param_meas) / sizeof (param_meas[0]); - -MICRO_MEASURE micro_meas[] = { - {"micro_prec", "Total relevant retrieved documents / Total retrieved documents", - 0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_ret)}, - {"micro_recall", "Total relevant retrieved documents / Total relevant documents", - 0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_rel)}, - {"micro_bpref", "Total correct preferences / Total possible preferences", - 0, offsetof(TREC_EVAL, bpref_num_correct), offsetof(TREC_EVAL, bpref_num_possible)}, -}; - -int num_micro_meas = sizeof (micro_meas) / sizeof (micro_meas[0]); - diff --git a/sm_cnn/trec_eval-8.0/print_meas.c b/sm_cnn/trec_eval-8.0/print_meas.c deleted file mode 100755 index e478e3c..0000000 --- a/sm_cnn/trec_eval-8.0/print_meas.c +++ /dev/null @@ -1,336 +0,0 @@ -#ifdef RCSID -static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $"; -#endif - -/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. - - Permission is granted for use of this file in unmodified form for - research purposes. Please contact the SMART project to obtain - permission for other uses. -*/ - -#include "common.h" -#include "sysfunc.h" -#include "buf.h" -#include "trec_eval.h" - -static SM_BUF internal_output = {0, 0, (char *) 0}; -int add_buf_string(); - -extern SINGLE_MEASURE sing_meas[]; -extern PARAMETERIZED_MEASURE param_meas[]; -extern MICRO_MEASURE micro_meas[]; -extern int num_param_meas, num_sing_meas, num_micro_meas; - -int -accumulate_results (query_eval, accum_eval) -TREC_EVAL *query_eval; -TREC_EVAL *accum_eval; -{ - long i,j; - float *float_query, *float_accum; - long *long_query, *long_accum; - - if (query_eval->num_ret <= 0) - return (0); - - accum_eval->num_queries++; - - for (i = 0; i < num_sing_meas; i++) { - if (sing_meas[i].is_long_flag) { - long_query = (long *) (((char *) query_eval) + - sing_meas[i].byte_offset); - long_accum = (long *) (((char *) accum_eval) + - sing_meas[i].byte_offset); - *long_accum += *long_query; - } - else { - float_query = (float *) (((char *) query_eval) + - sing_meas[i].byte_offset); - float_accum = (float *) (((char *) accum_eval) + - sing_meas[i].byte_offset); - *float_accum += *float_query; - } - } - - for (i = 0; i < num_param_meas; i++) { - for (j = 0; j < param_meas[i].num_values; j++) { - if (param_meas[i].is_long_flag) { - long_query = (long *) (((char *) query_eval) + - param_meas[i].byte_offset); - long_accum = (long *) (((char *) accum_eval) + - param_meas[i].byte_offset); - long_accum[j] += long_query[j]; - } - else { - float_query = (float *) (((char *) query_eval) + - param_meas[i].byte_offset); - float_accum = (float *) (((char *) accum_eval) + - param_meas[i].byte_offset); - float_accum[j] += float_query[j]; - } - } - } - - return (0); -} - -void -print_rel_trec_eval_list (is_single_query_flag, epi, eval, output) -long is_single_query_flag; -EVAL_PARAM_INFO *epi; -TREC_EVAL *eval; -SM_BUF *output; -{ - long i,j; - char temp_buf[1024]; - char q_buf[20]; - char name_buf[80]; - SM_BUF *out_p; - long long_eval; - float float_eval; - - if (output == NULL) { - out_p = &internal_output; - out_p->end = 0; - } - else - out_p = output; - - if (is_single_query_flag) { - (void) sprintf (q_buf, "%.20s", eval[0].qid); - } - else { - (void) sprintf (q_buf, "%s", "all"); - (void) sprintf (temp_buf, "%-15s\t%s\t%ld\n", - "num_q", q_buf, eval->num_queries); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - - for (i = 0; i < num_sing_meas; i++) { - if ((! sing_meas[i].print_short_flag) && (! epi->all_flag)) - continue; - if (sing_meas[i].print_time_flag && (!epi->time_flag)) - continue; - if (sing_meas[i].print_only_query_flag && (!is_single_query_flag)) - continue; - if (sing_meas[i].print_only_average_flag && (is_single_query_flag)) - continue; - if (sing_meas[i].is_long_flag) { - long_eval = *((long *) (((char *) eval) + - sing_meas[i].byte_offset)); - if (sing_meas[i].avg_results_flag) - long_eval /= eval->num_queries; - (void) sprintf (temp_buf, "%-15s\t%s\t%ld\n", - sing_meas[i].name, q_buf, long_eval); - } - else { - float_eval = *((float *) (((char *) eval) + - sing_meas[i].byte_offset)); - if (sing_meas[i].avg_results_flag) - float_eval /= eval->num_queries; - else if (sing_meas[i].avg_rel_results_flag && eval->num_rel > 0) - /* average over number of rel docs instead of number queries */ - float_eval /= eval->num_rel; - else if (sing_meas[i].gm_results_flag) { - /* computing geometric mean instead of mean */ - if (!is_single_query_flag && epi->average_complete_flag) - /* Must patch up averages for any missing queries, since */ - /* value of 0 means perfection */ - float_eval += (eval->num_queries - eval->num_orig_queries)* - log (MIN_GEO_MEAN); - float_eval = (float) exp ((double) (float_eval / - eval->num_queries)); - } - (void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n", - sing_meas[i].name, q_buf, float_eval); - } - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - - for (i = 0; i < num_param_meas; i++) { - if ((! param_meas[i].print_short_flag) && (! epi->all_flag)) - continue; - if (param_meas[i].print_time_flag && (!epi->time_flag)) - continue; - if (param_meas[i].print_only_query_flag && (!is_single_query_flag)) - continue; - if (param_meas[i].print_only_average_flag && (is_single_query_flag)) - continue; - for (j = 0; j < param_meas[i].num_values; j++) { - sprintf (name_buf, param_meas[i].format_string, - param_meas[i].get_param_str (epi, j)); - if (param_meas[i].is_long_flag) { - long_eval = ((long *) (((char *) eval) + - param_meas[i].byte_offset))[j]; - if (param_meas[i].avg_results_flag) - long_eval /= eval->num_queries; - (void) sprintf (temp_buf, "%-15s\t%s\t%ld\n", - name_buf, q_buf, long_eval); - } - else { - float_eval = ((float *) (((char *) eval) + - param_meas[i].byte_offset))[j]; - if (param_meas[i].avg_results_flag) - float_eval /= eval->num_queries; - (void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n", - name_buf, q_buf, float_eval); - } - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - } - - if (! is_single_query_flag) { - long denom_long_eval; - for (i = 0; i < num_micro_meas; i++) { - if ((! micro_meas[i].print_short_flag) && (! epi->all_flag)) - continue; - long_eval = *((long *) (((char *) eval) + - micro_meas[i].numerator_byte_offset)); - denom_long_eval = *((long *) (((char *) eval) + - micro_meas[i].denominator_byte_offset)); - float_eval = (float) long_eval / (float) denom_long_eval; - (void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n", - micro_meas[i].name, q_buf, float_eval); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - } - - if (output == NULL) { - (void) fwrite (out_p->buf, 1, out_p->end, stdout); - out_p->end = 0; - } -} - -static long cutoff[] = CUTOFF_VALUES; - -void -old_print_trec_eval_list (epi, eval, num_runs, output) -EVAL_PARAM_INFO *epi; -TREC_EVAL *eval; -int num_runs; -SM_BUF *output; -{ - long i,j; - char temp_buf[1024]; - SM_BUF *out_p; - - if (output == NULL) { - out_p = &internal_output; - out_p->end = 0; - } - else - out_p = output; - - /* Print total numbers retrieved/rel for all runs */ - if (UNDEF == add_buf_string("\nQueryid (Num):\t", out_p)) - return; - for (i = 0; i < num_runs; i++) { - if (UNDEF == add_buf_string (eval->qid, out_p)) - return; - } - if (UNDEF == add_buf_string("\nTotal number of documents over all queries", - out_p)) - return; - if (UNDEF == add_buf_string("\n Retrieved:", out_p)) - return; - for (i = 0; i < num_runs; i++) { - (void) sprintf (temp_buf, " %5ld", eval[i].num_ret); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - if (UNDEF == add_buf_string("\n Relevant: ", out_p)) - return; - for (i = 0; i < num_runs; i++) { - (void) sprintf (temp_buf, " %5ld", eval[i].num_rel); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - if (UNDEF == add_buf_string("\n Rel_ret: ", out_p)) - return; - for (i = 0; i < num_runs; i++) { - (void) sprintf (temp_buf, " %5ld", eval[i].num_rel_ret); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - - /* Print recall precision figures at NUM_RP_PTS recall levels */ - if (UNDEF == add_buf_string - ("\nInterpolated Recall - Precision Averages:", out_p)) - return; - for (j = 0; j < NUM_RP_PTS; j++) { - (void) sprintf (temp_buf, "\n at %4.2f ", - (float) j / (NUM_RP_PTS - 1)); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - for (i = 0; i < num_runs; i++) { - (void) sprintf (temp_buf, " %6.4f ", - eval[i].int_recall_precis[j] /eval[i].num_queries); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - } - - /* Print average recall precision and percentage improvement */ - (void) sprintf (temp_buf, - "\nAverage precision (non-interpolated) for all rel docs(averaged over queries)\n "); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - for (i = 0; i < num_runs; i++) { - (void) sprintf (temp_buf, " %6.4f ", - eval[i].av_recall_precis / eval[i].num_queries); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - if (num_runs > 1) { - (void) sprintf (temp_buf, "\n %% Change: "); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - for (i = 1; i < num_runs; i++) { - (void) sprintf (temp_buf, " %6.1f ", - (((eval[i].av_recall_precis / eval[i].num_queries)/ - (eval[0].av_recall_precis / eval[i].num_queries)) - - 1.0) * 100.0); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - } - (void) sprintf (temp_buf, "\nPrecision:"); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - for (j = 0; j < NUM_CUTOFF; j++) { - (void) sprintf (temp_buf, "\n At %4ld docs:", cutoff[j]); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - for (i = 0; i < num_runs; i++) { - (void) sprintf (temp_buf, " %6.4f", - eval[i].precis_cut[j] / eval[i].num_queries); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - } - - (void) sprintf (temp_buf, "\nR-Precision (precision after R (= num_rel for a query) docs retrieved):\n Exact: "); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - for (i = 0; i < num_runs; i++) { - (void) sprintf (temp_buf, " %6.4f", - eval[i].R_recall_precis / eval[i].num_queries); - if (UNDEF == add_buf_string (temp_buf, out_p)) - return; - } - - if (UNDEF == add_buf_string ("\n", out_p)) - return; - - if (output == NULL) { - (void) fwrite (out_p->buf, 1, out_p->end, stdout); - out_p->end = 0; - } - return; -} - diff --git a/sm_cnn/trec_eval-8.0/smart_error.h b/sm_cnn/trec_eval-8.0/smart_error.h deleted file mode 100755 index 78d7523..0000000 --- a/sm_cnn/trec_eval-8.0/smart_error.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef SMART_ERRORH -#define SMART_ERRORH -#include -#define SMART_MINERR 1000 -#define SM_INCON_ERR 1000 -#define SM_ILLSK_ERR 1001 -#define SM_ILLMD_ERR 1002 -#define SM_ILLPA_ERR 1003 -#define SMART_NUMERR 4 - - -extern int errno; -extern int smart_errno; /* If > 0 and <= sys_nerr then refers to */ - /* sys_errlist, else if >= smart_errmin */ - /* and <= smart_errmax, then smart_errlist */ -extern char *smart_message; /* Message to be printed (often filename) */ -extern char *smart_routine; /* Major routine issuing error message */ - - -#define set_error(n,m,r) { if (n > 0) smart_errno = n;\ - smart_message = m;\ - smart_routine = r; } -#define clr_err() smart_errno = errno = 0 -#endif /* SMART_ERRORH */ diff --git a/sm_cnn/trec_eval-8.0/sysfunc.h b/sm_cnn/trec_eval-8.0/sysfunc.h deleted file mode 100755 index 64021da..0000000 --- a/sm_cnn/trec_eval-8.0/sysfunc.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef SYSFUNCH -#define SYSFUNCH -/* Declarations of major functions within standard C libraries */ -/* Once all of the major systems get their act together (and I follow - suit!), this file should just include system header files from - /usr/include. Until then... */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* For time being, define Berkeley constructs in terms of SVR4 constructs*/ -#define bzero(dest,len) memset(dest,'\0',len) -#define bcopy(source,dest,len) memcpy(dest,source,len) -#define srandom(seed) srand(seed) -#define random() rand() - -/* ANSI should give us an offsetof suitable for the implementation; - * otherwise, try a non-portable but commonly supported definition - */ -#ifdef __STDC__ -#include -#endif -#ifndef offsetof -#define offsetof(type, member) ((size_t) \ - ((char *)&((type*)0)->member - (char *)(type *)0)) -#endif - -#endif /* SYSFUNCH */ diff --git a/sm_cnn/trec_eval-8.0/tr_vec.h b/sm_cnn/trec_eval-8.0/tr_vec.h deleted file mode 100755 index 4b5b868..0000000 --- a/sm_cnn/trec_eval-8.0/tr_vec.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef TR_VECH -#define TR_VECH -/* $Header: /home/smart/release/./src/h/tr_vec.h,v 10.1 91/11/05 23:47:35 smart Exp Locker: smart $*/ - -typedef struct { - long did; /* document id */ - long rank; /* Rank of this document */ - char action; /* what action a user has taken with doc */ - char rel; /* whether doc judged relevant(1) or not(0) */ - char iter; /* Number of feedback runs for this query */ - char trtup_unused; /* Presently unused field */ - float sim; /* similarity of did to qid */ -} TR_TUP; - -typedef struct { - char *qid; /* query id */ - long num_tr; /* Number of tuples for tr_vec */ - TR_TUP *tr; /* tuples. Invariant: tr sorted increasing did */ -} TR_VEC; - -#endif /* TR_VECH */ diff --git a/sm_cnn/trec_eval-8.0/trec_eval.c b/sm_cnn/trec_eval-8.0/trec_eval.c deleted file mode 100755 index b85b8a7..0000000 --- a/sm_cnn/trec_eval-8.0/trec_eval.c +++ /dev/null @@ -1,256 +0,0 @@ -static char *VersionID = VERSIONID; -/* "Version 7.3 trec_eval Dec 15, 2004"; */ - -/* Copyright (c) 2004, 2003, 1991, 1990, 1984 - Chris Buckley. */ - -/******************** PROCEDURE DESCRIPTION ************************ - *0 Take TREC results text file, TREC qrels file, and evaluate - *1 local.convert.obj.trec_eval - *2 trec_eval [-q] [-a] [-t] [-o] [-v] [-n num] trec_rel_file trec_top_file - - *7 Read text tuples from trec_top_file of the form - *7 030 Q0 ZF08-175-870 0 4238 prise1 - *7 qid iter docno rank sim run_id - *7 giving TREC document numbers (a string) retrieved by query qid - *7 (an integer) with similarity sim (a float). The other fields are ignored. - *7 Input is asssumed to be sorted numerically by qid. - *7 Sim is assumed to be higher for the docs to be retrieved first. - *7 Relevance for each docno to qid is determined from text_qrels_file, which - *7 consists of text tuples of the form - *7 qid iter docno rel - *7 giving TREC document numbers (a string) and their relevance to query qid - *7 (an integer). Tuples are asssumed to be sorted numerically by qid. - *7 The text tuples with relevence judgements are converted to TR_VEC form - *7 and then submitted to the evaluation routines. - *7 - *7 -q: In addition to summary evaluation, give evaluation for each query - *7 -a: Print all evaluation measures calculated, instead of just the - *7 official measures for TREC 2. - *7 -o: Print everything out in old, non-relational format - *7 -v: Print version number and exit - *7 -h: Print full help message and exit - *7 -t: Treat similarity as time that document retrieved. Compute - *7 several time-based measures after ranking docs by time retrieved - *7 (first doc (lowest sim) retrieved ranked highest). - *7 Only done if -a selected. - *7 -J: Calculate all measures only over judged documents that appear - *7 in qrels. (DO NOT USE) - *7 -n: following integer is the number of queries to average over. - *7 -ua: Value to use for 'a' coefficient of utility computation. - *7 -ub: Value to use for 'b' coefficient of utility computation. - *7 -uc: Value to use for 'c' coefficient of utility computation. - *7 -ud: Value to use for 'd' coefficient of utility computation. - *7 -N: Number of docs in collection - *7 -M:Max number of results to evaluate per topic - - *8 Procedure is to read all the docs retrieved for a query, and all the - *8 relevant docs for that query, - *8 sort and rank the retrieved docs by sim/docno, - *8 and look up docno in the relevant docs to determine relevance. - *8 The qid,did,rank,sim,rel fields of of TR_VEC are filled in; - *8 action,iter fields are set to 0. - *8 Queries for which there are no relevant docs are ignored completely. - -***********************************************************************/ - -#include "common.h" -#include "sysfunc.h" -#include "smart_error.h" -#include "tr_vec.h" -#include "trec_eval.h" -#include "buf.h" - -void print_error(); -void old_print_trec_eval_list(); -void print_rel_trec_eval_list(); - -int trec_eval_help(EVAL_PARAM_INFO *epi); -int accumulate_results (TREC_EVAL *query_eval, TREC_EVAL *accum_eval); -int get_top (char *trec_top_file, ALL_TREC_TOP *all_trec_top); -int get_qrels (char *text_qrels_file, ALL_TREC_QRELS *all_trec_qrels); -int form_trvec (EVAL_PARAM_INFO *ep, TREC_TOP *trec_top, - TREC_QRELS *trec_qrels, TR_VEC *tr_vec, long *num_rel); -int trvec_trec_eval (EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, - TREC_EVAL *eval, long num_rel, long num_nonrel); - -static char *usage = "Usage: trec_eval [-h] [-q] [-a] [-o] [-v] trec_rel_file trec_top_file\n\ - -h: Give full help information, including other options\n\ - -q: In addition to summary evaluation, give evaluation for each query\n\ - -a: Print all evaluation measures, instead of just official measures\n\ - -o: Print requested measures in old non-relational format\n"; - - -int -main (argc, argv) -int argc; -char *argv[]; -{ - char *trec_rel_file, *trec_top_file; - ALL_TREC_TOP all_trec_top; - ALL_TREC_QRELS all_trec_qrels; - TREC_EVAL accum_eval, query_eval; - TR_VEC tr_vec; - long num_rel; - long num_eval_q; - long i,j; - EVAL_PARAM_INFO epi; - - /* Initialize static info before getting program optional args */ - epi.query_flag = epi.all_flag = epi.time_flag = epi.average_complete_flag = 0; - epi.judged_docs_only_flag = 0; - epi.relation_flag = 1; - epi.utility_a = UTILITY_A; epi.utility_b = UTILITY_B; - epi.utility_c = UTILITY_C; epi.utility_d = UTILITY_D; - epi.num_docs_in_coll = 0; - epi.relevance_level = 1; - epi.max_num_docs_per_topic = MAXLONG; - - /* Should use getopts, but some people may not have it. */ - /* This keeps growing over the years. Should redo */ - while (argc > 1 && argv[1][0] == '-') { - if (argv[1][1] == 'q') - epi.query_flag++; - else if (argv[1][1] == 'v') { - fprintf (stderr, "trec_eval version %s\n", VersionID); - exit (0); - } - else if (argv[1][1] == 'h') { - (void) trec_eval_help(&epi); - exit (0); - } - else if (argv[1][1] == 'a') - epi.all_flag++; - else if (argv[1][1] == 'o') - epi.relation_flag = 0; - else if (argv[1][1] == 'c') { - epi.average_complete_flag++; - } - else if (argv[1][1] == 'l') { - epi.relevance_level = atol (&argv[1][2]); - } - else if (argv[1][1] == 'J') { - epi.judged_docs_only_flag++; - } - else if (argv[1][1] == 'N') - epi.num_docs_in_coll = atol (&argv[1][2]); - else if (argv[1][1] == 'M') - epi.max_num_docs_per_topic = atol (&argv[1][2]); - else if (argv[1][1] == 'U') { - if (argv[1][2] == 'a') - epi.utility_a = atof (&argv[1][3]); - else if (argv[1][2] == 'b') - epi.utility_b = atof (&argv[1][3]); - else if (argv[1][2] == 'c') - epi.utility_c = atof (&argv[1][3]); - else if (argv[1][2] == 'd') - epi.utility_d = atof (&argv[1][3]); - else { - (void) fputs (usage,stderr); - exit (1); - } - } - else if (argv[1][1] == 'T') - epi.time_flag++; - else { - (void) fputs (usage,stderr); - exit (1); - } - argc--; argv++; - } - - if (argc != 3) { - (void) fputs (usage,stderr); - exit (1); - } - - trec_rel_file = argv[1]; - trec_top_file = argv[2]; - - /* Get qrels and top results information for all queries from the - input text files */ - if (UNDEF == get_qrels (trec_rel_file, &all_trec_qrels) || - UNDEF == get_top (trec_top_file, &all_trec_top)) { - print_error ("trec_eval: input error", "Quit"); - exit (2); - } - - /* For each topic which has both qrels and top results information, - calculate, possibly print (if query_flag), and accumulate - evaluation measures. */ - num_eval_q = 0; - (void) memset ((void *) &accum_eval, 0, sizeof (TREC_EVAL)); - accum_eval.qid = "All"; - - for (i = 0; i < all_trec_top.num_q_tr; i++) { - /* Find rel info for this query (skip if no rel info) */ - for (j = 0; j < all_trec_qrels.num_q_qrels; j++) { - if (0 == strcmp (all_trec_top.trec_top[i].qid, - all_trec_qrels.trec_qrels[j].qid)) - break; - } - if (j >= all_trec_qrels.num_q_qrels) - continue; - - /* Form results/rel into SMART TR_VEC form */ - if (UNDEF == form_trvec (&epi, - &all_trec_top.trec_top[i], - &all_trec_qrels.trec_qrels[j], - &tr_vec, - &num_rel)) { - print_error ("trec_eval: form_tr_vec error", "Quit"); - exit (3); - } - - /* Evaluate results/rel for this query */ - if (UNDEF == trvec_trec_eval (&epi, - &tr_vec, - &query_eval, - num_rel, - all_trec_qrels.trec_qrels[j].num_text_qrels - num_rel)) { - print_error ("trec_eval: evaluation error", "Quit"); - exit (4); - } - - /* Print results for this query, if desired */ - if (epi.query_flag) { - if (epi.relation_flag) - print_rel_trec_eval_list (1, &epi, &query_eval, (SM_BUF *) NULL); - else - old_print_trec_eval_list (&epi, &query_eval, 1, (SM_BUF *) NULL); - } - - /* Accumulate results for later averaging */ - if (UNDEF == accumulate_results (&query_eval, &accum_eval)) { - print_error ("trec_eval: accumulation error", "Quit"); - exit (5); - } - - num_eval_q++; - } - - /******** REMOVE THIS ONCE WARNING FLAG ADDED */ - /* Warn if numq_flag_num < num_eval_q */ - if (num_eval_q == 0) { - set_error (SM_INCON_ERR, - "No queries with both results and relevance info", - "trec_eval"); - return (UNDEF); - print_error ("trec_eval", "Quit"); - exit (6); - } - - if (epi.average_complete_flag) { - /* Want to average over possibly missing queries. Pass in actual - * number of queries in num_orig_queries */ - accum_eval.num_orig_queries = accum_eval.num_queries; - accum_eval.num_queries = all_trec_qrels.num_q_qrels; - } - - /* Print final evaluation results */ - if (epi.relation_flag) - print_rel_trec_eval_list (0, &epi, &accum_eval, (SM_BUF *) NULL); - else - old_print_trec_eval_list (&epi, &accum_eval, 1, (SM_BUF *) NULL); - - exit (0); -} diff --git a/sm_cnn/trec_eval-8.0/trec_eval.h b/sm_cnn/trec_eval-8.0/trec_eval.h deleted file mode 100755 index 54859af..0000000 --- a/sm_cnn/trec_eval-8.0/trec_eval.h +++ /dev/null @@ -1,353 +0,0 @@ -#ifndef TRECEVALH -#define TRECEVALH - -/* Static state info; set at beginning, possibly from program options, */ -/* but then remains constant throughout. */ -typedef struct { - long query_flag; /* 0. If set, evaluation output will be - printed for each query, in addition - to summary at end. */ - long all_flag; /* 0. If set, all evaluation measures will - be printed instead of just the - final TREC 2 measures. */ - long time_flag; /* 0. If set, calculate time-based measures*/ - long relation_flag; /* 1. If set, print in relational form */ - long average_complete_flag; /* 0. If set, average over the complete set - of relevance judgements (qrels), instead - of the number of queries - in the intersection of qrels and result */ - long judged_docs_only_flag; /* 0. If set, throw out all unjudged docs - for the retrieved set before calculating - any measures. */ - double utility_a; /* UTILITY_A. Default utility values */ - double utility_b; /* UTILITY_B. Default utility values */ - double utility_c; /* UTILITY_C. Default utility values */ - double utility_d; /* UTILITY_D. Default utility values */ - long num_docs_in_coll; /* 0. number of docs in collection */ - long relevance_level; /* 1. In relevance judgements, the level at - which a doc is considered relevant for - this evaluation */ - long max_num_docs_per_topic; /* MAXLONG. evaluate only this many docs */ -} EVAL_PARAM_INFO; - -/* Measure characteristics (how to print them, average them). */ -/* List of measures is in measures.c */ -/* Three types of measures: - single measures - single measure and name - parameterized measures - arrays of a measure, whose measure name - depends on parameter (eg P5, P10) - micro measures - measures defined as the micro average over all - docs retrieved independent of topic. Only calculated - and printed for the "all" pseudo-query. - Eg micro_prec = num_rel_ret / num_ret -*/ -typedef struct { - char *name; - char *long_name; - unsigned char is_long_flag; /* otherwise float */ - unsigned char print_short_flag; /* if set, measure is always printed - (not just if all_flag set) */ - unsigned char print_time_flag; /* if set, measure is printed only - if time_flag is set */ - unsigned char print_only_query_flag; /* if set, measure is printed only - when printing individual query output*/ - unsigned char print_only_average_flag; /* if set, measure is printed only - when printing overall average output*/ - unsigned char avg_results_flag; /* if set, average results over queries */ - unsigned char avg_rel_results_flag;/* if set,average results over num_rel*/ - unsigned char gm_results_flag; /* if set, measure uses geometric mean. ie - exponentiate the average before - printing */ - long byte_offset; -} SINGLE_MEASURE; - -typedef struct { - char *long_name; - unsigned char is_long_flag; /* otherwise float */ - unsigned char print_short_flag; /* if set, print in short output */ - unsigned char print_time_flag; /* if set, measure is printed only - if time_flag is set */ - unsigned char print_only_query_flag; /* if set, measure is printed only - when printing individual query output*/ - unsigned char print_only_average_flag; /* if set, measure is printed only - when printing overall average output*/ - unsigned char avg_results_flag; /* if set, average results over queries */ - long byte_offset; - long num_values; - char *format_string; - char *long_format_string; - char *(*get_param_str) (EVAL_PARAM_INFO *ip, long index); -} PARAMETERIZED_MEASURE; - -typedef struct { - char *name; - char *long_name; - unsigned char print_short_flag; /* if set, measure is always printed - (not just if all_flag set) */ - long numerator_byte_offset; - long denominator_byte_offset; -} MICRO_MEASURE; - - -typedef struct { /* For each retrieved document result */ - char *docno; /* document id */ - float sim; /* score */ - long rank; /* rank assigned after breaking ties */ -} TEXT_TR; - -typedef struct { /* For each query in retrieved results */ - char *qid; /* query id */ - long num_text_tr; /* number of TEXT_TR results for query*/ - long max_num_text_tr; /* number results space reserved for */ - TEXT_TR *text_tr; /* Array of TEXT_TR results */ -} TREC_TOP; - -typedef struct { /* Overall retrieved results */ - char *run_id; /* run id */ - long num_q_tr; /* Number of TREC_TOP queries */ - long max_num_q_tr; /* Num queries space reserved for*/ - TREC_TOP *trec_top; /* Array of TREC_TOP query results */ -} ALL_TREC_TOP; - -typedef struct { /* For each relevance judgement */ - char *docno; /* document id */ - long rel; /* document judgement */ -} TEXT_QRELS; - -typedef struct { /* For each query in rel judgements */ - char *qid; /* query id */ - long num_text_qrels; /* number of judged documents */ - long max_num_text_qrels; /* Num docs space reserved for */ - TEXT_QRELS *text_qrels; /* Array of judged TEXT_QRELS */ -} TREC_QRELS; - -typedef struct { /* Overall relevance judgements */ - long num_q_qrels; /* Number of TREC_QRELS queries */ - long max_num_q_qrels; /* Num queries space reserved for */ - TREC_QRELS *trec_qrels; /* Array of TREC_QRELS queries */ -} ALL_TREC_QRELS; - - - -#define INIT_NUM_QUERIES 50 -#define INIT_NUM_RESULTS 1000 -#define INIT_NUM_RELS 2000 - -/* Set retrieval is based on contingency table: - relevant nonrelevant - retrieved a b - nonretrieved c d - - Often you see r == num_rel_ret == a - R == num_rel == a+c - n == num_ret == a+b - N == num_docs == a+b+c+d - Some of these definitions are used in comments below -*/ - - -/* ----------------------------------------------- */ -/* Defined constants that are collection/purpose dependent */ - -/* Number of cutoffs for recall,precision, and rel_precis measures. */ -/* CUTOFF_VALUES gives the number of retrieved docs that these */ -/* evaluation mesures are applied at. */ -#define NUM_CUTOFF 9 -#define CUTOFF_VALUES {5, 10, 15, 20, 30, 100, 200, 500, 1000} - -/* Maximum fallout value, expressed in number of non-rel docs retrieved. */ -/* (Make the approximation that number of non-rel docs in collection */ -/* is equal to the number of number of docs in collection) */ -#define MAX_FALL_RET 142 - -/* Maximum multiple of R (number of rel docs for this query) to calculate */ -/* R-based precision at */ -#define MAX_RPREC 2.0 - -#define MAX_TIME 300.0 -#define NUM_TIME_PTS 60 - -/* Set a maximum number of nonrel docs to be used for preference measures */ -#define PREF_TOP_NONREL_NUM 100 - -/* ----------------------------------------------- */ -/* Defined constants that are collection/purpose independent. If you - change these, you probably need to change comments and documentation, - and some variable names may not be appropriate any more! */ -#define NUM_RP_PTS 11 -#define THREE_PTS {2, 5, 8} -#define NUM_FR_PTS 11 -#define NUM_PREC_PTS 11 -#define UTILITY_A 1.0 -#define UTILITY_B -1.0 -#define UTILITY_C 0.0 -#define UTILITY_D 0.0 -#define MIN_GEO_MEAN .00001 - -typedef struct { - char *qid; /* query id */ - long num_queries; /* Number of queries for this eval */ - long num_orig_queries; /* Number of queries for this eval without - missing values, if using trec_eval -c */ - /* Summary Numbers over all queries */ - long num_rel; /* Number of relevant docs */ - long num_ret; /* Number of retrieved docs */ - long num_rel_ret; /* Number of relevant retrieved docs */ - float avg_doc_prec; /* Average of precision over all - relevant documents (query independent)*/ - - /* Measures after num_ret docs */ - float exact_recall; /* Recall after num_ret docs */ - float exact_precis; /* Precision after num_ret docs */ - float exact_rel_precis; /* Relative Precision (or recall) */ - /* Defined to be precision / max possible - precision */ - float exact_uap; /* Unranked Average Precision */ - /* Every rel doc in retrieved set gets - precision, every nonret rel doc gets 0. - Average over all rel docs */ - /* Note this = exact_recall * - exact_precision for a query */ - /* Preferred measure for evaluation of - unranked sets of arbitrary size. */ - float exact_rel_uap; /* Relative Unranked Average Precision */ - /* Above, but relativized given size of - retrieved set */ - /* If (nR) set num_ret to R - Then use uap formula */ - /* exact_rel_precis ** 2 */ - float exact_utility; /* From contingency table, by default: - UTILITY_A * a + UTILITY_B * b + - UTILITY_C * c + UTILITY_D * d. - By default, a-b (or r - (n-r)) */ - float recip_rank; /* reciprical rank of top retrieved - relevant document */ - long rank_first_rel; /* Rank of top retrieved rel doc. Set to - 0 if none. Unaveraged */ - - /* Measures after each document */ - float recall_cut[NUM_CUTOFF]; /* Recall after cutoff[i] docs */ - - float precis_cut[NUM_CUTOFF]; /* precision after cutoff[i] docs. If - less than cutoff[i] docs retrieved, - then assume an additional - cutoff[i]-num_ret non-relevant docs - are retrieved. */ - float rel_precis_cut[NUM_CUTOFF];/* Relative precision after cutoff[i] - docs. (Note relative precision is - identical to relative recall) */ - float uap_cut[NUM_CUTOFF]; /* uap (is recall * precision) after - cutoff[i] docs. Not recommended */ - float rel_uap_cut[NUM_CUTOFF]; /* rel_uap at cutoff[i] docs */ - float av_rel_precis; /* average (integral) of rel_precis - after each doc. Do not use if - number of docs retrieved varies */ - float av_rel_uap; /* average (integral) of rel_uap - after each doc. Do not use if - number of docs retrieved varies */ - - - /* Measures after each rel doc */ - float av_recall_precis; /* MAP! average(integral) of precision at - all rel doc ranks. THE MAJOR - EVALUATION MEASURE FOR RANKED DOCS */ - float int_av_recall_precis; /* Same as above, but the precision values - have been interpolated, so that prec(X) - is actually MAX prec(Y) for all - Y >= X */ - float int_recall_precis[NUM_RP_PTS];/* interpolated precision at - 0.1 increments of recall */ - float int_av3_recall_precis; /* interpolated average at 3 intermediate - points */ - float int_av11_recall_precis; /* interpolated average at NUM_RP_PTS - intermediate points (recall_level) */ - - /* Measures after each non-rel doc */ - float fall_recall[NUM_FR_PTS]; /* max recall after each non-rel doc, - at 11 points starting at 0.0 and - ending at MAX_FALL_RET /num_docs */ - float av_fall_recall; /* Average of fallout-recall, after each - non-rel doc until fallout of - MAX_FALL_RET / num_docs achieved */ - - /* Measures after R-related cutoffs. R is the number of relevant - docs for a particular query, but note that these cutoffs are after - R docs, whether relevant or non-relevant, have been retrieved. - R-related cutoffs are really only applicable to a situtation where - there are many relevant docs per query (or lots of queries). */ - float R_recall_precis; /* Recall or precision after R docs - (note they are equal at this point) */ - float av_R_precis; /* Average (or integral) of precision at - each doc until R docs have been - retrieved */ - float R_prec_cut[NUM_PREC_PTS]; /* Precision measured after multiples of - R docs have been retrieved. 10 - equal points, with max multiple - having value MAX_RPREC */ - float int_R_recall_precis; /* Interpolated precision after R docs - Prec(X) = MAX(prec(Y)) for all Y>=X */ - float int_av_R_precis; /* Interpolated */ - float int_R_prec_cut[NUM_PREC_PTS]; /* Interpolated */ - - /* Measures after particular time relative to size of eventual retrieved - set. Eg, precision is num_rel_so_far/num_ret - relprecision is num_rel_so_far/MIN(num_ret,num_rel) - uap is num_rel_so_far**2/(num_ret*MIN(num_ret,num_rel)) - reluap is relprecision * relprecision */ - float time_num_rel[NUM_TIME_PTS]; /* Number of rel docs in time bucket*/ - float time_num_nrel[NUM_TIME_PTS];/* Number of nrel docs in each bucket*/ - float time_cum_rel[NUM_TIME_PTS]; /* Cumulative time_num_rel */ - float time_precis[NUM_TIME_PTS]; /* First Precision in each bucket */ - float time_relprecis[NUM_TIME_PTS];/* First rel-Precision in each bucket */ - float time_uap[NUM_TIME_PTS]; /* First uap in bucket*/ - float time_reluap[NUM_TIME_PTS]; /* First relative uap in bucket*/ - float time_utility[NUM_TIME_PTS]; /* First Utility (default 1,-1,0,0) - in bucket */ - float av_time_precis; /* Sum (integral) of time_precis */ - float av_time_relprecis; /* Sum (integral) of time_relprecis */ - float av_time_uap; /* Sum (integral) of time_uap */ - float av_time_reluap; /* Sum (integral) of time_reluap */ - float av_time_utility; /* Sum (integral) of time_utility */ - float av_time_cum_rel; /* Sum (integral) of time_cum_rel */ - - /* Measures dependent on only judged documents */ - /* Binary Pref relations: fraction of nonrel documents retrieved after - each rel doc */ - float bpref; /* real BPREF. Top num_rel nonrel docs */ - float bpref_top5Rnonrel; /* Top 5 * num_rel nonrel docs */ - float bpref_top10Rnonrel; /* Top 10 * num_rel nonrel docs */ -/* float bpref_topRnonrel; * renamed as bpref */ - float bpref_allnonrel; /* all judged nonrel docs */ - float bpref_retnonrel; /* Only retrieved nonrel docs */ - float bpref_topnonrel; /* Top PREF_TOPNREL_NUM nonrel docs */ - float bpref_top50pRnonrel; /* Top 50 + num_rel nonrel docs */ - float bpref_top25pRnonrel; /* Top 25 + num_rel nonrel docs */ - float bpref_top10pRnonrel; /* Top 10 + num_rel nonrel docs. - Bad version used in SIGIR 2004 paper */ - float old_bpref_top10pRnonrel; /* bad old version. Top 10 + num_rel - nonrel docs. Used in SIGIR 2004 paper*/ - float bpref_top25p2Rnonrel; /* Top 25 + 2 * num_rel nonrel docs */ - float bpref_retall; /* Only retrieved rel,nonrel docs */ - float bpref_5; /* Only top 5 rel, top 5 nonrel */ - float bpref_10; /* Only top 10 rel, top 10 nonrel */ - float old_bpref; /* Bad old bpref. Top num_rel nonrel docs. - Only used retrieved nonrel docs. - Used in TREC 12,13, mention in - SIGIR 2004 paper */ - float bpref_num_all; /* num not retrieved before (all judged)*/ - float bpref_num_ret; /* num retrieved after */ - long bpref_num_correct; /* num correct preferences */ - long bpref_num_possible; /* num possible correct preferences */ - - /* Measures that use Geometric Mean - avg_Score = exp (SUM (log (MAX (query_score, .00001))) / N) - WARNING: Geometric Mean measures special cased for "trec_eval -c". - Works, but be careful when implementing new measure */ - float gm_ap; /* Geometric Mean version of MAP */ - float gm_bpref; /* Geometric Mean version of bpref. Note - bpref has lots of 0.0 values */ - -} TREC_EVAL; - -#endif /* TRECEVALH */ diff --git a/sm_cnn/trec_eval-8.0/trec_eval_help.c b/sm_cnn/trec_eval-8.0/trec_eval_help.c deleted file mode 100755 index eb7bf60..0000000 --- a/sm_cnn/trec_eval-8.0/trec_eval_help.c +++ /dev/null @@ -1,203 +0,0 @@ -/* Copyright (c) 2003, 1991, 1990, 1984 - Chris Buckley. */ - -#include "common.h" -#include "trec_eval.h" - -static char *help_message = -"trec_eval [-h] [-q] [-a] [-o] [-c] [-l [-N] [-M] [-Ua] [-Ub] [-Uc] [-Ud] [-T] trec_rel_file trec_top_file \n\ - \n\ -Calculate and print various evaluation measures, evaluating the results \n\ -in trec_top_file against the relevance judgements in trec_rel_file. \n\ - \n\ -There are a fair number of options, of which only the lower case options are \n\ -normally ever used. \n\ - -h: Print full help message and exit \n\ - -q: In addition to summary evaluation, give evaluation for each query \n\ - -a: Print all evaluation measures calculated, instead of just the \n\ - main official measures for TREC. \n\ - -o: Print everything out in old, nonrelational format (default is relational) \n\ - -c: Average over the complete set of queries in the relevance judgements \n\ - instead of the queries in the intersection of relevance judgements \n\ - and results. Missing queries will contribute a value of 0 to all \n\ - evaluation measures (which may or may not be reasonable for a \n\ - particular evaluation measure, but is reasonable for standard TREC \n\ - measures.) \n\ - -l: Num indicates the minimum relevance judgement value needed for \n\ - a document to be called relevant. (All measures used by TREC eval are \n\ - based on binary relevance). Used if trec_rel_file contains relevance \n\ - judged on a multi-relevance scale. Default is 1. \n\ - -N: Number of docs in collection \n\ - -M: Max number of docs per topic to use in evaluation (discard rest). \n\ - -Ua: Value to use for 'a' coefficient of utility computation. \n\ - relevant nonrelevant \n\ - retrieved a b \n\ - nonretrieved c d \n\ - -Ub: Value to use for 'b' coefficient of utility computation. \n\ - -Uc: Value to use for 'c' coefficient of utility computation. \n\ - -Ud: Value to use for 'd' coefficient of utility computation. \n\ - -J: Calculate all values only over the judged (either relevant or \n\ - nonrelevant) documents. All unjudged documents are removed from the \n\ - retrieved set before any calculations (possibly leaving an empty set). \n\ - DO NOT USE, unless you really know what you're doing - very easy to get \n\ - reasonable looking, but invalid, numbers. \n\ - -T: Treat similarity as time that document retrieved. Compute \n\ - several time-based measures after ranking docs by time retrieved \n\ - (first doc (lowest sim) retrieved ranked highest). \n\ - Only done if -a selected. \n\ - \n\ - \n\ -Read text tuples from trec_top_file of the form \n\ - 030 Q0 ZF08-175-870 0 4238 prise1 \n\ - qid iter docno rank sim run_id \n\ -giving TREC document numbers (a string) retrieved by query qid \n\ -(a string) with similarity sim (a float). The other fields are ignored, \n\ -with the exception that the run_id field of the last line is kept and \n\ -output. In particular, note that the rank field is ignored here; \n\ -internally ranks are assigned by sorting by the sim field with ties \n\ -broken deterministicly (using docno). \n\ -Sim is assumed to be higher for the docs to be retrieved first. \n\ -File may contain no NULL characters. \n\ -Lines may contain fields after the run_id; they are ignored. \n\ - \n\ -Relevance for each docno to qid is determined from text_qrels_file, which \n\ -consists of text tuples of the form \n\ - qid iter docno rel \n\ -giving TREC document numbers (docno, a string) and their relevance (rel, \n\ -an integer) to query qid (a string). iter string field is ignored. \n\ -Fields are separated by whitespace, string fields can contain no whitespace. \n\ -File may contain no NULL characters. \n\ - \n\ -The text tuples with relevance judgements are converted to TR_VEC form \n\ -and then submitted to the SMART evaluation routines. \n\ -The qid,did,rank,sim,rel fields of TR_VEC are filled in; \n\ -action,iter fields are set to 0. \n\ -The rel field is set to -1 if the document was not judged (not in \n\ -text_qrels_file). Most measures, but not all, will treat -1 the same as 0, \n\ -namely nonrelevant. Note that relevance_level is used to determine if the \n\ -document is relevant during score calculations. \n\ -Queries for which there are no relevant docs are ignored. \n\ -Warning: queries for which there are relevant docs but no retrieved docs \n\ -are also ignored by default. This allows systems to evaluate over subsets \n\ -of the relevant docs, but means if a system improperly retrieves no docs, \n\ -it will not be detected. Use the -c flag to avoid this behavior. \n\ - \n\ -EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. \n\ -Relational Format prints the same values, but all lines are of the form \n\ - measure_name query value \n\ - \n\ -1. Total number of documents over all queries \n\ - Retrieved: \n\ - Relevant: \n\ - Rel_ret: (relevant and retrieved) \n\ - These should be self-explanatory. All values are totals over all \n\ - queries being evaluated. \n\ -2. Interpolated Recall - Precision Averages: \n\ - at 0.00 \n\ - at 0.10 \n\ - ... \n\ - at 1.00 \n\ - See any standard IR text (especially by Salton) for more details of \n\ - recall-precision evaluation. Measures precision (percent of retrieved \n\ - docs that are relevant) at various recall levels (after a certain \n\ - percentage of all the relevant docs for that query have been retrieved). \n\ - 'Interpolated' means that, for example, precision at recall \n\ - 0.10 (ie, after 10% of rel docs for a query have been retrieved) is \n\ - taken to be MAXIMUM of precision at all recall points >= 0.10. \n\ - Values are averaged over all queries (for each of the 11 recall levels). \n\ - These values are used for Recall-Precision graphs. \n\ -3. Average precision (non-interpolated) over all rel docs \n\ - The precision is calculated after each relevant doc is retrieved. \n\ - If a relevant doc is not retrieved, its precision is 0.0. \n\ - All precision values are then averaged together to get a single number \n\ - for the performance of a query. Conceptually this is the area \n\ - underneath the recall-precision graph for the query. \n\ - The values are then averaged over all queries. \n\ -4. Precision: \n\ - at 5 docs \n\ - at 10 docs \n\ - ... \n\ - at 1000 docs \n\ - The precision (percent of retrieved docs that are relevant) after X \n\ - documents (whether relevant or nonrelevant) have been retrieved. \n\ - Values averaged over all queries. If X docs were not retrieved \n\ - for a query, then all missing docs are assumed to be non-relevant. \n\ -5. R-Precision (precision after R (= num_rel for a query) docs retrieved): \n\ - Measures precision (or recall, they're the same) after R docs \n\ - have been retrieved, where R is the total number of relevant docs \n\ - for a query. Thus if a query has 40 relevant docs, then precision \n\ - is measured after 40 docs, while if it has 600 relevant docs, precision \n\ - is measured after 600 docs. This avoids some of the averaging \n\ - problems of the 'precision at X docs' values in (4) above. \n\ - If R is greater than the number of docs retrieved for a query, then \n\ - the nonretrieved docs are all assumed to be nonrelevant. \n\ -"; - - -extern SINGLE_MEASURE sing_meas[]; -extern PARAMETERIZED_MEASURE param_meas[]; -extern MICRO_MEASURE micro_meas[]; -extern int num_param_meas, num_sing_meas, num_micro_meas; - -int -trec_eval_help(epi) -EVAL_PARAM_INFO *epi; -{ - long i, j; - char temp_buf1[30]; - char temp_buf2[80]; - - printf ("%s\n", help_message); - - printf ("Major measures (again) with their relational names:\n"); - for (i = 0; i < num_sing_meas; i++) { - if (sing_meas[i].print_short_flag) - printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name); - } - for (i = 0; i < num_param_meas; i++) { - if (param_meas[i].print_short_flag) { - for (j = 0; j < param_meas[i].num_values; j++) { - sprintf (temp_buf1, param_meas[i].format_string, - param_meas[i].get_param_str (epi, j)); - sprintf (temp_buf2, param_meas[i].long_format_string, - param_meas[i].get_param_str (epi, j)); - printf ("%-15s\t%s%s\n", temp_buf1, - param_meas[i].long_name, temp_buf2); - } - } - } - for (i = 0; i < num_micro_meas; i++) { - if (micro_meas[i].print_short_flag) - printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name); - } - - printf ("\n\nMinor measures with their relational names:\n"); - for (i = 0; i < num_sing_meas; i++) { - if (sing_meas[i].print_short_flag) - continue; - if (sing_meas[i].print_time_flag && (! epi->time_flag)) - continue; - if (! sing_meas[i].print_short_flag) - printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name); - } - for (i = 0; i < num_param_meas; i++) { - if (param_meas[i].print_short_flag) - continue; - if (param_meas[i].print_time_flag && (! epi->time_flag)) - continue; - for (j = 0; j < param_meas[i].num_values; j++) { - sprintf (temp_buf1, param_meas[i].format_string, - param_meas[i].get_param_str (epi, j)); - sprintf (temp_buf2, param_meas[i].long_format_string, - param_meas[i].get_param_str (epi, j)); - printf ("%-15s\t%s%s\n", temp_buf1, - param_meas[i].long_name, temp_buf2); - } - } - for (i = 0; i < num_micro_meas; i++) { - if (! micro_meas[i].print_short_flag) - printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name); - } - - return (1); -} - diff --git a/sm_cnn/trec_eval-8.0/trvec_teval.c b/sm_cnn/trec_eval-8.0/trvec_teval.c deleted file mode 100755 index e2d2891..0000000 --- a/sm_cnn/trec_eval-8.0/trvec_teval.c +++ /dev/null @@ -1,673 +0,0 @@ -#ifdef RCSID -static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/trvec_trec_eval.c,v 11.0 1992/07/21 18:20:35 chrisb Exp chrisb $"; -#endif - -/* Copyright (c) 2005 -*/ - -#include "common.h" -#include "sysfunc.h" -#include "smart_error.h" -#include "tr_vec.h" -#include "trec_eval.h" - -static int compare_iter_rank(); -static void calc_cutoff_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, - TREC_EVAL *eval, long num_rel, - long num_nonrel); -static void calc_bpref_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, - TREC_EVAL *eval, long num_rel, - long num_nonrel); -static void calc_average_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, - TREC_EVAL *eval, long num_rel, - long num_nonrel); -static void calc_exact_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, - TREC_EVAL *eval, long num_rel, - long num_nonrel); -static void calc_time_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, - TREC_EVAL *eval, long num_rel, - long num_nonrel); - -int -trvec_trec_eval (epi, tr_vec, eval, num_rel, num_nonrel) -EVAL_PARAM_INFO *epi; -TR_VEC *tr_vec; -TREC_EVAL *eval; -long num_rel; /* Number relevant judged */ -long num_nonrel; /* Number nonrelevant judged */ -{ - long j; - long max_iter; - - if (tr_vec == (TR_VEC *) NULL) - return (UNDEF); - - /* Initialize everything to 0 */ - bzero ((char *) eval, sizeof (TREC_EVAL)); - - eval->qid = tr_vec->qid; - eval->num_queries = 1; - - /* If no retrieved docs, then just return */ - if (tr_vec->num_tr == 0) { - return (0); - } - - eval->num_rel = num_rel; - - /* Evaluate only the docs on the last iteration of new_tr_vec */ - /* Sort the tr tuples for this query by decreasing iter and - increasing rank */ - qsort ((char *) tr_vec->tr, - (int) tr_vec->num_tr, - sizeof (TR_TUP), - compare_iter_rank); - - max_iter = tr_vec->tr[0].iter; - for (j = 0; j < tr_vec->num_tr; j++) { - if (tr_vec->tr[j].iter == max_iter) { - eval->num_ret++; - if (tr_vec->tr[j].rel >= epi->relevance_level) - eval->num_rel_ret++; - } - else { - if (tr_vec->tr[j].rel >= epi->relevance_level) - eval->num_rel--; - } - } - - /* Calculate cutoff measures, and those measures dependant on them */ - /* Also includes recip_rank and rank_first_rel */ - calc_cutoff_measures (epi, tr_vec, eval, num_rel, num_nonrel); - - /* Calculate bpref measures */ - calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel); - - /* Calculate measures that average over ret or rel docs */ - calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel); - - /* Calculate exact measures over entire retrieved sets */ - calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel); - - /* Calculate time measures, if wanted */ - if (epi->time_flag) - calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel); - - - return (1); -} - -static int -compare_iter_rank (tr1, tr2) -TR_TUP *tr1; -TR_TUP *tr2; -{ - if (tr1->iter > tr2->iter) - return (-1); - if (tr1->iter < tr2->iter) - return (1); - if (tr1->rank < tr2->rank) - return (-1); - if (tr1->rank > tr2->rank) - return (1); - return (0); -} - - -/* ********************************************************************* */ -/* calculate cutoff measures */ - /* cutoff values for recall precision output */ -static int cutoff[NUM_CUTOFF] = CUTOFF_VALUES; -static int three_pts[3] = THREE_PTS; - - -static void -calc_cutoff_measures(epi, tr_vec, eval, num_rel, num_nonrel) -EVAL_PARAM_INFO *epi; -TR_VEC *tr_vec; -TREC_EVAL *eval; -long num_rel; /* Number relevant judged */ -long num_nonrel; /* Number nonrelevant judged */ -{ - double recall, precis; /* current recall, precision values */ - double rel_precis, rel_uap;/* relative precision, uap values */ - double int_precis; /* current interpolated precision values */ - - long i,j; - - long cut_rp[NUM_RP_PTS]; /* number of rel docs needed to be retrieved - for each recall-prec cutoff */ - long cut_fr[NUM_FR_PTS]; /* number of non-rel docs needed to be - retrieved for each fall-recall cutoff */ - long cut_rprec[NUM_PREC_PTS]; /* Number of docs needed to be retrieved - for each R-based prec cutoff */ - long current_cutoff, current_cut_rp, current_cut_fr, current_cut_rprec; - - long rel_so_far = eval->num_rel_ret; - - /* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all - Y >= X) */ - int_precis = (float) rel_so_far / (float) eval->num_ret; - - /* Discover cutoff values for this query */ - current_cutoff = NUM_CUTOFF - 1; - while (current_cutoff > 0 && cutoff[current_cutoff] > eval->num_ret) - current_cutoff--; - for (i = 0; i < NUM_RP_PTS; i++) - cut_rp[i] = ((eval->num_rel * i) + NUM_RP_PTS - 2) / (NUM_RP_PTS - 1); - current_cut_rp = NUM_RP_PTS - 1; - while (current_cut_rp > 0 && cut_rp[current_cut_rp] > eval->num_rel_ret) - current_cut_rp--; - for (i = 0; i < NUM_FR_PTS; i++) - cut_fr[i] = ((MAX_FALL_RET * i) + NUM_FR_PTS - 2) / (NUM_FR_PTS - 1); - current_cut_fr = NUM_FR_PTS - 1; - while (current_cut_fr > 0 && cut_fr[current_cut_fr] > eval->num_ret - eval->num_rel_ret) - current_cut_fr--; - for (i = 1; i < NUM_PREC_PTS+1; i++) - cut_rprec[i-1] = ((MAX_RPREC * eval->num_rel * i) + NUM_PREC_PTS - 2) - / (NUM_PREC_PTS - 1); - current_cut_rprec = NUM_PREC_PTS - 1; - while (current_cut_rprec > 0 && cut_rprec[current_cut_rprec]>eval->num_ret) - current_cut_rprec--; - - /* Loop over all retrieved docs in reverse order */ - for (j = eval->num_ret; j > 0; j--) { - if (rel_so_far > 0) { - recall = (float) rel_so_far / (float) eval->num_rel; - precis = (float) rel_so_far / (float) j; - if (j > eval->num_rel) { - rel_precis = (float) rel_so_far / (float) eval->num_rel; - } - else { - rel_precis = (float) rel_so_far / (float) j; - } - } - else { - recall = 0.0; - precis = 0.0; - rel_precis = 0.0; - } - rel_uap = rel_precis * rel_precis; - if (int_precis < precis) - int_precis = precis; - while (j == cutoff[current_cutoff]) { - eval->recall_cut[current_cutoff] = recall; - eval->precis_cut[current_cutoff] = precis; - eval->rel_precis_cut[current_cutoff] = rel_precis; - eval->uap_cut[current_cutoff] = precis * recall; - eval->rel_uap_cut[current_cutoff] = rel_uap; - current_cutoff--; - } - - while (j == cut_rprec[current_cut_rprec]) { - eval->R_prec_cut[current_cut_rprec] = precis; - eval->int_R_prec_cut[current_cut_rprec] = int_precis; - current_cut_rprec--; - } - - if (j == eval->num_rel) { - eval->R_recall_precis = precis; - eval->int_R_recall_precis = int_precis; - } - - if (tr_vec->tr[j-1].rel >= epi->relevance_level) { - while (rel_so_far == cut_rp[current_cut_rp]) { - eval->int_recall_precis[current_cut_rp] = int_precis; - current_cut_rp--; - } - eval->recip_rank = 1.0 / (float) j; - eval->rank_first_rel = j; - rel_so_far--; - } - else { - /* Note: for fallout-recall, the recall at X non-rel docs - is used for the recall 'after' (X-1) non-rel docs. - Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for - Y retrieved docs where X-1 non-rel retrieved */ - while (current_cut_fr >= 0 && - j - rel_so_far == cut_fr[current_cut_fr] + 1) { - eval->fall_recall[current_cut_fr] = recall; - current_cut_fr--; - } - } - } - - /* Fill in the 0.0 value for recall-precision (== max precision - at any point in the retrieval ranking) */ - eval->int_recall_precis[0] = int_precis; - - /* Fill in those cutoff values and averages that were not achieved - because insufficient docs were retrieved. */ - for (i = 0; i < NUM_CUTOFF; i++) { - if (eval->num_ret < cutoff[i]) { - if (eval->num_rel_ret > 0) { - eval->recall_cut[i] = ((float) eval->num_rel_ret / - (float) eval->num_rel); - eval->precis_cut[i] = ((float) eval->num_rel_ret / - (float) cutoff[i]); - } - eval->rel_precis_cut[i] = (cutoff[i] < eval->num_rel) ? - eval->precis_cut[i] : - eval->recall_cut[i]; - eval->uap_cut[i] = eval->precis_cut[i] * - eval->recall_cut[i]; - eval->rel_uap_cut[i] = eval->precis_cut[i] * - eval->precis_cut[i]; - } - } - for (i = 0; i < NUM_FR_PTS; i++) { - if (eval->num_ret - eval->num_rel_ret < cut_fr[i]) { - if (eval->num_rel_ret > 0) - eval->fall_recall[i] = (float) eval->num_rel_ret / - (float) eval->num_rel; - } - } - for (i = 0; i < NUM_PREC_PTS; i++) { - if (eval->num_ret < cut_rprec[i]) { - eval->R_prec_cut[i] = (float) eval->num_rel_ret / - (float) cut_rprec[i]; - eval->int_R_prec_cut[i] = (float) eval->num_rel_ret / - (float) cut_rprec[i]; - } - } - - if (eval->num_rel > eval->num_ret) { - eval->R_recall_precis = (float) eval->num_rel_ret / - (float)eval->num_rel; - eval->int_R_recall_precis = (float) eval->num_rel_ret / - (float)eval->num_rel; - } - - /* Calculate other indirect evaluation measure averages. */ - /* average recall-precis of 3 and 11 intermediate points */ - eval->int_av3_recall_precis = - (eval->int_recall_precis[three_pts[0]] + - eval->int_recall_precis[three_pts[1]] + - eval->int_recall_precis[three_pts[2]]) / 3.0; - for (i = 0; i < NUM_RP_PTS; i++) { - eval->int_av11_recall_precis += eval->int_recall_precis[i]; - } - eval->int_av11_recall_precis /= NUM_RP_PTS; - -} - -static void -calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel) -EVAL_PARAM_INFO *epi; -TR_VEC *tr_vec; -TREC_EVAL *eval; -long num_rel; /* Number relevant judged */ -long num_nonrel; /* Number nonrelevant judged */ -{ - long j; - long nonrel_ret, nonrel_so_far, rel_so_far; - long pref_top_nonrel_num = PREF_TOP_NONREL_NUM; - long pref_top_50pRnonrel_num; - long pref_top_25pRnonrel_num; - long pref_top_25p2Rnonrel_num; - long pref_top_10pRnonrel_num; - long pref_top_Rnonrel_num; - long bounded_5R_nonrel_so_far, bounded_10R_nonrel_so_far; - - /* Calculate judgement based measures (dependent on only - judged docs; no assumption of non-relevance if not judged) */ - /* Binary Preference measures; here expressed as all docs with a higher - value of rel are to be preferred. Optimize by keeping track of nonrel - seen so far */ - pref_top_nonrel_num = PREF_TOP_NONREL_NUM; - pref_top_50pRnonrel_num = 50 + eval->num_rel; - pref_top_25pRnonrel_num = 25 + eval->num_rel; - pref_top_10pRnonrel_num = 10 + eval->num_rel; - pref_top_Rnonrel_num = eval->num_rel; - pref_top_25p2Rnonrel_num = 25 + (2 * eval->num_rel); - nonrel_ret = 0; - for (j = 0; j < tr_vec->num_tr; j++) { - if (tr_vec->tr[j].rel == 0) - nonrel_ret++; - } - nonrel_so_far = 0; - rel_so_far = 0; - bounded_5R_nonrel_so_far = 0; - bounded_10R_nonrel_so_far = 0; - for (j = 0; j < tr_vec->num_tr; j++) { - if (tr_vec->tr[j].rel == 0) { - if (nonrel_so_far < 5 * eval->num_rel) { - bounded_5R_nonrel_so_far++; - if (nonrel_so_far < 10 * eval->num_rel) { - bounded_10R_nonrel_so_far++; - } - } - nonrel_so_far++; - } - else if (tr_vec->tr[j].rel >= epi->relevance_level) { - rel_so_far++; - /* Add fraction of correct preferences. */ - /* Special case nonrel_so_far == 0 to avoid division by 0 */ - if (nonrel_so_far > 0) { - eval->bpref_allnonrel += 1.0 - (((float) nonrel_so_far) / - (float) num_nonrel); - eval->bpref_retnonrel += 1.0 - (((float) nonrel_so_far) / - (float) nonrel_ret); - eval->bpref_retall += 1.0 - (((float) nonrel_so_far) / - (float) nonrel_ret); - eval->bpref_num_correct += - MIN (num_nonrel, pref_top_Rnonrel_num) - - MIN (nonrel_so_far, pref_top_Rnonrel_num); - eval->bpref += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) / - (float) MIN (num_nonrel, pref_top_Rnonrel_num)); - eval->old_bpref += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) / - (float) MIN (nonrel_ret, pref_top_Rnonrel_num)); - eval->bpref_topnonrel += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_nonrel_num)) / - (float) MIN (num_nonrel, pref_top_nonrel_num)); - eval->bpref_top50pRnonrel += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_50pRnonrel_num)) / - (float) MIN (num_nonrel, pref_top_50pRnonrel_num)); - eval->bpref_top25pRnonrel += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_25pRnonrel_num)) / - (float) MIN (num_nonrel, pref_top_25pRnonrel_num)); - eval->bpref_top10pRnonrel += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) / - (float) MIN (num_nonrel, pref_top_10pRnonrel_num)); - eval->old_bpref_top10pRnonrel += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) / - (float) MIN (nonrel_ret, pref_top_10pRnonrel_num)); - eval->bpref_top25p2Rnonrel += 1.0 - - (((float) MIN (nonrel_so_far, pref_top_25p2Rnonrel_num)) / - (float) MIN (num_nonrel, pref_top_25p2Rnonrel_num)); - if (rel_so_far <= 5 && nonrel_so_far < 5) - eval->bpref_5 += 1.0 - (float) nonrel_so_far / - (float) MIN (num_nonrel, 5); - if (rel_so_far <= 10 && nonrel_so_far < 10) - eval->bpref_10 += 1.0 - (float) nonrel_so_far / - (float) MIN (num_nonrel, 10); - } - else { - eval->bpref += 1.0; - eval->old_bpref += 1.0; - eval->bpref_allnonrel += 1.0; - eval->bpref_retnonrel += 1.0; - eval->bpref_retall += 1.0; - eval->bpref_topnonrel += 1.0; - eval->bpref_top50pRnonrel += 1.0; - eval->bpref_top25pRnonrel += 1.0; - eval->bpref_top10pRnonrel += 1.0; - eval->old_bpref_top10pRnonrel += 1.0; - eval->bpref_top25p2Rnonrel += 1.0; - if (rel_so_far <= 5) - eval->bpref_5 += 1.0; - if (rel_so_far <= 10) - eval->bpref_10 += 1.0; - - } - eval->bpref_top5Rnonrel += 1.0 - - (((float) bounded_5R_nonrel_so_far) / - (float) MIN (num_nonrel, eval->num_rel * 5)); - eval->bpref_top10Rnonrel += 1.0 - - (((float) bounded_10R_nonrel_so_far) / - (float) MIN (num_nonrel, eval->num_rel * 10)); - eval->bpref_num_all += num_nonrel - nonrel_so_far; - eval->bpref_num_ret += nonrel_ret - nonrel_so_far; - } - } - if (eval->num_rel) { - eval->bpref /= eval->num_rel; - eval->old_bpref /= eval->num_rel; - eval->bpref_allnonrel /= eval->num_rel; - eval->bpref_retnonrel /= eval->num_rel; - eval->bpref_topnonrel /= eval->num_rel; - eval->bpref_top5Rnonrel /= eval->num_rel; - eval->bpref_top10Rnonrel /= eval->num_rel; - eval->bpref_top50pRnonrel /= eval->num_rel; - eval->bpref_top25pRnonrel /= eval->num_rel; - eval->bpref_top10pRnonrel /= eval->num_rel; - eval->old_bpref_top10pRnonrel /= eval->num_rel; - eval->bpref_top25p2Rnonrel /= eval->num_rel; - if (eval->num_rel_ret) { - eval->bpref_retall /= eval->num_rel_ret; - eval->bpref_5 /= MIN (rel_so_far, 5); - eval->bpref_10 /= MIN (rel_so_far, 10); - } - eval->bpref_num_possible = eval->num_rel * - MIN (num_nonrel, pref_top_Rnonrel_num); - } - /* For those bpref measure variants which use the geometric mean instead - of straight averages, compute them here. Original measure value - is constrained to be greater than MIN_GEO_MEAN (for time being .00001, - since trec_eval prints to four significant digits) */ - eval->gm_bpref = (float) log ((double)(MAX (eval->bpref, - MIN_GEO_MEAN))); -} - -static void -calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel) -EVAL_PARAM_INFO *epi; -TR_VEC *tr_vec; -TREC_EVAL *eval; -long num_rel; /* Number relevant judged */ -long num_nonrel; /* Number nonrelevant judged */ -{ - double recall, precis; /* current recall, precision values */ - double rel_precis, rel_uap;/* relative precision, uap values */ - double int_precis; /* current interpolated precision values */ - - long i,j; - long rel_so_far; - - /* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all - Y >= X) */ - rel_so_far = eval->num_rel_ret; - int_precis = (float) rel_so_far / (float) eval->num_ret; - - /* Loop over all retrieved docs in reverse order */ - for (j = eval->num_ret; j > 0; j--) { - if (rel_so_far > 0) { - recall = (float) rel_so_far / (float) eval->num_rel; - precis = (float) rel_so_far / (float) j; - if (j > eval->num_rel) { - rel_precis = (float) rel_so_far / (float) eval->num_rel; - } - else { - rel_precis = (float) rel_so_far / (float) j; - } - } - else { - recall = 0.0; - precis = 0.0; - rel_precis = 0.0; - } - rel_uap = rel_precis * rel_precis; - if (int_precis < precis) - int_precis = precis; - eval->av_rel_precis += rel_precis; - eval->av_rel_uap += rel_uap; - - if (j < eval->num_rel) { - eval->av_R_precis += precis; - eval->int_av_R_precis += int_precis; - } - - if (tr_vec->tr[j-1].rel >= epi->relevance_level) { - eval->int_av_recall_precis += int_precis; - eval->av_recall_precis += precis; - eval->avg_doc_prec += precis; - rel_so_far--; - } - else { - /* Note: for fallout-recall, the recall at X non-rel docs - is used for the recall 'after' (X-1) non-rel docs. - Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for - Y retrieved docs where X-1 non-rel retrieved */ - if (j - rel_so_far < MAX_FALL_RET) { - eval->av_fall_recall += recall; - } - } - } - - if (eval->num_ret - eval->num_rel_ret < MAX_FALL_RET) { - if (eval->num_rel_ret > 0) - eval->av_fall_recall += ((MAX_FALL_RET - - (eval->num_ret - eval->num_rel_ret)) - * ((float)eval->num_rel_ret / - (float)eval->num_rel)); - } - if (eval->num_rel > eval->num_ret) { - for (i = eval->num_ret; i < eval->num_rel; i++) { - eval->av_R_precis += (float) eval->num_rel_ret / - (float) i; - eval->int_av_R_precis += (float) eval->num_rel_ret / - (float) i; - } - } - - /* Calculate all the other averages */ - if (eval->num_rel_ret > 0) { - eval->av_recall_precis /= eval->num_rel; - eval->int_av_recall_precis /= eval->num_rel; - } - - eval->av_fall_recall /= MAX_FALL_RET; - - eval->av_rel_precis /= eval->num_ret; - eval->av_rel_uap /= eval->num_ret; - - if (eval->num_rel) { - eval->av_R_precis /= eval->num_rel; - eval->int_av_R_precis /= eval->num_rel; - } - - /* For those measure variants which use the geometric mean instead - of straight averages, compute them here. Original measure value - is constrained to be greater than MIN_GEO_MEAN (for time being .00001, - since trec_eval prints to four significant digits) */ - eval->gm_ap = (float) log ((double)(MAX (eval->av_recall_precis, - MIN_GEO_MEAN))); -} - -static void -calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel) -EVAL_PARAM_INFO *epi; -TR_VEC *tr_vec; -TREC_EVAL *eval; -long num_rel; /* Number relevant judged */ -long num_nonrel; /* Number nonrelevant judged */ -{ - - if (eval->num_rel) { - eval->exact_recall = (double) eval->num_rel_ret / eval->num_rel; - eval->exact_precis = (double) eval->num_rel_ret / eval->num_ret; - eval->exact_uap = eval->exact_recall * eval->exact_precis; - if (eval->num_rel > eval->num_ret) { - eval->exact_rel_precis = eval->exact_precis; - } - else { - eval->exact_rel_precis = eval->exact_recall; - } - eval->exact_rel_uap = eval->exact_precis * eval->exact_precis; - eval->exact_utility = - epi->utility_a * eval->num_rel_ret + - epi->utility_b * (eval->num_ret - eval->num_rel_ret) + - epi->utility_c * (eval->num_rel - eval->num_rel_ret) + - epi->utility_d * (epi->num_docs_in_coll + eval->num_rel_ret - - eval->num_ret - eval->num_rel); - } -} - -static void -calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel) -EVAL_PARAM_INFO *epi; -TR_VEC *tr_vec; -TREC_EVAL *eval; -long num_rel; /* Number relevant judged */ -long num_nonrel; /* Number nonrelevant judged */ -{ - double recall, precis; /* current recall, precision values */ - double rel_precis, rel_uap;/* relative precision, uap values */ - double int_precis; /* current interpolated precision values */ - - long i,j; - - long bucket; - long last_time_bucket = NUM_TIME_PTS; /* Last time bucket filled in */ - - long rel_so_far = eval->num_rel_ret; - long min_ret_rel = MIN(eval->num_rel, eval->num_ret); - - /* Loop over all retrieved docs in reverse order */ - for (j = eval->num_ret; j > 0; j--) { - if (rel_so_far > 0) { - recall = (float) rel_so_far / (float) eval->num_rel; - precis = (float) rel_so_far / (float) j; - if (j > eval->num_rel) { - rel_precis = (float) rel_so_far / (float) eval->num_rel; - } - else { - rel_precis = (float) rel_so_far / (float) j; - } - } - else { - recall = 0.0; - precis = 0.0; - rel_precis = 0.0; - } - rel_uap = rel_precis * rel_precis; - if (int_precis < precis) - int_precis = precis; - - bucket = tr_vec->tr[j-1].sim * - ((double) NUM_TIME_PTS / (double) MAX_TIME); - if (bucket < 0) bucket = 0; - if (bucket >= NUM_TIME_PTS) bucket = NUM_TIME_PTS-1; - if (tr_vec->tr[j-1].rel >= epi->relevance_level) - eval->time_num_rel[bucket]++; - else - eval->time_num_nrel[bucket]++; - eval->time_precis[bucket] = (float)rel_so_far / - (float) eval->num_ret; - eval->time_relprecis[bucket] = ((float)rel_so_far) / - (float) min_ret_rel; - eval->time_uap[bucket] = (float) rel_so_far * rel_so_far / - ((float) eval->num_ret * (float) min_ret_rel); - eval->time_reluap[bucket] = (float) rel_so_far * rel_so_far / - ((float) min_ret_rel * (float) min_ret_rel); - eval->time_utility[bucket] = - epi->utility_a * rel_so_far + - epi->utility_b * (j - rel_so_far) + - epi->utility_c * (eval->num_rel - rel_so_far) + - epi->utility_d * (epi->num_docs_in_coll + - rel_so_far - j - eval->num_rel); - - /* Need to fill in buckets up to last bucket */ - /* note assumes buckets are decreasing */ - /* Must do here since utility can be negative and zero - cannot be used as flag later */ - for (i = bucket+1; i < last_time_bucket; i++) { - eval->time_precis[i] = eval->time_precis[bucket]; - eval->time_relprecis[i] = eval->time_relprecis[bucket]; - eval->time_uap[i] = eval->time_uap[bucket]; - eval->time_reluap[i] = eval->time_reluap[bucket]; - eval->time_utility[i] = eval->time_utility[bucket]; - } - last_time_bucket = bucket; - } - - eval->time_cum_rel[0] = eval->time_num_rel[0]; - eval->av_time_cum_rel = eval->time_num_rel[0]; - for (i=1; i< NUM_TIME_PTS; i++) { - eval->time_cum_rel[i] = eval->time_cum_rel[i-1] + eval->time_num_rel[i]; - eval->av_time_cum_rel += eval->time_cum_rel[i]; - eval->av_time_precis += eval->time_precis[i]; - eval->av_time_relprecis += eval->time_relprecis[i]; - eval->av_time_uap += eval->time_uap[i]; - eval->av_time_reluap += eval->time_reluap[i]; - eval->av_time_utility += eval->time_utility[i]; - } - eval->av_time_cum_rel /= NUM_TIME_PTS; - eval->av_time_precis /= NUM_TIME_PTS; - eval->av_time_relprecis /= NUM_TIME_PTS; - eval->av_time_uap /= NUM_TIME_PTS; - eval->av_time_reluap /= NUM_TIME_PTS; - eval->av_time_utility /= NUM_TIME_PTS; -} diff --git a/sm_cnn/tutorial.ipynb b/sm_cnn/tutorial.ipynb deleted file mode 100644 index 488cfc4..0000000 --- a/sm_cnn/tutorial.ipynb +++ /dev/null @@ -1,714 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# SM CNN Model PyTorch Walkthrough" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This purpose of this notebook is to explain how to use PyTorch to implement the SM CNN Model for new PyTorch users. Here are the recommended prerequisites before reading this walkthrough:\n", - "\n", - "* Have knowledge of Convolutional Neural Networks. If not these are helpful slides: https://cs.uwaterloo.ca/~mli/Deep-Learning-2017-Lecture5CNN.ppt.\n", - "* Read the SM Model paper: http://dl.acm.org/citation.cfm?id=2767738\n", - "\n", - "The following is a slightly modified version of the SM CNN architecture that will be implemented in this tutorial. It does not have the bilinear similarity modeling component present in the original model by Severyn and MoschiŠtti. The following paper found removing this component actually improved answer selection effectiveness:\n", - "\n", - "Jinfeng Rao, Hua He, and Jimmy Lin. Experiments with Convolutional Neural Network Models for Answer Selection. *Proceedings of the 40th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2017)*, August 2017, Tokyo, Japan." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "![caption](files/nn-architecture.png)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the block below we define the model, with detailed explanations in comments. This model is slightly different from the model in model.py to keep the tutorial straightforward (e.g. ignore GPU code)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": true - }, - "outputs": [], - "source": [ - "class QAModel(nn.Module):\n", - " \"\"\"\n", - " All PyTorch models should subclass nn.Module, the base class for neural network modules.\n", - " \"\"\"\n", - "\n", - " def __init__(self, input_n_dim, filter_width, conv_filters=100,\n", - " no_ext_feats=False, ext_feats_size=4, n_classes=2):\n", - " \"\"\"\n", - " :param input_n_dim: the dimension of each word vector\n", - " :param filter_width: the width of each convolution filter\n", - " :param conv_filters: the number of convolution filters\n", - " :param no_ext_feats: no additional external features\n", - " :param ext_feats_size: number of external features to use\n", - " :param n_classes: number of label classes\n", - " \"\"\"\n", - " super(QAModel, self).__init__()\n", - "\n", - " self.no_ext_feats = no_ext_feats\n", - "\n", - " # self.conv_channels specify the dimension of the output of the convolution,\n", - " # i.e. the number of convolution feature maps\n", - " self.conv_channels = conv_filters\n", - " # the elements in the hidden layer consist of equal number of inputs from the query and document (hence the 2*)\n", - " # and optionally the additional features (ext_feats_size)\n", - " n_hidden = 2*self.conv_channels + (0 if no_ext_feats else ext_feats_size)\n", - "\n", - " # define the convolution for the question/query - 1D convolution followed by tanh nonlinear activation\n", - " # modules (nn.Conv1d and nn.Tanh) will be added in the order presented to the nn.Sequential container\n", - " self.conv_q = nn.Sequential(\n", - " # the first parameter specifies the input dimension, the second parameter specifies the output dimension\n", - " nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),\n", - " # tanh activation is used to allow the network to learn non-linear decision boundaries\n", - " nn.Tanh()\n", - " )\n", - "\n", - " # define the convolution for the answer/document\n", - " self.conv_a = nn.Sequential(\n", - " nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),\n", - " nn.Tanh()\n", - " )\n", - "\n", - " # combining the features from the question, answer, and external features if any into a single vector\n", - " # note PyTorch nn classes follow a similar signature - the first parameter specifies the input dimension,\n", - " # the second parameter specifies the output dimension\n", - " # nn.Linear applies a linear transformation: Ax + b, where A and b are learned parameters.\n", - " self.combined_feature_vector = nn.Linear(2*self.conv_channels + \\\n", - " (0 if no_ext_feats else ext_feats_size), n_hidden)\n", - "\n", - " # defining other layers used in the network, note they are not yet linked with each other yet\n", - " # tanh is a non-linear activation function\n", - " self.combined_features_activation = nn.Tanh()\n", - " # dropout is used to prevent overfitting and only used during training\n", - " # elements are randomly zeroed with probability 0.5 and all elements are scaled by a factor of 1/0.5 = 2\n", - " self.dropout = nn.Dropout(0.5)\n", - " # hidden layer is used to capture additional interactions between the components of the intermediate representation\n", - " self.hidden = nn.Linear(n_hidden, n_classes)\n", - " # softmax computes probability distributions\n", - " self.logsoftmax = nn.LogSoftmax()\n", - "\n", - "\n", - " def forward(self, question, answer, ext_feats):\n", - " \"\"\"\n", - " Defines the forward pass of the network. When the model is called, e.g. model(*args) the args\n", - " are actually passed to the forward method.\n", - " The question and answer tensors are 3-dimensional. The first dimension specifies the sentence - it\n", - " can be larger than 1 since multiple sentences can be batched together in one forward pass.\n", - " The second and third dimensions specify the dimension of the word vector and the number of tokens respectively.\n", - " \n", - " :param question: the sentence matrices of questions (queries). Note the plural form - this is explained above.\n", - " :param answer: the sentence matrices of answers (documents). Note the plural form - this is explained above.\n", - " :param ext_feats: the external features for the question-answer pairs.\n", - " :returns: the log-likelihood of the question-answer pairs belonging in each class.\n", - " \"\"\"\n", - " # feed the question sentence matrices through the conv_q layers.\n", - " # IMPORTANT: the second dimension of the question MUST match the the first argument\n", - " # the Conv1d instance created (input_n_dim). The first dimension of the question specifies\n", - " # the batch size (number of questions).\n", - " q = self.conv_q.forward(question)\n", - " # max pool using q.size()[2] as the window size, which is the length of each convolution feature map\n", - " q = F.max_pool1d(q, q.size()[2])\n", - " # reshape max pooled elements into a vector of length equal to the number of feature maps\n", - " # the max pooling takes one value (the max) out of each convolution feature map\n", - " q = q.view(-1, self.conv_channels)\n", - "\n", - " # feed the answer sentence matrices through the conv_a layers, similar to the previous part for the question.\n", - " a = self.conv_a.forward(answer)\n", - " a = F.max_pool1d(a, a.size()[2])\n", - " a = a.view(-1, self.conv_channels)\n", - "\n", - " # concatenate the outputs of the conv_q, conv_a layers together\n", - " # with optionally the ext_feats along the first dimension\n", - " x = None\n", - " if self.no_ext_feats:\n", - " x = torch.cat([q, a], 1)\n", - " else:\n", - " x = torch.cat([q, a, ext_feats], 1)\n", - "\n", - " # feed the concatenated feature vector through the rest of the network (starting with join layer in figure)\n", - " x = self.combined_feature_vector.forward(x)\n", - " x = self.combined_features_activation.forward(x)\n", - " x = self.dropout(x)\n", - " x = self.hidden(x)\n", - " x = self.logsoftmax(x)\n", - "\n", - " return x\n", - " \n", - " @staticmethod\n", - " def load(model_fname):\n", - " return torch.load(model_fname)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We now load a pre-trained model with one input and see what the model actually does. For this, you'll need to clone the `data` and `models` projects in https://github.com/castorini." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The cell below contains some bootstrapping code to prepare the data, load the model, etc.. It is not important to understand it just to see how the SM CNN model itself works." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING - WARNING: expecting a .gz file. Is the ../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin in the correct format?\n", - "/Users/michael/anaconda/lib/python3.6/site-packages/torch/serialization.py:284: SourceChangeWarning: source code of class 'model.QAModel' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n", - " warnings.warn(msg, SourceChangeWarning)\n" - ] - } - ], - "source": [ - "import os\n", - "import sys\n", - "\n", - "import numpy as np\n", - "\n", - "from train import Trainer\n", - "import utils\n", - "\n", - "torch.manual_seed(1234)\n", - "np.random.seed(1234)\n", - "\n", - "# cache word embeddings\n", - "word_vectors_file = '../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin'\n", - "cache_file = os.path.splitext(word_vectors_file)[0] + '.cache'\n", - "utils.cache_word_embeddings(word_vectors_file, cache_file)\n", - "\n", - "vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file)\n", - "\n", - "# loading a pre-trained model\n", - "trained_model = QAModel.load('../../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor')\n", - "evaluator = Trainer(trained_model, 0.001, 0.0, False, vec_dim)\n", - "\n", - "evaluator.load_input_data('../../data/TrecQA', cache_file, None, None, 'raw-dev')\n", - "\n", - "questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = evaluator.data_splits['raw-dev']\n", - "word_vectors = evaluator.embeddings\n", - "pair_idx = 100 # particular question/answer pair we are interested in\n", - "batch_inputs, batch_labels = evaluator.get_tensorized_inputs(\n", - " questions[pair_idx:pair_idx + 1],\n", - " sentences[pair_idx:pair_idx + 1],\n", - " labels[pair_idx:pair_idx + 1],\n", - " ext_feats[pair_idx:pair_idx + 1],\n", - " word_vectors, vec_dim\n", - ")\n", - "\n", - "xq, xa, x_ext_feats = batch_inputs[0]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "collapsed": true - }, - "source": [ - "The question we want to compute similarity for and its sentence matrix dimension is shown below. The first dimension is the batch size, which is one in this case. Hence, the first index can be thought of as an index into the particular single sentence matrix. For each sentence matrix, each column represents the word vector for the corresponding word/token in the sentence (5 in total)." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "where was durst born ?\n", - "torch.Size([1, 50, 5])\n" - ] - } - ], - "source": [ - "print(questions[pair_idx])\n", - "print(xq.size())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The answer we want to compute similarity for and its sentence matrix is:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "born in jacksonville , fla . , durst grew up in gastonia , n.c . , where his love of hip-hop music and break dancing made him an outcast .\n", - "torch.Size([1, 50, 30])\n" - ] - } - ], - "source": [ - "print(sentences[pair_idx])\n", - "print(xa.size())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We'll not use any external features for this example, so `x_ext_feats` is a vector of zeros." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Variable containing:\n", - " 0 0 0 0\n", - "[torch.FloatTensor of size 1x4]" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x_ext_feats" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let us step through the `forward` method of the model. Normally we'll just call `trained_model(xq, xa, x_ext_feats)` but to illustrate the steps we'll copy the lines here again and see what happens underneath the hood." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we want to compute convolutional feature maps for the question. We just call `forward` to make a forward pass. We get 100 convolutional feature maps of length 9 each. We have 5 tokens with a padding of 4 on each side, for a total width of 5+2*4 = 13. Our convolution filter width is 5. Hence, we have 13 - 5 + 1 = 9 total positions for the \"sliding window\"." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 100, 9])" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "q = trained_model.conv_q.forward(xq)\n", - "q.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then, we want to max-pool the convolutional feature maps. We take max element out of every convolutional feature map of length 9, getting back 100 elements." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "q.size()[2]: 9\n" - ] - }, - { - "data": { - "text/plain": [ - "torch.Size([1, 100, 1])" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "print('q.size()[2]:', q.size()[2])\n", - "# max pool using q.size()[2] as the window size, which is the length of each convolution feature map\n", - "q = F.max_pool1d(q, q.size()[2])\n", - "q.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we reshape `q` into a 1 x 100 vector. Using -1 automatically determines the dimension for that index." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 100])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "q = q.view(-1, trained_model.conv_channels)\n", - "q.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Similarly, we want to compute the max-pooled convolutional feature maps for the answer. This is a vector of length 100." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 100])" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a = trained_model.conv_a.forward(xa)\n", - "a = F.max_pool1d(a, a.size()[2])\n", - "a = a.view(-1, trained_model.conv_channels)\n", - "a.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we join the max-pooled results together with the external features. Note the pre-trained model was trained with external features so we must run the code path with external features, but we can use 0 as the inputs since this is only for demonstration purposes." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 204])" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x = torch.cat([q, a, x_ext_feats], 1)\n", - "x.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we forward pass the features through the join layer, getting 201 inputs into the hidden layer." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 201])" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x = trained_model.combined_feature_vector.forward(x)\n", - "x.size()" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 201])" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x = trained_model.combined_features_activation.forward(x)\n", - "x.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Note the activation doesn't change the dimensions. After activation we pass it through the Dropout layer, although this doesn't do anything since are not training the model." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "First 10 elements before Dropout [-0.12813647 0.01474741 -0.12794048 -0.13291343 -0.24393715 -0.00718142\n", - " -0.08802623 -0.08587593 0.23123664 0.02877411]\n", - "First 10 elements after Dropout [-0.12813647 0.01474741 -0.12794048 -0.13291343 -0.24393715 -0.00718142\n", - " -0.08802623 -0.08587593 0.23123664 0.02877411]\n" - ] - }, - { - "data": { - "text/plain": [ - "torch.Size([1, 201])" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "print('First 10 elements before Dropout', x[0, :10].data.numpy())\n", - "x = trained_model.dropout(x)\n", - "print('First 10 elements after Dropout', x[0, :10].data.numpy())\n", - "x.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we pass the elements through the hidden layer, outputing just 2 elements." - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "torch.Size([1, 2])" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x = trained_model.hidden(x)\n", - "x.size()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, we find the log-probabilities." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Variable containing:\n", - "-0.0051 -5.2729\n", - "[torch.FloatTensor of size 1x2]" - ] - }, - "execution_count": 16, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "x = trained_model.logsoftmax(x)\n", - "x" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Get the actual probabilities by using `exp`." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": { - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "Variable containing:\n", - " 0.9949 0.0051\n", - "[torch.FloatTensor of size 1x2]" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "torch.exp(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Hence, the probability of label 0 is 0.9949 while the probability of label 1 is 0.0051." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.0" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/sm_cnn/utils.py b/sm_cnn/utils.py deleted file mode 100644 index 926028c..0000000 --- a/sm_cnn/utils.py +++ /dev/null @@ -1,179 +0,0 @@ -# file input output -import os -import re -import string - -from gensim.models.keyedvectors import KeyedVectors -from nltk.corpus import stopwords -from nltk.stem import PorterStemmer -import numpy as np - -# logging setup -import logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) - -def logargs(func): - def inner(*args, **kwargs): - logger.info('%s : %s %s' % (func.__name__, args, kwargs)) - return func(*args, **kwargs) - return inner - - -def cache_word_embeddings(word_embeddings_file, cache_file): - if not word_embeddings_file.endswith('.gz'): - logger.warning('WARNING: expecting a .gz file. Is the {} in the correct \ - format?'.format(word_embeddings_file)) - - vocab_size, vec_dim = 0, 0 - - if not os.path.exists(cache_file): - # cache does not exist - if not os.path.exists(os.path.dirname(cache_file)): - # make cache folder if needed - os.mkdir(os.path.dirname(cache_file)) - logger.info('caching the word embeddings in np.memmap format') - wv = KeyedVectors.load_word2vec_format(word_embeddings_file, binary=True) - # print len(wv.syn0), wv.syn0.shape - # print len(wv.syn0norm) if wv.syn0norm else None - fp = np.memmap(cache_file, dtype=np.double, mode='w+', shape=wv.syn0.shape) - fp[:] = wv.syn0[:] - with open(cache_file + '.vocab', 'w', encoding='utf-8') as f: - logger.info('writing out vocab for {}'.format(word_embeddings_file)) - for _, w in sorted((voc.index, word) for word, voc in wv.vocab.items()): - print(w, file=f) - with open(cache_file + '.dimensions', 'w', encoding='utf-8') as f: - logger.info('writing out dimensions for {}'.format(word_embeddings_file)) - print(wv.syn0.shape[0], wv.syn0.shape[1], file=f) - vocab_size, vec_dim = wv.syn0.shape - del fp, wv - print('cached {} into {}'.format(word_embeddings_file, cache_file)) - - return vocab_size, vec_dim - - -def load_embedding_dimensions(cache_file): - vocab_size, vec_dim = 0, 0 - with open(cache_file + '.dimensions', encoding='utf-8') as d: - vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] - return vocab_size, vec_dim - - -def load_cached_embeddings(cache_file, vocab_list, w2v_dict, oov_vec=[]): - """ - w2v_dict is filled up as reference - """ - logger.debug('loading cached embeddings ') - - with open(cache_file + '.dimensions', encoding='utf-8') as d: - vocab_size, vec_dim = [int(e) for e in d.read().strip().split()] - - W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim)) - - with open(cache_file + '.vocab', encoding='utf-8') as f: - logger.debug('loading vocab') - w2v_vocab_list = map(str.strip, f.readlines()) - - vocab_dict = {w:k for k, w in enumerate(w2v_vocab_list)} - - # Read w2v for vocab appears in Q and A - for word in vocab_list: - if word in w2v_dict: - continue - if word in vocab_dict: - w2v_dict[word] = W[vocab_dict[word]] - else: - w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) \ - if len(oov_vec) == 0 else oov_vec - #w2v_dict[word] = W[vocab_dict["unk"]] - -def read_in_data(datapath, set_name, file, stop_and_stem=False, stop_punct=False, dash_split=False): - data = [] - with open(os.path.join(datapath, set_name, file), encoding='utf-8') as inf: - data = [line.strip() for line in inf.readlines()] - - if dash_split: - def split_hyphenated_words(sentence): - rtokens = [] - for term in sentence.split(): - for t in term.split('-'): - if t: - rtokens.append(t) - return ' '.join(rtokens) - data = [split_hyphenated_words(sentence) for sentence in data] - - if stop_punct: - regex = re.compile('[{}]'.format(re.escape(string.punctuation))) - def remove_punctuation(sentence): - rtokens = [] - for term in sentence.split(): - for t in regex.sub(' ', term).strip().split(): - if t: - rtokens.append(t) - return ' '.join(rtokens) - data = [remove_punctuation(sentence) for sentence in data] - - if stop_and_stem: - stemmer = PorterStemmer() - stoplist = set(stopwords.words('english')) - def stop_stem(sentence): - return ' '.join([stemmer.stem(word) for word in sentence.split() \ - if word not in stoplist]) - data = [stop_stem(sentence) for sentence in data] - return data - -def read_in_dataset(dataset_folder, set_folder, stop_punct=False, dash_split=False): - """ - read in the data to return (question, sentence, label) - set_folder = {train|dev|test} - """ - max_q = 0 - max_s = 0 - set_path = os.path.join(dataset_folder, set_folder) - - #questions = [line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines()] - questions = read_in_data(dataset_folder, set_folder, "a.toks", False, stop_punct, dash_split) - len_q_list = [len(q.split()) for q in questions] - - #sentences = [line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines()] - sentences = read_in_data(dataset_folder, set_folder, "b.toks", False, stop_punct, dash_split) - len_s_list = [len(s.split()) for s in sentences] - - #labels = [int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines()] - labels = [int(lbl) for lbl in read_in_data(dataset_folder, set_folder, "sim.txt")] - - #vocab = [line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines()] - #all_data = list(set(questions)) + list(set(sentences)) - all_data = questions + sentences - vocab_set = set() - for sentence in all_data: - for term in sentence.split(): - vocab_set.add(term) - vocab = sorted(list(vocab_set)) - - return [questions, sentences, labels, max(len_q_list), max(len_s_list), vocab] - - -def get_test_qids_labels(dataset_folder, set_folder): - set_path = os.path.join(dataset_folder, set_folder) - qids = [line.strip() for line in open(os.path.join(set_path, 'id.txt'), encoding='utf-8').readlines()] - labels = np.array([int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt'), encoding='utf-8').readlines()]) - return qids, labels - - -if __name__ == "__main__": - - vocab = ["unk", "idontreallythinkthiswordexists", "hello"] - - w2v_dict = {} - load_cached_embeddings("../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.cache", vocab, w2v_dict) - - for w, v in w2v_dict.iteritems(): - print(w) - print(v) diff --git a/sm_modified_cnn/wiki_dataset.py b/sm_cnn/wiki_dataset.py similarity index 67% rename from sm_modified_cnn/wiki_dataset.py rename to sm_cnn/wiki_dataset.py index 67cbbf9..8c23110 100644 --- a/sm_modified_cnn/wiki_dataset.py +++ b/sm_cnn/wiki_dataset.py @@ -1,16 +1,14 @@ from torchtext import data -import os class WikiDataset(data.TabularDataset): dirname = 'data' @classmethod - def splits(cls, question_id, question_field, answer_field, external_field, label_field, - train='train.tsv', validation='dev.tsv', test='test.tsv'): + def splits(cls, question_id, question_field, answer_field, external_field, label_field, root='.data', + train='wikiqa.train.tsv', validation='wikiqa.dev.tsv', test='wikiqa.test.tsv'): path = './data' - prefix_name = 'wikiqa.' return super(WikiDataset, cls).splits( - os.path.join(path, prefix_name), train, validation, test, + path, root, train, validation, test, format='TSV', fields=[('qid', question_id), ('label', label_field), ('question', question_field), ('answer', answer_field), ('ext_feat', external_field)] ) diff --git a/sm_modified_cnn/.gitignore b/sm_modified_cnn/.gitignore deleted file mode 100644 index 7483f8e..0000000 --- a/sm_modified_cnn/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*pyc -*.pt -text/ -trained_models/ -trec_eval-8.0/trec_eval.dSYM -data/ diff --git a/sm_modified_cnn/README.md b/sm_modified_cnn/README.md deleted file mode 100644 index f9bfc3d..0000000 --- a/sm_modified_cnn/README.md +++ /dev/null @@ -1,149 +0,0 @@ -## SM model - -#### References: -1. Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural -Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information -Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 - - -### Requirements -``` -nltk==3.2.2 -numpy==1.11.3 -pytorch==0.1.12 -gensim==1.0.1 -``` - -The code uses torchtext for text processing. Set torchtext: -```bash -git clone https://github.com/pytorch/text.git -cd text - -#use this commit number -git reset --hard 2980f1bc39ba6af332c5c2783da8bee109796d4c -python setup.py install -``` - -We use `trec_eval` for evaluation: - -```bash -cd eval -tar -xvf trec_eval.9.0.tar.gz -cd trec_eval.9.0 -make -cd ../.. -``` - - -### Setup -Clone and create the dataset: -```bash -git clone https://github.com/castorini/data.git -git clone https://github.com/castorini/Castor.git -``` - -You should you see the following tree: -``` -. -├── Castor -│   ├── README.md -│   ├── baseline_results.tsv -│   ├── idf_baseline -│   ├── kim_cnn -│   ├── mp_cnn -│   ├── setup.py -│   ├── sm_cnn -│   └── sm_modified_cnn -└── data - ├── GloVe - ├── ParagramEmbeddings - ├── README.md - ├── SimpleQuestions_v2 - ├── TrecQA - ├── WikiQA - ├── msrvid - ├── requirements.txt - ├── sick - ├── twitterPPDB - ├── utils - └── word2vec -``` - -To create the dataset: -```bash -cd Castor/sm_modified_cnn/ -./create_dataset.sh -``` - -### Training -Download the word2vec model from [here] (https://drive.google.com/file/d/0B2u_nClt6NbzUmhOZU55eEo4QWM/view?usp=sharing) -and copy it to the `data/` folder. - -You can train the SM model for the 4 following configurations: -1. __random__ - the word embedddings are initialized randomly and are tuned during training -2. __static__ - the word embeddings are static (Severyn and Moschitti, SIGIR'15) -3. __non-static__ - the word embeddings are tuned during training -4. __multichannel__ - contains static and non-static channels for question and answer conv layers - -To train on GPU 0 with static configuration: - -```bash -python train.py --mode static --gpu 0 -``` - -NB: pass `--no_cuda` to use CPU - -The trained model will be save to: -``` -saves/static_best_model.pt -``` - -### Testing the model - -``` -python main.py --trained_model saves/TREC/multichannel_best_model.pt -``` - -### Evaluation - -The performance on TrecQA dataset: - -### TrecQA: - -#### Best dev -Metric |rand |static|non-static|multichannel --------|------|------|----------|------------ -MAP |0.8096|0.8162|0.8387 | 0.8274 -MRR |0.8560|0.8918|0.9058 | 0.8818 - -#### Test -Metric |rand |static|non-static|multichannel --------|-------|------|----------|------------ -MAP |0.7441 |0.7524|0.7688 |0.7641 -MRR |0.8172 |0.8012|0.8144 |0.8174 - -### WikiQA: - -#### Best dev -Metric |rand |static|non-static|multichannel --------|------|------|----------|------------ -MAP |0.7109|0.7204|0.7049 | 0.7245 -MRR |0.7169|0.7234|0.7075 | 0.7259 - -#### Test -Metric |rand |static|non-static|multichannel --------|-------|------|----------|------------ -MAP |0.6313 |0.6378|0.6455 |0.6476 -MRR |0.6522 |0.6542|0.6689 |0.6646 - -NB: The results on WikiQA are based on the SM model hyperparameters. - - -### To create your own word2vec.pt file - -+ Download word2vec from [here](https://drive.google.com/drive/u/0/folders/0B-yipfgecoSBfkZlY2FFWEpDR3M4Qkw5U055MWJrenE5MTBFVXlpRnd0QjZaMDQxejh1cWs) -to the `data/` folder - -```bash -python utils.py --input data/aquaint+wiki.txt.gz.ndim=50.bin -``` \ No newline at end of file diff --git a/sm_modified_cnn/eval/gdeval.pl b/sm_modified_cnn/eval/gdeval.pl deleted file mode 100755 index 7ae7572..0000000 --- a/sm_modified_cnn/eval/gdeval.pl +++ /dev/null @@ -1,340 +0,0 @@ -#!/usr/bin/perl -w - -# Graded relevance assessment script for the TREC 2010 Web track -# Evalution measures are written to standard output in CSV format. -# -# Currently reports only NDCG and ERR -# (see http://learningtorankchallenge.yahoo.com/instructions.php) - -use constant LOGBASEDIV => log(2.0); - -# gloals -my $QRELS; -my $VERSION = "version 1.3 (Mon Apr 29 20:50:24 EDT 2013)"; -my $MAX_JUDGMENT = 4; # Maximum gain value allowed in qrels file. -my $K = 20; # Reporting depth for results. -my $USAGE = "usage: $0 [options] qrels run\n - options:\n - -c - Average over the complete set of topics in the relevance judgments - instead of the topics in the intersection of relevance judgments - and results.\n - -k value - Non-negative integer depth of ranking to evaluate in range [1,inf]. - Default value is k=@{[($K)]}.\n - -baseline BASELINE_RUN_FILE - Baseline run to use for risk-sensitive evaluation\n - -riskAlpha value - Non-negative Risk sensitivity value to use when doing risk-sensitive - evaluation. A baseline must still be specified. By default 0. - The final weight to downside changes in performance is (1+value).\n"; - - - -use strict 'vars'; - -{ # main block to scope variables - if ($#ARGV >= 0 && ($ARGV[0] eq "-v" || $ARGV[0] eq "-version")) { - print "$0: $VERSION\n"; - exit 0; - } - - my $baselineRun = undef; - my $riskAlpha = 0; - my $cflag = 0; - while ($#ARGV != 1) # should probably replace this with perl's argument parsing - { - if ($#ARGV >= 0 && $ARGV[0] eq "-help") { - print "$USAGE\n"; - exit 0; - } - elsif ($#ARGV >= 2 and ("-c" eq $ARGV[0])) - { - $cflag = 1; - shift @ARGV; - } - elsif ($#ARGV >= 3 and ("-k" eq $ARGV[0])) - { - $K = int($ARGV[1]); - die $USAGE if ($K < 1); -# print STDERR "k=$K\n"; - shift @ARGV; shift @ARGV; - } - elsif ($#ARGV >= 3 and ("-baseline" eq $ARGV[0])) - { - $baselineRun = $ARGV[1]; - shift @ARGV; shift @ARGV; - } - elsif ($#ARGV >= 3 and ("-riskAlpha" eq $ARGV[0])) - { - $riskAlpha = $ARGV[1]; - die $USAGE if ($riskAlpha < 0.0); - shift @ARGV; shift @ARGV; - } - else - { - die $USAGE; - } - } - - die $USAGE unless $#ARGV == 1; - $QRELS = $ARGV[0]; - my $run = $ARGV[1]; - - # Read qrels file, check format, and sort - my @qrels = (); - my %seen = (); - open (QRELS,"<$QRELS") || die "$0: cannot open \"$QRELS\": $!\n"; - while () { - s/[\r\n]//g; - my ($topic, $zero, $docno, $judgment) = split (' '); - $topic =~ s/^.*\-//; - die "$0: format error on line $. of \"$QRELS\"\n" - unless - $topic =~ /^[0-9]+$/ && $zero == 0 - && $judgment =~ /^-?[0-9]+$/ && $judgment <= $MAX_JUDGMENT; - if ($judgment > 0) { - $qrels[$#qrels + 1]= "$topic $docno $judgment"; - $seen{$topic} = 1; - } - } - close (QRELS); - @qrels = sort qrelsOrder (@qrels); - - # Process qrels: store judgments and compute ideal gains - my $topicCurrent = -1; - my %ideal = (); - my @gain = (); - my %judgment = (); - for (my $i = 0; $i <= $#qrels; $i++) { - my ($topic, $docno, $judgment) = split (' ', $qrels[$i]); - if ($topic != $topicCurrent) { - if ($topicCurrent >= 0) { - $ideal{$topicCurrent} = &dcg($K, @gain); - $#gain = -1; - } - $topicCurrent = $topic; - } - next if $judgment < 0; - $judgment{"$topic:$docno"} = $gain[$#gain + 1] = $judgment; - } - if ($topicCurrent >= 0) { - $ideal{$topicCurrent} = &dcg($K, @gain); - $#gain = -1; - } - - # process baseline if doing risk sensitive - my ($baseNDCGByTopic,$baseERRByTopic,$baserunname); - if (defined $baselineRun) - { - ($baseNDCGByTopic,$baseERRByTopic,$baserunname) = processRun($baselineRun,0,\%seen,\%ideal,\%judgment,$cflag,0); - } - - # process main run - processRun($run,1,\%seen,\%ideal,\%judgment,$cflag,defined($baselineRun),$riskAlpha,$baserunname,$baseNDCGByTopic,$baseERRByTopic); - - exit 0; - -} # end main block - -# comparison function for qrels: by topic then judgment -sub qrelsOrder { - my ($topicA, $docnoA, $judgmentA) = split (' ', $a); - my ($topicB, $docnoB, $judgmentB) = split (' ', $b); - - if ($topicA < $topicB) { - return -1; - } elsif ($topicA > $topicB) { - return 1; - } else { - return $judgmentB <=> $judgmentA; - } -} - -# comparison function for runs: by topic then score then docno -sub runOrder { - my ($topicA, $docnoA, $scoreA) = split (' ', $a); - my ($topicB, $docnoB, $scoreB) = split (' ', $b); - - if ($topicA < $topicB) { - return -1; - } elsif ($topicA > $topicB) { - return 1; - } elsif ($scoreA < $scoreB) { - return 1; - } elsif ($scoreA > $scoreB) { - return -1; - } elsif ($docnoA lt $docnoB) { - return 1; - } elsif ($docnoA gt $docnoB) { - return -1; - } else { - return 0; - } -} - -# compute DCG over a sorted array of gain values, reporting at depth $k -sub dcg { - my ($k, @gain) = @_; - my ($i, $score) = (0, 0); - - for ($i = 0; $i <= ($k <= $#gain ? $k - 1 : $#gain); $i++) { - $score += (2**$gain[$i] - 1)/(log ($i + 2)/ +LOGBASEDIV); - } - return $score; -} - -# compute ERR over a sorted array of gain values, reporting at depth $k -sub err { - my ($k, @gain) = @_; - my ($i, $score, $decay, $r); - - $score = 0.0; - $decay = 1.0; - for ($i = 0; $i <= ($k <= $#gain ? $k - 1 : $#gain); $i++) { - $r = (2**$gain[$i] - 1)/(2**$MAX_JUDGMENT); - $score += $r*$decay/($i + 1); - $decay *= (1 - $r); - } - return $score; -} - -sub riskWeighted -{ - my ($run,$base,$alpha) = @_; - if ($run < $base) - { - $run = (1+$alpha) * ($run - $base); - } - else - { - $run = $run - $base; - } - return $run; -} -# compute and report information for current topic -sub topicDone { - my ($printTopic, $runid, $topic, $pndcgTotal, $perrTotal, $ptopics, $pseen, $pideal, - $isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain) = @_; - my($ndcg, $err) = (0, 0); - if (exists($$pseen{$topic}) and defined($$pseen{$topic}) and $$pseen{$topic}) { - $ndcg = &dcg($K, @gain)/$$pideal{$topic}; - $err = &err ($K, @gain); - $ndcg = riskWeighted($ndcg,$baseNDCG,$riskAlpha) if ($isRiskSensitive); - $err = riskWeighted($err,$baseERR,$riskAlpha) if ($isRiskSensitive); - $$pndcgTotal += $ndcg; - $$perrTotal += $err; - $$ptopics++; - printf("$runid,$topic,%.5f,%.5f\n",$ndcg,$err) if ($printTopic); - return ($ndcg,$err); - } -} - -sub processRun -{ - my ($run,$printTopics,$pseen,$pideal,$pjudgment,$avgOverAllTopics,$isRiskSensitive,$riskAlpha,$baserunname,$baseNDCGByTopic,$baseERRByTopic) = @_; - my $ndcgByTopic = {()}; - my $errByTopic = {()}; - my $runid = "?????"; - my @run = (); - # Read run rile, check format, and sort - open (RUN,"<$run") || die "$0: cannot open \"$run\": $!\n"; - while () { - s/[\r\n]//g; - my ($topic, $q0, $docno, $rank, $score); - ($topic, $q0, $docno, $rank, $score, $runid) = split (' '); - $topic =~ s/^.*\-//; - die "$0: format error on line $. of \"$run\"\n" - unless - $topic =~ /^[0-9]+$/ && $q0 eq "Q0" && $rank =~ /^[0-9]+$/ && $runid; - $run[$#run + 1] = "$topic $docno $score"; - } - - @run = sort runOrder (@run); - - my %processed = (); - foreach my $topic (%$pseen) - { - $processed{$topic} = 0; - } - - if ($isRiskSensitive) - { - $runid = sprintf("%s (rel to. %s, rs=1+a, a=%s)",$runid,$baserunname,$riskAlpha); - } - - # Process runs: compute measures for each topic and average - my $ndcgTotal = 0; - my $errTotal = 0; - my $topics = 0; - print "runid,topic,ndcg\@$K,err\@$K\n" if ($printTopics); - my $topicCurrent = -1; - my @gain = (); - for (my $i = 0; $i <= $#run; $i++) { - my ($topic, $docno, $score) = split (' ', $run[$i]); - if ($topic != $topicCurrent) { - if ($topicCurrent >= 0) { - my ($baseNDCG,$baseERR) = 0; - if ($isRiskSensitive) - { - $baseNDCG = $$baseNDCGByTopic{$topicCurrent} if (exists($$baseNDCGByTopic{$topicCurrent}) and defined($$baseNDCGByTopic{$topicCurrent})); - $baseERR = $$baseERRByTopic{$topicCurrent} if (exists($$baseERRByTopic{$topicCurrent}) and defined($$baseERRByTopic{$topicCurrent})); - } - my ($ndcg,$err) = &topicDone ($printTopics, $runid, $topicCurrent, \$ndcgTotal, \$errTotal, \$topics, - $pseen, $pideal, $isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain); - $$ndcgByTopic{$topicCurrent} = $ndcg; - $$errByTopic{$topicCurrent} = $err; - $processed{$topicCurrent} = 1; - $#gain = -1; - } - $topicCurrent = $topic; - } - my $j = $$pjudgment{"$topic:$docno"}; - $j = 0 unless $j; - $gain[$#gain + 1] = $j; - } - if ($topicCurrent >= 0) { - my ($baseNDCG,$baseERR) = 0; - if ($isRiskSensitive) - { - $baseNDCG = $$baseNDCGByTopic{$topicCurrent} if (exists($$baseNDCGByTopic{$topicCurrent}) and defined($$baseNDCGByTopic{$topicCurrent})); - $baseERR = $$baseERRByTopic{$topicCurrent} if (exists($$baseERRByTopic{$topicCurrent}) and defined($$baseERRByTopic{$topicCurrent})); - } - my ($ndcg,$err) = &topicDone ($printTopics, $runid, $topicCurrent, \$ndcgTotal, \$errTotal, \$topics, - $pseen, $pideal, $isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain); - $$ndcgByTopic{$topicCurrent} = $ndcg; - $$errByTopic{$topicCurrent} = $err; - $processed{$topicCurrent} = 1; - $#gain = -1; - } - my $numTopics = $topics; # $topics has the number in the run (at this point) - if ($avgOverAllTopics) - { - $numTopics = scalar(keys %$pseen); # we want denominator to change whenever flag is on but only need to compute differences for risk - if ($isRiskSensitive) - { # need to process any topics that were missing from run - my ($baseNDCG,$baseERR) = 0; - my @gain = (); - foreach my $topicCurrent (sort {$a <=> $b} keys %processed) - { - next if ($processed{$topicCurrent}); - $baseNDCG = $$baseNDCGByTopic{$topicCurrent} if (exists($$baseNDCGByTopic{$topicCurrent}) and defined($$baseNDCGByTopic{$topicCurrent})); - $baseERR = $$baseERRByTopic{$topicCurrent} if (exists($$baseERRByTopic{$topicCurrent}) and defined($$baseERRByTopic{$topicCurrent})); - my ($ndcg,$err) = &topicDone ($printTopics, $runid, $topicCurrent, \$ndcgTotal, \$errTotal, \$topics, - $pseen, $pideal, $isRiskSensitive, $riskAlpha, $baseNDCG, $baseERR, @gain); - } - } - } - - my $ndcgAvg = $ndcgTotal; - my $errAvg = $errTotal; - if ($numTopics > 0) - { - $ndcgAvg /= $numTopics; - $errAvg /= $numTopics; - } - printf "$runid,amean,%.5f,%.5f\n",$ndcgAvg,$errAvg if ($printTopics); - - return ($ndcgByTopic,$errByTopic,$runid); - close(RUN); -} diff --git a/sm_modified_cnn/eval/trec_eval.9.0.tar.gz b/sm_modified_cnn/eval/trec_eval.9.0.tar.gz deleted file mode 100644 index 660de16..0000000 Binary files a/sm_modified_cnn/eval/trec_eval.9.0.tar.gz and /dev/null differ diff --git a/sm_modified_cnn/evaluate.py b/sm_modified_cnn/evaluate.py deleted file mode 100644 index 76ba31d..0000000 --- a/sm_modified_cnn/evaluate.py +++ /dev/null @@ -1,24 +0,0 @@ -import shlex -import subprocess - -def evaluate(instances, dataset, valid, config): - sorted_instances = sorted(instances, key=lambda x: (x[0])) - with open('{}.{}.{}.run.txt'.format(dataset, valid, config), 'w') as run, \ - open('{}.{}.{}.qrel.txt'.format(dataset, valid, config), 'w') as qrel: - i = 0 - for instance in sorted_instances: - qid, predicted, score, gold = instance[0], instance[1], instance[2], instance[3] - - # 32.1 0 1 0 0.13309887051582336 smmodel - run.write('{} 0 {} 0 {} sm_model\n'.format(qid, i, score)) - qrel.write('{} 0 {} {}\n'.format(qid, i, gold)) - i += 1 - - pargs = shlex.split("./eval/trec_eval.9.0/trec_eval -m map -m recip_rank {}.{}.{}.qrel.txt {}.{}.{}.run.txt" - .format(dataset, valid, config, dataset, valid, config)) - p = subprocess.Popen(pargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - pout, perr = p.communicate() - lines = pout.split(b'\n') - map = float(lines[0].strip().split()[-1]) - mrr = float(lines[1].strip().split()[-1]) - return map, mrr \ No newline at end of file diff --git a/sm_modified_cnn/main.py b/sm_modified_cnn/main.py deleted file mode 100644 index a7a97df..0000000 --- a/sm_modified_cnn/main.py +++ /dev/null @@ -1,105 +0,0 @@ -import numpy as np -import random -import logging - -import torch -from torchtext import data - -from args import get_args -from trec_dataset import TrecDataset -from wiki_dataset import WikiDataset -from evaluate import evaluate - -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) - -args = get_args() -config = args - -torch.manual_seed(args.seed) - -if not args.cuda: - args.gpu = -1 -if torch.cuda.is_available() and args.cuda: - logger.info("Note: You are using GPU for training") - torch.cuda.set_device(args.gpu) - torch.cuda.manual_seed(args.seed) -if torch.cuda.is_available() and not args.cuda: - logger.info("Warning: You have Cuda but do not use it. You are using CPU for training") -np.random.seed(args.seed) -random.seed(args.seed) - -QID = data.Field(sequential=False) -QUESTION = data.Field(batch_first=True) -ANSWER = data.Field(batch_first=True) -LABEL = data.Field(sequential=False) -EXTERNAL = data.Field(sequential=False, tensor_type=torch.FloatTensor, batch_first=True, use_vocab=False, - preprocessing=data.Pipeline(lambda x: x.split()), - postprocessing=data.Pipeline(lambda x, train: [float(y) for y in x])) -if config.dataset == 'trec': - train, dev, test = TrecDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) -elif config.dataset == 'wiki': - train, dev, test = WikiDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) -else: - print("Unsupported dataset") - exit() - -QID.build_vocab(train, dev, test) -QUESTION.build_vocab(train, dev, test) -ANSWER.build_vocab(train, dev, test) -LABEL.build_vocab(train, dev, test) - -train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False, - sort=False, shuffle=True) -dev_iter = data.Iterator(dev, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, - sort=False, shuffle=False) -test_iter = data.Iterator(test, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, - sort=False, shuffle=False) - -config.target_class = len(LABEL.vocab) -config.questions_num = len(QUESTION.vocab) -config.answers_num = len(ANSWER.vocab) -print("Label dict:", LABEL.vocab.itos) - -if args.cuda: - model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu)) -else: - model = torch.load(args.trained_model, map_location=lambda storage,location: storage) - -index2label = np.array(LABEL.vocab.itos) -index2qid = np.array(QID.vocab.itos) - -def predict(dataset, test_mode, dataset_iter): - model.eval() - dataset_iter.init_epoch() - - instance = [] - for dev_batch_idx, dev_batch in enumerate(dataset_iter): - qid_array = index2qid[np.transpose(dev_batch.qid.cpu().data.numpy())] - true_label_array = index2label[np.transpose(dev_batch.label.cpu().data.numpy())] - - scores = model(dev_batch) - - index_label = np.transpose(torch.max(scores, 1)[1].view(dev_batch.label.size()).cpu().data.numpy()) - label_array = index2label[index_label] - score_array = scores[:, 2].cpu().data.numpy() - # print and write the result - for i in range(dev_batch.batch_size): - this_qid, predicted_label, score, gold_label = qid_array[i], label_array[i], score_array[i], \ - true_label_array[i] - instance.append((this_qid, predicted_label, score, gold_label)) - - dev_map, dev_mrr = evaluate(instance, dataset, test_mode, config.mode) - print(dev_map, dev_mrr) - -# Run the model on the dev set -predict(config.dataset, 'dev', dataset_iter=dev_iter) - -# Run the model on the test set -predict(config.dataset, 'test', dataset_iter=test_iter) diff --git a/sm_modified_cnn/model.py b/sm_modified_cnn/model.py deleted file mode 100644 index ecde607..0000000 --- a/sm_modified_cnn/model.py +++ /dev/null @@ -1,83 +0,0 @@ -import torch -import torch.nn as nn - -import torch.nn.functional as F - -class SmPlusPlus(nn.Module): - def __init__(self, config): - super(SmPlusPlus, self).__init__() - output_channel = config.output_channel - questions_num = config.questions_num - answers_num = config.answers_num - words_dim = config.words_dim - filter_width = config.filter_width - self.mode = config.mode - - n_classes = config.target_class - ext_feats_size = 4 - - if self.mode == 'multichannel': - input_channel = 2 - else: - input_channel = 1 - - self.question_embed = nn.Embedding(questions_num, words_dim) - self.answer_embed = nn.Embedding(answers_num, words_dim) - self.static_question_embed = nn.Embedding(questions_num, words_dim) - self.nonstatic_question_embed = nn.Embedding(questions_num, words_dim) - self.static_answer_embed = nn.Embedding(answers_num, words_dim) - self.nonstatic_answer_embed = nn.Embedding(answers_num, words_dim) - self.static_question_embed.weight.requires_grad = False - self.static_answer_embed.weight.requires_grad = False - - self.conv_q = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0)) - self.conv_a = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0)) - - self.dropout = nn.Dropout(config.dropout) - n_hidden = 2 * output_channel + ext_feats_size - - self.combined_feature_vector = nn.Linear(n_hidden, n_hidden) - self.hidden = nn.Linear(n_hidden, n_classes) - - def forward(self, x): - x_question = x.question - x_answer = x.answer - x_ext = x.ext_feat - - if self.mode == 'rand': - question = self.question_embed(x_question).unsqueeze(1) - answer = self.answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) - x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] - x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling - # actual SM model mode (Severyn & Moschitti, 2015) - elif self.mode == 'static': - question = self.static_question_embed(x_question).unsqueeze(1) - answer = self.static_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) - x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] - x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling - elif self.mode == 'non-static': - question = self.nonstatic_question_embed(x_question).unsqueeze(1) - answer = self.nonstatic_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) - x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] - x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling - elif self.mode == 'multichannel': - question_static = self.static_question_embed(x_question) - answer_static = self.static_answer_embed(x_answer) - question_nonstatic = self.nonstatic_question_embed(x_question) - answer_nonstatic = self.nonstatic_answer_embed(x_answer) - question = torch.stack([question_static, question_nonstatic], dim=1) - answer = torch.stack([answer_static, answer_nonstatic], dim=1) - x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] - x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling - else: - print("Unsupported Mode") - exit() - - # append external features and feed to fc - x.append(x_ext) - x = torch.cat(x, 1) - - x = F.tanh(self.combined_feature_vector(x)) - x = self.dropout(x) - x = self.hidden(x) - return x \ No newline at end of file diff --git a/sm_modified_cnn/requirements.txt b/sm_modified_cnn/requirements.txt deleted file mode 100644 index a5fbe01..0000000 --- a/sm_modified_cnn/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -nltk==3.2.1 -numpy==1.11.3 -gensim==1.0.1 -pytorch==0.1.12 diff --git a/sm_modified_cnn/train.py b/sm_modified_cnn/train.py deleted file mode 100644 index 5f41de2..0000000 --- a/sm_modified_cnn/train.py +++ /dev/null @@ -1,205 +0,0 @@ -import time -import os -import numpy as np -import random - -import torch -import torch.nn as nn -from torchtext import data - -from args import get_args -from model import SmPlusPlus -from trec_dataset import TrecDataset -from wiki_dataset import WikiDataset -from evaluate import evaluate - -args = get_args() -config = args - -torch.manual_seed(args.seed) - -def set_vectors(field, vector_path): - if os.path.isfile(vector_path): - stoi, vectors, dim = torch.load(vector_path) - field.vocab.vectors = torch.Tensor(len(field.vocab), dim) - - for i, token in enumerate(field.vocab.itos): - wv_index = stoi.get(token, None) - if wv_index is not None: - field.vocab.vectors[i] = vectors[wv_index] - else: - # initialize with U(-0.25, 0.25) vectors - field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.25, 0.25) - else: - print("Error: Need word embedding pt file") - exit(1) - return field - -# Set default configuration in : args.py -args = get_args() -config = args - -# Set random seed for reproducibility -torch.manual_seed(args.seed) -if not args.cuda: - args.gpu = -1 -if torch.cuda.is_available() and args.cuda: - print("Note: You are using GPU for training") - torch.cuda.set_device(args.gpu) - torch.cuda.manual_seed(args.seed) -if torch.cuda.is_available() and not args.cuda: - print("You have Cuda but you're using CPU for training.") -np.random.seed(args.seed) -random.seed(args.seed) - -QID = data.Field(sequential=False) -QUESTION = data.Field(batch_first=True) -ANSWER = data.Field(batch_first=True) -LABEL = data.Field(sequential=False) -EXTERNAL = data.Field(sequential=False, tensor_type=torch.FloatTensor, batch_first=True, use_vocab=False, - preprocessing=data.Pipeline(lambda x: x.split()), - postprocessing=data.Pipeline(lambda x, train: [float(y) for y in x])) -if config.dataset == 'TREC': - train, dev, test = TrecDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) -elif config.dataset == 'wiki': - train, dev, test = WikiDataset.splits(QID, QUESTION, ANSWER, EXTERNAL, LABEL) -else: - print("Unsupported dataset") - exit() - -QID.build_vocab(train, dev, test) -QUESTION.build_vocab(train, dev, test) -ANSWER.build_vocab(train, dev, test) -LABEL.build_vocab(train, dev, test) - - -QUESTION = set_vectors(QUESTION, args.vector_cache) -ANSWER = set_vectors(ANSWER, args.vector_cache) - -train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False, - sort=False, shuffle=True) -dev_iter = data.Iterator(dev, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, - sort=False, shuffle=False) -test_iter = data.Iterator(test, batch_size=args.batch_size, device=args.gpu, train=False, repeat=False, - sort=False, shuffle=False) - -config.target_class = len(LABEL.vocab) -config.questions_num = len(QUESTION.vocab) -config.answers_num = len(ANSWER.vocab) - -print("Dataset {} Mode {}".format(args.dataset, args.mode)) -print("VOCAB num", len(QUESTION.vocab)) -print("LABEL.target_class:", len(LABEL.vocab)) -print("LABELS:", LABEL.vocab.itos) -print("Train instance", len(train)) -print("Dev instance", len(dev)) -print("Test instance", len(test)) - -if args.resume_snapshot: - if args.cuda: - model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage.cuda(args.gpu)) - else: - model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage) -else: - model = SmPlusPlus(config) - model.static_question_embed.weight.data.copy_(QUESTION.vocab.vectors) - model.nonstatic_question_embed.weight.data.copy_(QUESTION.vocab.vectors) - model.static_answer_embed.weight.data.copy_(ANSWER.vocab.vectors) - model.nonstatic_answer_embed.weight.data.copy_(ANSWER.vocab.vectors) - - if args.cuda: - model.cuda() - print("Shift model to GPU") - - -parameter = filter(lambda p: p.requires_grad, model.parameters()) - -# the SM model originally follows SGD but Adadelta is used here -optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay) -criterion = nn.CrossEntropyLoss() -early_stop = False -best_dev_map = 0 -iterations = 0 -iters_not_improved = 0 -epoch = 0 -start = time.time() -header = ' Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss Accuracy Dev/Accuracy' -dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(',')) -log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(',')) -os.makedirs(args.save_path, exist_ok=True) -os.makedirs(os.path.join(args.save_path, args.dataset), exist_ok=True) -print(header) - -index2label = np.array(LABEL.vocab.itos) -index2qid = np.array(QID.vocab.itos) -index2question = np.array(ANSWER.vocab.itos) - -while True: - if early_stop: - print("Early Stopping. Epoch: {}, Best Dev Acc: {}".format(epoch, best_dev_map)) - break - epoch += 1 - train_iter.init_epoch() - n_correct, n_total = 0, 0 - - for batch_idx, batch in enumerate(train_iter): - iterations += 1 - model.train(); optimizer.zero_grad() - scores = model(batch) - n_correct += (torch.max(scores, 1)[1].view(batch.label.size()).data == batch.label.data).sum() - n_total += batch.batch_size - train_acc = 100. * n_correct / n_total - - loss = criterion(scores, batch.label) - loss.backward() - optimizer.step() - - # Evaluate performance on validation set - if iterations % args.dev_every == 1: - # switch model into evaluation mode - model.eval() - dev_iter.init_epoch() - n_dev_correct = 0 - dev_losses = [] - instance = [] - for dev_batch_idx, dev_batch in enumerate(dev_iter): - qid_array = index2qid[np.transpose(dev_batch.qid.cpu().data.numpy())] - true_label_array = index2label[np.transpose(dev_batch.label.cpu().data.numpy())] - - scores = model(dev_batch) - n_dev_correct += (torch.max(scores, 1)[1].view(dev_batch.label.size()).data == dev_batch.label.data).sum() - dev_loss = criterion(scores, dev_batch.label) - dev_losses.append(dev_loss.data[0]) - index_label = np.transpose(torch.max(scores, 1)[1].view(dev_batch.label.size()).cpu().data.numpy()) - label_array = index2label[index_label] - # get the relevance scores - score_array = scores[:, 2].cpu().data.numpy() - for i in range(dev_batch.batch_size): - this_qid, predicted_label, score, gold_label = qid_array[i], label_array[i], score_array[i], true_label_array[i] - instance.append((this_qid, predicted_label, score, gold_label)) - - - dev_map, dev_mrr = evaluate(instance, config.dataset, 'valid', config.mode) - print(dev_log_template.format(time.time() - start, - epoch, iterations, 1 + batch_idx, len(train_iter), - 100. * (1 + batch_idx) / len(train_iter), loss.data[0], - sum(dev_losses) / len(dev_losses), train_acc, dev_map)) - - # Update validation results - if dev_map > best_dev_map: - iters_not_improved = 0 - best_dev_map = dev_map - snapshot_path = os.path.join(args.save_path, args.dataset, args.mode+'_best_model.pt') - torch.save(model, snapshot_path) - else: - iters_not_improved += 1 - if iters_not_improved >= args.patience: - early_stop = True - break - - if iterations % args.log_every == 1: - # print progress message - print(log_template.format(time.time() - start, - epoch, iterations, 1 + batch_idx, len(train_iter), - 100. * (1 + batch_idx) / len(train_iter), loss.data[0], ' ' * 8, - n_correct / n_total * 100, ' ' * 12)) diff --git a/sm_modified_cnn/utils.py b/sm_modified_cnn/utils.py deleted file mode 100644 index f83dd93..0000000 --- a/sm_modified_cnn/utils.py +++ /dev/null @@ -1,31 +0,0 @@ -from tqdm import tqdm -import torch - -from gensim.models.keyedvectors import KeyedVectors -from argparse import ArgumentParser - - -def convert(fname, save_file): - with open(fname, 'rb') as dim_file: - vocab_size, dim = (int(x) for x in dim_file.readline().split()) - - word_vectors = KeyedVectors.load_word2vec_format(fname, binary=True) - - print("Loading vectors from {}".format(fname)) - vectors = [] - for line in tqdm(word_vectors.syn0, total=len(word_vectors.syn0)): - vectors.extend(line.tolist()) - vectors = torch.Tensor(vectors).view(-1, dim) - - stoi = {word.strip():voc.index for word, voc in word_vectors.vocab.items()} - - print('saving vectors to', save_file) - torch.save((stoi, vectors, dim), save_file) - -if __name__ == '__main__': - parser = ArgumentParser(description='create word embedding') - parser.add_argument('--input', type=str, required=True) - parser.add_argument('--output', type=str, default='data/word2vec.trecqa.pt') - - args = parser.parse_args() - convert(args.input, args.output) \ No newline at end of file