From e7ca33de288137eee71480c91ce2053b9fa2aa37 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Thu, 5 Oct 2017 14:14:31 -0400 Subject: [PATCH] Add convolutional RNN for sentence classification (#57) * Add SST data preprocessing * Add ConvRNN model * Add LR scheduler * Add grid search on hyperparameters * Add random search * Add CLI options * Add usage to README.md * Refactor code * Fix randomized search parameters * Update README.md with results * Use Dataset and DataLoader --- conv_rnn/README.md | 23 + conv_rnn/data.py | 61 +++ conv_rnn/getData.sh | 7 + conv_rnn/model.py | 141 ++++++ conv_rnn/test.py | 36 ++ conv_rnn/train.py | 177 +++++++ sm_cnn/trec_eval-8.0/help.info | 264 ++++++++++ sm_cnn/trec_eval-8.0/measures.c | 282 +++++++++++ sm_cnn/trec_eval-8.0/print_meas.c | 336 +++++++++++++ sm_cnn/trec_eval-8.0/trec_eval.h | 353 ++++++++++++++ sm_cnn/trec_eval-8.0/trec_eval_help.c | 203 ++++++++ sm_cnn/trec_eval-8.0/trvec_teval.c | 673 ++++++++++++++++++++++++++ 12 files changed, 2556 insertions(+) create mode 100644 conv_rnn/README.md create mode 100644 conv_rnn/data.py create mode 100755 conv_rnn/getData.sh create mode 100644 conv_rnn/model.py create mode 100644 conv_rnn/test.py create mode 100644 conv_rnn/train.py create mode 100755 sm_cnn/trec_eval-8.0/help.info create mode 100755 sm_cnn/trec_eval-8.0/measures.c create mode 100755 sm_cnn/trec_eval-8.0/print_meas.c create mode 100755 sm_cnn/trec_eval-8.0/trec_eval.h create mode 100755 sm_cnn/trec_eval-8.0/trec_eval_help.c create mode 100755 sm_cnn/trec_eval-8.0/trvec_teval.c diff --git a/conv_rnn/README.md b/conv_rnn/README.md new file mode 100644 index 0000000..0224078 --- /dev/null +++ b/conv_rnn/README.md @@ -0,0 +1,23 @@ +## Convolutional RNN + +Implementation based on [[1]](http://dl.acm.org/citation.cfm?id=3098140). + +### Usage + +Run `./getData.sh` to fetch the data. The project structure should now look like this: + +``` +├── conv_rnn/ +│ ├── data/ +│ ├── saves/ +│ └── *.* +``` +You may then run `python train.py` and `python test.py` for training and testing, respectively. For more options, add the `-h` switch. + +### Empirical results +Best dev | Test +-- | -- +51.1 | 50.7 + +### References +[1] Chenglong Wang, Feijun Jiang, and Hongxia Yang. 2017. A Hybrid Framework for Text Modeling with Convolutional RNN. In Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD '17). diff --git a/conv_rnn/data.py b/conv_rnn/data.py new file mode 100644 index 0000000..868e32d --- /dev/null +++ b/conv_rnn/data.py @@ -0,0 +1,61 @@ +import os +import re + +import numpy as np +import torch.utils.data as data + +def sst_tokenize(sentence): + extraneous_pattern = re.compile(r"^(--lrb--|--rrb--|``|''|--|\.)$") + words = [] + for word in sentence.split(): + if re.match(extraneous_pattern, word): + continue + words.append(word) + return words + +class SSTEmbeddingLoader(object): + def __init__(self, dirname, fmt="stsa.fine.{}", word2vec_file="word2vec.sst-1"): + self.dirname = dirname + self.fmt = fmt + self.word2vec_file = word2vec_file + + def load_embed_data(self): + weights = [] + id_dict = {} + unk_vocab_set = set() + with open(os.path.join(self.dirname, self.word2vec_file)) as f: + for i, line in enumerate(f.readlines()): + word, vec = line.replace("\n", "").split(" ", 1) + word = word.replace("#", "") + vec = np.array([float(v) for v in vec.split(" ")]) + weights.append(vec) + id_dict[word] = i + with open(os.path.join(self.dirname, self.fmt.format("phrases.train"))) as f: + for line in f.readlines(): + for word in sst_tokenize(line): + if word not in id_dict and word not in unk_vocab_set: + unk_vocab_set.add(word) + return (id_dict, np.array(weights), list(unk_vocab_set)) + +class SSTDataset(data.Dataset): + def __init__(self, sentences): + super().__init__() + self.sentences = sentences + + def __len__(self): + return len(self.sentences) + + def __getitem__(self, index): + return self.sentences[index] + + @classmethod + def load_sst_sets(cls, dirname, fmt="stsa.fine.{}"): + set_names = ["phrases.train", "dev", "test"] + def read_set(name): + data_set = [] + with open(os.path.join(dirname, fmt.format(name))) as f: + for line in f.readlines(): + sentiment, sentence = line.replace("\n", "").split(" ", 1) + data_set.append((sentiment, sentence)) + return np.array(data_set) + return [cls(read_set(name)) for name in set_names] diff --git a/conv_rnn/getData.sh b/conv_rnn/getData.sh new file mode 100755 index 0000000..021afc1 --- /dev/null +++ b/conv_rnn/getData.sh @@ -0,0 +1,7 @@ +#!/bin/sh +mkdir -p data +mkdir -p saves +wget http://ocp59jkku.bkt.clouddn.com/sst-1.zip -P data/ +wget http://ocp59jkku.bkt.clouddn.com/sst-2.zip -P data/ +unzip data/sst-1.zip -d data/ +unzip data/sst-2.zip -d data/ diff --git a/conv_rnn/model.py b/conv_rnn/model.py new file mode 100644 index 0000000..0e16146 --- /dev/null +++ b/conv_rnn/model.py @@ -0,0 +1,141 @@ +import random + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as nn_func + +import data + +class ConvRNNModel(nn.Module): + def __init__(self, word_model, **config): + super().__init__() + embedding_dim = word_model.dim + self.word_model = word_model + self.hidden_size = config["hidden_size"] + fc_size = config["fc_size"] + self.batch_size = config["mbatch_size"] + dropout = config["dropout_prob"] + n_fmaps = config["n_feature_maps"] + self.rnn_type = config["rnn_type"] + + self.h_0_cache = torch.autograd.Variable(torch.zeros(2, self.batch_size, self.hidden_size)) + self.c_0_cache = torch.autograd.Variable(torch.zeros(2, self.batch_size, self.hidden_size)) + + self.no_cuda = config["no_cuda"] + if not self.no_cuda: + self.h_0_cache = self.h_0_cache.cuda() + self.c_0_cache = self.c_0_cache.cuda() + + if self.rnn_type.upper() == "LSTM": + self.bi_rnn = nn.LSTM(embedding_dim, self.hidden_size, 1, batch_first=True, bidirectional=True) + elif self.rnn_type.upper() == "GRU": + self.bi_rnn = nn.GRU(embedding_dim, self.hidden_size, 1, batch_first=True, bidirectional=True) + else: + raise ValueError("RNN type must be one of LSTM or GRU") + self.conv = nn.Conv2d(1, n_fmaps, (1, self.hidden_size * 2)) + if dropout: + self.dropout = nn.Dropout(dropout) + self.fc1 = nn.Linear(n_fmaps + 2 * self.hidden_size, fc_size) + self.fc2 = nn.Linear(fc_size, config["n_labels"]) + + def convert_dataset(self, dataset): + dataset = np.stack(dataset) + model_in = dataset[:, 1].reshape(-1) + model_out = dataset[:, 0].flatten().astype(np.int) + model_out = torch.autograd.Variable(torch.from_numpy(model_out)) + model_in = self.preprocess(model_in) + model_in = torch.autograd.Variable(model_in) + if not self.no_cuda: + model_out = model_out.cuda() + model_in = model_in.cuda() + return (model_in, model_out) + + def preprocess(self, sentences): + return torch.from_numpy(np.array(self.word_model.lookup(sentences))) + + def forward(self, x): + x = self.word_model(x) # shape: (batch, max sent, embed dim) + if x.size(0) == self.batch_size: + h_0 = self.h_0_cache + c_0 = self.c_0_cache + else: + h_0 = torch.autograd.Variable(torch.zeros(2, x.size(0), self.hidden_size)) + c_0 = torch.autograd.Variable(torch.zeros(2, x.size(0), self.hidden_size)) + if not self.no_cuda: + h_0 = h_0.cuda() + c_0 = c_0.cuda() + if self.rnn_type.upper() == "LSTM": + rnn_seq, rnn_out = self.bi_rnn(x, (h_0, c_0)) # shape: (batch, seq len, 2 * hidden_size), (2, batch, hidden_size) + rnn_out = rnn_out[0] # (h_0, c_0) + else: + rnn_seq, rnn_out = self.bi_rnn(x, h_0) # shape: (batch, 2, hidden_size) + rnn_out.data = rnn_out.data.permute(1, 0, 2) + x = self.conv(rnn_seq.unsqueeze(1)).squeeze(3) # shape: (batch, channels, seq len) + x = nn_func.relu(x) # shape: (batch, channels, seq len) + x = nn_func.max_pool1d(x, x.size(2)) # shape: (batch, channels) + out = [t.squeeze(1) for t in rnn_out.chunk(2, 1)] + out.append(x) + x = torch.cat(out, 1).squeeze(2) + if hasattr(self, "dropout"): + x = self.dropout(x) + x = nn_func.relu(self.fc1(x)) + return self.fc2(x) + +class WordEmbeddingModel(nn.Module): + def __init__(self, id_dict, weights, unknown_vocab=[], static=True, padding_idx=0): + super().__init__() + vocab_size = len(id_dict) + len(unknown_vocab) + self.lookup_table = id_dict + last_id = max(id_dict.values()) + for word in unknown_vocab: + last_id += 1 + self.lookup_table[word] = last_id + self.dim = weights.shape[1] + self.weights = np.concatenate((weights, np.random.rand(len(unknown_vocab), self.dim) / 2 - 0.25)) + self.padding_idx = padding_idx + self.embedding = nn.Embedding(vocab_size, self.dim, padding_idx=padding_idx) + self.embedding.weight.data.copy_(torch.from_numpy(self.weights)) + if static: + self.embedding.weight.requires_grad = False + + @classmethod + def make_random_model(cls, id_dict, unknown_vocab=[], dim=300): + weights = np.random.rand(len(id_dict), dim) - 0.5 + return cls(id_dict, weights, unknown_vocab, static=False) + + def forward(self, x): + return self.embedding(x) + + def lookup(self, sentences): + raise NotImplementedError + +class SSTWordEmbeddingModel(WordEmbeddingModel): + def __init__(self, id_dict, weights, unknown_vocab=[]): + super().__init__(id_dict, weights, unknown_vocab, padding_idx=16259) + + def lookup(self, sentences): + indices_list = [] + max_len = 0 + for sentence in sentences: + indices = [] + for word in data.sst_tokenize(sentence): + try: + index = self.lookup_table[word] + indices.append(index) + except KeyError: + continue + indices_list.append(indices) + if len(indices) > max_len: + max_len = len(indices) + for indices in indices_list: + indices.extend([self.padding_idx] * (max_len - len(indices))) + return indices_list + +def set_seed(seed=0, no_cuda=False): + np.random.seed(seed) + if not no_cuda: + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.manual_seed(seed) + random.seed(seed) diff --git a/conv_rnn/test.py b/conv_rnn/test.py new file mode 100644 index 0000000..53f66ee --- /dev/null +++ b/conv_rnn/test.py @@ -0,0 +1,36 @@ +import argparse +import os +import random + +import numpy as np +import torch +import torch.nn as nn + +import data +import model + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--no_cuda", action="store_true", default=False) + parser.add_argument("--input_file", default="saves/model.pt", type=str) + parser.add_argument("--data_dir", default="data", type=str) + parser.add_argument("--gpu_number", default=0, type=int) + args = parser.parse_args() + + model.set_seed(5, no_cuda=args.no_cuda) + data_loader = data.SSTDataLoader(args.data_dir) + conv_rnn = torch.load(args.input_file) + if not args.no_cuda: + torch.cuda.set_device(args.gpu_number) + conv_rnn.cuda() + _, _, test_set = data_loader.load_sst_sets() + + conv_rnn.eval() + test_in, test_out = conv_rnn.convert_dataset(test_set) + scores = conv_rnn(test_in) + n_correct = (torch.max(scores, 1)[1].view(len(test_set)).data == test_out.data).sum() + accuracy = n_correct / len(test_set) + print("Test set accuracy: {}".format(accuracy)) + +if __name__ == "__main__": + main() diff --git a/conv_rnn/train.py b/conv_rnn/train.py new file mode 100644 index 0000000..675529d --- /dev/null +++ b/conv_rnn/train.py @@ -0,0 +1,177 @@ +import argparse +import os +import random + +import numpy as np +import torch +import torch.nn as nn +from torch import utils + +import data +import model + +class RandomSearch(object): + def __init__(self, params): + self.params = params + + def __iter__(self): + param_space = list(GridSearch(self.params)) + random.shuffle(param_space) + for param in param_space: + yield param + +class GridSearch(object): + def __init__(self, params): + self.params = params + self.param_lengths = [len(param) for param in self.params] + self.indices = [1] * len(params) + + def _update(self, carry_idx): + if carry_idx >= len(self.params): + return True + if self.indices[carry_idx] < self.param_lengths[carry_idx]: + self.indices[carry_idx] += 1 + return False + else: + self.indices[carry_idx] = 1 + return False or self._update(carry_idx + 1) + + def __iter__(self): + self.stop_next = False + self.indices = [1] * len(self.params) + return self + + def __next__(self): + if self.stop_next: + raise StopIteration + result = [param[idx - 1] for param, idx in zip(self.params, self.indices)] + self.indices[0] += 1 + if self.indices[0] == self.param_lengths[0] + 1: + self.indices[0] = 1 + self.stop_next = self._update(1) + return result + +def train(**kwargs): + mbatch_size = kwargs["mbatch_size"] + n_epochs = kwargs["n_epochs"] + restore = kwargs["restore"] + verbose = not kwargs["quiet"] + lr = kwargs["lr"] + weight_decay = kwargs["weight_decay"] + gradient_clip = kwargs["gradient_clip"] + seed = kwargs["seed"] + + if not kwargs["no_cuda"]: + torch.cuda.set_device(kwargs["gpu_number"]) + model.set_seed(seed) + embed_loader = data.SSTEmbeddingLoader("data") + if restore: + conv_rnn = torch.load(kwargs["input_file"]) + else: + id_dict, weights, unk_vocab_list = embed_loader.load_embed_data() + word_model = model.SSTWordEmbeddingModel(id_dict, weights, unk_vocab_list) + if not kwargs["no_cuda"]: + word_model.cuda() + conv_rnn = model.ConvRNNModel(word_model, **kwargs) + if not kwargs["no_cuda"]: + conv_rnn.cuda() + + conv_rnn.train() + criterion = nn.CrossEntropyLoss() + parameters = list(filter(lambda p: p.requires_grad, conv_rnn.parameters())) + optimizer = torch.optim.Adadelta(parameters, lr=lr, weight_decay=weight_decay) + train_set, dev_set, test_set = data.SSTDataset.load_sst_sets("data") + + collate_fn = conv_rnn.convert_dataset + train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=mbatch_size, drop_last=True, + collate_fn=collate_fn) + dev_loader = utils.data.DataLoader(dev_set, batch_size=len(dev_set), collate_fn=collate_fn) + test_loader = utils.data.DataLoader(test_set, batch_size=len(test_set), collate_fn=collate_fn) + + def evaluate(loader, dev=True): + conv_rnn.eval() + for m_in, m_out in loader: + scores = conv_rnn(m_in) + loss = criterion(scores, m_out) + n_correct = (torch.max(scores, 1)[1].view(m_in.size(0)).data == m_out.data).sum() + accuracy = n_correct / m_in.size(0) + if dev and accuracy > evaluate.best_dev: + evaluate.best_dev = accuracy + torch.save(conv_rnn, kwargs["output_file"]) + if verbose: + print("{} set accuracy: {}, loss: {}".format("dev" if dev else "test", accuracy, loss.cpu().data[0])) + conv_rnn.train() + evaluate.best_dev = 0 + + for epoch in range(n_epochs): + optimizer.zero_grad() + print("Epoch number: {}".format(epoch), end="\r") + if verbose: + print() + i = 0 + for j, (train_in, train_out) in enumerate(train_loader): + if verbose and i % (mbatch_size * 10) == 0: + print("{} / {}".format(j * mbatch_size, len(train_set)), end="\r") + + if not kwargs["no_cuda"]: + train_in.cuda() + train_out.cuda() + + scores = conv_rnn(train_in) + loss = criterion(scores, train_out) + loss.backward() + torch.nn.utils.clip_grad_norm(parameters, gradient_clip) + optimizer.step() + i += mbatch_size + if i % (mbatch_size * 256) == 0: + evaluate(dev_loader) + evaluate(test_loader, dev=False) + return evaluate.best_dev + +def do_random_search(given_params): + test_grid = [[0.15, 0.2], [4, 5, 6], [150, 200], [3, 4, 5], [200, 300], [200, 250]] + max_params = None + max_acc = 0. + for args in RandomSearch(test_grid): + sf, gc, hid, seed, fc_size, fmaps = args + print("Testing {}".format(args)) + given_params.update(dict(n_epochs=7, quiet=True, gradient_clip=gc, hidden_Size=hid, seed=seed, + n_feature_maps=fmaps, fc_size=fc_size)) + dev_acc = train(**given_params) + print("Dev accuracy: {}".format(dev_acc)) + if dev_acc > max_acc: + print("Found current max") + max_acc = dev_acc + max_params = args + print("Best params: {}".format(max_params)) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--dropout_prob", default=0.5, type=float) + parser.add_argument("--fc_size", default=200, type=int) + parser.add_argument("--gpu_number", default=0, type=int) + parser.add_argument("--gradient_clip", default=5, type=float) + parser.add_argument("--hidden_size", default=200, type=int) + parser.add_argument("--input_file", default="saves/model.pt", type=str) + parser.add_argument("--lr", default=5E-2, type=float) + parser.add_argument("--mbatch_size", default=64, type=int) + parser.add_argument("--n_epochs", default=30, type=int) + parser.add_argument("--n_feature_maps", default=200, type=float) + parser.add_argument("--n_labels", default=5, type=int) + parser.add_argument("--no_cuda", action="store_true", default=False) + parser.add_argument("--output_file", default="saves/model.pt", type=str) + parser.add_argument("--random_search", action="store_true", default=False) + parser.add_argument("--restore", action="store_true", default=False) + parser.add_argument("--rnn_type", choices=["lstm", "gru"], default="lstm", type=str) + parser.add_argument("--seed", default=3, type=int) + parser.add_argument("--quiet", action="store_true", default=False) + parser.add_argument("--weight_decay", default=1E-3, type=float) + args = parser.parse_args() + if args.random_search: + do_random_search(vars(args)) + return + train(**vars(args)) + +if __name__ == "__main__": + main() + diff --git a/sm_cnn/trec_eval-8.0/help.info b/sm_cnn/trec_eval-8.0/help.info new file mode 100755 index 0000000..eb243c7 --- /dev/null +++ b/sm_cnn/trec_eval-8.0/help.info @@ -0,0 +1,264 @@ +trec_eval [-h] [-q] [-a] [-o] [-c] [-l [-N] [-M] [-Ua] [-Ub] [-Uc] [-Ud] [-T] trec_rel_file trec_top_file + +Calculate and print various evaluation measures, evaluating the results +in trec_top_file against the relevance judgements in trec_rel_file. + +There are a fair number of options, of which only the lower case options are +normally ever used. + -h: Print full help message and exit + -q: In addition to summary evaluation, give evaluation for each query + -a: Print all evaluation measures calculated, instead of just the + main official measures for TREC. + -o: Print everything out in old, nonrelational format (default is relational) + -c: Average over the complete set of queries in the relevance judgements + instead of the queries in the intersection of relevance judgements + and results. Missing queries will contribute a value of 0 to all + evaluation measures (which may or may not be reasonable for a + particular evaluation measure, but is reasonable for standard TREC + measures.) + -l: Num indicates the minimum relevance judgement value needed for + a document to be called relevant. (All measures used by TREC eval are + based on binary relevance). Used if trec_rel_file contains relevance + judged on a multi-relevance scale. Default is 1. + -N: Number of docs in collection + -M: Max number of docs per topic to use in evaluation (discard rest). + -Ua: Value to use for 'a' coefficient of utility computation. + relevant nonrelevant + retrieved a b + nonretrieved c d + -Ub: Value to use for 'b' coefficient of utility computation. + -Uc: Value to use for 'c' coefficient of utility computation. + -Ud: Value to use for 'd' coefficient of utility computation. + -J: Calculate all values only over the judged (either relevant or + nonrelevant) documents. All unjudged documents are removed from the + retrieved set before any calculations (possibly leaving an empty set). + DO NOT USE, unless you really know what you're doing - very easy to get + reasonable looking, but invalid, numbers. + -T: Treat similarity as time that document retrieved. Compute + several time-based measures after ranking docs by time retrieved + (first doc (lowest sim) retrieved ranked highest). + Only done if -a selected. + + +Read text tuples from trec_top_file of the form + 030 Q0 ZF08-175-870 0 4238 prise1 + qid iter docno rank sim run_id +giving TREC document numbers (a string) retrieved by query qid +(a string) with similarity sim (a float). The other fields are ignored, +with the exception that the run_id field of the last line is kept and +output. In particular, note that the rank field is ignored here; +internally ranks are assigned by sorting by the sim field with ties +broken deterministicly (using docno). +Sim is assumed to be higher for the docs to be retrieved first. +File may contain no NULL characters. +Lines may contain fields after the run_id; they are ignored. + +Relevance for each docno to qid is determined from text_qrels_file, which +consists of text tuples of the form + qid iter docno rel +giving TREC document numbers (docno, a string) and their relevance (rel, +an integer) to query qid (a string). iter string field is ignored. +Fields are separated by whitespace, string fields can contain no whitespace. +File may contain no NULL characters. + +The text tuples with relevance judgements are converted to TR_VEC form +and then submitted to the SMART evaluation routines. +The qid,did,rank,sim,rel fields of TR_VEC are filled in; +action,iter fields are set to 0. +The rel field is set to -1 if the document was not judged (not in +text_qrels_file). Most measures, but not all, will treat -1 the same as 0, +namely nonrelevant. Note that relevance_level is used to determine if the +document is relevant during score calculations. +Queries for which there are no relevant docs are ignored. +Warning: queries for which there are relevant docs but no retrieved docs +are also ignored by default. This allows systems to evaluate over subsets +of the relevant docs, but means if a system improperly retrieves no docs, +it will not be detected. Use the -c flag to avoid this behavior. + +EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. +Relational Format prints the same values, but all lines are of the form + measure_name query value + +1. Total number of documents over all queries + Retrieved: + Relevant: + Rel_ret: (relevant and retrieved) + These should be self-explanatory. All values are totals over all + queries being evaluated. +2. Interpolated Recall - Precision Averages: + at 0.00 + at 0.10 + ... + at 1.00 + See any standard IR text (especially by Salton) for more details of + recall-precision evaluation. Measures precision (percent of retrieved + docs that are relevant) at various recall levels (after a certain + percentage of all the relevant docs for that query have been retrieved). + 'Interpolated' means that, for example, precision at recall + 0.10 (ie, after 10% of rel docs for a query have been retrieved) is + taken to be MAXIMUM of precision at all recall points >= 0.10. + Values are averaged over all queries (for each of the 11 recall levels). + These values are used for Recall-Precision graphs. +3. Average precision (non-interpolated) over all rel docs + The precision is calculated after each relevant doc is retrieved. + If a relevant doc is not retrieved, its precision is 0.0. + All precision values are then averaged together to get a single number + for the performance of a query. Conceptually this is the area + underneath the recall-precision graph for the query. + The values are then averaged over all queries. +4. Precision: + at 5 docs + at 10 docs + ... + at 1000 docs + The precision (percent of retrieved docs that are relevant) after X + documents (whether relevant or nonrelevant) have been retrieved. + Values averaged over all queries. If X docs were not retrieved + for a query, then all missing docs are assumed to be non-relevant. +5. R-Precision (precision after R (= num_rel for a query) docs retrieved): + Measures precision (or recall, they're the same) after R docs + have been retrieved, where R is the total number of relevant docs + for a query. Thus if a query has 40 relevant docs, then precision + is measured after 40 docs, while if it has 600 relevant docs, precision + is measured after 600 docs. This avoids some of the averaging + problems of the 'precision at X docs' values in (4) above. + If R is greater than the number of docs retrieved for a query, then + the nonretrieved docs are all assumed to be nonrelevant. + +Major measures (again) with their relational names: +num_ret Total number of documents retrieved over all queries +num_rel Total number of relevant documents over all queries +num_rel_ret Total number of relevant documents retrieved over all queries +map Mean Average Precision (MAP) +gm_ap Average Precision. Geometric Mean, q_score=log(MAX(map,.00001)) +R-prec R-Precision (Precision after R (= num-rel for topic) documents retrieved) +bpref Binary Preference, top R judged nonrel +recip_rank Reciprical rank of top relevant document +ircl_prn.0.00 Interpolated Recall - Precision Averages at 0.00 recall +ircl_prn.0.10 Interpolated Recall - Precision Averages at 0.10 recall +ircl_prn.0.20 Interpolated Recall - Precision Averages at 0.20 recall +ircl_prn.0.30 Interpolated Recall - Precision Averages at 0.30 recall +ircl_prn.0.40 Interpolated Recall - Precision Averages at 0.40 recall +ircl_prn.0.50 Interpolated Recall - Precision Averages at 0.50 recall +ircl_prn.0.60 Interpolated Recall - Precision Averages at 0.60 recall +ircl_prn.0.70 Interpolated Recall - Precision Averages at 0.70 recall +ircl_prn.0.80 Interpolated Recall - Precision Averages at 0.80 recall +ircl_prn.0.90 Interpolated Recall - Precision Averages at 0.90 recall +ircl_prn.1.00 Interpolated Recall - Precision Averages at 1.00 recall +P5 Precision after 5 docs retrieved +P10 Precision after 10 docs retrieved +P15 Precision after 15 docs retrieved +P20 Precision after 20 docs retrieved +P30 Precision after 30 docs retrieved +P100 Precision after 100 docs retrieved +P200 Precision after 200 docs retrieved +P500 Precision after 500 docs retrieved +P1000 Precision after 1000 docs retrieved + + +Minor measures with their relational names: +exact_prec Exact Precision over retrieved set +exact_recall Exact Recall over retrieved set +11-pt_avg Average over all 11 points of recall-precision graph +3-pt_avg Average over 3 points of recall-precision graph +avg_doc_prec Rel doc precision averaged over all relevant docs (NOT over topics) +exact_relative_prec Exact relative precision +avg_relative_prec Average relative precision +exact_unranked_avg_prec Exact Unranked Average Precision +exact_relative_unranked_avg_prec Exact Relative Unranked Average Precision +map_at_R Average Precision over first R docs retrieved +int_map Interpolated Mean Average Precision +exact_int_R_rcl_prec Exact R-based-interpolated-Precision +int_map_at_R Average Interpolated Precision for first R docs retrieved +bpref_allnonrel Binary Preference, all judged nonrel +bpref_retnonrel Binary Preference, all retrieved judged nonrel +bpref_topnonrel Binary Preference, top 100 judged nonrel +bpref_top5Rnonrel Binary Preference, top 5R judged nonrel +bpref_top10Rnonrel Binary Preference, top 10R judged nonrel +bpref_top10pRnonrel Binary Preference, top 10 + R judged nonrel +bpref_top25pRnonrel Binary Preference, top 25 + R judged nonrel +bpref_top50pRnonrel Binary Preference, top 50 + R judged nonrel +bpref_top25p2Rnonrel Binary Preference, top 25 + 2*R judged nonrel +bpref_retall Binary Preference, Only retrieved judged rel and nonrel +bpref_5 Binary Preference, top 5 rel, top 5 nonrel +bpref_10 Binary Preference, top 10 rel, top 10 nonrel +bpref_num_all Binary Preference, Number not retrieved before (all judged) +bpref_num_ret Binary Preference, Number retrieved after +bpref_num_correct Binary Preference, Number correct preferences +bpref_num_possible Binary Preference, Number possible correct_preferences +old_bpref Buggy Version 7.3. Binary Preference, top R judged nonrel +old_bpref_top10pRnonrel Buggy Version 7.3. Binary Preference,top 10+R judged nonrel +gm_bpref Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001)) +rank_first_rel Rank of top relevant document (0 if none) +recall5 Recall after 5 docs retrieved +recall10 Recall after 10 docs retrieved +recall15 Recall after 15 docs retrieved +recall20 Recall after 20 docs retrieved +recall30 Recall after 30 docs retrieved +recall100 Recall after 100 docs retrieved +recall200 Recall after 200 docs retrieved +recall500 Recall after 500 docs retrieved +recall1000 Recall after 1000 docs retrieved +0.20R-prec R-based precision- precision after 0.20 * R docs retrieved +0.40R-prec R-based precision- precision after 0.40 * R docs retrieved +0.60R-prec R-based precision- precision after 0.60 * R docs retrieved +0.80R-prec R-based precision- precision after 0.80 * R docs retrieved +1.00R-prec R-based precision- precision after 1.00 * R docs retrieved +1.20R-prec R-based precision- precision after 1.20 * R docs retrieved +1.40R-prec R-based precision- precision after 1.40 * R docs retrieved +1.60R-prec R-based precision- precision after 1.60 * R docs retrieved +1.80R-prec R-based precision- precision after 1.80 * R docs retrieved +2.00R-prec R-based precision- precision after 2.00 * R docs retrieved +relative_prec5 Relative precision after 5 docs retrieved +relative_prec10 Relative precision after 10 docs retrieved +relative_prec15 Relative precision after 15 docs retrieved +relative_prec20 Relative precision after 20 docs retrieved +relative_prec30 Relative precision after 30 docs retrieved +relative_prec100 Relative precision after 100 docs retrieved +relative_prec200 Relative precision after 200 docs retrieved +relative_prec500 Relative precision after 500 docs retrieved +relative_prec1000 Relative precision after 1000 docs retrieved +unranked_avg_prec5 Unranked Average Precision after 5 docs retrieved +unranked_avg_prec10 Unranked Average Precision after 10 docs retrieved +unranked_avg_prec15 Unranked Average Precision after 15 docs retrieved +unranked_avg_prec20 Unranked Average Precision after 20 docs retrieved +unranked_avg_prec30 Unranked Average Precision after 30 docs retrieved +unranked_avg_prec100 Unranked Average Precision after 100 docs retrieved +unranked_avg_prec200 Unranked Average Precision after 200 docs retrieved +unranked_avg_prec500 Unranked Average Precision after 500 docs retrieved +unranked_avg_prec1000 Unranked Average Precision after 1000 docs retrieved +relative_unranked_avg_prec5 Relative Unranked Average Precision after 5 docs retrieved +relative_unranked_avg_prec10 Relative Unranked Average Precision after 10 docs retrieved +relative_unranked_avg_prec15 Relative Unranked Average Precision after 15 docs retrieved +relative_unranked_avg_prec20 Relative Unranked Average Precision after 20 docs retrieved +relative_unranked_avg_prec30 Relative Unranked Average Precision after 30 docs retrieved +relative_unranked_avg_prec100 Relative Unranked Average Precision after 100 docs retrieved +relative_unranked_avg_prec200 Relative Unranked Average Precision after 200 docs retrieved +relative_unranked_avg_prec500 Relative Unranked Average Precision after 500 docs retrieved +relative_unranked_avg_prec1000 Relative Unranked Average Precision after 1000 docs retrieved +utility_1.0_-1.0_0.0_0.0 Utility (a,b,c,d) Coefficients 1.0_-1.0_0.0_0.0 +rcl_at_142_nonrel Recall averaged at X nonrel docs X= 142 +fallout_recall_0 Fallout - Recall Averages- recall after 0 nonrel docs retrieved +fallout_recall_14 Fallout - Recall Averages- recall after 14 nonrel docs retrieved +fallout_recall_28 Fallout - Recall Averages- recall after 28 nonrel docs retrieved +fallout_recall_42 Fallout - Recall Averages- recall after 42 nonrel docs retrieved +fallout_recall_56 Fallout - Recall Averages- recall after 56 nonrel docs retrieved +fallout_recall_71 Fallout - Recall Averages- recall after 71 nonrel docs retrieved +fallout_recall_85 Fallout - Recall Averages- recall after 85 nonrel docs retrieved +fallout_recall_99 Fallout - Recall Averages- recall after 99 nonrel docs retrieved +fallout_recall_113 Fallout - Recall Averages- recall after 113 nonrel docs retrieved +fallout_recall_127 Fallout - Recall Averages- recall after 127 nonrel docs retrieved +fallout_recall_142 Fallout - Recall Averages- recall after 142 nonrel docs retrieved +int_0.20R-prec Interpolated R-based precision, after 0.20 * R docs retrieved +int_0.40R-prec Interpolated R-based precision, after 0.40 * R docs retrieved +int_0.60R-prec Interpolated R-based precision, after 0.60 * R docs retrieved +int_0.80R-prec Interpolated R-based precision, after 0.80 * R docs retrieved +int_1.00R-prec Interpolated R-based precision, after 1.00 * R docs retrieved +int_1.20R-prec Interpolated R-based precision, after 1.20 * R docs retrieved +int_1.40R-prec Interpolated R-based precision, after 1.40 * R docs retrieved +int_1.60R-prec Interpolated R-based precision, after 1.60 * R docs retrieved +int_1.80R-prec Interpolated R-based precision, after 1.80 * R docs retrieved +int_2.00R-prec Interpolated R-based precision, after 2.00 * R docs retrieved +micro_prec Total relevant retrieved documents / Total retrieved documents +micro_recall Total relevant retrieved documents / Total relevant documents +micro_bpref Total correct preferences / Total possible preferences diff --git a/sm_cnn/trec_eval-8.0/measures.c b/sm_cnn/trec_eval-8.0/measures.c new file mode 100755 index 0000000..8bf6263 --- /dev/null +++ b/sm_cnn/trec_eval-8.0/measures.c @@ -0,0 +1,282 @@ +#ifdef RCSID +static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $"; +#endif + +/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. + + Permission is granted for use of this file in unmodified form for + research purposes. Please contact the SMART project to obtain + permission for other uses. +*/ + +#include "common.h" +#include "sysfunc.h" +#include "buf.h" +#include "trec_eval.h" + +static long cutoff[] = CUTOFF_VALUES; +static char param_val[20]; +char *get_param_str_ircl_prn(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%4.2f", (float) index / (NUM_RP_PTS -1)); + return (param_val); +} +char *get_param_str_cutoff(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%ld", cutoff[index]); + return (param_val); +} +char *get_param_str_Rcutoff(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%4.2f", + (float) MAX_RPREC * (index+1) /(float) (NUM_PREC_PTS - 1)); + return (param_val); +} +char *get_param_str_utility(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f", + epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d); + return (param_val); +} +char *get_param_str_maxfallout(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%ld", (long) MAX_FALL_RET); + return (param_val); +} +char *get_param_str_fall_recall(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%ld", + (long) (MAX_FALL_RET * index) / (NUM_FR_PTS - 1)); + return (param_val); +} +char *get_param_str_time_cutoff(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%ld", + (long) (index * MAX_TIME / NUM_TIME_PTS)); + return (param_val); +} +char *get_param_str_time_utility_cutoff(epi, index) +EVAL_PARAM_INFO *epi; +long index; +{ + sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f-%ld", + epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d, + (long) (index * MAX_TIME / NUM_TIME_PTS)); + return (param_val); +} + +SINGLE_MEASURE sing_meas[] = { + {"num_ret", "Total number of documents retrieved over all queries", + 1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_ret)}, + {"num_rel", "Total number of relevant documents over all queries", + 1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel)}, + {"num_rel_ret", "Total number of relevant documents retrieved over all queries", + 1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel_ret)}, + {"map", "Mean Average Precision (MAP)", + 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_recall_precis)}, + {"gm_ap","Average Precision. Geometric Mean, q_score=log(MAX(map,.00001))", + 0, 1, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_ap)}, + {"R-prec", "R-Precision (Precision after R (= num-rel for topic) documents retrieved)", + 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, R_recall_precis)}, + {"bpref", "Binary Preference, top R judged nonrel", + 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref)}, + {"recip_rank", "Reciprical rank of top relevant document", + 0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, recip_rank)}, + /* end of short output measures (the major ones) */ + + {"exact_prec", "Exact Precision over retrieved set", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_precis)}, + {"exact_recall", "Exact Recall over retrieved set", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_recall)}, + {"11-pt_avg", "Average over all 11 points of recall-precision graph", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av11_recall_precis)}, + {"3-pt_avg", "Average over 3 points of recall-precision graph", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av3_recall_precis)}, + {"avg_doc_prec", "Rel doc precision averaged over all relevant docs (NOT over topics)", + 0, 0, 0, 0, 0, 0, 1, 0, offsetof(TREC_EVAL, avg_doc_prec)}, + {"exact_relative_prec", "Exact relative precision", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_precis)}, + {"avg_relative_prec", "Average relative precision", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_rel_precis)}, + {"exact_unranked_avg_prec", "Exact Unranked Average Precision", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_uap)}, + {"exact_relative_unranked_avg_prec", "Exact Relative Unranked Average Precision", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_uap)}, + {"map_at_R", "Average Precision over first R docs retrieved", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_R_precis)}, + {"int_map", "Interpolated Mean Average Precision", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_recall_precis)}, + {"exact_int_R_rcl_prec", "Exact R-based-interpolated-Precision", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_R_recall_precis)}, + {"int_map_at_R", "Average Interpolated Precision for first R docs retrieved", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_R_precis)}, + {"time_integral_prec", "Time: Average Integral Precision", + 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_precis)}, + {"time_integral_relative_prec", "Time: Average Integral Relative Precision", + 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_relprecis)}, + {"time_integral_uap", "Time: Average Integral Unranked Precision", + 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_uap)}, + {"time_integral_relative_uap", "Time: Average Integral Unranked Relative Precision", + 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_reluap)}, + {"time_integral_cum_rel", "Time: Average (Integral) cumulative number relevant", + 0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_cum_rel)}, + {"bpref_allnonrel", "Binary Preference, all judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_allnonrel)}, + {"bpref_retnonrel", "Binary Preference, all retrieved judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retnonrel)}, + {"bpref_topnonrel", "Binary Preference, top 100 judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_topnonrel)}, + {"bpref_top5Rnonrel", "Binary Preference, top 5R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top5Rnonrel)}, + {"bpref_top10Rnonrel", "Binary Preference, top 10R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10Rnonrel)}, + {"bpref_top10pRnonrel", "Binary Preference, top 10 + R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10pRnonrel)}, + {"bpref_top25pRnonrel", "Binary Preference, top 25 + R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25pRnonrel)}, + {"bpref_top50pRnonrel", "Binary Preference, top 50 + R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top50pRnonrel)}, + {"bpref_top25p2Rnonrel", "Binary Preference, top 25 + 2*R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25p2Rnonrel)}, + {"bpref_retall", "Binary Preference, Only retrieved judged rel and nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retall)}, + {"bpref_5", "Binary Preference, top 5 rel, top 5 nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_5)}, + {"bpref_10", "Binary Preference, top 10 rel, top 10 nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_10)}, + {"bpref_num_all", "Binary Preference, Number not retrieved before (all judged)", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_all)}, + {"bpref_num_ret", "Binary Preference, Number retrieved after", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_ret)}, + {"bpref_num_correct", "Binary Preference, Number correct preferences", + 1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_correct)}, + {"bpref_num_possible", "Binary Preference, Number possible correct_preferences", + 1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_possible)}, + {"old_bpref", "Buggy Version 7.3. Binary Preference, top R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref)}, + {"old_bpref_top10pRnonrel", "Buggy Version 7.3. Binary Preference,top 10+R judged nonrel", + 0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref_top10pRnonrel)}, + {"gm_bpref", "Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001))", + 0, 0, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_bpref)}, + {"rank_first_rel", "Rank of top relevant document (0 if none)", + 1, 0, 0, 1, 0, 0, 0, 0, offsetof(TREC_EVAL, rank_first_rel)}, +}; + +int num_sing_meas = sizeof (sing_meas) / sizeof (sing_meas[0]); + +PARAMETERIZED_MEASURE param_meas[] = { + {"Interpolated Recall - Precision Averages", + 0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, int_recall_precis[0]), NUM_RP_PTS, + "ircl_prn.%s", " at %s recall", + get_param_str_ircl_prn}, + {"Precision", + 0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, precis_cut[0]), NUM_CUTOFF, + "P%s", " after %s docs retrieved", + get_param_str_cutoff}, + /* end of short output measures (the major ones) */ + + {"Recall", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, recall_cut[0]), NUM_CUTOFF, + "recall%s", " after %s docs retrieved", + get_param_str_cutoff}, + {"R-based precision", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, R_prec_cut[0]), NUM_PREC_PTS-1, + "%sR-prec", "- precision after %s * R docs retrieved", + get_param_str_Rcutoff}, + {"Relative precision", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_precis_cut[0]), NUM_CUTOFF, + "relative_prec%s", " after %s docs retrieved", + get_param_str_cutoff}, + {"Unranked Average Precision", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, uap_cut[0]), NUM_CUTOFF, + "unranked_avg_prec%s", " after %s docs retrieved", + get_param_str_cutoff}, + {"Relative Unranked Average Precision", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_uap_cut[0]), NUM_CUTOFF, + "relative_unranked_avg_prec%s", " after %s docs retrieved", + get_param_str_cutoff}, + {"Utility (a,b,c,d)", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, exact_utility), 1, + "utility_%s", " Coefficients %s ", + get_param_str_utility}, + {"Recall averaged at X nonrel docs", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, av_fall_recall), 1, + "rcl_at_%s_nonrel", " X= %s ", + get_param_str_maxfallout}, + {"Fallout - Recall Averages", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, fall_recall[0]), NUM_FR_PTS, + "fallout_recall_%s", "- recall after %s nonrel docs retrieved", + get_param_str_fall_recall}, + {"Interpolated R-based precision,", + 0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, int_R_prec_cut[0]), NUM_PREC_PTS-1, + "int_%sR-prec", " after %s * R docs retrieved", + get_param_str_Rcutoff}, + {"Time: Utility (a,b,c,d):", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, av_time_utility), 1, + "time_integral_utility_%s", " Coefficients %s ", + get_param_str_utility}, + {"Time: num_rel at cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_rel[0]), NUM_TIME_PTS, + "time_num_rel_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: num_nonrel at cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_nrel[0]), NUM_TIME_PTS, + "time_num_nonrel_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: cumulative rel at cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_cum_rel[0]), NUM_TIME_PTS, + "time_cum_rel_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: precision at time cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS, + "time_precis_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: precision at time cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS, + "time_precis_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: relative precision at time cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_relprecis[0]), NUM_TIME_PTS, + "time_relative_precis_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: unranked precision at time cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_uap[0]), NUM_TIME_PTS, + "time_uap_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: relative unranked precision at time cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_reluap[0]), NUM_TIME_PTS, + "time_relative_uap_%s", " after %s seconds", + get_param_str_time_cutoff}, + {"Time: utility at time cutoff:", + 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_utility[0]), NUM_TIME_PTS, + "time_utility_%s", " after %s seconds", + get_param_str_time_utility_cutoff}, +}; + +int num_param_meas = sizeof (param_meas) / sizeof (param_meas[0]); + +MICRO_MEASURE micro_meas[] = { + {"micro_prec", "Total relevant retrieved documents / Total retrieved documents", + 0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_ret)}, + {"micro_recall", "Total relevant retrieved documents / Total relevant documents", + 0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_rel)}, + {"micro_bpref", "Total correct preferences / Total possible preferences", + 0, offsetof(TREC_EVAL, bpref_num_correct), offsetof(TREC_EVAL, bpref_num_possible)}, +}; + +int num_micro_meas = sizeof (micro_meas) / sizeof (micro_meas[0]); + diff --git a/sm_cnn/trec_eval-8.0/print_meas.c b/sm_cnn/trec_eval-8.0/print_meas.c new file mode 100755 index 0000000..e478e3c --- /dev/null +++ b/sm_cnn/trec_eval-8.0/print_meas.c @@ -0,0 +1,336 @@ +#ifdef RCSID +static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $"; +#endif + +/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. + + Permission is granted for use of this file in unmodified form for + research purposes. Please contact the SMART project to obtain + permission for other uses. +*/ + +#include "common.h" +#include "sysfunc.h" +#include "buf.h" +#include "trec_eval.h" + +static SM_BUF internal_output = {0, 0, (char *) 0}; +int add_buf_string(); + +extern SINGLE_MEASURE sing_meas[]; +extern PARAMETERIZED_MEASURE param_meas[]; +extern MICRO_MEASURE micro_meas[]; +extern int num_param_meas, num_sing_meas, num_micro_meas; + +int +accumulate_results (query_eval, accum_eval) +TREC_EVAL *query_eval; +TREC_EVAL *accum_eval; +{ + long i,j; + float *float_query, *float_accum; + long *long_query, *long_accum; + + if (query_eval->num_ret <= 0) + return (0); + + accum_eval->num_queries++; + + for (i = 0; i < num_sing_meas; i++) { + if (sing_meas[i].is_long_flag) { + long_query = (long *) (((char *) query_eval) + + sing_meas[i].byte_offset); + long_accum = (long *) (((char *) accum_eval) + + sing_meas[i].byte_offset); + *long_accum += *long_query; + } + else { + float_query = (float *) (((char *) query_eval) + + sing_meas[i].byte_offset); + float_accum = (float *) (((char *) accum_eval) + + sing_meas[i].byte_offset); + *float_accum += *float_query; + } + } + + for (i = 0; i < num_param_meas; i++) { + for (j = 0; j < param_meas[i].num_values; j++) { + if (param_meas[i].is_long_flag) { + long_query = (long *) (((char *) query_eval) + + param_meas[i].byte_offset); + long_accum = (long *) (((char *) accum_eval) + + param_meas[i].byte_offset); + long_accum[j] += long_query[j]; + } + else { + float_query = (float *) (((char *) query_eval) + + param_meas[i].byte_offset); + float_accum = (float *) (((char *) accum_eval) + + param_meas[i].byte_offset); + float_accum[j] += float_query[j]; + } + } + } + + return (0); +} + +void +print_rel_trec_eval_list (is_single_query_flag, epi, eval, output) +long is_single_query_flag; +EVAL_PARAM_INFO *epi; +TREC_EVAL *eval; +SM_BUF *output; +{ + long i,j; + char temp_buf[1024]; + char q_buf[20]; + char name_buf[80]; + SM_BUF *out_p; + long long_eval; + float float_eval; + + if (output == NULL) { + out_p = &internal_output; + out_p->end = 0; + } + else + out_p = output; + + if (is_single_query_flag) { + (void) sprintf (q_buf, "%.20s", eval[0].qid); + } + else { + (void) sprintf (q_buf, "%s", "all"); + (void) sprintf (temp_buf, "%-15s\t%s\t%ld\n", + "num_q", q_buf, eval->num_queries); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + + for (i = 0; i < num_sing_meas; i++) { + if ((! sing_meas[i].print_short_flag) && (! epi->all_flag)) + continue; + if (sing_meas[i].print_time_flag && (!epi->time_flag)) + continue; + if (sing_meas[i].print_only_query_flag && (!is_single_query_flag)) + continue; + if (sing_meas[i].print_only_average_flag && (is_single_query_flag)) + continue; + if (sing_meas[i].is_long_flag) { + long_eval = *((long *) (((char *) eval) + + sing_meas[i].byte_offset)); + if (sing_meas[i].avg_results_flag) + long_eval /= eval->num_queries; + (void) sprintf (temp_buf, "%-15s\t%s\t%ld\n", + sing_meas[i].name, q_buf, long_eval); + } + else { + float_eval = *((float *) (((char *) eval) + + sing_meas[i].byte_offset)); + if (sing_meas[i].avg_results_flag) + float_eval /= eval->num_queries; + else if (sing_meas[i].avg_rel_results_flag && eval->num_rel > 0) + /* average over number of rel docs instead of number queries */ + float_eval /= eval->num_rel; + else if (sing_meas[i].gm_results_flag) { + /* computing geometric mean instead of mean */ + if (!is_single_query_flag && epi->average_complete_flag) + /* Must patch up averages for any missing queries, since */ + /* value of 0 means perfection */ + float_eval += (eval->num_queries - eval->num_orig_queries)* + log (MIN_GEO_MEAN); + float_eval = (float) exp ((double) (float_eval / + eval->num_queries)); + } + (void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n", + sing_meas[i].name, q_buf, float_eval); + } + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + + for (i = 0; i < num_param_meas; i++) { + if ((! param_meas[i].print_short_flag) && (! epi->all_flag)) + continue; + if (param_meas[i].print_time_flag && (!epi->time_flag)) + continue; + if (param_meas[i].print_only_query_flag && (!is_single_query_flag)) + continue; + if (param_meas[i].print_only_average_flag && (is_single_query_flag)) + continue; + for (j = 0; j < param_meas[i].num_values; j++) { + sprintf (name_buf, param_meas[i].format_string, + param_meas[i].get_param_str (epi, j)); + if (param_meas[i].is_long_flag) { + long_eval = ((long *) (((char *) eval) + + param_meas[i].byte_offset))[j]; + if (param_meas[i].avg_results_flag) + long_eval /= eval->num_queries; + (void) sprintf (temp_buf, "%-15s\t%s\t%ld\n", + name_buf, q_buf, long_eval); + } + else { + float_eval = ((float *) (((char *) eval) + + param_meas[i].byte_offset))[j]; + if (param_meas[i].avg_results_flag) + float_eval /= eval->num_queries; + (void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n", + name_buf, q_buf, float_eval); + } + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + } + + if (! is_single_query_flag) { + long denom_long_eval; + for (i = 0; i < num_micro_meas; i++) { + if ((! micro_meas[i].print_short_flag) && (! epi->all_flag)) + continue; + long_eval = *((long *) (((char *) eval) + + micro_meas[i].numerator_byte_offset)); + denom_long_eval = *((long *) (((char *) eval) + + micro_meas[i].denominator_byte_offset)); + float_eval = (float) long_eval / (float) denom_long_eval; + (void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n", + micro_meas[i].name, q_buf, float_eval); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + } + + if (output == NULL) { + (void) fwrite (out_p->buf, 1, out_p->end, stdout); + out_p->end = 0; + } +} + +static long cutoff[] = CUTOFF_VALUES; + +void +old_print_trec_eval_list (epi, eval, num_runs, output) +EVAL_PARAM_INFO *epi; +TREC_EVAL *eval; +int num_runs; +SM_BUF *output; +{ + long i,j; + char temp_buf[1024]; + SM_BUF *out_p; + + if (output == NULL) { + out_p = &internal_output; + out_p->end = 0; + } + else + out_p = output; + + /* Print total numbers retrieved/rel for all runs */ + if (UNDEF == add_buf_string("\nQueryid (Num):\t", out_p)) + return; + for (i = 0; i < num_runs; i++) { + if (UNDEF == add_buf_string (eval->qid, out_p)) + return; + } + if (UNDEF == add_buf_string("\nTotal number of documents over all queries", + out_p)) + return; + if (UNDEF == add_buf_string("\n Retrieved:", out_p)) + return; + for (i = 0; i < num_runs; i++) { + (void) sprintf (temp_buf, " %5ld", eval[i].num_ret); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + if (UNDEF == add_buf_string("\n Relevant: ", out_p)) + return; + for (i = 0; i < num_runs; i++) { + (void) sprintf (temp_buf, " %5ld", eval[i].num_rel); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + if (UNDEF == add_buf_string("\n Rel_ret: ", out_p)) + return; + for (i = 0; i < num_runs; i++) { + (void) sprintf (temp_buf, " %5ld", eval[i].num_rel_ret); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + + /* Print recall precision figures at NUM_RP_PTS recall levels */ + if (UNDEF == add_buf_string + ("\nInterpolated Recall - Precision Averages:", out_p)) + return; + for (j = 0; j < NUM_RP_PTS; j++) { + (void) sprintf (temp_buf, "\n at %4.2f ", + (float) j / (NUM_RP_PTS - 1)); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + for (i = 0; i < num_runs; i++) { + (void) sprintf (temp_buf, " %6.4f ", + eval[i].int_recall_precis[j] /eval[i].num_queries); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + } + + /* Print average recall precision and percentage improvement */ + (void) sprintf (temp_buf, + "\nAverage precision (non-interpolated) for all rel docs(averaged over queries)\n "); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + for (i = 0; i < num_runs; i++) { + (void) sprintf (temp_buf, " %6.4f ", + eval[i].av_recall_precis / eval[i].num_queries); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + if (num_runs > 1) { + (void) sprintf (temp_buf, "\n %% Change: "); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + for (i = 1; i < num_runs; i++) { + (void) sprintf (temp_buf, " %6.1f ", + (((eval[i].av_recall_precis / eval[i].num_queries)/ + (eval[0].av_recall_precis / eval[i].num_queries)) + - 1.0) * 100.0); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + } + (void) sprintf (temp_buf, "\nPrecision:"); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + for (j = 0; j < NUM_CUTOFF; j++) { + (void) sprintf (temp_buf, "\n At %4ld docs:", cutoff[j]); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + for (i = 0; i < num_runs; i++) { + (void) sprintf (temp_buf, " %6.4f", + eval[i].precis_cut[j] / eval[i].num_queries); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + } + + (void) sprintf (temp_buf, "\nR-Precision (precision after R (= num_rel for a query) docs retrieved):\n Exact: "); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + for (i = 0; i < num_runs; i++) { + (void) sprintf (temp_buf, " %6.4f", + eval[i].R_recall_precis / eval[i].num_queries); + if (UNDEF == add_buf_string (temp_buf, out_p)) + return; + } + + if (UNDEF == add_buf_string ("\n", out_p)) + return; + + if (output == NULL) { + (void) fwrite (out_p->buf, 1, out_p->end, stdout); + out_p->end = 0; + } + return; +} + diff --git a/sm_cnn/trec_eval-8.0/trec_eval.h b/sm_cnn/trec_eval-8.0/trec_eval.h new file mode 100755 index 0000000..54859af --- /dev/null +++ b/sm_cnn/trec_eval-8.0/trec_eval.h @@ -0,0 +1,353 @@ +#ifndef TRECEVALH +#define TRECEVALH + +/* Static state info; set at beginning, possibly from program options, */ +/* but then remains constant throughout. */ +typedef struct { + long query_flag; /* 0. If set, evaluation output will be + printed for each query, in addition + to summary at end. */ + long all_flag; /* 0. If set, all evaluation measures will + be printed instead of just the + final TREC 2 measures. */ + long time_flag; /* 0. If set, calculate time-based measures*/ + long relation_flag; /* 1. If set, print in relational form */ + long average_complete_flag; /* 0. If set, average over the complete set + of relevance judgements (qrels), instead + of the number of queries + in the intersection of qrels and result */ + long judged_docs_only_flag; /* 0. If set, throw out all unjudged docs + for the retrieved set before calculating + any measures. */ + double utility_a; /* UTILITY_A. Default utility values */ + double utility_b; /* UTILITY_B. Default utility values */ + double utility_c; /* UTILITY_C. Default utility values */ + double utility_d; /* UTILITY_D. Default utility values */ + long num_docs_in_coll; /* 0. number of docs in collection */ + long relevance_level; /* 1. In relevance judgements, the level at + which a doc is considered relevant for + this evaluation */ + long max_num_docs_per_topic; /* MAXLONG. evaluate only this many docs */ +} EVAL_PARAM_INFO; + +/* Measure characteristics (how to print them, average them). */ +/* List of measures is in measures.c */ +/* Three types of measures: + single measures - single measure and name + parameterized measures - arrays of a measure, whose measure name + depends on parameter (eg P5, P10) + micro measures - measures defined as the micro average over all + docs retrieved independent of topic. Only calculated + and printed for the "all" pseudo-query. + Eg micro_prec = num_rel_ret / num_ret +*/ +typedef struct { + char *name; + char *long_name; + unsigned char is_long_flag; /* otherwise float */ + unsigned char print_short_flag; /* if set, measure is always printed + (not just if all_flag set) */ + unsigned char print_time_flag; /* if set, measure is printed only + if time_flag is set */ + unsigned char print_only_query_flag; /* if set, measure is printed only + when printing individual query output*/ + unsigned char print_only_average_flag; /* if set, measure is printed only + when printing overall average output*/ + unsigned char avg_results_flag; /* if set, average results over queries */ + unsigned char avg_rel_results_flag;/* if set,average results over num_rel*/ + unsigned char gm_results_flag; /* if set, measure uses geometric mean. ie + exponentiate the average before + printing */ + long byte_offset; +} SINGLE_MEASURE; + +typedef struct { + char *long_name; + unsigned char is_long_flag; /* otherwise float */ + unsigned char print_short_flag; /* if set, print in short output */ + unsigned char print_time_flag; /* if set, measure is printed only + if time_flag is set */ + unsigned char print_only_query_flag; /* if set, measure is printed only + when printing individual query output*/ + unsigned char print_only_average_flag; /* if set, measure is printed only + when printing overall average output*/ + unsigned char avg_results_flag; /* if set, average results over queries */ + long byte_offset; + long num_values; + char *format_string; + char *long_format_string; + char *(*get_param_str) (EVAL_PARAM_INFO *ip, long index); +} PARAMETERIZED_MEASURE; + +typedef struct { + char *name; + char *long_name; + unsigned char print_short_flag; /* if set, measure is always printed + (not just if all_flag set) */ + long numerator_byte_offset; + long denominator_byte_offset; +} MICRO_MEASURE; + + +typedef struct { /* For each retrieved document result */ + char *docno; /* document id */ + float sim; /* score */ + long rank; /* rank assigned after breaking ties */ +} TEXT_TR; + +typedef struct { /* For each query in retrieved results */ + char *qid; /* query id */ + long num_text_tr; /* number of TEXT_TR results for query*/ + long max_num_text_tr; /* number results space reserved for */ + TEXT_TR *text_tr; /* Array of TEXT_TR results */ +} TREC_TOP; + +typedef struct { /* Overall retrieved results */ + char *run_id; /* run id */ + long num_q_tr; /* Number of TREC_TOP queries */ + long max_num_q_tr; /* Num queries space reserved for*/ + TREC_TOP *trec_top; /* Array of TREC_TOP query results */ +} ALL_TREC_TOP; + +typedef struct { /* For each relevance judgement */ + char *docno; /* document id */ + long rel; /* document judgement */ +} TEXT_QRELS; + +typedef struct { /* For each query in rel judgements */ + char *qid; /* query id */ + long num_text_qrels; /* number of judged documents */ + long max_num_text_qrels; /* Num docs space reserved for */ + TEXT_QRELS *text_qrels; /* Array of judged TEXT_QRELS */ +} TREC_QRELS; + +typedef struct { /* Overall relevance judgements */ + long num_q_qrels; /* Number of TREC_QRELS queries */ + long max_num_q_qrels; /* Num queries space reserved for */ + TREC_QRELS *trec_qrels; /* Array of TREC_QRELS queries */ +} ALL_TREC_QRELS; + + + +#define INIT_NUM_QUERIES 50 +#define INIT_NUM_RESULTS 1000 +#define INIT_NUM_RELS 2000 + +/* Set retrieval is based on contingency table: + relevant nonrelevant + retrieved a b + nonretrieved c d + + Often you see r == num_rel_ret == a + R == num_rel == a+c + n == num_ret == a+b + N == num_docs == a+b+c+d + Some of these definitions are used in comments below +*/ + + +/* ----------------------------------------------- */ +/* Defined constants that are collection/purpose dependent */ + +/* Number of cutoffs for recall,precision, and rel_precis measures. */ +/* CUTOFF_VALUES gives the number of retrieved docs that these */ +/* evaluation mesures are applied at. */ +#define NUM_CUTOFF 9 +#define CUTOFF_VALUES {5, 10, 15, 20, 30, 100, 200, 500, 1000} + +/* Maximum fallout value, expressed in number of non-rel docs retrieved. */ +/* (Make the approximation that number of non-rel docs in collection */ +/* is equal to the number of number of docs in collection) */ +#define MAX_FALL_RET 142 + +/* Maximum multiple of R (number of rel docs for this query) to calculate */ +/* R-based precision at */ +#define MAX_RPREC 2.0 + +#define MAX_TIME 300.0 +#define NUM_TIME_PTS 60 + +/* Set a maximum number of nonrel docs to be used for preference measures */ +#define PREF_TOP_NONREL_NUM 100 + +/* ----------------------------------------------- */ +/* Defined constants that are collection/purpose independent. If you + change these, you probably need to change comments and documentation, + and some variable names may not be appropriate any more! */ +#define NUM_RP_PTS 11 +#define THREE_PTS {2, 5, 8} +#define NUM_FR_PTS 11 +#define NUM_PREC_PTS 11 +#define UTILITY_A 1.0 +#define UTILITY_B -1.0 +#define UTILITY_C 0.0 +#define UTILITY_D 0.0 +#define MIN_GEO_MEAN .00001 + +typedef struct { + char *qid; /* query id */ + long num_queries; /* Number of queries for this eval */ + long num_orig_queries; /* Number of queries for this eval without + missing values, if using trec_eval -c */ + /* Summary Numbers over all queries */ + long num_rel; /* Number of relevant docs */ + long num_ret; /* Number of retrieved docs */ + long num_rel_ret; /* Number of relevant retrieved docs */ + float avg_doc_prec; /* Average of precision over all + relevant documents (query independent)*/ + + /* Measures after num_ret docs */ + float exact_recall; /* Recall after num_ret docs */ + float exact_precis; /* Precision after num_ret docs */ + float exact_rel_precis; /* Relative Precision (or recall) */ + /* Defined to be precision / max possible + precision */ + float exact_uap; /* Unranked Average Precision */ + /* Every rel doc in retrieved set gets + precision, every nonret rel doc gets 0. + Average over all rel docs */ + /* Note this = exact_recall * + exact_precision for a query */ + /* Preferred measure for evaluation of + unranked sets of arbitrary size. */ + float exact_rel_uap; /* Relative Unranked Average Precision */ + /* Above, but relativized given size of + retrieved set */ + /* If (nR) set num_ret to R + Then use uap formula */ + /* exact_rel_precis ** 2 */ + float exact_utility; /* From contingency table, by default: + UTILITY_A * a + UTILITY_B * b + + UTILITY_C * c + UTILITY_D * d. + By default, a-b (or r - (n-r)) */ + float recip_rank; /* reciprical rank of top retrieved + relevant document */ + long rank_first_rel; /* Rank of top retrieved rel doc. Set to + 0 if none. Unaveraged */ + + /* Measures after each document */ + float recall_cut[NUM_CUTOFF]; /* Recall after cutoff[i] docs */ + + float precis_cut[NUM_CUTOFF]; /* precision after cutoff[i] docs. If + less than cutoff[i] docs retrieved, + then assume an additional + cutoff[i]-num_ret non-relevant docs + are retrieved. */ + float rel_precis_cut[NUM_CUTOFF];/* Relative precision after cutoff[i] + docs. (Note relative precision is + identical to relative recall) */ + float uap_cut[NUM_CUTOFF]; /* uap (is recall * precision) after + cutoff[i] docs. Not recommended */ + float rel_uap_cut[NUM_CUTOFF]; /* rel_uap at cutoff[i] docs */ + float av_rel_precis; /* average (integral) of rel_precis + after each doc. Do not use if + number of docs retrieved varies */ + float av_rel_uap; /* average (integral) of rel_uap + after each doc. Do not use if + number of docs retrieved varies */ + + + /* Measures after each rel doc */ + float av_recall_precis; /* MAP! average(integral) of precision at + all rel doc ranks. THE MAJOR + EVALUATION MEASURE FOR RANKED DOCS */ + float int_av_recall_precis; /* Same as above, but the precision values + have been interpolated, so that prec(X) + is actually MAX prec(Y) for all + Y >= X */ + float int_recall_precis[NUM_RP_PTS];/* interpolated precision at + 0.1 increments of recall */ + float int_av3_recall_precis; /* interpolated average at 3 intermediate + points */ + float int_av11_recall_precis; /* interpolated average at NUM_RP_PTS + intermediate points (recall_level) */ + + /* Measures after each non-rel doc */ + float fall_recall[NUM_FR_PTS]; /* max recall after each non-rel doc, + at 11 points starting at 0.0 and + ending at MAX_FALL_RET /num_docs */ + float av_fall_recall; /* Average of fallout-recall, after each + non-rel doc until fallout of + MAX_FALL_RET / num_docs achieved */ + + /* Measures after R-related cutoffs. R is the number of relevant + docs for a particular query, but note that these cutoffs are after + R docs, whether relevant or non-relevant, have been retrieved. + R-related cutoffs are really only applicable to a situtation where + there are many relevant docs per query (or lots of queries). */ + float R_recall_precis; /* Recall or precision after R docs + (note they are equal at this point) */ + float av_R_precis; /* Average (or integral) of precision at + each doc until R docs have been + retrieved */ + float R_prec_cut[NUM_PREC_PTS]; /* Precision measured after multiples of + R docs have been retrieved. 10 + equal points, with max multiple + having value MAX_RPREC */ + float int_R_recall_precis; /* Interpolated precision after R docs + Prec(X) = MAX(prec(Y)) for all Y>=X */ + float int_av_R_precis; /* Interpolated */ + float int_R_prec_cut[NUM_PREC_PTS]; /* Interpolated */ + + /* Measures after particular time relative to size of eventual retrieved + set. Eg, precision is num_rel_so_far/num_ret + relprecision is num_rel_so_far/MIN(num_ret,num_rel) + uap is num_rel_so_far**2/(num_ret*MIN(num_ret,num_rel)) + reluap is relprecision * relprecision */ + float time_num_rel[NUM_TIME_PTS]; /* Number of rel docs in time bucket*/ + float time_num_nrel[NUM_TIME_PTS];/* Number of nrel docs in each bucket*/ + float time_cum_rel[NUM_TIME_PTS]; /* Cumulative time_num_rel */ + float time_precis[NUM_TIME_PTS]; /* First Precision in each bucket */ + float time_relprecis[NUM_TIME_PTS];/* First rel-Precision in each bucket */ + float time_uap[NUM_TIME_PTS]; /* First uap in bucket*/ + float time_reluap[NUM_TIME_PTS]; /* First relative uap in bucket*/ + float time_utility[NUM_TIME_PTS]; /* First Utility (default 1,-1,0,0) + in bucket */ + float av_time_precis; /* Sum (integral) of time_precis */ + float av_time_relprecis; /* Sum (integral) of time_relprecis */ + float av_time_uap; /* Sum (integral) of time_uap */ + float av_time_reluap; /* Sum (integral) of time_reluap */ + float av_time_utility; /* Sum (integral) of time_utility */ + float av_time_cum_rel; /* Sum (integral) of time_cum_rel */ + + /* Measures dependent on only judged documents */ + /* Binary Pref relations: fraction of nonrel documents retrieved after + each rel doc */ + float bpref; /* real BPREF. Top num_rel nonrel docs */ + float bpref_top5Rnonrel; /* Top 5 * num_rel nonrel docs */ + float bpref_top10Rnonrel; /* Top 10 * num_rel nonrel docs */ +/* float bpref_topRnonrel; * renamed as bpref */ + float bpref_allnonrel; /* all judged nonrel docs */ + float bpref_retnonrel; /* Only retrieved nonrel docs */ + float bpref_topnonrel; /* Top PREF_TOPNREL_NUM nonrel docs */ + float bpref_top50pRnonrel; /* Top 50 + num_rel nonrel docs */ + float bpref_top25pRnonrel; /* Top 25 + num_rel nonrel docs */ + float bpref_top10pRnonrel; /* Top 10 + num_rel nonrel docs. + Bad version used in SIGIR 2004 paper */ + float old_bpref_top10pRnonrel; /* bad old version. Top 10 + num_rel + nonrel docs. Used in SIGIR 2004 paper*/ + float bpref_top25p2Rnonrel; /* Top 25 + 2 * num_rel nonrel docs */ + float bpref_retall; /* Only retrieved rel,nonrel docs */ + float bpref_5; /* Only top 5 rel, top 5 nonrel */ + float bpref_10; /* Only top 10 rel, top 10 nonrel */ + float old_bpref; /* Bad old bpref. Top num_rel nonrel docs. + Only used retrieved nonrel docs. + Used in TREC 12,13, mention in + SIGIR 2004 paper */ + float bpref_num_all; /* num not retrieved before (all judged)*/ + float bpref_num_ret; /* num retrieved after */ + long bpref_num_correct; /* num correct preferences */ + long bpref_num_possible; /* num possible correct preferences */ + + /* Measures that use Geometric Mean + avg_Score = exp (SUM (log (MAX (query_score, .00001))) / N) + WARNING: Geometric Mean measures special cased for "trec_eval -c". + Works, but be careful when implementing new measure */ + float gm_ap; /* Geometric Mean version of MAP */ + float gm_bpref; /* Geometric Mean version of bpref. Note + bpref has lots of 0.0 values */ + +} TREC_EVAL; + +#endif /* TRECEVALH */ diff --git a/sm_cnn/trec_eval-8.0/trec_eval_help.c b/sm_cnn/trec_eval-8.0/trec_eval_help.c new file mode 100755 index 0000000..eb7bf60 --- /dev/null +++ b/sm_cnn/trec_eval-8.0/trec_eval_help.c @@ -0,0 +1,203 @@ +/* Copyright (c) 2003, 1991, 1990, 1984 - Chris Buckley. */ + +#include "common.h" +#include "trec_eval.h" + +static char *help_message = +"trec_eval [-h] [-q] [-a] [-o] [-c] [-l [-N] [-M] [-Ua] [-Ub] [-Uc] [-Ud] [-T] trec_rel_file trec_top_file \n\ + \n\ +Calculate and print various evaluation measures, evaluating the results \n\ +in trec_top_file against the relevance judgements in trec_rel_file. \n\ + \n\ +There are a fair number of options, of which only the lower case options are \n\ +normally ever used. \n\ + -h: Print full help message and exit \n\ + -q: In addition to summary evaluation, give evaluation for each query \n\ + -a: Print all evaluation measures calculated, instead of just the \n\ + main official measures for TREC. \n\ + -o: Print everything out in old, nonrelational format (default is relational) \n\ + -c: Average over the complete set of queries in the relevance judgements \n\ + instead of the queries in the intersection of relevance judgements \n\ + and results. Missing queries will contribute a value of 0 to all \n\ + evaluation measures (which may or may not be reasonable for a \n\ + particular evaluation measure, but is reasonable for standard TREC \n\ + measures.) \n\ + -l: Num indicates the minimum relevance judgement value needed for \n\ + a document to be called relevant. (All measures used by TREC eval are \n\ + based on binary relevance). Used if trec_rel_file contains relevance \n\ + judged on a multi-relevance scale. Default is 1. \n\ + -N: Number of docs in collection \n\ + -M: Max number of docs per topic to use in evaluation (discard rest). \n\ + -Ua: Value to use for 'a' coefficient of utility computation. \n\ + relevant nonrelevant \n\ + retrieved a b \n\ + nonretrieved c d \n\ + -Ub: Value to use for 'b' coefficient of utility computation. \n\ + -Uc: Value to use for 'c' coefficient of utility computation. \n\ + -Ud: Value to use for 'd' coefficient of utility computation. \n\ + -J: Calculate all values only over the judged (either relevant or \n\ + nonrelevant) documents. All unjudged documents are removed from the \n\ + retrieved set before any calculations (possibly leaving an empty set). \n\ + DO NOT USE, unless you really know what you're doing - very easy to get \n\ + reasonable looking, but invalid, numbers. \n\ + -T: Treat similarity as time that document retrieved. Compute \n\ + several time-based measures after ranking docs by time retrieved \n\ + (first doc (lowest sim) retrieved ranked highest). \n\ + Only done if -a selected. \n\ + \n\ + \n\ +Read text tuples from trec_top_file of the form \n\ + 030 Q0 ZF08-175-870 0 4238 prise1 \n\ + qid iter docno rank sim run_id \n\ +giving TREC document numbers (a string) retrieved by query qid \n\ +(a string) with similarity sim (a float). The other fields are ignored, \n\ +with the exception that the run_id field of the last line is kept and \n\ +output. In particular, note that the rank field is ignored here; \n\ +internally ranks are assigned by sorting by the sim field with ties \n\ +broken deterministicly (using docno). \n\ +Sim is assumed to be higher for the docs to be retrieved first. \n\ +File may contain no NULL characters. \n\ +Lines may contain fields after the run_id; they are ignored. \n\ + \n\ +Relevance for each docno to qid is determined from text_qrels_file, which \n\ +consists of text tuples of the form \n\ + qid iter docno rel \n\ +giving TREC document numbers (docno, a string) and their relevance (rel, \n\ +an integer) to query qid (a string). iter string field is ignored. \n\ +Fields are separated by whitespace, string fields can contain no whitespace. \n\ +File may contain no NULL characters. \n\ + \n\ +The text tuples with relevance judgements are converted to TR_VEC form \n\ +and then submitted to the SMART evaluation routines. \n\ +The qid,did,rank,sim,rel fields of TR_VEC are filled in; \n\ +action,iter fields are set to 0. \n\ +The rel field is set to -1 if the document was not judged (not in \n\ +text_qrels_file). Most measures, but not all, will treat -1 the same as 0, \n\ +namely nonrelevant. Note that relevance_level is used to determine if the \n\ +document is relevant during score calculations. \n\ +Queries for which there are no relevant docs are ignored. \n\ +Warning: queries for which there are relevant docs but no retrieved docs \n\ +are also ignored by default. This allows systems to evaluate over subsets \n\ +of the relevant docs, but means if a system improperly retrieves no docs, \n\ +it will not be detected. Use the -c flag to avoid this behavior. \n\ + \n\ +EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. \n\ +Relational Format prints the same values, but all lines are of the form \n\ + measure_name query value \n\ + \n\ +1. Total number of documents over all queries \n\ + Retrieved: \n\ + Relevant: \n\ + Rel_ret: (relevant and retrieved) \n\ + These should be self-explanatory. All values are totals over all \n\ + queries being evaluated. \n\ +2. Interpolated Recall - Precision Averages: \n\ + at 0.00 \n\ + at 0.10 \n\ + ... \n\ + at 1.00 \n\ + See any standard IR text (especially by Salton) for more details of \n\ + recall-precision evaluation. Measures precision (percent of retrieved \n\ + docs that are relevant) at various recall levels (after a certain \n\ + percentage of all the relevant docs for that query have been retrieved). \n\ + 'Interpolated' means that, for example, precision at recall \n\ + 0.10 (ie, after 10% of rel docs for a query have been retrieved) is \n\ + taken to be MAXIMUM of precision at all recall points >= 0.10. \n\ + Values are averaged over all queries (for each of the 11 recall levels). \n\ + These values are used for Recall-Precision graphs. \n\ +3. Average precision (non-interpolated) over all rel docs \n\ + The precision is calculated after each relevant doc is retrieved. \n\ + If a relevant doc is not retrieved, its precision is 0.0. \n\ + All precision values are then averaged together to get a single number \n\ + for the performance of a query. Conceptually this is the area \n\ + underneath the recall-precision graph for the query. \n\ + The values are then averaged over all queries. \n\ +4. Precision: \n\ + at 5 docs \n\ + at 10 docs \n\ + ... \n\ + at 1000 docs \n\ + The precision (percent of retrieved docs that are relevant) after X \n\ + documents (whether relevant or nonrelevant) have been retrieved. \n\ + Values averaged over all queries. If X docs were not retrieved \n\ + for a query, then all missing docs are assumed to be non-relevant. \n\ +5. R-Precision (precision after R (= num_rel for a query) docs retrieved): \n\ + Measures precision (or recall, they're the same) after R docs \n\ + have been retrieved, where R is the total number of relevant docs \n\ + for a query. Thus if a query has 40 relevant docs, then precision \n\ + is measured after 40 docs, while if it has 600 relevant docs, precision \n\ + is measured after 600 docs. This avoids some of the averaging \n\ + problems of the 'precision at X docs' values in (4) above. \n\ + If R is greater than the number of docs retrieved for a query, then \n\ + the nonretrieved docs are all assumed to be nonrelevant. \n\ +"; + + +extern SINGLE_MEASURE sing_meas[]; +extern PARAMETERIZED_MEASURE param_meas[]; +extern MICRO_MEASURE micro_meas[]; +extern int num_param_meas, num_sing_meas, num_micro_meas; + +int +trec_eval_help(epi) +EVAL_PARAM_INFO *epi; +{ + long i, j; + char temp_buf1[30]; + char temp_buf2[80]; + + printf ("%s\n", help_message); + + printf ("Major measures (again) with their relational names:\n"); + for (i = 0; i < num_sing_meas; i++) { + if (sing_meas[i].print_short_flag) + printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name); + } + for (i = 0; i < num_param_meas; i++) { + if (param_meas[i].print_short_flag) { + for (j = 0; j < param_meas[i].num_values; j++) { + sprintf (temp_buf1, param_meas[i].format_string, + param_meas[i].get_param_str (epi, j)); + sprintf (temp_buf2, param_meas[i].long_format_string, + param_meas[i].get_param_str (epi, j)); + printf ("%-15s\t%s%s\n", temp_buf1, + param_meas[i].long_name, temp_buf2); + } + } + } + for (i = 0; i < num_micro_meas; i++) { + if (micro_meas[i].print_short_flag) + printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name); + } + + printf ("\n\nMinor measures with their relational names:\n"); + for (i = 0; i < num_sing_meas; i++) { + if (sing_meas[i].print_short_flag) + continue; + if (sing_meas[i].print_time_flag && (! epi->time_flag)) + continue; + if (! sing_meas[i].print_short_flag) + printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name); + } + for (i = 0; i < num_param_meas; i++) { + if (param_meas[i].print_short_flag) + continue; + if (param_meas[i].print_time_flag && (! epi->time_flag)) + continue; + for (j = 0; j < param_meas[i].num_values; j++) { + sprintf (temp_buf1, param_meas[i].format_string, + param_meas[i].get_param_str (epi, j)); + sprintf (temp_buf2, param_meas[i].long_format_string, + param_meas[i].get_param_str (epi, j)); + printf ("%-15s\t%s%s\n", temp_buf1, + param_meas[i].long_name, temp_buf2); + } + } + for (i = 0; i < num_micro_meas; i++) { + if (! micro_meas[i].print_short_flag) + printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name); + } + + return (1); +} + diff --git a/sm_cnn/trec_eval-8.0/trvec_teval.c b/sm_cnn/trec_eval-8.0/trvec_teval.c new file mode 100755 index 0000000..e2d2891 --- /dev/null +++ b/sm_cnn/trec_eval-8.0/trvec_teval.c @@ -0,0 +1,673 @@ +#ifdef RCSID +static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/trvec_trec_eval.c,v 11.0 1992/07/21 18:20:35 chrisb Exp chrisb $"; +#endif + +/* Copyright (c) 2005 +*/ + +#include "common.h" +#include "sysfunc.h" +#include "smart_error.h" +#include "tr_vec.h" +#include "trec_eval.h" + +static int compare_iter_rank(); +static void calc_cutoff_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, + TREC_EVAL *eval, long num_rel, + long num_nonrel); +static void calc_bpref_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, + TREC_EVAL *eval, long num_rel, + long num_nonrel); +static void calc_average_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, + TREC_EVAL *eval, long num_rel, + long num_nonrel); +static void calc_exact_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, + TREC_EVAL *eval, long num_rel, + long num_nonrel); +static void calc_time_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec, + TREC_EVAL *eval, long num_rel, + long num_nonrel); + +int +trvec_trec_eval (epi, tr_vec, eval, num_rel, num_nonrel) +EVAL_PARAM_INFO *epi; +TR_VEC *tr_vec; +TREC_EVAL *eval; +long num_rel; /* Number relevant judged */ +long num_nonrel; /* Number nonrelevant judged */ +{ + long j; + long max_iter; + + if (tr_vec == (TR_VEC *) NULL) + return (UNDEF); + + /* Initialize everything to 0 */ + bzero ((char *) eval, sizeof (TREC_EVAL)); + + eval->qid = tr_vec->qid; + eval->num_queries = 1; + + /* If no retrieved docs, then just return */ + if (tr_vec->num_tr == 0) { + return (0); + } + + eval->num_rel = num_rel; + + /* Evaluate only the docs on the last iteration of new_tr_vec */ + /* Sort the tr tuples for this query by decreasing iter and + increasing rank */ + qsort ((char *) tr_vec->tr, + (int) tr_vec->num_tr, + sizeof (TR_TUP), + compare_iter_rank); + + max_iter = tr_vec->tr[0].iter; + for (j = 0; j < tr_vec->num_tr; j++) { + if (tr_vec->tr[j].iter == max_iter) { + eval->num_ret++; + if (tr_vec->tr[j].rel >= epi->relevance_level) + eval->num_rel_ret++; + } + else { + if (tr_vec->tr[j].rel >= epi->relevance_level) + eval->num_rel--; + } + } + + /* Calculate cutoff measures, and those measures dependant on them */ + /* Also includes recip_rank and rank_first_rel */ + calc_cutoff_measures (epi, tr_vec, eval, num_rel, num_nonrel); + + /* Calculate bpref measures */ + calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel); + + /* Calculate measures that average over ret or rel docs */ + calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel); + + /* Calculate exact measures over entire retrieved sets */ + calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel); + + /* Calculate time measures, if wanted */ + if (epi->time_flag) + calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel); + + + return (1); +} + +static int +compare_iter_rank (tr1, tr2) +TR_TUP *tr1; +TR_TUP *tr2; +{ + if (tr1->iter > tr2->iter) + return (-1); + if (tr1->iter < tr2->iter) + return (1); + if (tr1->rank < tr2->rank) + return (-1); + if (tr1->rank > tr2->rank) + return (1); + return (0); +} + + +/* ********************************************************************* */ +/* calculate cutoff measures */ + /* cutoff values for recall precision output */ +static int cutoff[NUM_CUTOFF] = CUTOFF_VALUES; +static int three_pts[3] = THREE_PTS; + + +static void +calc_cutoff_measures(epi, tr_vec, eval, num_rel, num_nonrel) +EVAL_PARAM_INFO *epi; +TR_VEC *tr_vec; +TREC_EVAL *eval; +long num_rel; /* Number relevant judged */ +long num_nonrel; /* Number nonrelevant judged */ +{ + double recall, precis; /* current recall, precision values */ + double rel_precis, rel_uap;/* relative precision, uap values */ + double int_precis; /* current interpolated precision values */ + + long i,j; + + long cut_rp[NUM_RP_PTS]; /* number of rel docs needed to be retrieved + for each recall-prec cutoff */ + long cut_fr[NUM_FR_PTS]; /* number of non-rel docs needed to be + retrieved for each fall-recall cutoff */ + long cut_rprec[NUM_PREC_PTS]; /* Number of docs needed to be retrieved + for each R-based prec cutoff */ + long current_cutoff, current_cut_rp, current_cut_fr, current_cut_rprec; + + long rel_so_far = eval->num_rel_ret; + + /* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all + Y >= X) */ + int_precis = (float) rel_so_far / (float) eval->num_ret; + + /* Discover cutoff values for this query */ + current_cutoff = NUM_CUTOFF - 1; + while (current_cutoff > 0 && cutoff[current_cutoff] > eval->num_ret) + current_cutoff--; + for (i = 0; i < NUM_RP_PTS; i++) + cut_rp[i] = ((eval->num_rel * i) + NUM_RP_PTS - 2) / (NUM_RP_PTS - 1); + current_cut_rp = NUM_RP_PTS - 1; + while (current_cut_rp > 0 && cut_rp[current_cut_rp] > eval->num_rel_ret) + current_cut_rp--; + for (i = 0; i < NUM_FR_PTS; i++) + cut_fr[i] = ((MAX_FALL_RET * i) + NUM_FR_PTS - 2) / (NUM_FR_PTS - 1); + current_cut_fr = NUM_FR_PTS - 1; + while (current_cut_fr > 0 && cut_fr[current_cut_fr] > eval->num_ret - eval->num_rel_ret) + current_cut_fr--; + for (i = 1; i < NUM_PREC_PTS+1; i++) + cut_rprec[i-1] = ((MAX_RPREC * eval->num_rel * i) + NUM_PREC_PTS - 2) + / (NUM_PREC_PTS - 1); + current_cut_rprec = NUM_PREC_PTS - 1; + while (current_cut_rprec > 0 && cut_rprec[current_cut_rprec]>eval->num_ret) + current_cut_rprec--; + + /* Loop over all retrieved docs in reverse order */ + for (j = eval->num_ret; j > 0; j--) { + if (rel_so_far > 0) { + recall = (float) rel_so_far / (float) eval->num_rel; + precis = (float) rel_so_far / (float) j; + if (j > eval->num_rel) { + rel_precis = (float) rel_so_far / (float) eval->num_rel; + } + else { + rel_precis = (float) rel_so_far / (float) j; + } + } + else { + recall = 0.0; + precis = 0.0; + rel_precis = 0.0; + } + rel_uap = rel_precis * rel_precis; + if (int_precis < precis) + int_precis = precis; + while (j == cutoff[current_cutoff]) { + eval->recall_cut[current_cutoff] = recall; + eval->precis_cut[current_cutoff] = precis; + eval->rel_precis_cut[current_cutoff] = rel_precis; + eval->uap_cut[current_cutoff] = precis * recall; + eval->rel_uap_cut[current_cutoff] = rel_uap; + current_cutoff--; + } + + while (j == cut_rprec[current_cut_rprec]) { + eval->R_prec_cut[current_cut_rprec] = precis; + eval->int_R_prec_cut[current_cut_rprec] = int_precis; + current_cut_rprec--; + } + + if (j == eval->num_rel) { + eval->R_recall_precis = precis; + eval->int_R_recall_precis = int_precis; + } + + if (tr_vec->tr[j-1].rel >= epi->relevance_level) { + while (rel_so_far == cut_rp[current_cut_rp]) { + eval->int_recall_precis[current_cut_rp] = int_precis; + current_cut_rp--; + } + eval->recip_rank = 1.0 / (float) j; + eval->rank_first_rel = j; + rel_so_far--; + } + else { + /* Note: for fallout-recall, the recall at X non-rel docs + is used for the recall 'after' (X-1) non-rel docs. + Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for + Y retrieved docs where X-1 non-rel retrieved */ + while (current_cut_fr >= 0 && + j - rel_so_far == cut_fr[current_cut_fr] + 1) { + eval->fall_recall[current_cut_fr] = recall; + current_cut_fr--; + } + } + } + + /* Fill in the 0.0 value for recall-precision (== max precision + at any point in the retrieval ranking) */ + eval->int_recall_precis[0] = int_precis; + + /* Fill in those cutoff values and averages that were not achieved + because insufficient docs were retrieved. */ + for (i = 0; i < NUM_CUTOFF; i++) { + if (eval->num_ret < cutoff[i]) { + if (eval->num_rel_ret > 0) { + eval->recall_cut[i] = ((float) eval->num_rel_ret / + (float) eval->num_rel); + eval->precis_cut[i] = ((float) eval->num_rel_ret / + (float) cutoff[i]); + } + eval->rel_precis_cut[i] = (cutoff[i] < eval->num_rel) ? + eval->precis_cut[i] : + eval->recall_cut[i]; + eval->uap_cut[i] = eval->precis_cut[i] * + eval->recall_cut[i]; + eval->rel_uap_cut[i] = eval->precis_cut[i] * + eval->precis_cut[i]; + } + } + for (i = 0; i < NUM_FR_PTS; i++) { + if (eval->num_ret - eval->num_rel_ret < cut_fr[i]) { + if (eval->num_rel_ret > 0) + eval->fall_recall[i] = (float) eval->num_rel_ret / + (float) eval->num_rel; + } + } + for (i = 0; i < NUM_PREC_PTS; i++) { + if (eval->num_ret < cut_rprec[i]) { + eval->R_prec_cut[i] = (float) eval->num_rel_ret / + (float) cut_rprec[i]; + eval->int_R_prec_cut[i] = (float) eval->num_rel_ret / + (float) cut_rprec[i]; + } + } + + if (eval->num_rel > eval->num_ret) { + eval->R_recall_precis = (float) eval->num_rel_ret / + (float)eval->num_rel; + eval->int_R_recall_precis = (float) eval->num_rel_ret / + (float)eval->num_rel; + } + + /* Calculate other indirect evaluation measure averages. */ + /* average recall-precis of 3 and 11 intermediate points */ + eval->int_av3_recall_precis = + (eval->int_recall_precis[three_pts[0]] + + eval->int_recall_precis[three_pts[1]] + + eval->int_recall_precis[three_pts[2]]) / 3.0; + for (i = 0; i < NUM_RP_PTS; i++) { + eval->int_av11_recall_precis += eval->int_recall_precis[i]; + } + eval->int_av11_recall_precis /= NUM_RP_PTS; + +} + +static void +calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel) +EVAL_PARAM_INFO *epi; +TR_VEC *tr_vec; +TREC_EVAL *eval; +long num_rel; /* Number relevant judged */ +long num_nonrel; /* Number nonrelevant judged */ +{ + long j; + long nonrel_ret, nonrel_so_far, rel_so_far; + long pref_top_nonrel_num = PREF_TOP_NONREL_NUM; + long pref_top_50pRnonrel_num; + long pref_top_25pRnonrel_num; + long pref_top_25p2Rnonrel_num; + long pref_top_10pRnonrel_num; + long pref_top_Rnonrel_num; + long bounded_5R_nonrel_so_far, bounded_10R_nonrel_so_far; + + /* Calculate judgement based measures (dependent on only + judged docs; no assumption of non-relevance if not judged) */ + /* Binary Preference measures; here expressed as all docs with a higher + value of rel are to be preferred. Optimize by keeping track of nonrel + seen so far */ + pref_top_nonrel_num = PREF_TOP_NONREL_NUM; + pref_top_50pRnonrel_num = 50 + eval->num_rel; + pref_top_25pRnonrel_num = 25 + eval->num_rel; + pref_top_10pRnonrel_num = 10 + eval->num_rel; + pref_top_Rnonrel_num = eval->num_rel; + pref_top_25p2Rnonrel_num = 25 + (2 * eval->num_rel); + nonrel_ret = 0; + for (j = 0; j < tr_vec->num_tr; j++) { + if (tr_vec->tr[j].rel == 0) + nonrel_ret++; + } + nonrel_so_far = 0; + rel_so_far = 0; + bounded_5R_nonrel_so_far = 0; + bounded_10R_nonrel_so_far = 0; + for (j = 0; j < tr_vec->num_tr; j++) { + if (tr_vec->tr[j].rel == 0) { + if (nonrel_so_far < 5 * eval->num_rel) { + bounded_5R_nonrel_so_far++; + if (nonrel_so_far < 10 * eval->num_rel) { + bounded_10R_nonrel_so_far++; + } + } + nonrel_so_far++; + } + else if (tr_vec->tr[j].rel >= epi->relevance_level) { + rel_so_far++; + /* Add fraction of correct preferences. */ + /* Special case nonrel_so_far == 0 to avoid division by 0 */ + if (nonrel_so_far > 0) { + eval->bpref_allnonrel += 1.0 - (((float) nonrel_so_far) / + (float) num_nonrel); + eval->bpref_retnonrel += 1.0 - (((float) nonrel_so_far) / + (float) nonrel_ret); + eval->bpref_retall += 1.0 - (((float) nonrel_so_far) / + (float) nonrel_ret); + eval->bpref_num_correct += + MIN (num_nonrel, pref_top_Rnonrel_num) - + MIN (nonrel_so_far, pref_top_Rnonrel_num); + eval->bpref += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) / + (float) MIN (num_nonrel, pref_top_Rnonrel_num)); + eval->old_bpref += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) / + (float) MIN (nonrel_ret, pref_top_Rnonrel_num)); + eval->bpref_topnonrel += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_nonrel_num)) / + (float) MIN (num_nonrel, pref_top_nonrel_num)); + eval->bpref_top50pRnonrel += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_50pRnonrel_num)) / + (float) MIN (num_nonrel, pref_top_50pRnonrel_num)); + eval->bpref_top25pRnonrel += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_25pRnonrel_num)) / + (float) MIN (num_nonrel, pref_top_25pRnonrel_num)); + eval->bpref_top10pRnonrel += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) / + (float) MIN (num_nonrel, pref_top_10pRnonrel_num)); + eval->old_bpref_top10pRnonrel += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) / + (float) MIN (nonrel_ret, pref_top_10pRnonrel_num)); + eval->bpref_top25p2Rnonrel += 1.0 - + (((float) MIN (nonrel_so_far, pref_top_25p2Rnonrel_num)) / + (float) MIN (num_nonrel, pref_top_25p2Rnonrel_num)); + if (rel_so_far <= 5 && nonrel_so_far < 5) + eval->bpref_5 += 1.0 - (float) nonrel_so_far / + (float) MIN (num_nonrel, 5); + if (rel_so_far <= 10 && nonrel_so_far < 10) + eval->bpref_10 += 1.0 - (float) nonrel_so_far / + (float) MIN (num_nonrel, 10); + } + else { + eval->bpref += 1.0; + eval->old_bpref += 1.0; + eval->bpref_allnonrel += 1.0; + eval->bpref_retnonrel += 1.0; + eval->bpref_retall += 1.0; + eval->bpref_topnonrel += 1.0; + eval->bpref_top50pRnonrel += 1.0; + eval->bpref_top25pRnonrel += 1.0; + eval->bpref_top10pRnonrel += 1.0; + eval->old_bpref_top10pRnonrel += 1.0; + eval->bpref_top25p2Rnonrel += 1.0; + if (rel_so_far <= 5) + eval->bpref_5 += 1.0; + if (rel_so_far <= 10) + eval->bpref_10 += 1.0; + + } + eval->bpref_top5Rnonrel += 1.0 - + (((float) bounded_5R_nonrel_so_far) / + (float) MIN (num_nonrel, eval->num_rel * 5)); + eval->bpref_top10Rnonrel += 1.0 - + (((float) bounded_10R_nonrel_so_far) / + (float) MIN (num_nonrel, eval->num_rel * 10)); + eval->bpref_num_all += num_nonrel - nonrel_so_far; + eval->bpref_num_ret += nonrel_ret - nonrel_so_far; + } + } + if (eval->num_rel) { + eval->bpref /= eval->num_rel; + eval->old_bpref /= eval->num_rel; + eval->bpref_allnonrel /= eval->num_rel; + eval->bpref_retnonrel /= eval->num_rel; + eval->bpref_topnonrel /= eval->num_rel; + eval->bpref_top5Rnonrel /= eval->num_rel; + eval->bpref_top10Rnonrel /= eval->num_rel; + eval->bpref_top50pRnonrel /= eval->num_rel; + eval->bpref_top25pRnonrel /= eval->num_rel; + eval->bpref_top10pRnonrel /= eval->num_rel; + eval->old_bpref_top10pRnonrel /= eval->num_rel; + eval->bpref_top25p2Rnonrel /= eval->num_rel; + if (eval->num_rel_ret) { + eval->bpref_retall /= eval->num_rel_ret; + eval->bpref_5 /= MIN (rel_so_far, 5); + eval->bpref_10 /= MIN (rel_so_far, 10); + } + eval->bpref_num_possible = eval->num_rel * + MIN (num_nonrel, pref_top_Rnonrel_num); + } + /* For those bpref measure variants which use the geometric mean instead + of straight averages, compute them here. Original measure value + is constrained to be greater than MIN_GEO_MEAN (for time being .00001, + since trec_eval prints to four significant digits) */ + eval->gm_bpref = (float) log ((double)(MAX (eval->bpref, + MIN_GEO_MEAN))); +} + +static void +calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel) +EVAL_PARAM_INFO *epi; +TR_VEC *tr_vec; +TREC_EVAL *eval; +long num_rel; /* Number relevant judged */ +long num_nonrel; /* Number nonrelevant judged */ +{ + double recall, precis; /* current recall, precision values */ + double rel_precis, rel_uap;/* relative precision, uap values */ + double int_precis; /* current interpolated precision values */ + + long i,j; + long rel_so_far; + + /* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all + Y >= X) */ + rel_so_far = eval->num_rel_ret; + int_precis = (float) rel_so_far / (float) eval->num_ret; + + /* Loop over all retrieved docs in reverse order */ + for (j = eval->num_ret; j > 0; j--) { + if (rel_so_far > 0) { + recall = (float) rel_so_far / (float) eval->num_rel; + precis = (float) rel_so_far / (float) j; + if (j > eval->num_rel) { + rel_precis = (float) rel_so_far / (float) eval->num_rel; + } + else { + rel_precis = (float) rel_so_far / (float) j; + } + } + else { + recall = 0.0; + precis = 0.0; + rel_precis = 0.0; + } + rel_uap = rel_precis * rel_precis; + if (int_precis < precis) + int_precis = precis; + eval->av_rel_precis += rel_precis; + eval->av_rel_uap += rel_uap; + + if (j < eval->num_rel) { + eval->av_R_precis += precis; + eval->int_av_R_precis += int_precis; + } + + if (tr_vec->tr[j-1].rel >= epi->relevance_level) { + eval->int_av_recall_precis += int_precis; + eval->av_recall_precis += precis; + eval->avg_doc_prec += precis; + rel_so_far--; + } + else { + /* Note: for fallout-recall, the recall at X non-rel docs + is used for the recall 'after' (X-1) non-rel docs. + Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for + Y retrieved docs where X-1 non-rel retrieved */ + if (j - rel_so_far < MAX_FALL_RET) { + eval->av_fall_recall += recall; + } + } + } + + if (eval->num_ret - eval->num_rel_ret < MAX_FALL_RET) { + if (eval->num_rel_ret > 0) + eval->av_fall_recall += ((MAX_FALL_RET - + (eval->num_ret - eval->num_rel_ret)) + * ((float)eval->num_rel_ret / + (float)eval->num_rel)); + } + if (eval->num_rel > eval->num_ret) { + for (i = eval->num_ret; i < eval->num_rel; i++) { + eval->av_R_precis += (float) eval->num_rel_ret / + (float) i; + eval->int_av_R_precis += (float) eval->num_rel_ret / + (float) i; + } + } + + /* Calculate all the other averages */ + if (eval->num_rel_ret > 0) { + eval->av_recall_precis /= eval->num_rel; + eval->int_av_recall_precis /= eval->num_rel; + } + + eval->av_fall_recall /= MAX_FALL_RET; + + eval->av_rel_precis /= eval->num_ret; + eval->av_rel_uap /= eval->num_ret; + + if (eval->num_rel) { + eval->av_R_precis /= eval->num_rel; + eval->int_av_R_precis /= eval->num_rel; + } + + /* For those measure variants which use the geometric mean instead + of straight averages, compute them here. Original measure value + is constrained to be greater than MIN_GEO_MEAN (for time being .00001, + since trec_eval prints to four significant digits) */ + eval->gm_ap = (float) log ((double)(MAX (eval->av_recall_precis, + MIN_GEO_MEAN))); +} + +static void +calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel) +EVAL_PARAM_INFO *epi; +TR_VEC *tr_vec; +TREC_EVAL *eval; +long num_rel; /* Number relevant judged */ +long num_nonrel; /* Number nonrelevant judged */ +{ + + if (eval->num_rel) { + eval->exact_recall = (double) eval->num_rel_ret / eval->num_rel; + eval->exact_precis = (double) eval->num_rel_ret / eval->num_ret; + eval->exact_uap = eval->exact_recall * eval->exact_precis; + if (eval->num_rel > eval->num_ret) { + eval->exact_rel_precis = eval->exact_precis; + } + else { + eval->exact_rel_precis = eval->exact_recall; + } + eval->exact_rel_uap = eval->exact_precis * eval->exact_precis; + eval->exact_utility = + epi->utility_a * eval->num_rel_ret + + epi->utility_b * (eval->num_ret - eval->num_rel_ret) + + epi->utility_c * (eval->num_rel - eval->num_rel_ret) + + epi->utility_d * (epi->num_docs_in_coll + eval->num_rel_ret + - eval->num_ret - eval->num_rel); + } +} + +static void +calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel) +EVAL_PARAM_INFO *epi; +TR_VEC *tr_vec; +TREC_EVAL *eval; +long num_rel; /* Number relevant judged */ +long num_nonrel; /* Number nonrelevant judged */ +{ + double recall, precis; /* current recall, precision values */ + double rel_precis, rel_uap;/* relative precision, uap values */ + double int_precis; /* current interpolated precision values */ + + long i,j; + + long bucket; + long last_time_bucket = NUM_TIME_PTS; /* Last time bucket filled in */ + + long rel_so_far = eval->num_rel_ret; + long min_ret_rel = MIN(eval->num_rel, eval->num_ret); + + /* Loop over all retrieved docs in reverse order */ + for (j = eval->num_ret; j > 0; j--) { + if (rel_so_far > 0) { + recall = (float) rel_so_far / (float) eval->num_rel; + precis = (float) rel_so_far / (float) j; + if (j > eval->num_rel) { + rel_precis = (float) rel_so_far / (float) eval->num_rel; + } + else { + rel_precis = (float) rel_so_far / (float) j; + } + } + else { + recall = 0.0; + precis = 0.0; + rel_precis = 0.0; + } + rel_uap = rel_precis * rel_precis; + if (int_precis < precis) + int_precis = precis; + + bucket = tr_vec->tr[j-1].sim * + ((double) NUM_TIME_PTS / (double) MAX_TIME); + if (bucket < 0) bucket = 0; + if (bucket >= NUM_TIME_PTS) bucket = NUM_TIME_PTS-1; + if (tr_vec->tr[j-1].rel >= epi->relevance_level) + eval->time_num_rel[bucket]++; + else + eval->time_num_nrel[bucket]++; + eval->time_precis[bucket] = (float)rel_so_far / + (float) eval->num_ret; + eval->time_relprecis[bucket] = ((float)rel_so_far) / + (float) min_ret_rel; + eval->time_uap[bucket] = (float) rel_so_far * rel_so_far / + ((float) eval->num_ret * (float) min_ret_rel); + eval->time_reluap[bucket] = (float) rel_so_far * rel_so_far / + ((float) min_ret_rel * (float) min_ret_rel); + eval->time_utility[bucket] = + epi->utility_a * rel_so_far + + epi->utility_b * (j - rel_so_far) + + epi->utility_c * (eval->num_rel - rel_so_far) + + epi->utility_d * (epi->num_docs_in_coll + + rel_so_far - j - eval->num_rel); + + /* Need to fill in buckets up to last bucket */ + /* note assumes buckets are decreasing */ + /* Must do here since utility can be negative and zero + cannot be used as flag later */ + for (i = bucket+1; i < last_time_bucket; i++) { + eval->time_precis[i] = eval->time_precis[bucket]; + eval->time_relprecis[i] = eval->time_relprecis[bucket]; + eval->time_uap[i] = eval->time_uap[bucket]; + eval->time_reluap[i] = eval->time_reluap[bucket]; + eval->time_utility[i] = eval->time_utility[bucket]; + } + last_time_bucket = bucket; + } + + eval->time_cum_rel[0] = eval->time_num_rel[0]; + eval->av_time_cum_rel = eval->time_num_rel[0]; + for (i=1; i< NUM_TIME_PTS; i++) { + eval->time_cum_rel[i] = eval->time_cum_rel[i-1] + eval->time_num_rel[i]; + eval->av_time_cum_rel += eval->time_cum_rel[i]; + eval->av_time_precis += eval->time_precis[i]; + eval->av_time_relprecis += eval->time_relprecis[i]; + eval->av_time_uap += eval->time_uap[i]; + eval->av_time_reluap += eval->time_reluap[i]; + eval->av_time_utility += eval->time_utility[i]; + } + eval->av_time_cum_rel /= NUM_TIME_PTS; + eval->av_time_precis /= NUM_TIME_PTS; + eval->av_time_relprecis /= NUM_TIME_PTS; + eval->av_time_uap /= NUM_TIME_PTS; + eval->av_time_reluap /= NUM_TIME_PTS; + eval->av_time_utility /= NUM_TIME_PTS; +}