mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
Add convolutional RNN for sentence classification (#57)
* Add SST data preprocessing * Add ConvRNN model * Add LR scheduler * Add grid search on hyperparameters * Add random search * Add CLI options * Add usage to README.md * Refactor code * Fix randomized search parameters * Update README.md with results * Use Dataset and DataLoader
This commit is contained in:
Ralph Tang
@@ -0,0 +1,23 @@
|
||||
## Convolutional RNN
|
||||
|
||||
Implementation based on [[1]](http://dl.acm.org/citation.cfm?id=3098140).
|
||||
|
||||
### Usage
|
||||
|
||||
Run `./getData.sh` to fetch the data. The project structure should now look like this:
|
||||
|
||||
```
|
||||
├── conv_rnn/
|
||||
│ ├── data/
|
||||
│ ├── saves/
|
||||
│ └── *.*
|
||||
```
|
||||
You may then run `python train.py` and `python test.py` for training and testing, respectively. For more options, add the `-h` switch.
|
||||
|
||||
### Empirical results
|
||||
Best dev | Test
|
||||
-- | --
|
||||
51.1 | 50.7
|
||||
|
||||
### References
|
||||
[1] Chenglong Wang, Feijun Jiang, and Hongxia Yang. 2017. A Hybrid Framework for Text Modeling with Convolutional RNN. In Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD '17).
|
||||
@@ -0,0 +1,61 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
import torch.utils.data as data
|
||||
|
||||
def sst_tokenize(sentence):
|
||||
extraneous_pattern = re.compile(r"^(--lrb--|--rrb--|``|''|--|\.)$")
|
||||
words = []
|
||||
for word in sentence.split():
|
||||
if re.match(extraneous_pattern, word):
|
||||
continue
|
||||
words.append(word)
|
||||
return words
|
||||
|
||||
class SSTEmbeddingLoader(object):
|
||||
def __init__(self, dirname, fmt="stsa.fine.{}", word2vec_file="word2vec.sst-1"):
|
||||
self.dirname = dirname
|
||||
self.fmt = fmt
|
||||
self.word2vec_file = word2vec_file
|
||||
|
||||
def load_embed_data(self):
|
||||
weights = []
|
||||
id_dict = {}
|
||||
unk_vocab_set = set()
|
||||
with open(os.path.join(self.dirname, self.word2vec_file)) as f:
|
||||
for i, line in enumerate(f.readlines()):
|
||||
word, vec = line.replace("\n", "").split(" ", 1)
|
||||
word = word.replace("#", "")
|
||||
vec = np.array([float(v) for v in vec.split(" ")])
|
||||
weights.append(vec)
|
||||
id_dict[word] = i
|
||||
with open(os.path.join(self.dirname, self.fmt.format("phrases.train"))) as f:
|
||||
for line in f.readlines():
|
||||
for word in sst_tokenize(line):
|
||||
if word not in id_dict and word not in unk_vocab_set:
|
||||
unk_vocab_set.add(word)
|
||||
return (id_dict, np.array(weights), list(unk_vocab_set))
|
||||
|
||||
class SSTDataset(data.Dataset):
|
||||
def __init__(self, sentences):
|
||||
super().__init__()
|
||||
self.sentences = sentences
|
||||
|
||||
def __len__(self):
|
||||
return len(self.sentences)
|
||||
|
||||
def __getitem__(self, index):
|
||||
return self.sentences[index]
|
||||
|
||||
@classmethod
|
||||
def load_sst_sets(cls, dirname, fmt="stsa.fine.{}"):
|
||||
set_names = ["phrases.train", "dev", "test"]
|
||||
def read_set(name):
|
||||
data_set = []
|
||||
with open(os.path.join(dirname, fmt.format(name))) as f:
|
||||
for line in f.readlines():
|
||||
sentiment, sentence = line.replace("\n", "").split(" ", 1)
|
||||
data_set.append((sentiment, sentence))
|
||||
return np.array(data_set)
|
||||
return [cls(read_set(name)) for name in set_names]
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
mkdir -p data
|
||||
mkdir -p saves
|
||||
wget http://ocp59jkku.bkt.clouddn.com/sst-1.zip -P data/
|
||||
wget http://ocp59jkku.bkt.clouddn.com/sst-2.zip -P data/
|
||||
unzip data/sst-1.zip -d data/
|
||||
unzip data/sst-2.zip -d data/
|
||||
@@ -0,0 +1,141 @@
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as nn_func
|
||||
|
||||
import data
|
||||
|
||||
class ConvRNNModel(nn.Module):
|
||||
def __init__(self, word_model, **config):
|
||||
super().__init__()
|
||||
embedding_dim = word_model.dim
|
||||
self.word_model = word_model
|
||||
self.hidden_size = config["hidden_size"]
|
||||
fc_size = config["fc_size"]
|
||||
self.batch_size = config["mbatch_size"]
|
||||
dropout = config["dropout_prob"]
|
||||
n_fmaps = config["n_feature_maps"]
|
||||
self.rnn_type = config["rnn_type"]
|
||||
|
||||
self.h_0_cache = torch.autograd.Variable(torch.zeros(2, self.batch_size, self.hidden_size))
|
||||
self.c_0_cache = torch.autograd.Variable(torch.zeros(2, self.batch_size, self.hidden_size))
|
||||
|
||||
self.no_cuda = config["no_cuda"]
|
||||
if not self.no_cuda:
|
||||
self.h_0_cache = self.h_0_cache.cuda()
|
||||
self.c_0_cache = self.c_0_cache.cuda()
|
||||
|
||||
if self.rnn_type.upper() == "LSTM":
|
||||
self.bi_rnn = nn.LSTM(embedding_dim, self.hidden_size, 1, batch_first=True, bidirectional=True)
|
||||
elif self.rnn_type.upper() == "GRU":
|
||||
self.bi_rnn = nn.GRU(embedding_dim, self.hidden_size, 1, batch_first=True, bidirectional=True)
|
||||
else:
|
||||
raise ValueError("RNN type must be one of LSTM or GRU")
|
||||
self.conv = nn.Conv2d(1, n_fmaps, (1, self.hidden_size * 2))
|
||||
if dropout:
|
||||
self.dropout = nn.Dropout(dropout)
|
||||
self.fc1 = nn.Linear(n_fmaps + 2 * self.hidden_size, fc_size)
|
||||
self.fc2 = nn.Linear(fc_size, config["n_labels"])
|
||||
|
||||
def convert_dataset(self, dataset):
|
||||
dataset = np.stack(dataset)
|
||||
model_in = dataset[:, 1].reshape(-1)
|
||||
model_out = dataset[:, 0].flatten().astype(np.int)
|
||||
model_out = torch.autograd.Variable(torch.from_numpy(model_out))
|
||||
model_in = self.preprocess(model_in)
|
||||
model_in = torch.autograd.Variable(model_in)
|
||||
if not self.no_cuda:
|
||||
model_out = model_out.cuda()
|
||||
model_in = model_in.cuda()
|
||||
return (model_in, model_out)
|
||||
|
||||
def preprocess(self, sentences):
|
||||
return torch.from_numpy(np.array(self.word_model.lookup(sentences)))
|
||||
|
||||
def forward(self, x):
|
||||
x = self.word_model(x) # shape: (batch, max sent, embed dim)
|
||||
if x.size(0) == self.batch_size:
|
||||
h_0 = self.h_0_cache
|
||||
c_0 = self.c_0_cache
|
||||
else:
|
||||
h_0 = torch.autograd.Variable(torch.zeros(2, x.size(0), self.hidden_size))
|
||||
c_0 = torch.autograd.Variable(torch.zeros(2, x.size(0), self.hidden_size))
|
||||
if not self.no_cuda:
|
||||
h_0 = h_0.cuda()
|
||||
c_0 = c_0.cuda()
|
||||
if self.rnn_type.upper() == "LSTM":
|
||||
rnn_seq, rnn_out = self.bi_rnn(x, (h_0, c_0)) # shape: (batch, seq len, 2 * hidden_size), (2, batch, hidden_size)
|
||||
rnn_out = rnn_out[0] # (h_0, c_0)
|
||||
else:
|
||||
rnn_seq, rnn_out = self.bi_rnn(x, h_0) # shape: (batch, 2, hidden_size)
|
||||
rnn_out.data = rnn_out.data.permute(1, 0, 2)
|
||||
x = self.conv(rnn_seq.unsqueeze(1)).squeeze(3) # shape: (batch, channels, seq len)
|
||||
x = nn_func.relu(x) # shape: (batch, channels, seq len)
|
||||
x = nn_func.max_pool1d(x, x.size(2)) # shape: (batch, channels)
|
||||
out = [t.squeeze(1) for t in rnn_out.chunk(2, 1)]
|
||||
out.append(x)
|
||||
x = torch.cat(out, 1).squeeze(2)
|
||||
if hasattr(self, "dropout"):
|
||||
x = self.dropout(x)
|
||||
x = nn_func.relu(self.fc1(x))
|
||||
return self.fc2(x)
|
||||
|
||||
class WordEmbeddingModel(nn.Module):
|
||||
def __init__(self, id_dict, weights, unknown_vocab=[], static=True, padding_idx=0):
|
||||
super().__init__()
|
||||
vocab_size = len(id_dict) + len(unknown_vocab)
|
||||
self.lookup_table = id_dict
|
||||
last_id = max(id_dict.values())
|
||||
for word in unknown_vocab:
|
||||
last_id += 1
|
||||
self.lookup_table[word] = last_id
|
||||
self.dim = weights.shape[1]
|
||||
self.weights = np.concatenate((weights, np.random.rand(len(unknown_vocab), self.dim) / 2 - 0.25))
|
||||
self.padding_idx = padding_idx
|
||||
self.embedding = nn.Embedding(vocab_size, self.dim, padding_idx=padding_idx)
|
||||
self.embedding.weight.data.copy_(torch.from_numpy(self.weights))
|
||||
if static:
|
||||
self.embedding.weight.requires_grad = False
|
||||
|
||||
@classmethod
|
||||
def make_random_model(cls, id_dict, unknown_vocab=[], dim=300):
|
||||
weights = np.random.rand(len(id_dict), dim) - 0.5
|
||||
return cls(id_dict, weights, unknown_vocab, static=False)
|
||||
|
||||
def forward(self, x):
|
||||
return self.embedding(x)
|
||||
|
||||
def lookup(self, sentences):
|
||||
raise NotImplementedError
|
||||
|
||||
class SSTWordEmbeddingModel(WordEmbeddingModel):
|
||||
def __init__(self, id_dict, weights, unknown_vocab=[]):
|
||||
super().__init__(id_dict, weights, unknown_vocab, padding_idx=16259)
|
||||
|
||||
def lookup(self, sentences):
|
||||
indices_list = []
|
||||
max_len = 0
|
||||
for sentence in sentences:
|
||||
indices = []
|
||||
for word in data.sst_tokenize(sentence):
|
||||
try:
|
||||
index = self.lookup_table[word]
|
||||
indices.append(index)
|
||||
except KeyError:
|
||||
continue
|
||||
indices_list.append(indices)
|
||||
if len(indices) > max_len:
|
||||
max_len = len(indices)
|
||||
for indices in indices_list:
|
||||
indices.extend([self.padding_idx] * (max_len - len(indices)))
|
||||
return indices_list
|
||||
|
||||
def set_seed(seed=0, no_cuda=False):
|
||||
np.random.seed(seed)
|
||||
if not no_cuda:
|
||||
torch.cuda.manual_seed(seed)
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
torch.manual_seed(seed)
|
||||
random.seed(seed)
|
||||
@@ -0,0 +1,36 @@
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import data
|
||||
import model
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--no_cuda", action="store_true", default=False)
|
||||
parser.add_argument("--input_file", default="saves/model.pt", type=str)
|
||||
parser.add_argument("--data_dir", default="data", type=str)
|
||||
parser.add_argument("--gpu_number", default=0, type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
model.set_seed(5, no_cuda=args.no_cuda)
|
||||
data_loader = data.SSTDataLoader(args.data_dir)
|
||||
conv_rnn = torch.load(args.input_file)
|
||||
if not args.no_cuda:
|
||||
torch.cuda.set_device(args.gpu_number)
|
||||
conv_rnn.cuda()
|
||||
_, _, test_set = data_loader.load_sst_sets()
|
||||
|
||||
conv_rnn.eval()
|
||||
test_in, test_out = conv_rnn.convert_dataset(test_set)
|
||||
scores = conv_rnn(test_in)
|
||||
n_correct = (torch.max(scores, 1)[1].view(len(test_set)).data == test_out.data).sum()
|
||||
accuracy = n_correct / len(test_set)
|
||||
print("Test set accuracy: {}".format(accuracy))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,177 @@
|
||||
import argparse
|
||||
import os
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch import utils
|
||||
|
||||
import data
|
||||
import model
|
||||
|
||||
class RandomSearch(object):
|
||||
def __init__(self, params):
|
||||
self.params = params
|
||||
|
||||
def __iter__(self):
|
||||
param_space = list(GridSearch(self.params))
|
||||
random.shuffle(param_space)
|
||||
for param in param_space:
|
||||
yield param
|
||||
|
||||
class GridSearch(object):
|
||||
def __init__(self, params):
|
||||
self.params = params
|
||||
self.param_lengths = [len(param) for param in self.params]
|
||||
self.indices = [1] * len(params)
|
||||
|
||||
def _update(self, carry_idx):
|
||||
if carry_idx >= len(self.params):
|
||||
return True
|
||||
if self.indices[carry_idx] < self.param_lengths[carry_idx]:
|
||||
self.indices[carry_idx] += 1
|
||||
return False
|
||||
else:
|
||||
self.indices[carry_idx] = 1
|
||||
return False or self._update(carry_idx + 1)
|
||||
|
||||
def __iter__(self):
|
||||
self.stop_next = False
|
||||
self.indices = [1] * len(self.params)
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self.stop_next:
|
||||
raise StopIteration
|
||||
result = [param[idx - 1] for param, idx in zip(self.params, self.indices)]
|
||||
self.indices[0] += 1
|
||||
if self.indices[0] == self.param_lengths[0] + 1:
|
||||
self.indices[0] = 1
|
||||
self.stop_next = self._update(1)
|
||||
return result
|
||||
|
||||
def train(**kwargs):
|
||||
mbatch_size = kwargs["mbatch_size"]
|
||||
n_epochs = kwargs["n_epochs"]
|
||||
restore = kwargs["restore"]
|
||||
verbose = not kwargs["quiet"]
|
||||
lr = kwargs["lr"]
|
||||
weight_decay = kwargs["weight_decay"]
|
||||
gradient_clip = kwargs["gradient_clip"]
|
||||
seed = kwargs["seed"]
|
||||
|
||||
if not kwargs["no_cuda"]:
|
||||
torch.cuda.set_device(kwargs["gpu_number"])
|
||||
model.set_seed(seed)
|
||||
embed_loader = data.SSTEmbeddingLoader("data")
|
||||
if restore:
|
||||
conv_rnn = torch.load(kwargs["input_file"])
|
||||
else:
|
||||
id_dict, weights, unk_vocab_list = embed_loader.load_embed_data()
|
||||
word_model = model.SSTWordEmbeddingModel(id_dict, weights, unk_vocab_list)
|
||||
if not kwargs["no_cuda"]:
|
||||
word_model.cuda()
|
||||
conv_rnn = model.ConvRNNModel(word_model, **kwargs)
|
||||
if not kwargs["no_cuda"]:
|
||||
conv_rnn.cuda()
|
||||
|
||||
conv_rnn.train()
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
parameters = list(filter(lambda p: p.requires_grad, conv_rnn.parameters()))
|
||||
optimizer = torch.optim.Adadelta(parameters, lr=lr, weight_decay=weight_decay)
|
||||
train_set, dev_set, test_set = data.SSTDataset.load_sst_sets("data")
|
||||
|
||||
collate_fn = conv_rnn.convert_dataset
|
||||
train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=mbatch_size, drop_last=True,
|
||||
collate_fn=collate_fn)
|
||||
dev_loader = utils.data.DataLoader(dev_set, batch_size=len(dev_set), collate_fn=collate_fn)
|
||||
test_loader = utils.data.DataLoader(test_set, batch_size=len(test_set), collate_fn=collate_fn)
|
||||
|
||||
def evaluate(loader, dev=True):
|
||||
conv_rnn.eval()
|
||||
for m_in, m_out in loader:
|
||||
scores = conv_rnn(m_in)
|
||||
loss = criterion(scores, m_out)
|
||||
n_correct = (torch.max(scores, 1)[1].view(m_in.size(0)).data == m_out.data).sum()
|
||||
accuracy = n_correct / m_in.size(0)
|
||||
if dev and accuracy > evaluate.best_dev:
|
||||
evaluate.best_dev = accuracy
|
||||
torch.save(conv_rnn, kwargs["output_file"])
|
||||
if verbose:
|
||||
print("{} set accuracy: {}, loss: {}".format("dev" if dev else "test", accuracy, loss.cpu().data[0]))
|
||||
conv_rnn.train()
|
||||
evaluate.best_dev = 0
|
||||
|
||||
for epoch in range(n_epochs):
|
||||
optimizer.zero_grad()
|
||||
print("Epoch number: {}".format(epoch), end="\r")
|
||||
if verbose:
|
||||
print()
|
||||
i = 0
|
||||
for j, (train_in, train_out) in enumerate(train_loader):
|
||||
if verbose and i % (mbatch_size * 10) == 0:
|
||||
print("{} / {}".format(j * mbatch_size, len(train_set)), end="\r")
|
||||
|
||||
if not kwargs["no_cuda"]:
|
||||
train_in.cuda()
|
||||
train_out.cuda()
|
||||
|
||||
scores = conv_rnn(train_in)
|
||||
loss = criterion(scores, train_out)
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm(parameters, gradient_clip)
|
||||
optimizer.step()
|
||||
i += mbatch_size
|
||||
if i % (mbatch_size * 256) == 0:
|
||||
evaluate(dev_loader)
|
||||
evaluate(test_loader, dev=False)
|
||||
return evaluate.best_dev
|
||||
|
||||
def do_random_search(given_params):
|
||||
test_grid = [[0.15, 0.2], [4, 5, 6], [150, 200], [3, 4, 5], [200, 300], [200, 250]]
|
||||
max_params = None
|
||||
max_acc = 0.
|
||||
for args in RandomSearch(test_grid):
|
||||
sf, gc, hid, seed, fc_size, fmaps = args
|
||||
print("Testing {}".format(args))
|
||||
given_params.update(dict(n_epochs=7, quiet=True, gradient_clip=gc, hidden_Size=hid, seed=seed,
|
||||
n_feature_maps=fmaps, fc_size=fc_size))
|
||||
dev_acc = train(**given_params)
|
||||
print("Dev accuracy: {}".format(dev_acc))
|
||||
if dev_acc > max_acc:
|
||||
print("Found current max")
|
||||
max_acc = dev_acc
|
||||
max_params = args
|
||||
print("Best params: {}".format(max_params))
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dropout_prob", default=0.5, type=float)
|
||||
parser.add_argument("--fc_size", default=200, type=int)
|
||||
parser.add_argument("--gpu_number", default=0, type=int)
|
||||
parser.add_argument("--gradient_clip", default=5, type=float)
|
||||
parser.add_argument("--hidden_size", default=200, type=int)
|
||||
parser.add_argument("--input_file", default="saves/model.pt", type=str)
|
||||
parser.add_argument("--lr", default=5E-2, type=float)
|
||||
parser.add_argument("--mbatch_size", default=64, type=int)
|
||||
parser.add_argument("--n_epochs", default=30, type=int)
|
||||
parser.add_argument("--n_feature_maps", default=200, type=float)
|
||||
parser.add_argument("--n_labels", default=5, type=int)
|
||||
parser.add_argument("--no_cuda", action="store_true", default=False)
|
||||
parser.add_argument("--output_file", default="saves/model.pt", type=str)
|
||||
parser.add_argument("--random_search", action="store_true", default=False)
|
||||
parser.add_argument("--restore", action="store_true", default=False)
|
||||
parser.add_argument("--rnn_type", choices=["lstm", "gru"], default="lstm", type=str)
|
||||
parser.add_argument("--seed", default=3, type=int)
|
||||
parser.add_argument("--quiet", action="store_true", default=False)
|
||||
parser.add_argument("--weight_decay", default=1E-3, type=float)
|
||||
args = parser.parse_args()
|
||||
if args.random_search:
|
||||
do_random_search(vars(args))
|
||||
return
|
||||
train(**vars(args))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Executable
+264
@@ -0,0 +1,264 @@
|
||||
trec_eval [-h] [-q] [-a] [-o] [-c] [-l<num> [-N<num>] [-M<num>] [-Ua<num>] [-Ub<num>] [-Uc<num>] [-Ud<num>] [-T] trec_rel_file trec_top_file
|
||||
|
||||
Calculate and print various evaluation measures, evaluating the results
|
||||
in trec_top_file against the relevance judgements in trec_rel_file.
|
||||
|
||||
There are a fair number of options, of which only the lower case options are
|
||||
normally ever used.
|
||||
-h: Print full help message and exit
|
||||
-q: In addition to summary evaluation, give evaluation for each query
|
||||
-a: Print all evaluation measures calculated, instead of just the
|
||||
main official measures for TREC.
|
||||
-o: Print everything out in old, nonrelational format (default is relational)
|
||||
-c: Average over the complete set of queries in the relevance judgements
|
||||
instead of the queries in the intersection of relevance judgements
|
||||
and results. Missing queries will contribute a value of 0 to all
|
||||
evaluation measures (which may or may not be reasonable for a
|
||||
particular evaluation measure, but is reasonable for standard TREC
|
||||
measures.)
|
||||
-l<num>: Num indicates the minimum relevance judgement value needed for
|
||||
a document to be called relevant. (All measures used by TREC eval are
|
||||
based on binary relevance). Used if trec_rel_file contains relevance
|
||||
judged on a multi-relevance scale. Default is 1.
|
||||
-N<num>: Number of docs in collection
|
||||
-M<num>: Max number of docs per topic to use in evaluation (discard rest).
|
||||
-Ua<num>: Value to use for 'a' coefficient of utility computation.
|
||||
relevant nonrelevant
|
||||
retrieved a b
|
||||
nonretrieved c d
|
||||
-Ub<num>: Value to use for 'b' coefficient of utility computation.
|
||||
-Uc<num>: Value to use for 'c' coefficient of utility computation.
|
||||
-Ud<num>: Value to use for 'd' coefficient of utility computation.
|
||||
-J: Calculate all values only over the judged (either relevant or
|
||||
nonrelevant) documents. All unjudged documents are removed from the
|
||||
retrieved set before any calculations (possibly leaving an empty set).
|
||||
DO NOT USE, unless you really know what you're doing - very easy to get
|
||||
reasonable looking, but invalid, numbers.
|
||||
-T: Treat similarity as time that document retrieved. Compute
|
||||
several time-based measures after ranking docs by time retrieved
|
||||
(first doc (lowest sim) retrieved ranked highest).
|
||||
Only done if -a selected.
|
||||
|
||||
|
||||
Read text tuples from trec_top_file of the form
|
||||
030 Q0 ZF08-175-870 0 4238 prise1
|
||||
qid iter docno rank sim run_id
|
||||
giving TREC document numbers (a string) retrieved by query qid
|
||||
(a string) with similarity sim (a float). The other fields are ignored,
|
||||
with the exception that the run_id field of the last line is kept and
|
||||
output. In particular, note that the rank field is ignored here;
|
||||
internally ranks are assigned by sorting by the sim field with ties
|
||||
broken deterministicly (using docno).
|
||||
Sim is assumed to be higher for the docs to be retrieved first.
|
||||
File may contain no NULL characters.
|
||||
Lines may contain fields after the run_id; they are ignored.
|
||||
|
||||
Relevance for each docno to qid is determined from text_qrels_file, which
|
||||
consists of text tuples of the form
|
||||
qid iter docno rel
|
||||
giving TREC document numbers (docno, a string) and their relevance (rel,
|
||||
an integer) to query qid (a string). iter string field is ignored.
|
||||
Fields are separated by whitespace, string fields can contain no whitespace.
|
||||
File may contain no NULL characters.
|
||||
|
||||
The text tuples with relevance judgements are converted to TR_VEC form
|
||||
and then submitted to the SMART evaluation routines.
|
||||
The qid,did,rank,sim,rel fields of TR_VEC are filled in;
|
||||
action,iter fields are set to 0.
|
||||
The rel field is set to -1 if the document was not judged (not in
|
||||
text_qrels_file). Most measures, but not all, will treat -1 the same as 0,
|
||||
namely nonrelevant. Note that relevance_level is used to determine if the
|
||||
document is relevant during score calculations.
|
||||
Queries for which there are no relevant docs are ignored.
|
||||
Warning: queries for which there are relevant docs but no retrieved docs
|
||||
are also ignored by default. This allows systems to evaluate over subsets
|
||||
of the relevant docs, but means if a system improperly retrieves no docs,
|
||||
it will not be detected. Use the -c flag to avoid this behavior.
|
||||
|
||||
EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT.
|
||||
Relational Format prints the same values, but all lines are of the form
|
||||
measure_name query value
|
||||
|
||||
1. Total number of documents over all queries
|
||||
Retrieved:
|
||||
Relevant:
|
||||
Rel_ret: (relevant and retrieved)
|
||||
These should be self-explanatory. All values are totals over all
|
||||
queries being evaluated.
|
||||
2. Interpolated Recall - Precision Averages:
|
||||
at 0.00
|
||||
at 0.10
|
||||
...
|
||||
at 1.00
|
||||
See any standard IR text (especially by Salton) for more details of
|
||||
recall-precision evaluation. Measures precision (percent of retrieved
|
||||
docs that are relevant) at various recall levels (after a certain
|
||||
percentage of all the relevant docs for that query have been retrieved).
|
||||
'Interpolated' means that, for example, precision at recall
|
||||
0.10 (ie, after 10% of rel docs for a query have been retrieved) is
|
||||
taken to be MAXIMUM of precision at all recall points >= 0.10.
|
||||
Values are averaged over all queries (for each of the 11 recall levels).
|
||||
These values are used for Recall-Precision graphs.
|
||||
3. Average precision (non-interpolated) over all rel docs
|
||||
The precision is calculated after each relevant doc is retrieved.
|
||||
If a relevant doc is not retrieved, its precision is 0.0.
|
||||
All precision values are then averaged together to get a single number
|
||||
for the performance of a query. Conceptually this is the area
|
||||
underneath the recall-precision graph for the query.
|
||||
The values are then averaged over all queries.
|
||||
4. Precision:
|
||||
at 5 docs
|
||||
at 10 docs
|
||||
...
|
||||
at 1000 docs
|
||||
The precision (percent of retrieved docs that are relevant) after X
|
||||
documents (whether relevant or nonrelevant) have been retrieved.
|
||||
Values averaged over all queries. If X docs were not retrieved
|
||||
for a query, then all missing docs are assumed to be non-relevant.
|
||||
5. R-Precision (precision after R (= num_rel for a query) docs retrieved):
|
||||
Measures precision (or recall, they're the same) after R docs
|
||||
have been retrieved, where R is the total number of relevant docs
|
||||
for a query. Thus if a query has 40 relevant docs, then precision
|
||||
is measured after 40 docs, while if it has 600 relevant docs, precision
|
||||
is measured after 600 docs. This avoids some of the averaging
|
||||
problems of the 'precision at X docs' values in (4) above.
|
||||
If R is greater than the number of docs retrieved for a query, then
|
||||
the nonretrieved docs are all assumed to be nonrelevant.
|
||||
|
||||
Major measures (again) with their relational names:
|
||||
num_ret Total number of documents retrieved over all queries
|
||||
num_rel Total number of relevant documents over all queries
|
||||
num_rel_ret Total number of relevant documents retrieved over all queries
|
||||
map Mean Average Precision (MAP)
|
||||
gm_ap Average Precision. Geometric Mean, q_score=log(MAX(map,.00001))
|
||||
R-prec R-Precision (Precision after R (= num-rel for topic) documents retrieved)
|
||||
bpref Binary Preference, top R judged nonrel
|
||||
recip_rank Reciprical rank of top relevant document
|
||||
ircl_prn.0.00 Interpolated Recall - Precision Averages at 0.00 recall
|
||||
ircl_prn.0.10 Interpolated Recall - Precision Averages at 0.10 recall
|
||||
ircl_prn.0.20 Interpolated Recall - Precision Averages at 0.20 recall
|
||||
ircl_prn.0.30 Interpolated Recall - Precision Averages at 0.30 recall
|
||||
ircl_prn.0.40 Interpolated Recall - Precision Averages at 0.40 recall
|
||||
ircl_prn.0.50 Interpolated Recall - Precision Averages at 0.50 recall
|
||||
ircl_prn.0.60 Interpolated Recall - Precision Averages at 0.60 recall
|
||||
ircl_prn.0.70 Interpolated Recall - Precision Averages at 0.70 recall
|
||||
ircl_prn.0.80 Interpolated Recall - Precision Averages at 0.80 recall
|
||||
ircl_prn.0.90 Interpolated Recall - Precision Averages at 0.90 recall
|
||||
ircl_prn.1.00 Interpolated Recall - Precision Averages at 1.00 recall
|
||||
P5 Precision after 5 docs retrieved
|
||||
P10 Precision after 10 docs retrieved
|
||||
P15 Precision after 15 docs retrieved
|
||||
P20 Precision after 20 docs retrieved
|
||||
P30 Precision after 30 docs retrieved
|
||||
P100 Precision after 100 docs retrieved
|
||||
P200 Precision after 200 docs retrieved
|
||||
P500 Precision after 500 docs retrieved
|
||||
P1000 Precision after 1000 docs retrieved
|
||||
|
||||
|
||||
Minor measures with their relational names:
|
||||
exact_prec Exact Precision over retrieved set
|
||||
exact_recall Exact Recall over retrieved set
|
||||
11-pt_avg Average over all 11 points of recall-precision graph
|
||||
3-pt_avg Average over 3 points of recall-precision graph
|
||||
avg_doc_prec Rel doc precision averaged over all relevant docs (NOT over topics)
|
||||
exact_relative_prec Exact relative precision
|
||||
avg_relative_prec Average relative precision
|
||||
exact_unranked_avg_prec Exact Unranked Average Precision
|
||||
exact_relative_unranked_avg_prec Exact Relative Unranked Average Precision
|
||||
map_at_R Average Precision over first R docs retrieved
|
||||
int_map Interpolated Mean Average Precision
|
||||
exact_int_R_rcl_prec Exact R-based-interpolated-Precision
|
||||
int_map_at_R Average Interpolated Precision for first R docs retrieved
|
||||
bpref_allnonrel Binary Preference, all judged nonrel
|
||||
bpref_retnonrel Binary Preference, all retrieved judged nonrel
|
||||
bpref_topnonrel Binary Preference, top 100 judged nonrel
|
||||
bpref_top5Rnonrel Binary Preference, top 5R judged nonrel
|
||||
bpref_top10Rnonrel Binary Preference, top 10R judged nonrel
|
||||
bpref_top10pRnonrel Binary Preference, top 10 + R judged nonrel
|
||||
bpref_top25pRnonrel Binary Preference, top 25 + R judged nonrel
|
||||
bpref_top50pRnonrel Binary Preference, top 50 + R judged nonrel
|
||||
bpref_top25p2Rnonrel Binary Preference, top 25 + 2*R judged nonrel
|
||||
bpref_retall Binary Preference, Only retrieved judged rel and nonrel
|
||||
bpref_5 Binary Preference, top 5 rel, top 5 nonrel
|
||||
bpref_10 Binary Preference, top 10 rel, top 10 nonrel
|
||||
bpref_num_all Binary Preference, Number not retrieved before (all judged)
|
||||
bpref_num_ret Binary Preference, Number retrieved after
|
||||
bpref_num_correct Binary Preference, Number correct preferences
|
||||
bpref_num_possible Binary Preference, Number possible correct_preferences
|
||||
old_bpref Buggy Version 7.3. Binary Preference, top R judged nonrel
|
||||
old_bpref_top10pRnonrel Buggy Version 7.3. Binary Preference,top 10+R judged nonrel
|
||||
gm_bpref Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001))
|
||||
rank_first_rel Rank of top relevant document (0 if none)
|
||||
recall5 Recall after 5 docs retrieved
|
||||
recall10 Recall after 10 docs retrieved
|
||||
recall15 Recall after 15 docs retrieved
|
||||
recall20 Recall after 20 docs retrieved
|
||||
recall30 Recall after 30 docs retrieved
|
||||
recall100 Recall after 100 docs retrieved
|
||||
recall200 Recall after 200 docs retrieved
|
||||
recall500 Recall after 500 docs retrieved
|
||||
recall1000 Recall after 1000 docs retrieved
|
||||
0.20R-prec R-based precision- precision after 0.20 * R docs retrieved
|
||||
0.40R-prec R-based precision- precision after 0.40 * R docs retrieved
|
||||
0.60R-prec R-based precision- precision after 0.60 * R docs retrieved
|
||||
0.80R-prec R-based precision- precision after 0.80 * R docs retrieved
|
||||
1.00R-prec R-based precision- precision after 1.00 * R docs retrieved
|
||||
1.20R-prec R-based precision- precision after 1.20 * R docs retrieved
|
||||
1.40R-prec R-based precision- precision after 1.40 * R docs retrieved
|
||||
1.60R-prec R-based precision- precision after 1.60 * R docs retrieved
|
||||
1.80R-prec R-based precision- precision after 1.80 * R docs retrieved
|
||||
2.00R-prec R-based precision- precision after 2.00 * R docs retrieved
|
||||
relative_prec5 Relative precision after 5 docs retrieved
|
||||
relative_prec10 Relative precision after 10 docs retrieved
|
||||
relative_prec15 Relative precision after 15 docs retrieved
|
||||
relative_prec20 Relative precision after 20 docs retrieved
|
||||
relative_prec30 Relative precision after 30 docs retrieved
|
||||
relative_prec100 Relative precision after 100 docs retrieved
|
||||
relative_prec200 Relative precision after 200 docs retrieved
|
||||
relative_prec500 Relative precision after 500 docs retrieved
|
||||
relative_prec1000 Relative precision after 1000 docs retrieved
|
||||
unranked_avg_prec5 Unranked Average Precision after 5 docs retrieved
|
||||
unranked_avg_prec10 Unranked Average Precision after 10 docs retrieved
|
||||
unranked_avg_prec15 Unranked Average Precision after 15 docs retrieved
|
||||
unranked_avg_prec20 Unranked Average Precision after 20 docs retrieved
|
||||
unranked_avg_prec30 Unranked Average Precision after 30 docs retrieved
|
||||
unranked_avg_prec100 Unranked Average Precision after 100 docs retrieved
|
||||
unranked_avg_prec200 Unranked Average Precision after 200 docs retrieved
|
||||
unranked_avg_prec500 Unranked Average Precision after 500 docs retrieved
|
||||
unranked_avg_prec1000 Unranked Average Precision after 1000 docs retrieved
|
||||
relative_unranked_avg_prec5 Relative Unranked Average Precision after 5 docs retrieved
|
||||
relative_unranked_avg_prec10 Relative Unranked Average Precision after 10 docs retrieved
|
||||
relative_unranked_avg_prec15 Relative Unranked Average Precision after 15 docs retrieved
|
||||
relative_unranked_avg_prec20 Relative Unranked Average Precision after 20 docs retrieved
|
||||
relative_unranked_avg_prec30 Relative Unranked Average Precision after 30 docs retrieved
|
||||
relative_unranked_avg_prec100 Relative Unranked Average Precision after 100 docs retrieved
|
||||
relative_unranked_avg_prec200 Relative Unranked Average Precision after 200 docs retrieved
|
||||
relative_unranked_avg_prec500 Relative Unranked Average Precision after 500 docs retrieved
|
||||
relative_unranked_avg_prec1000 Relative Unranked Average Precision after 1000 docs retrieved
|
||||
utility_1.0_-1.0_0.0_0.0 Utility (a,b,c,d) Coefficients 1.0_-1.0_0.0_0.0
|
||||
rcl_at_142_nonrel Recall averaged at X nonrel docs X= 142
|
||||
fallout_recall_0 Fallout - Recall Averages- recall after 0 nonrel docs retrieved
|
||||
fallout_recall_14 Fallout - Recall Averages- recall after 14 nonrel docs retrieved
|
||||
fallout_recall_28 Fallout - Recall Averages- recall after 28 nonrel docs retrieved
|
||||
fallout_recall_42 Fallout - Recall Averages- recall after 42 nonrel docs retrieved
|
||||
fallout_recall_56 Fallout - Recall Averages- recall after 56 nonrel docs retrieved
|
||||
fallout_recall_71 Fallout - Recall Averages- recall after 71 nonrel docs retrieved
|
||||
fallout_recall_85 Fallout - Recall Averages- recall after 85 nonrel docs retrieved
|
||||
fallout_recall_99 Fallout - Recall Averages- recall after 99 nonrel docs retrieved
|
||||
fallout_recall_113 Fallout - Recall Averages- recall after 113 nonrel docs retrieved
|
||||
fallout_recall_127 Fallout - Recall Averages- recall after 127 nonrel docs retrieved
|
||||
fallout_recall_142 Fallout - Recall Averages- recall after 142 nonrel docs retrieved
|
||||
int_0.20R-prec Interpolated R-based precision, after 0.20 * R docs retrieved
|
||||
int_0.40R-prec Interpolated R-based precision, after 0.40 * R docs retrieved
|
||||
int_0.60R-prec Interpolated R-based precision, after 0.60 * R docs retrieved
|
||||
int_0.80R-prec Interpolated R-based precision, after 0.80 * R docs retrieved
|
||||
int_1.00R-prec Interpolated R-based precision, after 1.00 * R docs retrieved
|
||||
int_1.20R-prec Interpolated R-based precision, after 1.20 * R docs retrieved
|
||||
int_1.40R-prec Interpolated R-based precision, after 1.40 * R docs retrieved
|
||||
int_1.60R-prec Interpolated R-based precision, after 1.60 * R docs retrieved
|
||||
int_1.80R-prec Interpolated R-based precision, after 1.80 * R docs retrieved
|
||||
int_2.00R-prec Interpolated R-based precision, after 2.00 * R docs retrieved
|
||||
micro_prec Total relevant retrieved documents / Total retrieved documents
|
||||
micro_recall Total relevant retrieved documents / Total relevant documents
|
||||
micro_bpref Total correct preferences / Total possible preferences
|
||||
Executable
+282
@@ -0,0 +1,282 @@
|
||||
#ifdef RCSID
|
||||
static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $";
|
||||
#endif
|
||||
|
||||
/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley.
|
||||
|
||||
Permission is granted for use of this file in unmodified form for
|
||||
research purposes. Please contact the SMART project to obtain
|
||||
permission for other uses.
|
||||
*/
|
||||
|
||||
#include "common.h"
|
||||
#include "sysfunc.h"
|
||||
#include "buf.h"
|
||||
#include "trec_eval.h"
|
||||
|
||||
static long cutoff[] = CUTOFF_VALUES;
|
||||
static char param_val[20];
|
||||
char *get_param_str_ircl_prn(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%4.2f", (float) index / (NUM_RP_PTS -1));
|
||||
return (param_val);
|
||||
}
|
||||
char *get_param_str_cutoff(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%ld", cutoff[index]);
|
||||
return (param_val);
|
||||
}
|
||||
char *get_param_str_Rcutoff(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%4.2f",
|
||||
(float) MAX_RPREC * (index+1) /(float) (NUM_PREC_PTS - 1));
|
||||
return (param_val);
|
||||
}
|
||||
char *get_param_str_utility(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f",
|
||||
epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d);
|
||||
return (param_val);
|
||||
}
|
||||
char *get_param_str_maxfallout(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%ld", (long) MAX_FALL_RET);
|
||||
return (param_val);
|
||||
}
|
||||
char *get_param_str_fall_recall(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%ld",
|
||||
(long) (MAX_FALL_RET * index) / (NUM_FR_PTS - 1));
|
||||
return (param_val);
|
||||
}
|
||||
char *get_param_str_time_cutoff(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%ld",
|
||||
(long) (index * MAX_TIME / NUM_TIME_PTS));
|
||||
return (param_val);
|
||||
}
|
||||
char *get_param_str_time_utility_cutoff(epi, index)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
long index;
|
||||
{
|
||||
sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f-%ld",
|
||||
epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d,
|
||||
(long) (index * MAX_TIME / NUM_TIME_PTS));
|
||||
return (param_val);
|
||||
}
|
||||
|
||||
SINGLE_MEASURE sing_meas[] = {
|
||||
{"num_ret", "Total number of documents retrieved over all queries",
|
||||
1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_ret)},
|
||||
{"num_rel", "Total number of relevant documents over all queries",
|
||||
1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel)},
|
||||
{"num_rel_ret", "Total number of relevant documents retrieved over all queries",
|
||||
1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel_ret)},
|
||||
{"map", "Mean Average Precision (MAP)",
|
||||
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_recall_precis)},
|
||||
{"gm_ap","Average Precision. Geometric Mean, q_score=log(MAX(map,.00001))",
|
||||
0, 1, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_ap)},
|
||||
{"R-prec", "R-Precision (Precision after R (= num-rel for topic) documents retrieved)",
|
||||
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, R_recall_precis)},
|
||||
{"bpref", "Binary Preference, top R judged nonrel",
|
||||
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref)},
|
||||
{"recip_rank", "Reciprical rank of top relevant document",
|
||||
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, recip_rank)},
|
||||
/* end of short output measures (the major ones) */
|
||||
|
||||
{"exact_prec", "Exact Precision over retrieved set",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_precis)},
|
||||
{"exact_recall", "Exact Recall over retrieved set",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_recall)},
|
||||
{"11-pt_avg", "Average over all 11 points of recall-precision graph",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av11_recall_precis)},
|
||||
{"3-pt_avg", "Average over 3 points of recall-precision graph",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av3_recall_precis)},
|
||||
{"avg_doc_prec", "Rel doc precision averaged over all relevant docs (NOT over topics)",
|
||||
0, 0, 0, 0, 0, 0, 1, 0, offsetof(TREC_EVAL, avg_doc_prec)},
|
||||
{"exact_relative_prec", "Exact relative precision",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_precis)},
|
||||
{"avg_relative_prec", "Average relative precision",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_rel_precis)},
|
||||
{"exact_unranked_avg_prec", "Exact Unranked Average Precision",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_uap)},
|
||||
{"exact_relative_unranked_avg_prec", "Exact Relative Unranked Average Precision",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_uap)},
|
||||
{"map_at_R", "Average Precision over first R docs retrieved",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_R_precis)},
|
||||
{"int_map", "Interpolated Mean Average Precision",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_recall_precis)},
|
||||
{"exact_int_R_rcl_prec", "Exact R-based-interpolated-Precision",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_R_recall_precis)},
|
||||
{"int_map_at_R", "Average Interpolated Precision for first R docs retrieved",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_R_precis)},
|
||||
{"time_integral_prec", "Time: Average Integral Precision",
|
||||
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_precis)},
|
||||
{"time_integral_relative_prec", "Time: Average Integral Relative Precision",
|
||||
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_relprecis)},
|
||||
{"time_integral_uap", "Time: Average Integral Unranked Precision",
|
||||
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_uap)},
|
||||
{"time_integral_relative_uap", "Time: Average Integral Unranked Relative Precision",
|
||||
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_reluap)},
|
||||
{"time_integral_cum_rel", "Time: Average (Integral) cumulative number relevant",
|
||||
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_cum_rel)},
|
||||
{"bpref_allnonrel", "Binary Preference, all judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_allnonrel)},
|
||||
{"bpref_retnonrel", "Binary Preference, all retrieved judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retnonrel)},
|
||||
{"bpref_topnonrel", "Binary Preference, top 100 judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_topnonrel)},
|
||||
{"bpref_top5Rnonrel", "Binary Preference, top 5R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top5Rnonrel)},
|
||||
{"bpref_top10Rnonrel", "Binary Preference, top 10R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10Rnonrel)},
|
||||
{"bpref_top10pRnonrel", "Binary Preference, top 10 + R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10pRnonrel)},
|
||||
{"bpref_top25pRnonrel", "Binary Preference, top 25 + R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25pRnonrel)},
|
||||
{"bpref_top50pRnonrel", "Binary Preference, top 50 + R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top50pRnonrel)},
|
||||
{"bpref_top25p2Rnonrel", "Binary Preference, top 25 + 2*R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25p2Rnonrel)},
|
||||
{"bpref_retall", "Binary Preference, Only retrieved judged rel and nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retall)},
|
||||
{"bpref_5", "Binary Preference, top 5 rel, top 5 nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_5)},
|
||||
{"bpref_10", "Binary Preference, top 10 rel, top 10 nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_10)},
|
||||
{"bpref_num_all", "Binary Preference, Number not retrieved before (all judged)",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_all)},
|
||||
{"bpref_num_ret", "Binary Preference, Number retrieved after",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_ret)},
|
||||
{"bpref_num_correct", "Binary Preference, Number correct preferences",
|
||||
1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_correct)},
|
||||
{"bpref_num_possible", "Binary Preference, Number possible correct_preferences",
|
||||
1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_possible)},
|
||||
{"old_bpref", "Buggy Version 7.3. Binary Preference, top R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref)},
|
||||
{"old_bpref_top10pRnonrel", "Buggy Version 7.3. Binary Preference,top 10+R judged nonrel",
|
||||
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref_top10pRnonrel)},
|
||||
{"gm_bpref", "Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001))",
|
||||
0, 0, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_bpref)},
|
||||
{"rank_first_rel", "Rank of top relevant document (0 if none)",
|
||||
1, 0, 0, 1, 0, 0, 0, 0, offsetof(TREC_EVAL, rank_first_rel)},
|
||||
};
|
||||
|
||||
int num_sing_meas = sizeof (sing_meas) / sizeof (sing_meas[0]);
|
||||
|
||||
PARAMETERIZED_MEASURE param_meas[] = {
|
||||
{"Interpolated Recall - Precision Averages",
|
||||
0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, int_recall_precis[0]), NUM_RP_PTS,
|
||||
"ircl_prn.%s", " at %s recall",
|
||||
get_param_str_ircl_prn},
|
||||
{"Precision",
|
||||
0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, precis_cut[0]), NUM_CUTOFF,
|
||||
"P%s", " after %s docs retrieved",
|
||||
get_param_str_cutoff},
|
||||
/* end of short output measures (the major ones) */
|
||||
|
||||
{"Recall",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, recall_cut[0]), NUM_CUTOFF,
|
||||
"recall%s", " after %s docs retrieved",
|
||||
get_param_str_cutoff},
|
||||
{"R-based precision",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, R_prec_cut[0]), NUM_PREC_PTS-1,
|
||||
"%sR-prec", "- precision after %s * R docs retrieved",
|
||||
get_param_str_Rcutoff},
|
||||
{"Relative precision",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_precis_cut[0]), NUM_CUTOFF,
|
||||
"relative_prec%s", " after %s docs retrieved",
|
||||
get_param_str_cutoff},
|
||||
{"Unranked Average Precision",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, uap_cut[0]), NUM_CUTOFF,
|
||||
"unranked_avg_prec%s", " after %s docs retrieved",
|
||||
get_param_str_cutoff},
|
||||
{"Relative Unranked Average Precision",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_uap_cut[0]), NUM_CUTOFF,
|
||||
"relative_unranked_avg_prec%s", " after %s docs retrieved",
|
||||
get_param_str_cutoff},
|
||||
{"Utility (a,b,c,d)",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, exact_utility), 1,
|
||||
"utility_%s", " Coefficients %s ",
|
||||
get_param_str_utility},
|
||||
{"Recall averaged at X nonrel docs",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, av_fall_recall), 1,
|
||||
"rcl_at_%s_nonrel", " X= %s ",
|
||||
get_param_str_maxfallout},
|
||||
{"Fallout - Recall Averages",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, fall_recall[0]), NUM_FR_PTS,
|
||||
"fallout_recall_%s", "- recall after %s nonrel docs retrieved",
|
||||
get_param_str_fall_recall},
|
||||
{"Interpolated R-based precision,",
|
||||
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, int_R_prec_cut[0]), NUM_PREC_PTS-1,
|
||||
"int_%sR-prec", " after %s * R docs retrieved",
|
||||
get_param_str_Rcutoff},
|
||||
{"Time: Utility (a,b,c,d):",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, av_time_utility), 1,
|
||||
"time_integral_utility_%s", " Coefficients %s ",
|
||||
get_param_str_utility},
|
||||
{"Time: num_rel at cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_rel[0]), NUM_TIME_PTS,
|
||||
"time_num_rel_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: num_nonrel at cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_nrel[0]), NUM_TIME_PTS,
|
||||
"time_num_nonrel_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: cumulative rel at cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_cum_rel[0]), NUM_TIME_PTS,
|
||||
"time_cum_rel_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: precision at time cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS,
|
||||
"time_precis_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: precision at time cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS,
|
||||
"time_precis_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: relative precision at time cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_relprecis[0]), NUM_TIME_PTS,
|
||||
"time_relative_precis_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: unranked precision at time cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_uap[0]), NUM_TIME_PTS,
|
||||
"time_uap_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: relative unranked precision at time cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_reluap[0]), NUM_TIME_PTS,
|
||||
"time_relative_uap_%s", " after %s seconds",
|
||||
get_param_str_time_cutoff},
|
||||
{"Time: utility at time cutoff:",
|
||||
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_utility[0]), NUM_TIME_PTS,
|
||||
"time_utility_%s", " after %s seconds",
|
||||
get_param_str_time_utility_cutoff},
|
||||
};
|
||||
|
||||
int num_param_meas = sizeof (param_meas) / sizeof (param_meas[0]);
|
||||
|
||||
MICRO_MEASURE micro_meas[] = {
|
||||
{"micro_prec", "Total relevant retrieved documents / Total retrieved documents",
|
||||
0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_ret)},
|
||||
{"micro_recall", "Total relevant retrieved documents / Total relevant documents",
|
||||
0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_rel)},
|
||||
{"micro_bpref", "Total correct preferences / Total possible preferences",
|
||||
0, offsetof(TREC_EVAL, bpref_num_correct), offsetof(TREC_EVAL, bpref_num_possible)},
|
||||
};
|
||||
|
||||
int num_micro_meas = sizeof (micro_meas) / sizeof (micro_meas[0]);
|
||||
|
||||
Executable
+336
@@ -0,0 +1,336 @@
|
||||
#ifdef RCSID
|
||||
static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $";
|
||||
#endif
|
||||
|
||||
/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley.
|
||||
|
||||
Permission is granted for use of this file in unmodified form for
|
||||
research purposes. Please contact the SMART project to obtain
|
||||
permission for other uses.
|
||||
*/
|
||||
|
||||
#include "common.h"
|
||||
#include "sysfunc.h"
|
||||
#include "buf.h"
|
||||
#include "trec_eval.h"
|
||||
|
||||
static SM_BUF internal_output = {0, 0, (char *) 0};
|
||||
int add_buf_string();
|
||||
|
||||
extern SINGLE_MEASURE sing_meas[];
|
||||
extern PARAMETERIZED_MEASURE param_meas[];
|
||||
extern MICRO_MEASURE micro_meas[];
|
||||
extern int num_param_meas, num_sing_meas, num_micro_meas;
|
||||
|
||||
int
|
||||
accumulate_results (query_eval, accum_eval)
|
||||
TREC_EVAL *query_eval;
|
||||
TREC_EVAL *accum_eval;
|
||||
{
|
||||
long i,j;
|
||||
float *float_query, *float_accum;
|
||||
long *long_query, *long_accum;
|
||||
|
||||
if (query_eval->num_ret <= 0)
|
||||
return (0);
|
||||
|
||||
accum_eval->num_queries++;
|
||||
|
||||
for (i = 0; i < num_sing_meas; i++) {
|
||||
if (sing_meas[i].is_long_flag) {
|
||||
long_query = (long *) (((char *) query_eval) +
|
||||
sing_meas[i].byte_offset);
|
||||
long_accum = (long *) (((char *) accum_eval) +
|
||||
sing_meas[i].byte_offset);
|
||||
*long_accum += *long_query;
|
||||
}
|
||||
else {
|
||||
float_query = (float *) (((char *) query_eval) +
|
||||
sing_meas[i].byte_offset);
|
||||
float_accum = (float *) (((char *) accum_eval) +
|
||||
sing_meas[i].byte_offset);
|
||||
*float_accum += *float_query;
|
||||
}
|
||||
}
|
||||
|
||||
for (i = 0; i < num_param_meas; i++) {
|
||||
for (j = 0; j < param_meas[i].num_values; j++) {
|
||||
if (param_meas[i].is_long_flag) {
|
||||
long_query = (long *) (((char *) query_eval) +
|
||||
param_meas[i].byte_offset);
|
||||
long_accum = (long *) (((char *) accum_eval) +
|
||||
param_meas[i].byte_offset);
|
||||
long_accum[j] += long_query[j];
|
||||
}
|
||||
else {
|
||||
float_query = (float *) (((char *) query_eval) +
|
||||
param_meas[i].byte_offset);
|
||||
float_accum = (float *) (((char *) accum_eval) +
|
||||
param_meas[i].byte_offset);
|
||||
float_accum[j] += float_query[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
void
|
||||
print_rel_trec_eval_list (is_single_query_flag, epi, eval, output)
|
||||
long is_single_query_flag;
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TREC_EVAL *eval;
|
||||
SM_BUF *output;
|
||||
{
|
||||
long i,j;
|
||||
char temp_buf[1024];
|
||||
char q_buf[20];
|
||||
char name_buf[80];
|
||||
SM_BUF *out_p;
|
||||
long long_eval;
|
||||
float float_eval;
|
||||
|
||||
if (output == NULL) {
|
||||
out_p = &internal_output;
|
||||
out_p->end = 0;
|
||||
}
|
||||
else
|
||||
out_p = output;
|
||||
|
||||
if (is_single_query_flag) {
|
||||
(void) sprintf (q_buf, "%.20s", eval[0].qid);
|
||||
}
|
||||
else {
|
||||
(void) sprintf (q_buf, "%s", "all");
|
||||
(void) sprintf (temp_buf, "%-15s\t%s\t%ld\n",
|
||||
"num_q", q_buf, eval->num_queries);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < num_sing_meas; i++) {
|
||||
if ((! sing_meas[i].print_short_flag) && (! epi->all_flag))
|
||||
continue;
|
||||
if (sing_meas[i].print_time_flag && (!epi->time_flag))
|
||||
continue;
|
||||
if (sing_meas[i].print_only_query_flag && (!is_single_query_flag))
|
||||
continue;
|
||||
if (sing_meas[i].print_only_average_flag && (is_single_query_flag))
|
||||
continue;
|
||||
if (sing_meas[i].is_long_flag) {
|
||||
long_eval = *((long *) (((char *) eval) +
|
||||
sing_meas[i].byte_offset));
|
||||
if (sing_meas[i].avg_results_flag)
|
||||
long_eval /= eval->num_queries;
|
||||
(void) sprintf (temp_buf, "%-15s\t%s\t%ld\n",
|
||||
sing_meas[i].name, q_buf, long_eval);
|
||||
}
|
||||
else {
|
||||
float_eval = *((float *) (((char *) eval) +
|
||||
sing_meas[i].byte_offset));
|
||||
if (sing_meas[i].avg_results_flag)
|
||||
float_eval /= eval->num_queries;
|
||||
else if (sing_meas[i].avg_rel_results_flag && eval->num_rel > 0)
|
||||
/* average over number of rel docs instead of number queries */
|
||||
float_eval /= eval->num_rel;
|
||||
else if (sing_meas[i].gm_results_flag) {
|
||||
/* computing geometric mean instead of mean */
|
||||
if (!is_single_query_flag && epi->average_complete_flag)
|
||||
/* Must patch up averages for any missing queries, since */
|
||||
/* value of 0 means perfection */
|
||||
float_eval += (eval->num_queries - eval->num_orig_queries)*
|
||||
log (MIN_GEO_MEAN);
|
||||
float_eval = (float) exp ((double) (float_eval /
|
||||
eval->num_queries));
|
||||
}
|
||||
(void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n",
|
||||
sing_meas[i].name, q_buf, float_eval);
|
||||
}
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
|
||||
for (i = 0; i < num_param_meas; i++) {
|
||||
if ((! param_meas[i].print_short_flag) && (! epi->all_flag))
|
||||
continue;
|
||||
if (param_meas[i].print_time_flag && (!epi->time_flag))
|
||||
continue;
|
||||
if (param_meas[i].print_only_query_flag && (!is_single_query_flag))
|
||||
continue;
|
||||
if (param_meas[i].print_only_average_flag && (is_single_query_flag))
|
||||
continue;
|
||||
for (j = 0; j < param_meas[i].num_values; j++) {
|
||||
sprintf (name_buf, param_meas[i].format_string,
|
||||
param_meas[i].get_param_str (epi, j));
|
||||
if (param_meas[i].is_long_flag) {
|
||||
long_eval = ((long *) (((char *) eval) +
|
||||
param_meas[i].byte_offset))[j];
|
||||
if (param_meas[i].avg_results_flag)
|
||||
long_eval /= eval->num_queries;
|
||||
(void) sprintf (temp_buf, "%-15s\t%s\t%ld\n",
|
||||
name_buf, q_buf, long_eval);
|
||||
}
|
||||
else {
|
||||
float_eval = ((float *) (((char *) eval) +
|
||||
param_meas[i].byte_offset))[j];
|
||||
if (param_meas[i].avg_results_flag)
|
||||
float_eval /= eval->num_queries;
|
||||
(void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n",
|
||||
name_buf, q_buf, float_eval);
|
||||
}
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (! is_single_query_flag) {
|
||||
long denom_long_eval;
|
||||
for (i = 0; i < num_micro_meas; i++) {
|
||||
if ((! micro_meas[i].print_short_flag) && (! epi->all_flag))
|
||||
continue;
|
||||
long_eval = *((long *) (((char *) eval) +
|
||||
micro_meas[i].numerator_byte_offset));
|
||||
denom_long_eval = *((long *) (((char *) eval) +
|
||||
micro_meas[i].denominator_byte_offset));
|
||||
float_eval = (float) long_eval / (float) denom_long_eval;
|
||||
(void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n",
|
||||
micro_meas[i].name, q_buf, float_eval);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (output == NULL) {
|
||||
(void) fwrite (out_p->buf, 1, out_p->end, stdout);
|
||||
out_p->end = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static long cutoff[] = CUTOFF_VALUES;
|
||||
|
||||
void
|
||||
old_print_trec_eval_list (epi, eval, num_runs, output)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TREC_EVAL *eval;
|
||||
int num_runs;
|
||||
SM_BUF *output;
|
||||
{
|
||||
long i,j;
|
||||
char temp_buf[1024];
|
||||
SM_BUF *out_p;
|
||||
|
||||
if (output == NULL) {
|
||||
out_p = &internal_output;
|
||||
out_p->end = 0;
|
||||
}
|
||||
else
|
||||
out_p = output;
|
||||
|
||||
/* Print total numbers retrieved/rel for all runs */
|
||||
if (UNDEF == add_buf_string("\nQueryid (Num):\t", out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
if (UNDEF == add_buf_string (eval->qid, out_p))
|
||||
return;
|
||||
}
|
||||
if (UNDEF == add_buf_string("\nTotal number of documents over all queries",
|
||||
out_p))
|
||||
return;
|
||||
if (UNDEF == add_buf_string("\n Retrieved:", out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %5ld", eval[i].num_ret);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
if (UNDEF == add_buf_string("\n Relevant: ", out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %5ld", eval[i].num_rel);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
if (UNDEF == add_buf_string("\n Rel_ret: ", out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %5ld", eval[i].num_rel_ret);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
|
||||
/* Print recall precision figures at NUM_RP_PTS recall levels */
|
||||
if (UNDEF == add_buf_string
|
||||
("\nInterpolated Recall - Precision Averages:", out_p))
|
||||
return;
|
||||
for (j = 0; j < NUM_RP_PTS; j++) {
|
||||
(void) sprintf (temp_buf, "\n at %4.2f ",
|
||||
(float) j / (NUM_RP_PTS - 1));
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %6.4f ",
|
||||
eval[i].int_recall_precis[j] /eval[i].num_queries);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/* Print average recall precision and percentage improvement */
|
||||
(void) sprintf (temp_buf,
|
||||
"\nAverage precision (non-interpolated) for all rel docs(averaged over queries)\n ");
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %6.4f ",
|
||||
eval[i].av_recall_precis / eval[i].num_queries);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
if (num_runs > 1) {
|
||||
(void) sprintf (temp_buf, "\n %% Change: ");
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
for (i = 1; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %6.1f ",
|
||||
(((eval[i].av_recall_precis / eval[i].num_queries)/
|
||||
(eval[0].av_recall_precis / eval[i].num_queries))
|
||||
- 1.0) * 100.0);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
}
|
||||
(void) sprintf (temp_buf, "\nPrecision:");
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
for (j = 0; j < NUM_CUTOFF; j++) {
|
||||
(void) sprintf (temp_buf, "\n At %4ld docs:", cutoff[j]);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %6.4f",
|
||||
eval[i].precis_cut[j] / eval[i].num_queries);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
(void) sprintf (temp_buf, "\nR-Precision (precision after R (= num_rel for a query) docs retrieved):\n Exact: ");
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
for (i = 0; i < num_runs; i++) {
|
||||
(void) sprintf (temp_buf, " %6.4f",
|
||||
eval[i].R_recall_precis / eval[i].num_queries);
|
||||
if (UNDEF == add_buf_string (temp_buf, out_p))
|
||||
return;
|
||||
}
|
||||
|
||||
if (UNDEF == add_buf_string ("\n", out_p))
|
||||
return;
|
||||
|
||||
if (output == NULL) {
|
||||
(void) fwrite (out_p->buf, 1, out_p->end, stdout);
|
||||
out_p->end = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
Executable
+353
@@ -0,0 +1,353 @@
|
||||
#ifndef TRECEVALH
|
||||
#define TRECEVALH
|
||||
|
||||
/* Static state info; set at beginning, possibly from program options, */
|
||||
/* but then remains constant throughout. */
|
||||
typedef struct {
|
||||
long query_flag; /* 0. If set, evaluation output will be
|
||||
printed for each query, in addition
|
||||
to summary at end. */
|
||||
long all_flag; /* 0. If set, all evaluation measures will
|
||||
be printed instead of just the
|
||||
final TREC 2 measures. */
|
||||
long time_flag; /* 0. If set, calculate time-based measures*/
|
||||
long relation_flag; /* 1. If set, print in relational form */
|
||||
long average_complete_flag; /* 0. If set, average over the complete set
|
||||
of relevance judgements (qrels), instead
|
||||
of the number of queries
|
||||
in the intersection of qrels and result */
|
||||
long judged_docs_only_flag; /* 0. If set, throw out all unjudged docs
|
||||
for the retrieved set before calculating
|
||||
any measures. */
|
||||
double utility_a; /* UTILITY_A. Default utility values */
|
||||
double utility_b; /* UTILITY_B. Default utility values */
|
||||
double utility_c; /* UTILITY_C. Default utility values */
|
||||
double utility_d; /* UTILITY_D. Default utility values */
|
||||
long num_docs_in_coll; /* 0. number of docs in collection */
|
||||
long relevance_level; /* 1. In relevance judgements, the level at
|
||||
which a doc is considered relevant for
|
||||
this evaluation */
|
||||
long max_num_docs_per_topic; /* MAXLONG. evaluate only this many docs */
|
||||
} EVAL_PARAM_INFO;
|
||||
|
||||
/* Measure characteristics (how to print them, average them). */
|
||||
/* List of measures is in measures.c */
|
||||
/* Three types of measures:
|
||||
single measures - single measure and name
|
||||
parameterized measures - arrays of a measure, whose measure name
|
||||
depends on parameter (eg P5, P10)
|
||||
micro measures - measures defined as the micro average over all
|
||||
docs retrieved independent of topic. Only calculated
|
||||
and printed for the "all" pseudo-query.
|
||||
Eg micro_prec = num_rel_ret / num_ret
|
||||
*/
|
||||
typedef struct {
|
||||
char *name;
|
||||
char *long_name;
|
||||
unsigned char is_long_flag; /* otherwise float */
|
||||
unsigned char print_short_flag; /* if set, measure is always printed
|
||||
(not just if all_flag set) */
|
||||
unsigned char print_time_flag; /* if set, measure is printed only
|
||||
if time_flag is set */
|
||||
unsigned char print_only_query_flag; /* if set, measure is printed only
|
||||
when printing individual query output*/
|
||||
unsigned char print_only_average_flag; /* if set, measure is printed only
|
||||
when printing overall average output*/
|
||||
unsigned char avg_results_flag; /* if set, average results over queries */
|
||||
unsigned char avg_rel_results_flag;/* if set,average results over num_rel*/
|
||||
unsigned char gm_results_flag; /* if set, measure uses geometric mean. ie
|
||||
exponentiate the average before
|
||||
printing */
|
||||
long byte_offset;
|
||||
} SINGLE_MEASURE;
|
||||
|
||||
typedef struct {
|
||||
char *long_name;
|
||||
unsigned char is_long_flag; /* otherwise float */
|
||||
unsigned char print_short_flag; /* if set, print in short output */
|
||||
unsigned char print_time_flag; /* if set, measure is printed only
|
||||
if time_flag is set */
|
||||
unsigned char print_only_query_flag; /* if set, measure is printed only
|
||||
when printing individual query output*/
|
||||
unsigned char print_only_average_flag; /* if set, measure is printed only
|
||||
when printing overall average output*/
|
||||
unsigned char avg_results_flag; /* if set, average results over queries */
|
||||
long byte_offset;
|
||||
long num_values;
|
||||
char *format_string;
|
||||
char *long_format_string;
|
||||
char *(*get_param_str) (EVAL_PARAM_INFO *ip, long index);
|
||||
} PARAMETERIZED_MEASURE;
|
||||
|
||||
typedef struct {
|
||||
char *name;
|
||||
char *long_name;
|
||||
unsigned char print_short_flag; /* if set, measure is always printed
|
||||
(not just if all_flag set) */
|
||||
long numerator_byte_offset;
|
||||
long denominator_byte_offset;
|
||||
} MICRO_MEASURE;
|
||||
|
||||
|
||||
typedef struct { /* For each retrieved document result */
|
||||
char *docno; /* document id */
|
||||
float sim; /* score */
|
||||
long rank; /* rank assigned after breaking ties */
|
||||
} TEXT_TR;
|
||||
|
||||
typedef struct { /* For each query in retrieved results */
|
||||
char *qid; /* query id */
|
||||
long num_text_tr; /* number of TEXT_TR results for query*/
|
||||
long max_num_text_tr; /* number results space reserved for */
|
||||
TEXT_TR *text_tr; /* Array of TEXT_TR results */
|
||||
} TREC_TOP;
|
||||
|
||||
typedef struct { /* Overall retrieved results */
|
||||
char *run_id; /* run id */
|
||||
long num_q_tr; /* Number of TREC_TOP queries */
|
||||
long max_num_q_tr; /* Num queries space reserved for*/
|
||||
TREC_TOP *trec_top; /* Array of TREC_TOP query results */
|
||||
} ALL_TREC_TOP;
|
||||
|
||||
typedef struct { /* For each relevance judgement */
|
||||
char *docno; /* document id */
|
||||
long rel; /* document judgement */
|
||||
} TEXT_QRELS;
|
||||
|
||||
typedef struct { /* For each query in rel judgements */
|
||||
char *qid; /* query id */
|
||||
long num_text_qrels; /* number of judged documents */
|
||||
long max_num_text_qrels; /* Num docs space reserved for */
|
||||
TEXT_QRELS *text_qrels; /* Array of judged TEXT_QRELS */
|
||||
} TREC_QRELS;
|
||||
|
||||
typedef struct { /* Overall relevance judgements */
|
||||
long num_q_qrels; /* Number of TREC_QRELS queries */
|
||||
long max_num_q_qrels; /* Num queries space reserved for */
|
||||
TREC_QRELS *trec_qrels; /* Array of TREC_QRELS queries */
|
||||
} ALL_TREC_QRELS;
|
||||
|
||||
|
||||
|
||||
#define INIT_NUM_QUERIES 50
|
||||
#define INIT_NUM_RESULTS 1000
|
||||
#define INIT_NUM_RELS 2000
|
||||
|
||||
/* Set retrieval is based on contingency table:
|
||||
relevant nonrelevant
|
||||
retrieved a b
|
||||
nonretrieved c d
|
||||
|
||||
Often you see r == num_rel_ret == a
|
||||
R == num_rel == a+c
|
||||
n == num_ret == a+b
|
||||
N == num_docs == a+b+c+d
|
||||
Some of these definitions are used in comments below
|
||||
*/
|
||||
|
||||
|
||||
/* ----------------------------------------------- */
|
||||
/* Defined constants that are collection/purpose dependent */
|
||||
|
||||
/* Number of cutoffs for recall,precision, and rel_precis measures. */
|
||||
/* CUTOFF_VALUES gives the number of retrieved docs that these */
|
||||
/* evaluation mesures are applied at. */
|
||||
#define NUM_CUTOFF 9
|
||||
#define CUTOFF_VALUES {5, 10, 15, 20, 30, 100, 200, 500, 1000}
|
||||
|
||||
/* Maximum fallout value, expressed in number of non-rel docs retrieved. */
|
||||
/* (Make the approximation that number of non-rel docs in collection */
|
||||
/* is equal to the number of number of docs in collection) */
|
||||
#define MAX_FALL_RET 142
|
||||
|
||||
/* Maximum multiple of R (number of rel docs for this query) to calculate */
|
||||
/* R-based precision at */
|
||||
#define MAX_RPREC 2.0
|
||||
|
||||
#define MAX_TIME 300.0
|
||||
#define NUM_TIME_PTS 60
|
||||
|
||||
/* Set a maximum number of nonrel docs to be used for preference measures */
|
||||
#define PREF_TOP_NONREL_NUM 100
|
||||
|
||||
/* ----------------------------------------------- */
|
||||
/* Defined constants that are collection/purpose independent. If you
|
||||
change these, you probably need to change comments and documentation,
|
||||
and some variable names may not be appropriate any more! */
|
||||
#define NUM_RP_PTS 11
|
||||
#define THREE_PTS {2, 5, 8}
|
||||
#define NUM_FR_PTS 11
|
||||
#define NUM_PREC_PTS 11
|
||||
#define UTILITY_A 1.0
|
||||
#define UTILITY_B -1.0
|
||||
#define UTILITY_C 0.0
|
||||
#define UTILITY_D 0.0
|
||||
#define MIN_GEO_MEAN .00001
|
||||
|
||||
typedef struct {
|
||||
char *qid; /* query id */
|
||||
long num_queries; /* Number of queries for this eval */
|
||||
long num_orig_queries; /* Number of queries for this eval without
|
||||
missing values, if using trec_eval -c */
|
||||
/* Summary Numbers over all queries */
|
||||
long num_rel; /* Number of relevant docs */
|
||||
long num_ret; /* Number of retrieved docs */
|
||||
long num_rel_ret; /* Number of relevant retrieved docs */
|
||||
float avg_doc_prec; /* Average of precision over all
|
||||
relevant documents (query independent)*/
|
||||
|
||||
/* Measures after num_ret docs */
|
||||
float exact_recall; /* Recall after num_ret docs */
|
||||
float exact_precis; /* Precision after num_ret docs */
|
||||
float exact_rel_precis; /* Relative Precision (or recall) */
|
||||
/* Defined to be precision / max possible
|
||||
precision */
|
||||
float exact_uap; /* Unranked Average Precision */
|
||||
/* Every rel doc in retrieved set gets
|
||||
precision, every nonret rel doc gets 0.
|
||||
Average over all rel docs */
|
||||
/* Note this = exact_recall *
|
||||
exact_precision for a query */
|
||||
/* Preferred measure for evaluation of
|
||||
unranked sets of arbitrary size. */
|
||||
float exact_rel_uap; /* Relative Unranked Average Precision */
|
||||
/* Above, but relativized given size of
|
||||
retrieved set */
|
||||
/* If (n<R) set num_rel to n
|
||||
If (n>R) set num_ret to R
|
||||
Then use uap formula */
|
||||
/* exact_rel_precis ** 2 */
|
||||
float exact_utility; /* From contingency table, by default:
|
||||
UTILITY_A * a + UTILITY_B * b +
|
||||
UTILITY_C * c + UTILITY_D * d.
|
||||
By default, a-b (or r - (n-r)) */
|
||||
float recip_rank; /* reciprical rank of top retrieved
|
||||
relevant document */
|
||||
long rank_first_rel; /* Rank of top retrieved rel doc. Set to
|
||||
0 if none. Unaveraged */
|
||||
|
||||
/* Measures after each document */
|
||||
float recall_cut[NUM_CUTOFF]; /* Recall after cutoff[i] docs */
|
||||
|
||||
float precis_cut[NUM_CUTOFF]; /* precision after cutoff[i] docs. If
|
||||
less than cutoff[i] docs retrieved,
|
||||
then assume an additional
|
||||
cutoff[i]-num_ret non-relevant docs
|
||||
are retrieved. */
|
||||
float rel_precis_cut[NUM_CUTOFF];/* Relative precision after cutoff[i]
|
||||
docs. (Note relative precision is
|
||||
identical to relative recall) */
|
||||
float uap_cut[NUM_CUTOFF]; /* uap (is recall * precision) after
|
||||
cutoff[i] docs. Not recommended */
|
||||
float rel_uap_cut[NUM_CUTOFF]; /* rel_uap at cutoff[i] docs */
|
||||
float av_rel_precis; /* average (integral) of rel_precis
|
||||
after each doc. Do not use if
|
||||
number of docs retrieved varies */
|
||||
float av_rel_uap; /* average (integral) of rel_uap
|
||||
after each doc. Do not use if
|
||||
number of docs retrieved varies */
|
||||
|
||||
|
||||
/* Measures after each rel doc */
|
||||
float av_recall_precis; /* MAP! average(integral) of precision at
|
||||
all rel doc ranks. THE MAJOR
|
||||
EVALUATION MEASURE FOR RANKED DOCS */
|
||||
float int_av_recall_precis; /* Same as above, but the precision values
|
||||
have been interpolated, so that prec(X)
|
||||
is actually MAX prec(Y) for all
|
||||
Y >= X */
|
||||
float int_recall_precis[NUM_RP_PTS];/* interpolated precision at
|
||||
0.1 increments of recall */
|
||||
float int_av3_recall_precis; /* interpolated average at 3 intermediate
|
||||
points */
|
||||
float int_av11_recall_precis; /* interpolated average at NUM_RP_PTS
|
||||
intermediate points (recall_level) */
|
||||
|
||||
/* Measures after each non-rel doc */
|
||||
float fall_recall[NUM_FR_PTS]; /* max recall after each non-rel doc,
|
||||
at 11 points starting at 0.0 and
|
||||
ending at MAX_FALL_RET /num_docs */
|
||||
float av_fall_recall; /* Average of fallout-recall, after each
|
||||
non-rel doc until fallout of
|
||||
MAX_FALL_RET / num_docs achieved */
|
||||
|
||||
/* Measures after R-related cutoffs. R is the number of relevant
|
||||
docs for a particular query, but note that these cutoffs are after
|
||||
R docs, whether relevant or non-relevant, have been retrieved.
|
||||
R-related cutoffs are really only applicable to a situtation where
|
||||
there are many relevant docs per query (or lots of queries). */
|
||||
float R_recall_precis; /* Recall or precision after R docs
|
||||
(note they are equal at this point) */
|
||||
float av_R_precis; /* Average (or integral) of precision at
|
||||
each doc until R docs have been
|
||||
retrieved */
|
||||
float R_prec_cut[NUM_PREC_PTS]; /* Precision measured after multiples of
|
||||
R docs have been retrieved. 10
|
||||
equal points, with max multiple
|
||||
having value MAX_RPREC */
|
||||
float int_R_recall_precis; /* Interpolated precision after R docs
|
||||
Prec(X) = MAX(prec(Y)) for all Y>=X */
|
||||
float int_av_R_precis; /* Interpolated */
|
||||
float int_R_prec_cut[NUM_PREC_PTS]; /* Interpolated */
|
||||
|
||||
/* Measures after particular time relative to size of eventual retrieved
|
||||
set. Eg, precision is num_rel_so_far/num_ret
|
||||
relprecision is num_rel_so_far/MIN(num_ret,num_rel)
|
||||
uap is num_rel_so_far**2/(num_ret*MIN(num_ret,num_rel))
|
||||
reluap is relprecision * relprecision */
|
||||
float time_num_rel[NUM_TIME_PTS]; /* Number of rel docs in time bucket*/
|
||||
float time_num_nrel[NUM_TIME_PTS];/* Number of nrel docs in each bucket*/
|
||||
float time_cum_rel[NUM_TIME_PTS]; /* Cumulative time_num_rel */
|
||||
float time_precis[NUM_TIME_PTS]; /* First Precision in each bucket */
|
||||
float time_relprecis[NUM_TIME_PTS];/* First rel-Precision in each bucket */
|
||||
float time_uap[NUM_TIME_PTS]; /* First uap in bucket*/
|
||||
float time_reluap[NUM_TIME_PTS]; /* First relative uap in bucket*/
|
||||
float time_utility[NUM_TIME_PTS]; /* First Utility (default 1,-1,0,0)
|
||||
in bucket */
|
||||
float av_time_precis; /* Sum (integral) of time_precis */
|
||||
float av_time_relprecis; /* Sum (integral) of time_relprecis */
|
||||
float av_time_uap; /* Sum (integral) of time_uap */
|
||||
float av_time_reluap; /* Sum (integral) of time_reluap */
|
||||
float av_time_utility; /* Sum (integral) of time_utility */
|
||||
float av_time_cum_rel; /* Sum (integral) of time_cum_rel */
|
||||
|
||||
/* Measures dependent on only judged documents */
|
||||
/* Binary Pref relations: fraction of nonrel documents retrieved after
|
||||
each rel doc */
|
||||
float bpref; /* real BPREF. Top num_rel nonrel docs */
|
||||
float bpref_top5Rnonrel; /* Top 5 * num_rel nonrel docs */
|
||||
float bpref_top10Rnonrel; /* Top 10 * num_rel nonrel docs */
|
||||
/* float bpref_topRnonrel; * renamed as bpref */
|
||||
float bpref_allnonrel; /* all judged nonrel docs */
|
||||
float bpref_retnonrel; /* Only retrieved nonrel docs */
|
||||
float bpref_topnonrel; /* Top PREF_TOPNREL_NUM nonrel docs */
|
||||
float bpref_top50pRnonrel; /* Top 50 + num_rel nonrel docs */
|
||||
float bpref_top25pRnonrel; /* Top 25 + num_rel nonrel docs */
|
||||
float bpref_top10pRnonrel; /* Top 10 + num_rel nonrel docs.
|
||||
Bad version used in SIGIR 2004 paper */
|
||||
float old_bpref_top10pRnonrel; /* bad old version. Top 10 + num_rel
|
||||
nonrel docs. Used in SIGIR 2004 paper*/
|
||||
float bpref_top25p2Rnonrel; /* Top 25 + 2 * num_rel nonrel docs */
|
||||
float bpref_retall; /* Only retrieved rel,nonrel docs */
|
||||
float bpref_5; /* Only top 5 rel, top 5 nonrel */
|
||||
float bpref_10; /* Only top 10 rel, top 10 nonrel */
|
||||
float old_bpref; /* Bad old bpref. Top num_rel nonrel docs.
|
||||
Only used retrieved nonrel docs.
|
||||
Used in TREC 12,13, mention in
|
||||
SIGIR 2004 paper */
|
||||
float bpref_num_all; /* num not retrieved before (all judged)*/
|
||||
float bpref_num_ret; /* num retrieved after */
|
||||
long bpref_num_correct; /* num correct preferences */
|
||||
long bpref_num_possible; /* num possible correct preferences */
|
||||
|
||||
/* Measures that use Geometric Mean
|
||||
avg_Score = exp (SUM (log (MAX (query_score, .00001))) / N)
|
||||
WARNING: Geometric Mean measures special cased for "trec_eval -c".
|
||||
Works, but be careful when implementing new measure */
|
||||
float gm_ap; /* Geometric Mean version of MAP */
|
||||
float gm_bpref; /* Geometric Mean version of bpref. Note
|
||||
bpref has lots of 0.0 values */
|
||||
|
||||
} TREC_EVAL;
|
||||
|
||||
#endif /* TRECEVALH */
|
||||
Executable
+203
@@ -0,0 +1,203 @@
|
||||
/* Copyright (c) 2003, 1991, 1990, 1984 - Chris Buckley. */
|
||||
|
||||
#include "common.h"
|
||||
#include "trec_eval.h"
|
||||
|
||||
static char *help_message =
|
||||
"trec_eval [-h] [-q] [-a] [-o] [-c] [-l<num> [-N<num>] [-M<num>] [-Ua<num>] [-Ub<num>] [-Uc<num>] [-Ud<num>] [-T] trec_rel_file trec_top_file \n\
|
||||
\n\
|
||||
Calculate and print various evaluation measures, evaluating the results \n\
|
||||
in trec_top_file against the relevance judgements in trec_rel_file. \n\
|
||||
\n\
|
||||
There are a fair number of options, of which only the lower case options are \n\
|
||||
normally ever used. \n\
|
||||
-h: Print full help message and exit \n\
|
||||
-q: In addition to summary evaluation, give evaluation for each query \n\
|
||||
-a: Print all evaluation measures calculated, instead of just the \n\
|
||||
main official measures for TREC. \n\
|
||||
-o: Print everything out in old, nonrelational format (default is relational) \n\
|
||||
-c: Average over the complete set of queries in the relevance judgements \n\
|
||||
instead of the queries in the intersection of relevance judgements \n\
|
||||
and results. Missing queries will contribute a value of 0 to all \n\
|
||||
evaluation measures (which may or may not be reasonable for a \n\
|
||||
particular evaluation measure, but is reasonable for standard TREC \n\
|
||||
measures.) \n\
|
||||
-l<num>: Num indicates the minimum relevance judgement value needed for \n\
|
||||
a document to be called relevant. (All measures used by TREC eval are \n\
|
||||
based on binary relevance). Used if trec_rel_file contains relevance \n\
|
||||
judged on a multi-relevance scale. Default is 1. \n\
|
||||
-N<num>: Number of docs in collection \n\
|
||||
-M<num>: Max number of docs per topic to use in evaluation (discard rest). \n\
|
||||
-Ua<num>: Value to use for 'a' coefficient of utility computation. \n\
|
||||
relevant nonrelevant \n\
|
||||
retrieved a b \n\
|
||||
nonretrieved c d \n\
|
||||
-Ub<num>: Value to use for 'b' coefficient of utility computation. \n\
|
||||
-Uc<num>: Value to use for 'c' coefficient of utility computation. \n\
|
||||
-Ud<num>: Value to use for 'd' coefficient of utility computation. \n\
|
||||
-J: Calculate all values only over the judged (either relevant or \n\
|
||||
nonrelevant) documents. All unjudged documents are removed from the \n\
|
||||
retrieved set before any calculations (possibly leaving an empty set). \n\
|
||||
DO NOT USE, unless you really know what you're doing - very easy to get \n\
|
||||
reasonable looking, but invalid, numbers. \n\
|
||||
-T: Treat similarity as time that document retrieved. Compute \n\
|
||||
several time-based measures after ranking docs by time retrieved \n\
|
||||
(first doc (lowest sim) retrieved ranked highest). \n\
|
||||
Only done if -a selected. \n\
|
||||
\n\
|
||||
\n\
|
||||
Read text tuples from trec_top_file of the form \n\
|
||||
030 Q0 ZF08-175-870 0 4238 prise1 \n\
|
||||
qid iter docno rank sim run_id \n\
|
||||
giving TREC document numbers (a string) retrieved by query qid \n\
|
||||
(a string) with similarity sim (a float). The other fields are ignored, \n\
|
||||
with the exception that the run_id field of the last line is kept and \n\
|
||||
output. In particular, note that the rank field is ignored here; \n\
|
||||
internally ranks are assigned by sorting by the sim field with ties \n\
|
||||
broken deterministicly (using docno). \n\
|
||||
Sim is assumed to be higher for the docs to be retrieved first. \n\
|
||||
File may contain no NULL characters. \n\
|
||||
Lines may contain fields after the run_id; they are ignored. \n\
|
||||
\n\
|
||||
Relevance for each docno to qid is determined from text_qrels_file, which \n\
|
||||
consists of text tuples of the form \n\
|
||||
qid iter docno rel \n\
|
||||
giving TREC document numbers (docno, a string) and their relevance (rel, \n\
|
||||
an integer) to query qid (a string). iter string field is ignored. \n\
|
||||
Fields are separated by whitespace, string fields can contain no whitespace. \n\
|
||||
File may contain no NULL characters. \n\
|
||||
\n\
|
||||
The text tuples with relevance judgements are converted to TR_VEC form \n\
|
||||
and then submitted to the SMART evaluation routines. \n\
|
||||
The qid,did,rank,sim,rel fields of TR_VEC are filled in; \n\
|
||||
action,iter fields are set to 0. \n\
|
||||
The rel field is set to -1 if the document was not judged (not in \n\
|
||||
text_qrels_file). Most measures, but not all, will treat -1 the same as 0, \n\
|
||||
namely nonrelevant. Note that relevance_level is used to determine if the \n\
|
||||
document is relevant during score calculations. \n\
|
||||
Queries for which there are no relevant docs are ignored. \n\
|
||||
Warning: queries for which there are relevant docs but no retrieved docs \n\
|
||||
are also ignored by default. This allows systems to evaluate over subsets \n\
|
||||
of the relevant docs, but means if a system improperly retrieves no docs, \n\
|
||||
it will not be detected. Use the -c flag to avoid this behavior. \n\
|
||||
\n\
|
||||
EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. \n\
|
||||
Relational Format prints the same values, but all lines are of the form \n\
|
||||
measure_name query value \n\
|
||||
\n\
|
||||
1. Total number of documents over all queries \n\
|
||||
Retrieved: \n\
|
||||
Relevant: \n\
|
||||
Rel_ret: (relevant and retrieved) \n\
|
||||
These should be self-explanatory. All values are totals over all \n\
|
||||
queries being evaluated. \n\
|
||||
2. Interpolated Recall - Precision Averages: \n\
|
||||
at 0.00 \n\
|
||||
at 0.10 \n\
|
||||
... \n\
|
||||
at 1.00 \n\
|
||||
See any standard IR text (especially by Salton) for more details of \n\
|
||||
recall-precision evaluation. Measures precision (percent of retrieved \n\
|
||||
docs that are relevant) at various recall levels (after a certain \n\
|
||||
percentage of all the relevant docs for that query have been retrieved). \n\
|
||||
'Interpolated' means that, for example, precision at recall \n\
|
||||
0.10 (ie, after 10% of rel docs for a query have been retrieved) is \n\
|
||||
taken to be MAXIMUM of precision at all recall points >= 0.10. \n\
|
||||
Values are averaged over all queries (for each of the 11 recall levels). \n\
|
||||
These values are used for Recall-Precision graphs. \n\
|
||||
3. Average precision (non-interpolated) over all rel docs \n\
|
||||
The precision is calculated after each relevant doc is retrieved. \n\
|
||||
If a relevant doc is not retrieved, its precision is 0.0. \n\
|
||||
All precision values are then averaged together to get a single number \n\
|
||||
for the performance of a query. Conceptually this is the area \n\
|
||||
underneath the recall-precision graph for the query. \n\
|
||||
The values are then averaged over all queries. \n\
|
||||
4. Precision: \n\
|
||||
at 5 docs \n\
|
||||
at 10 docs \n\
|
||||
... \n\
|
||||
at 1000 docs \n\
|
||||
The precision (percent of retrieved docs that are relevant) after X \n\
|
||||
documents (whether relevant or nonrelevant) have been retrieved. \n\
|
||||
Values averaged over all queries. If X docs were not retrieved \n\
|
||||
for a query, then all missing docs are assumed to be non-relevant. \n\
|
||||
5. R-Precision (precision after R (= num_rel for a query) docs retrieved): \n\
|
||||
Measures precision (or recall, they're the same) after R docs \n\
|
||||
have been retrieved, where R is the total number of relevant docs \n\
|
||||
for a query. Thus if a query has 40 relevant docs, then precision \n\
|
||||
is measured after 40 docs, while if it has 600 relevant docs, precision \n\
|
||||
is measured after 600 docs. This avoids some of the averaging \n\
|
||||
problems of the 'precision at X docs' values in (4) above. \n\
|
||||
If R is greater than the number of docs retrieved for a query, then \n\
|
||||
the nonretrieved docs are all assumed to be nonrelevant. \n\
|
||||
";
|
||||
|
||||
|
||||
extern SINGLE_MEASURE sing_meas[];
|
||||
extern PARAMETERIZED_MEASURE param_meas[];
|
||||
extern MICRO_MEASURE micro_meas[];
|
||||
extern int num_param_meas, num_sing_meas, num_micro_meas;
|
||||
|
||||
int
|
||||
trec_eval_help(epi)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
{
|
||||
long i, j;
|
||||
char temp_buf1[30];
|
||||
char temp_buf2[80];
|
||||
|
||||
printf ("%s\n", help_message);
|
||||
|
||||
printf ("Major measures (again) with their relational names:\n");
|
||||
for (i = 0; i < num_sing_meas; i++) {
|
||||
if (sing_meas[i].print_short_flag)
|
||||
printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name);
|
||||
}
|
||||
for (i = 0; i < num_param_meas; i++) {
|
||||
if (param_meas[i].print_short_flag) {
|
||||
for (j = 0; j < param_meas[i].num_values; j++) {
|
||||
sprintf (temp_buf1, param_meas[i].format_string,
|
||||
param_meas[i].get_param_str (epi, j));
|
||||
sprintf (temp_buf2, param_meas[i].long_format_string,
|
||||
param_meas[i].get_param_str (epi, j));
|
||||
printf ("%-15s\t%s%s\n", temp_buf1,
|
||||
param_meas[i].long_name, temp_buf2);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (i = 0; i < num_micro_meas; i++) {
|
||||
if (micro_meas[i].print_short_flag)
|
||||
printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name);
|
||||
}
|
||||
|
||||
printf ("\n\nMinor measures with their relational names:\n");
|
||||
for (i = 0; i < num_sing_meas; i++) {
|
||||
if (sing_meas[i].print_short_flag)
|
||||
continue;
|
||||
if (sing_meas[i].print_time_flag && (! epi->time_flag))
|
||||
continue;
|
||||
if (! sing_meas[i].print_short_flag)
|
||||
printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name);
|
||||
}
|
||||
for (i = 0; i < num_param_meas; i++) {
|
||||
if (param_meas[i].print_short_flag)
|
||||
continue;
|
||||
if (param_meas[i].print_time_flag && (! epi->time_flag))
|
||||
continue;
|
||||
for (j = 0; j < param_meas[i].num_values; j++) {
|
||||
sprintf (temp_buf1, param_meas[i].format_string,
|
||||
param_meas[i].get_param_str (epi, j));
|
||||
sprintf (temp_buf2, param_meas[i].long_format_string,
|
||||
param_meas[i].get_param_str (epi, j));
|
||||
printf ("%-15s\t%s%s\n", temp_buf1,
|
||||
param_meas[i].long_name, temp_buf2);
|
||||
}
|
||||
}
|
||||
for (i = 0; i < num_micro_meas; i++) {
|
||||
if (! micro_meas[i].print_short_flag)
|
||||
printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name);
|
||||
}
|
||||
|
||||
return (1);
|
||||
}
|
||||
|
||||
Executable
+673
@@ -0,0 +1,673 @@
|
||||
#ifdef RCSID
|
||||
static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/trvec_trec_eval.c,v 11.0 1992/07/21 18:20:35 chrisb Exp chrisb $";
|
||||
#endif
|
||||
|
||||
/* Copyright (c) 2005
|
||||
*/
|
||||
|
||||
#include "common.h"
|
||||
#include "sysfunc.h"
|
||||
#include "smart_error.h"
|
||||
#include "tr_vec.h"
|
||||
#include "trec_eval.h"
|
||||
|
||||
static int compare_iter_rank();
|
||||
static void calc_cutoff_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
|
||||
TREC_EVAL *eval, long num_rel,
|
||||
long num_nonrel);
|
||||
static void calc_bpref_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
|
||||
TREC_EVAL *eval, long num_rel,
|
||||
long num_nonrel);
|
||||
static void calc_average_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
|
||||
TREC_EVAL *eval, long num_rel,
|
||||
long num_nonrel);
|
||||
static void calc_exact_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
|
||||
TREC_EVAL *eval, long num_rel,
|
||||
long num_nonrel);
|
||||
static void calc_time_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
|
||||
TREC_EVAL *eval, long num_rel,
|
||||
long num_nonrel);
|
||||
|
||||
int
|
||||
trvec_trec_eval (epi, tr_vec, eval, num_rel, num_nonrel)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TR_VEC *tr_vec;
|
||||
TREC_EVAL *eval;
|
||||
long num_rel; /* Number relevant judged */
|
||||
long num_nonrel; /* Number nonrelevant judged */
|
||||
{
|
||||
long j;
|
||||
long max_iter;
|
||||
|
||||
if (tr_vec == (TR_VEC *) NULL)
|
||||
return (UNDEF);
|
||||
|
||||
/* Initialize everything to 0 */
|
||||
bzero ((char *) eval, sizeof (TREC_EVAL));
|
||||
|
||||
eval->qid = tr_vec->qid;
|
||||
eval->num_queries = 1;
|
||||
|
||||
/* If no retrieved docs, then just return */
|
||||
if (tr_vec->num_tr == 0) {
|
||||
return (0);
|
||||
}
|
||||
|
||||
eval->num_rel = num_rel;
|
||||
|
||||
/* Evaluate only the docs on the last iteration of new_tr_vec */
|
||||
/* Sort the tr tuples for this query by decreasing iter and
|
||||
increasing rank */
|
||||
qsort ((char *) tr_vec->tr,
|
||||
(int) tr_vec->num_tr,
|
||||
sizeof (TR_TUP),
|
||||
compare_iter_rank);
|
||||
|
||||
max_iter = tr_vec->tr[0].iter;
|
||||
for (j = 0; j < tr_vec->num_tr; j++) {
|
||||
if (tr_vec->tr[j].iter == max_iter) {
|
||||
eval->num_ret++;
|
||||
if (tr_vec->tr[j].rel >= epi->relevance_level)
|
||||
eval->num_rel_ret++;
|
||||
}
|
||||
else {
|
||||
if (tr_vec->tr[j].rel >= epi->relevance_level)
|
||||
eval->num_rel--;
|
||||
}
|
||||
}
|
||||
|
||||
/* Calculate cutoff measures, and those measures dependant on them */
|
||||
/* Also includes recip_rank and rank_first_rel */
|
||||
calc_cutoff_measures (epi, tr_vec, eval, num_rel, num_nonrel);
|
||||
|
||||
/* Calculate bpref measures */
|
||||
calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel);
|
||||
|
||||
/* Calculate measures that average over ret or rel docs */
|
||||
calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel);
|
||||
|
||||
/* Calculate exact measures over entire retrieved sets */
|
||||
calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel);
|
||||
|
||||
/* Calculate time measures, if wanted */
|
||||
if (epi->time_flag)
|
||||
calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel);
|
||||
|
||||
|
||||
return (1);
|
||||
}
|
||||
|
||||
static int
|
||||
compare_iter_rank (tr1, tr2)
|
||||
TR_TUP *tr1;
|
||||
TR_TUP *tr2;
|
||||
{
|
||||
if (tr1->iter > tr2->iter)
|
||||
return (-1);
|
||||
if (tr1->iter < tr2->iter)
|
||||
return (1);
|
||||
if (tr1->rank < tr2->rank)
|
||||
return (-1);
|
||||
if (tr1->rank > tr2->rank)
|
||||
return (1);
|
||||
return (0);
|
||||
}
|
||||
|
||||
|
||||
/* ********************************************************************* */
|
||||
/* calculate cutoff measures */
|
||||
/* cutoff values for recall precision output */
|
||||
static int cutoff[NUM_CUTOFF] = CUTOFF_VALUES;
|
||||
static int three_pts[3] = THREE_PTS;
|
||||
|
||||
|
||||
static void
|
||||
calc_cutoff_measures(epi, tr_vec, eval, num_rel, num_nonrel)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TR_VEC *tr_vec;
|
||||
TREC_EVAL *eval;
|
||||
long num_rel; /* Number relevant judged */
|
||||
long num_nonrel; /* Number nonrelevant judged */
|
||||
{
|
||||
double recall, precis; /* current recall, precision values */
|
||||
double rel_precis, rel_uap;/* relative precision, uap values */
|
||||
double int_precis; /* current interpolated precision values */
|
||||
|
||||
long i,j;
|
||||
|
||||
long cut_rp[NUM_RP_PTS]; /* number of rel docs needed to be retrieved
|
||||
for each recall-prec cutoff */
|
||||
long cut_fr[NUM_FR_PTS]; /* number of non-rel docs needed to be
|
||||
retrieved for each fall-recall cutoff */
|
||||
long cut_rprec[NUM_PREC_PTS]; /* Number of docs needed to be retrieved
|
||||
for each R-based prec cutoff */
|
||||
long current_cutoff, current_cut_rp, current_cut_fr, current_cut_rprec;
|
||||
|
||||
long rel_so_far = eval->num_rel_ret;
|
||||
|
||||
/* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all
|
||||
Y >= X) */
|
||||
int_precis = (float) rel_so_far / (float) eval->num_ret;
|
||||
|
||||
/* Discover cutoff values for this query */
|
||||
current_cutoff = NUM_CUTOFF - 1;
|
||||
while (current_cutoff > 0 && cutoff[current_cutoff] > eval->num_ret)
|
||||
current_cutoff--;
|
||||
for (i = 0; i < NUM_RP_PTS; i++)
|
||||
cut_rp[i] = ((eval->num_rel * i) + NUM_RP_PTS - 2) / (NUM_RP_PTS - 1);
|
||||
current_cut_rp = NUM_RP_PTS - 1;
|
||||
while (current_cut_rp > 0 && cut_rp[current_cut_rp] > eval->num_rel_ret)
|
||||
current_cut_rp--;
|
||||
for (i = 0; i < NUM_FR_PTS; i++)
|
||||
cut_fr[i] = ((MAX_FALL_RET * i) + NUM_FR_PTS - 2) / (NUM_FR_PTS - 1);
|
||||
current_cut_fr = NUM_FR_PTS - 1;
|
||||
while (current_cut_fr > 0 && cut_fr[current_cut_fr] > eval->num_ret - eval->num_rel_ret)
|
||||
current_cut_fr--;
|
||||
for (i = 1; i < NUM_PREC_PTS+1; i++)
|
||||
cut_rprec[i-1] = ((MAX_RPREC * eval->num_rel * i) + NUM_PREC_PTS - 2)
|
||||
/ (NUM_PREC_PTS - 1);
|
||||
current_cut_rprec = NUM_PREC_PTS - 1;
|
||||
while (current_cut_rprec > 0 && cut_rprec[current_cut_rprec]>eval->num_ret)
|
||||
current_cut_rprec--;
|
||||
|
||||
/* Loop over all retrieved docs in reverse order */
|
||||
for (j = eval->num_ret; j > 0; j--) {
|
||||
if (rel_so_far > 0) {
|
||||
recall = (float) rel_so_far / (float) eval->num_rel;
|
||||
precis = (float) rel_so_far / (float) j;
|
||||
if (j > eval->num_rel) {
|
||||
rel_precis = (float) rel_so_far / (float) eval->num_rel;
|
||||
}
|
||||
else {
|
||||
rel_precis = (float) rel_so_far / (float) j;
|
||||
}
|
||||
}
|
||||
else {
|
||||
recall = 0.0;
|
||||
precis = 0.0;
|
||||
rel_precis = 0.0;
|
||||
}
|
||||
rel_uap = rel_precis * rel_precis;
|
||||
if (int_precis < precis)
|
||||
int_precis = precis;
|
||||
while (j == cutoff[current_cutoff]) {
|
||||
eval->recall_cut[current_cutoff] = recall;
|
||||
eval->precis_cut[current_cutoff] = precis;
|
||||
eval->rel_precis_cut[current_cutoff] = rel_precis;
|
||||
eval->uap_cut[current_cutoff] = precis * recall;
|
||||
eval->rel_uap_cut[current_cutoff] = rel_uap;
|
||||
current_cutoff--;
|
||||
}
|
||||
|
||||
while (j == cut_rprec[current_cut_rprec]) {
|
||||
eval->R_prec_cut[current_cut_rprec] = precis;
|
||||
eval->int_R_prec_cut[current_cut_rprec] = int_precis;
|
||||
current_cut_rprec--;
|
||||
}
|
||||
|
||||
if (j == eval->num_rel) {
|
||||
eval->R_recall_precis = precis;
|
||||
eval->int_R_recall_precis = int_precis;
|
||||
}
|
||||
|
||||
if (tr_vec->tr[j-1].rel >= epi->relevance_level) {
|
||||
while (rel_so_far == cut_rp[current_cut_rp]) {
|
||||
eval->int_recall_precis[current_cut_rp] = int_precis;
|
||||
current_cut_rp--;
|
||||
}
|
||||
eval->recip_rank = 1.0 / (float) j;
|
||||
eval->rank_first_rel = j;
|
||||
rel_so_far--;
|
||||
}
|
||||
else {
|
||||
/* Note: for fallout-recall, the recall at X non-rel docs
|
||||
is used for the recall 'after' (X-1) non-rel docs.
|
||||
Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for
|
||||
Y retrieved docs where X-1 non-rel retrieved */
|
||||
while (current_cut_fr >= 0 &&
|
||||
j - rel_so_far == cut_fr[current_cut_fr] + 1) {
|
||||
eval->fall_recall[current_cut_fr] = recall;
|
||||
current_cut_fr--;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Fill in the 0.0 value for recall-precision (== max precision
|
||||
at any point in the retrieval ranking) */
|
||||
eval->int_recall_precis[0] = int_precis;
|
||||
|
||||
/* Fill in those cutoff values and averages that were not achieved
|
||||
because insufficient docs were retrieved. */
|
||||
for (i = 0; i < NUM_CUTOFF; i++) {
|
||||
if (eval->num_ret < cutoff[i]) {
|
||||
if (eval->num_rel_ret > 0) {
|
||||
eval->recall_cut[i] = ((float) eval->num_rel_ret /
|
||||
(float) eval->num_rel);
|
||||
eval->precis_cut[i] = ((float) eval->num_rel_ret /
|
||||
(float) cutoff[i]);
|
||||
}
|
||||
eval->rel_precis_cut[i] = (cutoff[i] < eval->num_rel) ?
|
||||
eval->precis_cut[i] :
|
||||
eval->recall_cut[i];
|
||||
eval->uap_cut[i] = eval->precis_cut[i] *
|
||||
eval->recall_cut[i];
|
||||
eval->rel_uap_cut[i] = eval->precis_cut[i] *
|
||||
eval->precis_cut[i];
|
||||
}
|
||||
}
|
||||
for (i = 0; i < NUM_FR_PTS; i++) {
|
||||
if (eval->num_ret - eval->num_rel_ret < cut_fr[i]) {
|
||||
if (eval->num_rel_ret > 0)
|
||||
eval->fall_recall[i] = (float) eval->num_rel_ret /
|
||||
(float) eval->num_rel;
|
||||
}
|
||||
}
|
||||
for (i = 0; i < NUM_PREC_PTS; i++) {
|
||||
if (eval->num_ret < cut_rprec[i]) {
|
||||
eval->R_prec_cut[i] = (float) eval->num_rel_ret /
|
||||
(float) cut_rprec[i];
|
||||
eval->int_R_prec_cut[i] = (float) eval->num_rel_ret /
|
||||
(float) cut_rprec[i];
|
||||
}
|
||||
}
|
||||
|
||||
if (eval->num_rel > eval->num_ret) {
|
||||
eval->R_recall_precis = (float) eval->num_rel_ret /
|
||||
(float)eval->num_rel;
|
||||
eval->int_R_recall_precis = (float) eval->num_rel_ret /
|
||||
(float)eval->num_rel;
|
||||
}
|
||||
|
||||
/* Calculate other indirect evaluation measure averages. */
|
||||
/* average recall-precis of 3 and 11 intermediate points */
|
||||
eval->int_av3_recall_precis =
|
||||
(eval->int_recall_precis[three_pts[0]] +
|
||||
eval->int_recall_precis[three_pts[1]] +
|
||||
eval->int_recall_precis[three_pts[2]]) / 3.0;
|
||||
for (i = 0; i < NUM_RP_PTS; i++) {
|
||||
eval->int_av11_recall_precis += eval->int_recall_precis[i];
|
||||
}
|
||||
eval->int_av11_recall_precis /= NUM_RP_PTS;
|
||||
|
||||
}
|
||||
|
||||
static void
|
||||
calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TR_VEC *tr_vec;
|
||||
TREC_EVAL *eval;
|
||||
long num_rel; /* Number relevant judged */
|
||||
long num_nonrel; /* Number nonrelevant judged */
|
||||
{
|
||||
long j;
|
||||
long nonrel_ret, nonrel_so_far, rel_so_far;
|
||||
long pref_top_nonrel_num = PREF_TOP_NONREL_NUM;
|
||||
long pref_top_50pRnonrel_num;
|
||||
long pref_top_25pRnonrel_num;
|
||||
long pref_top_25p2Rnonrel_num;
|
||||
long pref_top_10pRnonrel_num;
|
||||
long pref_top_Rnonrel_num;
|
||||
long bounded_5R_nonrel_so_far, bounded_10R_nonrel_so_far;
|
||||
|
||||
/* Calculate judgement based measures (dependent on only
|
||||
judged docs; no assumption of non-relevance if not judged) */
|
||||
/* Binary Preference measures; here expressed as all docs with a higher
|
||||
value of rel are to be preferred. Optimize by keeping track of nonrel
|
||||
seen so far */
|
||||
pref_top_nonrel_num = PREF_TOP_NONREL_NUM;
|
||||
pref_top_50pRnonrel_num = 50 + eval->num_rel;
|
||||
pref_top_25pRnonrel_num = 25 + eval->num_rel;
|
||||
pref_top_10pRnonrel_num = 10 + eval->num_rel;
|
||||
pref_top_Rnonrel_num = eval->num_rel;
|
||||
pref_top_25p2Rnonrel_num = 25 + (2 * eval->num_rel);
|
||||
nonrel_ret = 0;
|
||||
for (j = 0; j < tr_vec->num_tr; j++) {
|
||||
if (tr_vec->tr[j].rel == 0)
|
||||
nonrel_ret++;
|
||||
}
|
||||
nonrel_so_far = 0;
|
||||
rel_so_far = 0;
|
||||
bounded_5R_nonrel_so_far = 0;
|
||||
bounded_10R_nonrel_so_far = 0;
|
||||
for (j = 0; j < tr_vec->num_tr; j++) {
|
||||
if (tr_vec->tr[j].rel == 0) {
|
||||
if (nonrel_so_far < 5 * eval->num_rel) {
|
||||
bounded_5R_nonrel_so_far++;
|
||||
if (nonrel_so_far < 10 * eval->num_rel) {
|
||||
bounded_10R_nonrel_so_far++;
|
||||
}
|
||||
}
|
||||
nonrel_so_far++;
|
||||
}
|
||||
else if (tr_vec->tr[j].rel >= epi->relevance_level) {
|
||||
rel_so_far++;
|
||||
/* Add fraction of correct preferences. */
|
||||
/* Special case nonrel_so_far == 0 to avoid division by 0 */
|
||||
if (nonrel_so_far > 0) {
|
||||
eval->bpref_allnonrel += 1.0 - (((float) nonrel_so_far) /
|
||||
(float) num_nonrel);
|
||||
eval->bpref_retnonrel += 1.0 - (((float) nonrel_so_far) /
|
||||
(float) nonrel_ret);
|
||||
eval->bpref_retall += 1.0 - (((float) nonrel_so_far) /
|
||||
(float) nonrel_ret);
|
||||
eval->bpref_num_correct +=
|
||||
MIN (num_nonrel, pref_top_Rnonrel_num) -
|
||||
MIN (nonrel_so_far, pref_top_Rnonrel_num);
|
||||
eval->bpref += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) /
|
||||
(float) MIN (num_nonrel, pref_top_Rnonrel_num));
|
||||
eval->old_bpref += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) /
|
||||
(float) MIN (nonrel_ret, pref_top_Rnonrel_num));
|
||||
eval->bpref_topnonrel += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_nonrel_num)) /
|
||||
(float) MIN (num_nonrel, pref_top_nonrel_num));
|
||||
eval->bpref_top50pRnonrel += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_50pRnonrel_num)) /
|
||||
(float) MIN (num_nonrel, pref_top_50pRnonrel_num));
|
||||
eval->bpref_top25pRnonrel += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_25pRnonrel_num)) /
|
||||
(float) MIN (num_nonrel, pref_top_25pRnonrel_num));
|
||||
eval->bpref_top10pRnonrel += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) /
|
||||
(float) MIN (num_nonrel, pref_top_10pRnonrel_num));
|
||||
eval->old_bpref_top10pRnonrel += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) /
|
||||
(float) MIN (nonrel_ret, pref_top_10pRnonrel_num));
|
||||
eval->bpref_top25p2Rnonrel += 1.0 -
|
||||
(((float) MIN (nonrel_so_far, pref_top_25p2Rnonrel_num)) /
|
||||
(float) MIN (num_nonrel, pref_top_25p2Rnonrel_num));
|
||||
if (rel_so_far <= 5 && nonrel_so_far < 5)
|
||||
eval->bpref_5 += 1.0 - (float) nonrel_so_far /
|
||||
(float) MIN (num_nonrel, 5);
|
||||
if (rel_so_far <= 10 && nonrel_so_far < 10)
|
||||
eval->bpref_10 += 1.0 - (float) nonrel_so_far /
|
||||
(float) MIN (num_nonrel, 10);
|
||||
}
|
||||
else {
|
||||
eval->bpref += 1.0;
|
||||
eval->old_bpref += 1.0;
|
||||
eval->bpref_allnonrel += 1.0;
|
||||
eval->bpref_retnonrel += 1.0;
|
||||
eval->bpref_retall += 1.0;
|
||||
eval->bpref_topnonrel += 1.0;
|
||||
eval->bpref_top50pRnonrel += 1.0;
|
||||
eval->bpref_top25pRnonrel += 1.0;
|
||||
eval->bpref_top10pRnonrel += 1.0;
|
||||
eval->old_bpref_top10pRnonrel += 1.0;
|
||||
eval->bpref_top25p2Rnonrel += 1.0;
|
||||
if (rel_so_far <= 5)
|
||||
eval->bpref_5 += 1.0;
|
||||
if (rel_so_far <= 10)
|
||||
eval->bpref_10 += 1.0;
|
||||
|
||||
}
|
||||
eval->bpref_top5Rnonrel += 1.0 -
|
||||
(((float) bounded_5R_nonrel_so_far) /
|
||||
(float) MIN (num_nonrel, eval->num_rel * 5));
|
||||
eval->bpref_top10Rnonrel += 1.0 -
|
||||
(((float) bounded_10R_nonrel_so_far) /
|
||||
(float) MIN (num_nonrel, eval->num_rel * 10));
|
||||
eval->bpref_num_all += num_nonrel - nonrel_so_far;
|
||||
eval->bpref_num_ret += nonrel_ret - nonrel_so_far;
|
||||
}
|
||||
}
|
||||
if (eval->num_rel) {
|
||||
eval->bpref /= eval->num_rel;
|
||||
eval->old_bpref /= eval->num_rel;
|
||||
eval->bpref_allnonrel /= eval->num_rel;
|
||||
eval->bpref_retnonrel /= eval->num_rel;
|
||||
eval->bpref_topnonrel /= eval->num_rel;
|
||||
eval->bpref_top5Rnonrel /= eval->num_rel;
|
||||
eval->bpref_top10Rnonrel /= eval->num_rel;
|
||||
eval->bpref_top50pRnonrel /= eval->num_rel;
|
||||
eval->bpref_top25pRnonrel /= eval->num_rel;
|
||||
eval->bpref_top10pRnonrel /= eval->num_rel;
|
||||
eval->old_bpref_top10pRnonrel /= eval->num_rel;
|
||||
eval->bpref_top25p2Rnonrel /= eval->num_rel;
|
||||
if (eval->num_rel_ret) {
|
||||
eval->bpref_retall /= eval->num_rel_ret;
|
||||
eval->bpref_5 /= MIN (rel_so_far, 5);
|
||||
eval->bpref_10 /= MIN (rel_so_far, 10);
|
||||
}
|
||||
eval->bpref_num_possible = eval->num_rel *
|
||||
MIN (num_nonrel, pref_top_Rnonrel_num);
|
||||
}
|
||||
/* For those bpref measure variants which use the geometric mean instead
|
||||
of straight averages, compute them here. Original measure value
|
||||
is constrained to be greater than MIN_GEO_MEAN (for time being .00001,
|
||||
since trec_eval prints to four significant digits) */
|
||||
eval->gm_bpref = (float) log ((double)(MAX (eval->bpref,
|
||||
MIN_GEO_MEAN)));
|
||||
}
|
||||
|
||||
static void
|
||||
calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TR_VEC *tr_vec;
|
||||
TREC_EVAL *eval;
|
||||
long num_rel; /* Number relevant judged */
|
||||
long num_nonrel; /* Number nonrelevant judged */
|
||||
{
|
||||
double recall, precis; /* current recall, precision values */
|
||||
double rel_precis, rel_uap;/* relative precision, uap values */
|
||||
double int_precis; /* current interpolated precision values */
|
||||
|
||||
long i,j;
|
||||
long rel_so_far;
|
||||
|
||||
/* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all
|
||||
Y >= X) */
|
||||
rel_so_far = eval->num_rel_ret;
|
||||
int_precis = (float) rel_so_far / (float) eval->num_ret;
|
||||
|
||||
/* Loop over all retrieved docs in reverse order */
|
||||
for (j = eval->num_ret; j > 0; j--) {
|
||||
if (rel_so_far > 0) {
|
||||
recall = (float) rel_so_far / (float) eval->num_rel;
|
||||
precis = (float) rel_so_far / (float) j;
|
||||
if (j > eval->num_rel) {
|
||||
rel_precis = (float) rel_so_far / (float) eval->num_rel;
|
||||
}
|
||||
else {
|
||||
rel_precis = (float) rel_so_far / (float) j;
|
||||
}
|
||||
}
|
||||
else {
|
||||
recall = 0.0;
|
||||
precis = 0.0;
|
||||
rel_precis = 0.0;
|
||||
}
|
||||
rel_uap = rel_precis * rel_precis;
|
||||
if (int_precis < precis)
|
||||
int_precis = precis;
|
||||
eval->av_rel_precis += rel_precis;
|
||||
eval->av_rel_uap += rel_uap;
|
||||
|
||||
if (j < eval->num_rel) {
|
||||
eval->av_R_precis += precis;
|
||||
eval->int_av_R_precis += int_precis;
|
||||
}
|
||||
|
||||
if (tr_vec->tr[j-1].rel >= epi->relevance_level) {
|
||||
eval->int_av_recall_precis += int_precis;
|
||||
eval->av_recall_precis += precis;
|
||||
eval->avg_doc_prec += precis;
|
||||
rel_so_far--;
|
||||
}
|
||||
else {
|
||||
/* Note: for fallout-recall, the recall at X non-rel docs
|
||||
is used for the recall 'after' (X-1) non-rel docs.
|
||||
Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for
|
||||
Y retrieved docs where X-1 non-rel retrieved */
|
||||
if (j - rel_so_far < MAX_FALL_RET) {
|
||||
eval->av_fall_recall += recall;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (eval->num_ret - eval->num_rel_ret < MAX_FALL_RET) {
|
||||
if (eval->num_rel_ret > 0)
|
||||
eval->av_fall_recall += ((MAX_FALL_RET -
|
||||
(eval->num_ret - eval->num_rel_ret))
|
||||
* ((float)eval->num_rel_ret /
|
||||
(float)eval->num_rel));
|
||||
}
|
||||
if (eval->num_rel > eval->num_ret) {
|
||||
for (i = eval->num_ret; i < eval->num_rel; i++) {
|
||||
eval->av_R_precis += (float) eval->num_rel_ret /
|
||||
(float) i;
|
||||
eval->int_av_R_precis += (float) eval->num_rel_ret /
|
||||
(float) i;
|
||||
}
|
||||
}
|
||||
|
||||
/* Calculate all the other averages */
|
||||
if (eval->num_rel_ret > 0) {
|
||||
eval->av_recall_precis /= eval->num_rel;
|
||||
eval->int_av_recall_precis /= eval->num_rel;
|
||||
}
|
||||
|
||||
eval->av_fall_recall /= MAX_FALL_RET;
|
||||
|
||||
eval->av_rel_precis /= eval->num_ret;
|
||||
eval->av_rel_uap /= eval->num_ret;
|
||||
|
||||
if (eval->num_rel) {
|
||||
eval->av_R_precis /= eval->num_rel;
|
||||
eval->int_av_R_precis /= eval->num_rel;
|
||||
}
|
||||
|
||||
/* For those measure variants which use the geometric mean instead
|
||||
of straight averages, compute them here. Original measure value
|
||||
is constrained to be greater than MIN_GEO_MEAN (for time being .00001,
|
||||
since trec_eval prints to four significant digits) */
|
||||
eval->gm_ap = (float) log ((double)(MAX (eval->av_recall_precis,
|
||||
MIN_GEO_MEAN)));
|
||||
}
|
||||
|
||||
static void
|
||||
calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TR_VEC *tr_vec;
|
||||
TREC_EVAL *eval;
|
||||
long num_rel; /* Number relevant judged */
|
||||
long num_nonrel; /* Number nonrelevant judged */
|
||||
{
|
||||
|
||||
if (eval->num_rel) {
|
||||
eval->exact_recall = (double) eval->num_rel_ret / eval->num_rel;
|
||||
eval->exact_precis = (double) eval->num_rel_ret / eval->num_ret;
|
||||
eval->exact_uap = eval->exact_recall * eval->exact_precis;
|
||||
if (eval->num_rel > eval->num_ret) {
|
||||
eval->exact_rel_precis = eval->exact_precis;
|
||||
}
|
||||
else {
|
||||
eval->exact_rel_precis = eval->exact_recall;
|
||||
}
|
||||
eval->exact_rel_uap = eval->exact_precis * eval->exact_precis;
|
||||
eval->exact_utility =
|
||||
epi->utility_a * eval->num_rel_ret +
|
||||
epi->utility_b * (eval->num_ret - eval->num_rel_ret) +
|
||||
epi->utility_c * (eval->num_rel - eval->num_rel_ret) +
|
||||
epi->utility_d * (epi->num_docs_in_coll + eval->num_rel_ret
|
||||
- eval->num_ret - eval->num_rel);
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel)
|
||||
EVAL_PARAM_INFO *epi;
|
||||
TR_VEC *tr_vec;
|
||||
TREC_EVAL *eval;
|
||||
long num_rel; /* Number relevant judged */
|
||||
long num_nonrel; /* Number nonrelevant judged */
|
||||
{
|
||||
double recall, precis; /* current recall, precision values */
|
||||
double rel_precis, rel_uap;/* relative precision, uap values */
|
||||
double int_precis; /* current interpolated precision values */
|
||||
|
||||
long i,j;
|
||||
|
||||
long bucket;
|
||||
long last_time_bucket = NUM_TIME_PTS; /* Last time bucket filled in */
|
||||
|
||||
long rel_so_far = eval->num_rel_ret;
|
||||
long min_ret_rel = MIN(eval->num_rel, eval->num_ret);
|
||||
|
||||
/* Loop over all retrieved docs in reverse order */
|
||||
for (j = eval->num_ret; j > 0; j--) {
|
||||
if (rel_so_far > 0) {
|
||||
recall = (float) rel_so_far / (float) eval->num_rel;
|
||||
precis = (float) rel_so_far / (float) j;
|
||||
if (j > eval->num_rel) {
|
||||
rel_precis = (float) rel_so_far / (float) eval->num_rel;
|
||||
}
|
||||
else {
|
||||
rel_precis = (float) rel_so_far / (float) j;
|
||||
}
|
||||
}
|
||||
else {
|
||||
recall = 0.0;
|
||||
precis = 0.0;
|
||||
rel_precis = 0.0;
|
||||
}
|
||||
rel_uap = rel_precis * rel_precis;
|
||||
if (int_precis < precis)
|
||||
int_precis = precis;
|
||||
|
||||
bucket = tr_vec->tr[j-1].sim *
|
||||
((double) NUM_TIME_PTS / (double) MAX_TIME);
|
||||
if (bucket < 0) bucket = 0;
|
||||
if (bucket >= NUM_TIME_PTS) bucket = NUM_TIME_PTS-1;
|
||||
if (tr_vec->tr[j-1].rel >= epi->relevance_level)
|
||||
eval->time_num_rel[bucket]++;
|
||||
else
|
||||
eval->time_num_nrel[bucket]++;
|
||||
eval->time_precis[bucket] = (float)rel_so_far /
|
||||
(float) eval->num_ret;
|
||||
eval->time_relprecis[bucket] = ((float)rel_so_far) /
|
||||
(float) min_ret_rel;
|
||||
eval->time_uap[bucket] = (float) rel_so_far * rel_so_far /
|
||||
((float) eval->num_ret * (float) min_ret_rel);
|
||||
eval->time_reluap[bucket] = (float) rel_so_far * rel_so_far /
|
||||
((float) min_ret_rel * (float) min_ret_rel);
|
||||
eval->time_utility[bucket] =
|
||||
epi->utility_a * rel_so_far +
|
||||
epi->utility_b * (j - rel_so_far) +
|
||||
epi->utility_c * (eval->num_rel - rel_so_far) +
|
||||
epi->utility_d * (epi->num_docs_in_coll +
|
||||
rel_so_far - j - eval->num_rel);
|
||||
|
||||
/* Need to fill in buckets up to last bucket */
|
||||
/* note assumes buckets are decreasing */
|
||||
/* Must do here since utility can be negative and zero
|
||||
cannot be used as flag later */
|
||||
for (i = bucket+1; i < last_time_bucket; i++) {
|
||||
eval->time_precis[i] = eval->time_precis[bucket];
|
||||
eval->time_relprecis[i] = eval->time_relprecis[bucket];
|
||||
eval->time_uap[i] = eval->time_uap[bucket];
|
||||
eval->time_reluap[i] = eval->time_reluap[bucket];
|
||||
eval->time_utility[i] = eval->time_utility[bucket];
|
||||
}
|
||||
last_time_bucket = bucket;
|
||||
}
|
||||
|
||||
eval->time_cum_rel[0] = eval->time_num_rel[0];
|
||||
eval->av_time_cum_rel = eval->time_num_rel[0];
|
||||
for (i=1; i< NUM_TIME_PTS; i++) {
|
||||
eval->time_cum_rel[i] = eval->time_cum_rel[i-1] + eval->time_num_rel[i];
|
||||
eval->av_time_cum_rel += eval->time_cum_rel[i];
|
||||
eval->av_time_precis += eval->time_precis[i];
|
||||
eval->av_time_relprecis += eval->time_relprecis[i];
|
||||
eval->av_time_uap += eval->time_uap[i];
|
||||
eval->av_time_reluap += eval->time_reluap[i];
|
||||
eval->av_time_utility += eval->time_utility[i];
|
||||
}
|
||||
eval->av_time_cum_rel /= NUM_TIME_PTS;
|
||||
eval->av_time_precis /= NUM_TIME_PTS;
|
||||
eval->av_time_relprecis /= NUM_TIME_PTS;
|
||||
eval->av_time_uap /= NUM_TIME_PTS;
|
||||
eval->av_time_reluap /= NUM_TIME_PTS;
|
||||
eval->av_time_utility /= NUM_TIME_PTS;
|
||||
}
|
||||
Reference in New Issue
Block a user