wassname/keras-language-modeling
1
0
Fork 0
mirror of https://github.com/wassname/keras-language-modeling.git synced 2026-09-09 11:25:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

added trecqa eval

Browse Source
This commit is contained in:
codekansas
2016-04-27 00:03:23 -04:00
parent 3075ffcf54
commit 76bdb1323a
36 changed files with 15453 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,6 +1,7 @@
# data / models (also potentially very large)
data/
models/
treq_eval*
# pyc files aren't necessary
*.pyc
insuranceqa.py
+7 -14
View File
@@ -116,7 +116,7 @@ def get_data(f_name):
def get_accurate_percentage(model, questions, good_answers, bad_answers, n_eval=512):
if n_eval != 'all':
if n_eval != -1:
questions = questions[-n_eval:]
good_answers = good_answers[-n_eval:]
bad_answers = bad_answers[-n_eval:]
@@ -180,11 +180,11 @@ data_sets = [
q_data, ag_data, ab_data, targets = get_data(data_sets[0])
qv_data, avg_data, avb_data, v_targets = get_data(data_sets[1])
test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
test_model.load_weights(os.path.join(models_path, 'iqa_model_for_training_iter_900.h5'))
# found through experimentation that ~24 epochs generalized the best
print('Fitting model')
for i in range(10000):
for i in range(401, 10000):
print('----- %d -----' % i)
np.random.shuffle(ab_data)
train_model.fit([q_data, ag_data, ab_data], targets, nb_epoch=1, batch_size=128, validation_data=[[qv_data, avg_data, avb_data], v_targets], shuffle=True)
@@ -192,25 +192,18 @@ for i in range(10000):
if i % 100 == 0:
train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training_iter_%d.h5' % i), overwrite=True)
test_model.save_weights(os.path.join(models_path, 'iqa_model_for_training_iter_%d.h5' % i), overwrite=True)
print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval=-1)))
eq_data, ea_data, en_good = get_eval(data_sets[1])
print('MRR: {}'.format(get_mrr(test_model, eq_data, ea_data, en_good)))
train_model.save_weights(os.path.join(models_path, 'iqa_model_for_training.h5'), overwrite=True)
test_model.save_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'), overwrite=True)
test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
import keras.backend as K
get_attention = K.function([test_model.layers[0].input, test_model.layers[1].input], [test_model.layers[3].get_output_at(0)])
attention = get_attention([q_data[:20], ag_data[:20]])[0]
for i in range(20):
print('----- %d -----' % i)
print(revert(q_data[i]))
print(revert(ag_data[i]))
print([np.linalg.norm(x) for x in attention[i]])
# the model actually did really well, predicted correct vs. incorrect answer 85% of the time on the validation set
test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval='all')))
print('Percent correct: {}'.format(get_accurate_percentage(test_model, q_data, ag_data, ab_data, n_eval=-1)))
q_data, a_data, n_good = get_eval(data_sets[1])
test_model.load_weights(os.path.join(models_path, 'iqa_model_for_prediction.h5'))
keras_attention_model.py
+21 -8
View File
@@ -33,48 +33,61 @@ def make_model(maxlen_question, maxlen_answer, n_words, n_lstm_dims=141, n_embed
embedding = Embedding(n_words, n_embed_dims)
# forward and backward lstms
f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=False)
b_lstm = LSTM(n_lstm_dims, name='bq_lstm', go_backwards=True, consume_less='mem', return_sequences=False)
f_lstm = LSTM(n_lstm_dims, name='fq_lstm', consume_less='mem', return_sequences=True)
b_lstm = LSTM(n_lstm_dims, name='bq_lstm', go_backwards=True, consume_less='mem', return_sequences=True)
f_lstm_2 = LSTM(n_lstm_dims, consume_less='mem', return_sequences=False)
b_lstm_2 = LSTM(n_lstm_dims, go_backwards=True, consume_less='mem', return_sequences=False)
# question part
q_emb = embedding(question)
q_emb = Dropout(0.25)(q_emb)
q_emb = Convolution1D(nb_filter=64, filter_length=5)(q_emb)
q_emb = MaxPooling1D(pool_length=2)(q_emb)
q_fl = f_lstm(q_emb)
q_bl = b_lstm(q_emb)
q_out = merge([q_fl, q_bl], mode='concat', concat_axis=-1)
q_out = Dropout(0.25)(q_out)
q_fl = f_lstm_2(q_out)
q_bl = b_lstm_2(q_out)
q_out = merge([q_fl, q_bl], mode='concat', concat_axis=-1)
# q_out = Permute((2, 1))(q_out)
# q_out = Convolution1D(nb_filter=64, filter_length=5)(q_out)
# q_out = MaxPooling1D(2)(q_out)
# q_out = Flatten()(q_out)
# forward and backward attention lstms (paying attention to q_out)
f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=False)
b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, consume_less='mem', return_sequences=False)
f_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=True)
b_lstm_attention = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, consume_less='mem', return_sequences=True)
f_lstm_attention_2 = AttentionLSTM(n_lstm_dims, q_out, consume_less='mem', return_sequences=False)
b_lstm_attention_2 = AttentionLSTM(n_lstm_dims, q_out, go_backwards=True, consume_less='mem', return_sequences=False)
conv = Convolution1D(nb_filter=64, filter_length=5)
# answer part
ag_emb = embedding(answer_good)
ag_emb = Dropout(0.25)(ag_emb)
ag_emb = conv(ag_emb)
ag_emb = MaxPooling1D(pool_length=2)(ag_emb)
ag_fl = f_lstm_attention(ag_emb)
ag_bl = b_lstm_attention(ag_emb)
ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=-1)
ag_out = Dropout(0.25)(ag_out)
ag_fl = f_lstm_attention_2(ag_out)
ag_bl = b_lstm_attention_2(ag_out)
ag_out = merge([ag_fl, ag_bl], mode='concat', concat_axis=-1)
# ag_out = Permute((2, 1))(ag_out)
# ag_out = conv(ag_out)
# ag_out = MaxPooling1D(2)(ag_out)
# ag_out = Flatten()(ag_out)
ab_emb = embedding(answer_bad)
ab_emb = Dropout(0.25)(ab_emb)
ab_emb = conv(ab_emb)
ab_emb = MaxPooling1D(pool_length=2)(ab_emb)
ab_fl = f_lstm_attention(ab_emb)
ab_bl = b_lstm_attention(ab_emb)
ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=-1)
ab_out = Dropout(0.25)(ab_out)
ab_fl = f_lstm_attention_2(ab_out)
ab_bl = b_lstm_attention_2(ab_out)
ab_out = merge([ab_fl, ab_bl], mode='concat', concat_axis=-1)
# ab_out = Permute((2, 1))(ab_out)
# ab_out = conv(ab_out)
# ab_out = MaxPooling1D(2)(ab_out)
@@ -86,7 +99,7 @@ def make_model(maxlen_question, maxlen_answer, n_words, n_lstm_dims=141, n_embed
good_out = merge([q_out, ag_out], name='good', mode='cos', dot_axes=1)
bad_out = merge([q_out, ab_out], name='bad', mode='cos', dot_axes=1)
target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.3 - x[0] + x[1]), output_shape=lambda x: x[0])
target = merge([good_out, bad_out], name='target', mode=lambda x: K.maximum(1e-3, 0.2 - x[0] + x[1]), output_shape=lambda x: x[0])
train_model = Model(input=[question, answer_good, answer_bad], output=target)
test_model = Model(input=[question, answer_good], output=good_out)
trec_eval.8.1/Changelog
+59
View File
@@ -0,0 +1,59 @@
Version 8.1, Added infAP, minor bug fixes
7/24/06 Improved infAP comments (implementation verified by Yilmaz).
trec_eval_help.c: allow longer measure explanations.
6/27/06 get_opt.c Fixed error message
6/22/06 Added measure infAP (Aslam et al) to allow judging only sample
of pools. -1 in qrels file interpreted as pool doc not judged.
6/22/06 trvec_teval.c: fixed bugs in calculation of bpref if multiple
relevance levels were used and a non-default relevance level
was given. (Eg. A doc with rel level of 2 was counted as unjudged
rather than judged nonrel if a relevance level of 3 was needed
to consider relevant.)
4/5/06 Changed comments in README, trec_eval.c, trec_eval_help.c files
which incorrectly claimed queries with no relevant docs are
ignored (this was true with very old versions of trec_eval). Now
reads that queries with no relevance information are ignored.
Giorgio Di Nunzio and Nicola Ferro,
------------------------------------------------------------------------------
Version 8.0, full bpref bug fix, see file bpref_bug. I decided to up the
version number since bpref results are incompatible with previous
results (though the changes are small).
11/8/05: Bpref_bug: New file explaining bug and impact (conclusions after
rerunning all of SIGIR 2004 bpref paper experiments).
11/5/05: Added new measures: micro_prec, micro_recall, micro_bpref. I thought
I had an application for micro_bpref averaging (summing components of
measure over all docs (ignoring topics) and then computing measure),
but micro_bpref still proved a rotten measure. Left code in case
someone ever actually finds an application for valid micro averaging.
11/5/05: Added new measures: old_bpref, old_bpref_top10pRnonrel. These are
the old buggy measures included only for backward comparisons.
11/5/05: trvec_teval.c: Broke apart old trvec_trec_eval to calculate
different types of measures separately. Very hard to decipher
old code (though still difficult with new code) since parts of
the calculations for a measure were so far apart.
------------------------------------------------------------------------------
Version 7.4, minor changes from 7.3
11/4/05: trvec_teval.c: fixed bpref bug if very low (< R) numbers of non-rel
judgements available (divided by num_nonrel_ret instead of
num_nonrel). (pointed out by Ian Soboroff).
11/3/05: trvec_teval.c: bpref_10, bpref_5 had zero division problems if
no rel docs were retrieved. (pointed out by Ian Soboroff).
10/23/05: form_trvec.c: Added check for duplicate docno's in results and qrels.
(pointed out by Shlomo Geva. Default behavior used to be that
duplicate result docno's were always non-rel, but that changed in
later versions, so had better test explicitly for it and complain).
10/23/05: README: sample invocation of trec_eval had arguments reversed.
(pointed out by Carol Peters).
10/23/05: moved gm_ap to be a major measure (always printed). changed
measures.c, test/out*, README
------------------------------------------------------------------------------
Version 7.3, a reasonably major rewrite from earlier versions in terms
of internal structure and default output format (now relational), but
the input format and measures calculated remain the same (or at least
upward compatible).
trec_eval.8.1/Makefile
+146
View File
@@ -0,0 +1,146 @@
BIN = /home/smart/bin
H = .
VERSIONID = 8.1
# gcc
CC = gcc
CFLAGS = -g -I$H -O3 -Wall -DVERSIONID=\"$(VERSIONID)\"
CFLAGS = -g -I$H -Wall -DVERSIONID=\"$(VERSIONID)\"
# cc
###CC = cc
###CFLAGS = -I$H -g -DVERSIONID=\"$(VERSIONID)\"
# Other macros used in some or all makefiles
INSTALL = /bin/mv
OBJS = trec_eval.o get_qrels.o get_top.o form_trvec.o measures.o print_meas.o\
trvec_teval.o buf_util.o error_msgs.o \
trec_eval_help.o
SRCS = trec_eval.c get_qrels.c get_top.c form_trvec.c measures.c print_meas.c\
trvec_teval.c buf_util.c error_msgs.c \
trec_eval_help.c
SRCH = common.h trec_eval.h smart_error.h sysfunc.h tr_vec.h buf.h
SRCOTHER = README Makefile test bpref_bug Changelog
trec_eval: $(SRCS) Makefile $(SRCH)
$(CC) $(CFLAGS) -o trec_eval $(SRCS) -lm
install: $(BIN)/trec_eval
quicktest: trec_eval
./trec_eval test/qrels.test test/results.test | diff - test/out.test
./trec_eval -a test/qrels.test test/results.test | diff - test/out.test.a
./trec_eval -a -q test/qrels.test test/results.test | diff - test/out.test.aq
./trec_eval -a -q -c test/qrels.test test/results.trunc | diff - test/out.test.aqc
./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc | diff - test/out.test.aqcM
./trec_eval -a -q -l2 test/qrels.rel_level test/results.test | diff - test/out.test.aql
/bin/echo "Test succeeeded"
longtest: trec_eval
/bin/rm -rf test.long; mkdir test.long
./trec_eval test/qrels.test test/results.test > test.long/out.test
./trec_eval -a test/qrels.test test/results.test > test.long/out.test.a
./trec_eval -a -q test/qrels.test test/results.test > test.long/out.test.aq
./trec_eval -a -q -c test/qrels.test test/results.trunc > test.long/out.test.aqc
./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc > test.long/out.test.aqcM
./trec_eval -a -q -l2 test/qrels.rel_level test/results.test > test.long/out.test.aql
diff test.long test
$(BIN)/trec_eval: trec_eval
if [ -f $@ ]; then $(INSTALL) $@ $@.old; fi;
$(INSTALL) trec_eval $@
##4##########################################################################
##5##########################################################################
# All code below this line (except for automatically created dependencies)
# is independent of this particular makefile, and should not be changed!
#############################################################################
#########################################################################
# Odds and ends #
#########################################################################
clean semiclean:
/bin/rm -f *.o *.BAK *~ trec_eval trec_eval.*.tar out.trec_eval Makefile.bak
tar:
-/bin/rm -rf ./trec_eval.$(VERSIONID)
mkdir trec_eval.$(VERSIONID)
cp -rp $(SRCOTHER) $(SRCS) $(SRCH) trec_eval.$(VERSIONID)
tar cf - ./trec_eval.$(VERSIONID) > trec_eval.$(VERSIONID).tar
lint:
lint $(SRCS)
#########################################################################
# Determining program dependencies #
#########################################################################
depend:
grep '^#[ ]*include' *.c \
| sed -e 's?:[^"]*"\([^"]*\)".*?: \$H/\1?' \
-e '/</d' \
-e '/functions.h/d' \
-e 's/\.c/.o/' \
-e 's/\.y/.o/' \
-e 's/\.l/.o/' \
> makedep
echo '/^# DO NOT DELETE THIS LINE/+2,$$d' >eddep
echo '$$r makedep' >>eddep
echo 'w' >>eddep
cp Makefile Makefile.bak
ed - Makefile < eddep
/bin/rm eddep makedep
echo '# DEPENDENCIES MUST END AT END OF FILE' >> Makefile
echo '# IF YOU PUT STUFF HERE IT WILL GO AWAY' >> Makefile
echo '# see make depend above' >> Makefile
# DO NOT DELETE THIS LINE -- make depend uses it
buf_util.o: ./common.h
buf_util.o: ./sysfunc.h
buf_util.o: ./buf.h
error_msgs.o: ./smart_error.h
error_msgs.o: ./sysfunc.h
form_trvec.o: ./common.h
form_trvec.o: ./sysfunc.h
form_trvec.o: ./smart_error.h
form_trvec.o: ./tr_vec.h
form_trvec.o: ./trec_eval.h
form_trvec.o: ./buf.h
get_qrels.o: ./common.h
get_qrels.o: ./sysfunc.h
get_qrels.o: ./smart_error.h
get_qrels.o: ./trec_eval.h
get_top.o: ./common.h
get_top.o: ./sysfunc.h
get_top.o: ./smart_error.h
get_top.o: ./trec_eval.h
measures.o: ./common.h
measures.o: ./sysfunc.h
measures.o: ./buf.h
measures.o: ./trec_eval.h
print_meas.o: ./common.h
print_meas.o: ./sysfunc.h
print_meas.o: ./buf.h
print_meas.o: ./trec_eval.h
trec_eval.o: ./common.h
trec_eval.o: ./sysfunc.h
trec_eval.o: ./smart_error.h
trec_eval.o: ./tr_vec.h
trec_eval.o: ./trec_eval.h
trec_eval.o: ./buf.h
trec_eval_help.o: ./common.h
trvec_teval.o: ./common.h
trvec_teval.o: ./sysfunc.h
trvec_teval.o: ./smart_error.h
trvec_teval.o: ./tr_vec.h
trvec_teval.o: ./trec_eval.h
# DEPENDENCIES MUST END AT END OF FILE
# IF YOU PUT STUFF HERE IT WILL GO AWAY
# see make depend above
trec_eval.8.1/README
+356
View File
@@ -0,0 +1,356 @@
trec_eval is the standard tool used by the TREC community for
evaluating an ad hoc retrieval run, given the results file and a
standard set of judged results.
------------------------------------------------------------------------------
Installation: Should be as easy as typing "make" in the source directory,
if gcc is available. Otherwise, comment out the gcc lines (lines 5-6) and
uncomment out the cc lines (lines 9-10)
If you wish the trec_eval binary to be placed in a standard location, alter
the first line of Makefile appropriately.
------------------------------------------------------------------------------
Testing: sample input and output files are included in the directory test.
"make quicktest" will perform some sample simple evaluations and compare
the results.
------------------------------------------------------------------------------
Usage: Most options can be ignored. The only one most folks will need
is the "-q" flag, to indicate whether to output results for individual
queries as well as the averages over all queries. Official TREC usage
might be something like
trec_eval -q -c -M1000 official_qrels submitted_results
to ensure correct evaluation if submitted_results doesn't have results
for all queries, or returns more than 1000 documents per query.
------------------------------------------------------------------------------
Change Log (only recent)
------------------------------------------------------------------------------
Version 8.1, Added infAP, minor bug fixes
7/24/06 Improved infAP comments (implementation verified by Yilmaz).
trec_eval_help.c: allow longer measure explanations.
6/27/06 get_opt.c Fixed error message
6/22/06 Added measure infAP (Aslam et al) to allow judging only sample
of pools. -1 for rel in qrels file interpreted as pool doc not judged.
6/22/06 trvec_teval.c: fixed bugs in calculation of bpref if multiple
relevance levels were used and a non-default relevance level
was given. (Eg. A doc with rel level of 2 was counted as unjudged
rather than judged nonrel if a relevance level of 3 was needed
to consider relevant.)
4/5/06 Changed comments in README, trec_eval.c, trec_eval_help.c files
which incorrectly claimed queries with no relevant docs are
ignored (this was true with very old versions of trec_eval). Now
reads that queries with no relevance information are ignored.
Giorgio Di Nunzio and Nicola Ferro,
------------------------------------------------------------------------------
Version 8.0, full bpref bug fix, see file bpref_bug. I decided to up the
version number since bpref results are incompatible with previous
results (though the changes are small).
------------------------------------------------------------------------------
------------------------------------------------------------------------------
Files:
Makefile Compile and test trec_eval
README This file
test Collection of sample input and output for trec_eval
trec_eval.c Main procedure
get_qrels.c Called by main to read the standard judged documents (qrels)
get_top.c Called by main to read the results file to be evaluated
form_trvec.c Called by main to put the results and qrels for an individual
query in the proper format to be evaluated.
trvec_teval.c Called by main to evaluate an individual query
print_meas.c Called by main to print an evaluated query, and to accumulate
the results for later averaging over the queries.
measures.c Description of the measures used by printing.
trec_eval_help.c Descriptions of trec_eval, the output, and the measures.
trec_eval.h Basic evaluation structures.
bpref_bug: Description of bug in bpref that existed in trec_eval versions 6
through 7.3.
The rest of the files are small utility portions from SMART.
tr_vec.h
smart_error.h
sysfunc.h
buf.h
common.h
buf_util.c
error_msgs.c
------------------------------------------------------------------------------
The rest of this file consists of information printed by "trec_eval -h":
(If you REALLY want a complete list of measures calculated, you can add the
time based measures and run "trec_eval -T -h".)
trec_eval [-h] [-q] [-a] [-o] [-c] [-l<num> [-N<num>] [-M<num>] [-Ua<num>] [-Ub<num>] [-Uc<num>] [-Ud<num>] [-T] trec_rel_file trec_top_file
Calculate and print various evaluation measures, evaluating the results
in trec_top_file against the relevance judgements in trec_rel_file.
There are a fair number of options, of which only the lower case options are
normally ever used.
-h: Print full help message and exit
-q: In addition to summary evaluation, give evaluation for each query
-a: Print all evaluation measures calculated, instead of just the
main official measures for TREC.
-o: Print everything out in old, nonrelational format (default is relational)
-c: Average over the complete set of queries in the relevance judgements
instead of the queries in the intersection of relevance judgements
and results. Missing queries will contribute a value of 0 to all
evaluation measures (which may or may not be reasonable for a
particular evaluation measure, but is reasonable for standard TREC
measures.)
-l<num>: Num indicates the minimum relevance judgement value needed for
a document to be called relevant. (All measures used by TREC eval are
based on binary relevance). Used if trec_rel_file contains relevance
judged on a multi-relevance scale. Default is 1.
-N<num>: Number of docs in collection
-M<num>: Max number of docs per topic to use in evaluation (discard rest).
-Ua<num>: Value to use for 'a' coefficient of utility computation.
relevant nonrelevant
retrieved a b
nonretrieved c d
-Ub<num>: Value to use for 'b' coefficient of utility computation.
-Uc<num>: Value to use for 'c' coefficient of utility computation.
-Ud<num>: Value to use for 'd' coefficient of utility computation.
-J: Calculate all values only over the judged (either relevant or
nonrelevant) documents. All unjudged documents are removed from the
retrieved set before any calculations (possibly leaving an empty set).
DO NOT USE, unless you really know what you're doing - very easy to get
reasonable looking, but invalid, numbers.
-T: Treat similarity as time that document retrieved. Compute
several time-based measures after ranking docs by time retrieved
(first doc (lowest sim) retrieved ranked highest).
Only done if -a selected.
Read text tuples from trec_top_file of the form
030 Q0 ZF08-175-870 0 4238 prise1
qid iter docno rank sim run_id
giving TREC document numbers (a string) retrieved by query qid
(a string) with similarity sim (a float). The other fields are ignored,
with the exception that the run_id field of the last line is kept and
output. In particular, note that the rank field is ignored here;
internally ranks are assigned by sorting by the sim field with ties
broken deterministicly (using docno).
Sim is assumed to be higher for the docs to be retrieved first.
File may contain no NULL characters.
Lines may contain fields after the run_id; they are ignored.
Relevance for each docno to qid is determined from text_qrels_file, which
consists of text tuples of the form
qid iter docno rel
giving TREC document numbers (docno, a string) and their relevance (rel,
a non-negative integer less than 128, or -1 (unjudged))
to query qid (a string). iter string field is ignored.
Fields are separated by whitespace, string fields can contain no whitespace.
File may contain no NULL characters.
The text tuples with relevance judgements are converted to TR_VEC form
and then submitted to the SMART evaluation routines.
The did,rank,sim fields of TR_VEC are filled in from trec_top_file;
action,iter fields are set to 0.
The rel field is set to -1 if the document was not in the pool (not in
text_qrels_file) or -2 if the document was in the pool but unjudged (some
measures (infAP) allow the pool to be sampled instead of judged fully).
Otherwise it is set to the value in text_qrels_file.
Most measures, but not all, will treat -1 or -2 the same as 0,
namely nonrelevant. Note that relevance_level is used to
determine if the document is relevant during score calculations.
Queries for which there is no relevance information are ignored.
Warning: queries for which there are relevant docs but no retrieved docs
are also ignored by default. This allows systems to evaluate over subsets
of the relevant docs, but means if a system improperly retrieves no docs,
it will not be detected. Use the -c flag to avoid this behavior.
EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT.
Relational Format prints the same values, but all lines are of the form
measure_name query value
1. Total number of documents over all queries
Retrieved:
Relevant:
Rel_ret: (relevant and retrieved)
These should be self-explanatory. All values are totals over all
queries being evaluated.
2. Interpolated Recall - Precision Averages:
at 0.00
at 0.10
...
at 1.00
See any standard IR text (especially by Salton) for more details of
recall-precision evaluation. Measures precision (percent of retrieved
docs that are relevant) at various recall levels (after a certain
percentage of all the relevant docs for that query have been retrieved).
'Interpolated' means that, for example, precision at recall
0.10 (ie, after 10% of rel docs for a query have been retrieved) is
taken to be MAXIMUM of precision at all recall points >= 0.10.
Values are averaged over all queries (for each of the 11 recall levels).
These values are used for Recall-Precision graphs.
3. Average precision (non-interpolated) over all rel docs
The precision is calculated after each relevant doc is retrieved.
If a relevant doc is not retrieved, its precision is 0.0.
All precision values are then averaged together to get a single number
for the performance of a query. Conceptually this is the area
underneath the recall-precision graph for the query.
The values are then averaged over all queries.
4. Precision:
at 5 docs
at 10 docs
...
at 1000 docs
The precision (percent of retrieved docs that are relevant) after X
documents (whether relevant or nonrelevant) have been retrieved.
Values averaged over all queries. If X docs were not retrieved
for a query, then all missing docs are assumed to be non-relevant.
5. R-Precision (precision after R (= num_rel for a query) docs retrieved):
Measures precision (or recall, they're the same) after R docs
have been retrieved, where R is the total number of relevant docs
for a query. Thus if a query has 40 relevant docs, then precision
is measured after 40 docs, while if it has 600 relevant docs, precision
is measured after 600 docs. This avoids some of the averaging
problems of the 'precision at X docs' values in (4) above.
If R is greater than the number of docs retrieved for a query, then
the nonretrieved docs are all assumed to be nonrelevant.
Major measures (again) with their relational names:
num_ret Total number of documents retrieved over all queries
num_rel Total number of relevant documents over all queries
num_rel_ret Total number of relevant documents retrieved over all queries
map Mean Average Precision (MAP)
gm_ap Average Precision. Geometric Mean, q_score=log(MAX(map,.00001))
R-prec R-Precision (Precision after R (= num-rel for topic) documents retrieved)
bpref Binary Preference, top R judged nonrel
recip_rank Reciprical rank of top relevant document
ircl_prn.0.00 Interpolated Recall - Precision Averages at 0.00 recall
ircl_prn.0.10 Interpolated Recall - Precision Averages at 0.10 recall
ircl_prn.0.20 Interpolated Recall - Precision Averages at 0.20 recall
ircl_prn.0.30 Interpolated Recall - Precision Averages at 0.30 recall
ircl_prn.0.40 Interpolated Recall - Precision Averages at 0.40 recall
ircl_prn.0.50 Interpolated Recall - Precision Averages at 0.50 recall
ircl_prn.0.60 Interpolated Recall - Precision Averages at 0.60 recall
ircl_prn.0.70 Interpolated Recall - Precision Averages at 0.70 recall
ircl_prn.0.80 Interpolated Recall - Precision Averages at 0.80 recall
ircl_prn.0.90 Interpolated Recall - Precision Averages at 0.90 recall
ircl_prn.1.00 Interpolated Recall - Precision Averages at 1.00 recall
P5 Precision after 5 docs retrieved
P10 Precision after 10 docs retrieved
P15 Precision after 15 docs retrieved
P20 Precision after 20 docs retrieved
P30 Precision after 30 docs retrieved
P100 Precision after 100 docs retrieved
P200 Precision after 200 docs retrieved
P500 Precision after 500 docs retrieved
P1000 Precision after 1000 docs retrieved
Minor measures with their relational names:
num_nonrel_judged_ret Total number of judged non-relevant documents retrieved over all queries
exact_prec Exact Precision over retrieved set
exact_recall Exact Recall over retrieved set
11-pt_avg Average over all 11 points of recall-precision graph
3-pt_avg Average over 3 points of recall-precision graph
avg_doc_prec Rel doc precision averaged over all relevant docs (NOT over topics)
exact_relative_prec Exact relative precision
avg_relative_prec Average relative precision
exact_unranked_avg_prec Exact Unranked Average Precision
exact_relative_unranked_avg_prec Exact Relative Unranked Average Precision
map_at_R Average Precision over first R docs retrieved
int_map Interpolated Mean Average Precision
exact_int_R_rcl_prec Exact R-based-interpolated-Precision
int_map_at_R Average Interpolated Precision for first R docs retrieved
bpref_allnonrel Binary Preference, all judged nonrel
bpref_retnonrel Binary Preference, all retrieved judged nonrel
bpref_topnonrel Binary Preference, top 100 judged nonrel
bpref_top5Rnonrel Binary Preference, top 5R judged nonrel
bpref_top10Rnonrel Binary Preference, top 10R judged nonrel
bpref_top10pRnonrel Binary Preference, top 10 + R judged nonrel
bpref_top25pRnonrel Binary Preference, top 25 + R judged nonrel
bpref_top50pRnonrel Binary Preference, top 50 + R judged nonrel
bpref_top25p2Rnonrel Binary Preference, top 25 + 2*R judged nonrel
bpref_retall Binary Preference, Only retrieved judged rel and nonrel
bpref_5 Binary Preference, top 5 rel, top 5 nonrel
bpref_10 Binary Preference, top 10 rel, top 10 nonrel
bpref_num_all Binary Preference, Number not retrieved before (all judged)
bpref_num_ret Binary Preference, Number retrieved after
bpref_num_correct Binary Preference, Number correct preferences
bpref_num_possible Binary Preference, Number possible correct_preferences
old_bpref Buggy Version 7.3. Binary Preference, top R judged nonrel
old_bpref_top10pRnonrel Buggy Version 7.3. Binary Preference,top 10+R judged nonrel
infAP Inferred AP. Calculate AP using only a judged random sample of the pool, averaging in unpooled documents as nonrel.
gm_bpref Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001))
rank_first_rel Rank of top relevant document (0 if none)
recall5 Recall after 5 docs retrieved
recall10 Recall after 10 docs retrieved
recall15 Recall after 15 docs retrieved
recall20 Recall after 20 docs retrieved
recall30 Recall after 30 docs retrieved
recall100 Recall after 100 docs retrieved
recall200 Recall after 200 docs retrieved
recall500 Recall after 500 docs retrieved
recall1000 Recall after 1000 docs retrieved
0.20R-prec R-based precision- precision after 0.20 * R docs retrieved
0.40R-prec R-based precision- precision after 0.40 * R docs retrieved
0.60R-prec R-based precision- precision after 0.60 * R docs retrieved
0.80R-prec R-based precision- precision after 0.80 * R docs retrieved
1.00R-prec R-based precision- precision after 1.00 * R docs retrieved
1.20R-prec R-based precision- precision after 1.20 * R docs retrieved
1.40R-prec R-based precision- precision after 1.40 * R docs retrieved
1.60R-prec R-based precision- precision after 1.60 * R docs retrieved
1.80R-prec R-based precision- precision after 1.80 * R docs retrieved
2.00R-prec R-based precision- precision after 2.00 * R docs retrieved
relative_prec5 Relative precision after 5 docs retrieved
relative_prec10 Relative precision after 10 docs retrieved
relative_prec15 Relative precision after 15 docs retrieved
relative_prec20 Relative precision after 20 docs retrieved
relative_prec30 Relative precision after 30 docs retrieved
relative_prec100 Relative precision after 100 docs retrieved
relative_prec200 Relative precision after 200 docs retrieved
relative_prec500 Relative precision after 500 docs retrieved
relative_prec1000 Relative precision after 1000 docs retrieved
unranked_avg_prec5 Unranked Average Precision after 5 docs retrieved
unranked_avg_prec10 Unranked Average Precision after 10 docs retrieved
unranked_avg_prec15 Unranked Average Precision after 15 docs retrieved
unranked_avg_prec20 Unranked Average Precision after 20 docs retrieved
unranked_avg_prec30 Unranked Average Precision after 30 docs retrieved
unranked_avg_prec100 Unranked Average Precision after 100 docs retrieved
unranked_avg_prec200 Unranked Average Precision after 200 docs retrieved
unranked_avg_prec500 Unranked Average Precision after 500 docs retrieved
unranked_avg_prec1000 Unranked Average Precision after 1000 docs retrieved
relative_unranked_avg_prec5 Relative Unranked Average Precision after 5 docs retrieved
relative_unranked_avg_prec10 Relative Unranked Average Precision after 10 docs retrieved
relative_unranked_avg_prec15 Relative Unranked Average Precision after 15 docs retrieved
relative_unranked_avg_prec20 Relative Unranked Average Precision after 20 docs retrieved
relative_unranked_avg_prec30 Relative Unranked Average Precision after 30 docs retrieved
relative_unranked_avg_prec100 Relative Unranked Average Precision after 100 docs retrieved
relative_unranked_avg_prec200 Relative Unranked Average Precision after 200 docs retrieved
relative_unranked_avg_prec500 Relative Unranked Average Precision after 500 docs retrieved
relative_unranked_avg_prec1000 Relative Unranked Average Precision after 1000 docs retrieved
utility_1.0_-1.0_0.0_0.0 Utility (a,b,c,d) Coefficients 1.0_-1.0_0.0_0.0
rcl_at_142_nonrel Recall averaged at X nonrel docs X= 142
fallout_recall_0 Fallout - Recall Averages- recall after 0 nonrel docs retrieved
fallout_recall_14 Fallout - Recall Averages- recall after 14 nonrel docs retrieved
fallout_recall_28 Fallout - Recall Averages- recall after 28 nonrel docs retrieved
fallout_recall_42 Fallout - Recall Averages- recall after 42 nonrel docs retrieved
fallout_recall_56 Fallout - Recall Averages- recall after 56 nonrel docs retrieved
fallout_recall_71 Fallout - Recall Averages- recall after 71 nonrel docs retrieved
fallout_recall_85 Fallout - Recall Averages- recall after 85 nonrel docs retrieved
fallout_recall_99 Fallout - Recall Averages- recall after 99 nonrel docs retrieved
fallout_recall_113 Fallout - Recall Averages- recall after 113 nonrel docs retrieved
fallout_recall_127 Fallout - Recall Averages- recall after 127 nonrel docs retrieved
fallout_recall_142 Fallout - Recall Averages- recall after 142 nonrel docs retrieved
int_0.20R-prec Interpolated R-based precision, after 0.20 * R docs retrieved
int_0.40R-prec Interpolated R-based precision, after 0.40 * R docs retrieved
int_0.60R-prec Interpolated R-based precision, after 0.60 * R docs retrieved
int_0.80R-prec Interpolated R-based precision, after 0.80 * R docs retrieved
int_1.00R-prec Interpolated R-based precision, after 1.00 * R docs retrieved
int_1.20R-prec Interpolated R-based precision, after 1.20 * R docs retrieved
int_1.40R-prec Interpolated R-based precision, after 1.40 * R docs retrieved
int_1.60R-prec Interpolated R-based precision, after 1.60 * R docs retrieved
int_1.80R-prec Interpolated R-based precision, after 1.80 * R docs retrieved
int_2.00R-prec Interpolated R-based precision, after 2.00 * R docs retrieved
micro_prec Total relevant retrieved documents / Total retrieved documents
micro_recall Total relevant retrieved documents / Total relevant documents
micro_bpref Total correct preferences / Total possible preferences
trec_eval.8.1/bpref_bug
+176
View File
@@ -0,0 +1,176 @@
November 8, 2005
We found a bug in the calculation of the bpref measure within trec_eval.
BUG DESCRIPTION: The bpref measure calculates the fraction of
preferences between pairs of judged relevant and non-relevant
documents that were correctly ordered in a document ranking.
When a run does not retrieve R judged non-relevant documents,
only the retrieved non-relevant documents were considered. Thus
a (worst case) run which retrieved only 5 judged documents, the
first non-relevant and the following 4 relevant, would have a
score of 0.0 since the fraction of correct preferences among the
retrieved judged documents was 0.0. However, the retrieved judged
relevant documents should have been counted as being preferred
over any judged non-relevant document that wasn't retrieved. If
the nonretrieved documents included 3 judged non-relevant
documents and 2 judged relevant documents, then the bpref score
should be 0.5. (= ((4 * (3/4)) + 2 * (0/4)) / 6).
BUG IMPACT: Almost no impact for standard TREC-type ad hoc runs
(retrieve 1000 documents). Topics with large numbers of relevant
documents (eg, over 300) had their scores artificially depressed
for those topics, and thus performance with the corrected bpref
will be higher on those topics. Kendall tau of system rankings
show very strong (.95 - .98) agreement between the buggy and new
bpref.
There may be more impact for non-standard environments where the
number of retrieved judged documents is small. Eg, I've been
told the 2005 Terabyte efficiency track (only retrieve 20
documents) is more strongly affected.
BUG FIX: Version 8.0, available from the usual places at NIST,
implements the corrected bpref calculations. It also adds the
measures "old_bpref" and "old_bpref_top10pRnonrel" that calculate
the buggy numbers for comparisons with old results (the latter
measure was used in the SIGIR 2004 bpref paper). People using
bpref should switch to Version 8.0 or higher as soon as possible.
BUG APOLOGY: I want to apologize to the community for the error.
Doing research and using new measures is hard enough without
having to worry about buggy implementations!
Chris Buckley
#######################################################################
For those of you working with bpref who want to know more details
about the bug and its effects, here's a fuller version of above.
BUG DESCRIPTION: Here's code, pseudo_code, and comments comparing
old_bpref (the buggy version) and bpref on a single topic:
long nonrel_so_far; /* Number of non-relevant documents seen while
going through the ranking */
long num_nonrel; /* Number of judged non-relevant documents */
long nonrel_ret; /* Number of retrieved judged non-relevant documents */
long pref_top_Rnonrel_num; /* set to R (eval->num_rel) */
nonrel_so_far = 0;
foreach doc in retrieved documents (sorted in decreasing score) {
if (doc is not relevant)
nonrel_so_far++;
else {
/* Add fraction of correct preferences for this doc */
if (nonrel_so_far) {
-->new eval->bpref += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) /
(float) MIN (num_nonrel, pref_top_Rnonrel_num));
-->buggy eval->old_bpref += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) /
(float) MIN (nonrel_ret, pref_top_Rnonrel_num));
}
else {
eval->bpref += 1.0;
eval->old_bpref += 1.0;
}
}
}
if (eval->num_rel) {
eval->bpref /= eval->num_rel;
eval->old_bpref /= eval->num_rel;
}
BUG HOW IT HAPPENED:The first versions of bpref I wrote were
defined only on the retrieved documents. When I switched to
variants using the TREC standard of considering all documents
(which improved the measures greatly), I overlooked making the
needed change to the denominator in the code above.
BUG HOW DISCOVERED: On November 4, Ian Soboroff (NIST) was
looking at the bpref code and couldn't understand the "corner"
conditions in the code. We talked some, I stared at the code a
couple of minutes, and then went "OOPS", (or words to that
effect).
CHANGES CAUSED BY BUG: I made the obvious changes to the bpref
code, and also revamped the whole structure of the main function,
to at least break apart the major different kinds of measures
(eg, compute the cutoff measures, the per document average
measures, the bpref measures, and the time measures separately.)
It was impossible to understand the function and now it's merely
very difficult! I should probably rewrite it to compute most
measures separately; execution speed is no longer the critical
factor it once was.
BUG IMPACT VALIDATION: I reran the complete set of runs that went
into the Buckley, Voorhees SIGIR 2004 bpref paper. That included
comparing all systems in tasks in TREC 8, TREC 10, and TREC 12 at
various levels of completeness of the document set and relevance
levels. In the comparisons of bpref system rankings versus
original MAP rankings, the Kendall Tau scores of the two versions
of bpref were basically identical. They did not vary from each
other by more than .01, except when only using 1% or 2% of the
judgements in which case it was less than .03. (I was actually
expecting much greater differences when using very small number
of relevant and non-relevant documents. But it looks like it was
the same for all systems.)
Using full information, the actual scores of the old and new bprefs
were pretty much the same when averaged over all systems, except for 3
topics in TREC 10. Here's the topics with the top differences in old
and new bpref scores when averaged over all systems:
qid old_bpref bpref diff
541 0.189271 0.324097 -0.134826
544 0.555475 0.633235 -0.07776
549 0.278118 0.321332 -0.043214
530 0.391955 0.394681 -0.002726
519 0.132094 0.132671 -0.000577
509 0.266160 0.266544 -0.000384
547 0.167335 0.167504 -0.000169
511 0.303595 0.303679 -8.4e-05
501 0.175508 0.175508 0 ... all other topics tied at 0
Here's the number of relevant documents per topic
num_rel 541 372
num_rel 549 367
num_rel 544 324
num_rel 511 165
num_rel 519 149
num_rel 547 144
num_rel 509 140
num_rel 530 124
num_rel 527 93
Clearly there's a big impact in scores on the 3 topics with over
300 relevant documents, a small impact on the 5 topics with
between 100 and 200 relevant documents, and no impact on the
rest.
For TREC 8, there was 1 topic with a diff greater than .01 (.029)
and 23 topics that had any differences at all. For TREC 12,
there was a small impact on 32/100 topics with the largest being
.008.
Overall, I conclude that there's a minor impact on the standard
TREC 1000 document evaluations due to the buggy bpref on topics
which have hundreds of relevant documents. The average scores of
all systems will change because of these topics, but it should
not have an important effect on system ranking (except possibly
for systems which consistently retrieve less than 1000
documents).
I sent trec_eval Version 8.0beta to Ian to run on this year's bpref
oriented runs. His report was that there was strong agreement in
Kendall Tau between the buggy and nonbuggy versions, and when you
compared each against MAP, they were within .016 of each other (closer
depending on task). It actually was the buggy version that tracked
MAP slightly more closely, perhaps indicating that MAP emphasize
topics with lots of relevant documents a bit less than bpref.
Ian reported the runs on Terabyte efficiency track, where systems only
retrieved 20 documents per topic (running 50,000 topics but evaluating
over 50), had much larger bpref differences in score (the new bpref
average scores being 50% higher than the old), but still had a 90%
Kendall Tau between the versions; about the same that either had with
MAP. That's good enough to reassure me that most conclusions people
have reached in experiments with bpref will still be valid, though
the numbers will have to be redone.
trec_eval.8.1/buf.h
+13
View File
@@ -0,0 +1,13 @@
#ifndef BUFH
#define BUFH
/* $Header: /home/smart/release/src/h/buf.h,v 11.0 1992/07/21 18:18:32 chrisb Exp $*/
/* structure used for passing around text (buf) which possibly includes
NULLs. see buf_util.c for add_buf(). */
typedef struct {
int size;
int end;
char *buf;
} SM_BUF;
#endif /* BUFH */
trec_eval.8.1/buf_util.c
+86
View File
@@ -0,0 +1,86 @@
#ifdef RCSID
static char rcsid[] = "$Header: /home/smart/release/src/libgeneral/buf_util.c,v 11.0 1992/07/21 18:21:04 chrisb Exp $";
#endif
/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley.
Permission is granted for use of this file in unmodified form for
research purposes. Please contact the SMART project to obtain
permission for other uses.
*/
/******************** PROCEDURE DESCRIPTION ************************
*0 Utility procedure to add the memory contents of new.buf to result.buf
*2 add_buf (new, result)
*3 SM_BUF *new;
*3 SM_BUF *result;
*7 Both new and result are of type
*7 typedef struct {
*7 int size; * allocated space for buf *
*7 int end; * end of valid data in buf *
*7 char *buf; * buffer of arbitrary data *
*7 } SM_BUF;
*7
*7 Append the data in new to the end of the data in result. The data can
*7 be arbitrary data, eg, include '\0's.
*7 Return UNDEF if can't allocate enough space for the result, 0 otherwise.
***********************************************************************/
#include "common.h"
#include "sysfunc.h"
#include "buf.h"
int
add_buf (new, result)
SM_BUF *new, *result;
{
if (result->size == 0) {
if (NULL == (result->buf = malloc ((unsigned) new->end * 2 + 1)))
return (UNDEF);
result->size = 2 * new->end + 1;
result->end = 0;
}
else if (new->end >= result->size - result->end) {
if (NULL == (result->buf =
realloc (result->buf,
(unsigned) result->size * 2 + new->end)))
return (UNDEF);
result->size += result->size + new->end;
}
bcopy (new->buf, &result->buf[result->end], new->end);
result->end += new->end;
return (0);
}
/******************** PROCEDURE DESCRIPTION ************************
*0 Utility procedure to add the string new to result.buf
*2 add_buf_string (new, result)
*3 char *new;
*3 SM_BUF *result;
*7 Result is of type
*7 typedef struct {
*7 int size; * allocated space for buf *
*7 int end; * end of valid data in buf *
*7 char *buf; * buffer of arbitrary data *
*7 } SM_BUF;
*7
*7 Append the data in new to the end of the data in result.
*7 Return UNDEF if can't allocate enough space for the result, 0 otherwise.
***********************************************************************/
int
add_buf_string (new, result)
char *new;
SM_BUF *result;
{
SM_BUF temp_buf;
temp_buf.end = strlen (new);
temp_buf.buf = new;
return (add_buf (&temp_buf, result));
}
trec_eval.8.1/common.h
+31
View File
@@ -0,0 +1,31 @@
#ifndef COMMONH
#define COMMONH
#include <stdio.h>
#ifndef FALSE
#define FALSE 0
#endif
#ifndef TRUE
#define TRUE 1
#endif
#define UNDEF -1
#define MAX(A,B) ((A) > (B) ? (A) : (B))
#define MIN(A,B) ((A) > (B) ? (B) : (A))
#ifndef MAXLONG
#define MAXLONG 2147483647L /* largest long int. no. */
#endif
/*
* Some useful macros for making malloc et al easier to use.
* Macros handle the casting and the like that's needed.
*/
#define Malloc(n,type) (type *) malloc( (unsigned) ((n)*sizeof(type)))
#define Realloc(loc,n,type) (type *) realloc( (char *)(loc), \
(unsigned) ((n)*sizeof(type)))
#define Free(loc) (void) free( (char *)(loc) )
#endif /* COMMONH */
trec_eval.8.1/error_msgs.c
+93
View File
@@ -0,0 +1,93 @@
#ifdef RCSID
static char rcsid[] = "$Header: /home/smart/release/./src/libgeneral/error_msgs.c,v 10.1 91/11/05 23:49:06 smart Exp Locker: smart $";
#endif
/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley.
Permission is granted for use of this file in unmodified form for
research purposes. Please contact the SMART project to obtain
permission for other uses.
*/
/******************** PROCEDURE DESCRIPTION ************************
*0 print a SMART error message
*2 print_error (new_routine, new_message)
*3 char *new_routine;
*3 char *new_message;
*6 Global UNIX variables errno, sys_nerr, sys_errlist are used, as well
*6 as SMART global variables smart_errlist and smart_errno;
*7 Print an error message to stderr. At point of error determination,
*7 either smart_errno should be set, or (if UNIX library error) errno will
*7 be set. If smart_errno is set, then the routine name that detected the
*7 error and a message are printed. In addition, the routine name that prints
*7 the error and it's message (eg action to be taken) are printed.
*9 smart_errno should be more widely used, in particular to locate the
*9 procedure the error occurs in. Many errors can only get "located"
*9 by setting trace.
***********************************************************************/
#include <stdio.h>
#include "smart_error.h"
#include "sysfunc.h"
/* Declarations of external variables defined in "smart_error.h" */
int smart_errno; /* If > 0 and <= sys_nerr then refers to */
/* sys_errlist, else if >= smart_errmin */
/* and <= smart_errmax, then smart_errlist */
char *smart_message; /* Message to be printed (often filename) */
char *smart_routine; /* Major routine issuing error message */
extern int errno;
char *smart_errlist[] = {
"Inconsistency check",
"Illegal value for seek",
"Illegal mode for object",
"Illegal parameter value"
};
void
print_error (new_routine, new_message)
char *new_routine;
char *new_message;
{
if (smart_errno > 0 && smart_errno < SMART_MINERR) {
(void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n",
new_routine,
smart_routine,
smart_message,
strerror(smart_errno),
new_message);
}
else if (smart_errno >= SMART_MINERR &&
smart_errno < SMART_MINERR + SMART_NUMERR) {
(void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n",
new_routine,
smart_routine,
smart_message,
smart_errlist[smart_errno - SMART_MINERR],
new_message);
}
else if (smart_errno == 0 && errno != 0) {
/* Presumably error detected directly by new_routine */
/* after system call */
(void) fprintf (stderr, "%s: '%s' - %s\n",
new_routine,
strerror(errno),
new_message);
}
else {
(void) fprintf (stderr, "%s: Undetermined error detected - %s\n",
new_routine,
new_message);
}
/* Reset the global error indicators */
errno = 0;
smart_errno = 0;
smart_message = NULL;
smart_routine = NULL;
}
trec_eval.8.1/form_trvec.c
+213
View File
@@ -0,0 +1,213 @@
#include "common.h"
#include "sysfunc.h"
#include "smart_error.h"
#include "tr_vec.h"
#include "trec_eval.h"
#include "buf.h"
static int comp_tr_tup_rank(), comp_tr_tup_did(), comp_tr_docno(),
comp_qrels_docno(), comp_sim_docno(), comp_negsim_docno();
/* Space reserved for output TR_TUP tuples */
static TR_TUP *start_tr_tup;
static long max_tr_tup = 0;
/* Takes the top docs and pool docs for a query, and returns a
tr_vec object giving the relevance valuse for all top docs.
Relevance value is
value in trec_qrels if docno is in trec_qrels and was judged
RELVALUE_NONPOOL (-1) if docno is not in trec_qrels
RELVALUE_UNJUDGED (-2) if docno is in trec_qrels and was not judged
*/
int
form_trvec (epi, trec_top, trec_qrels, tr_vec, num_rel)
EVAL_PARAM_INFO *epi;
TREC_TOP *trec_top;
TREC_QRELS *trec_qrels;
TR_VEC *tr_vec;
long *num_rel;
{
TR_TUP *tr_tup;
TEXT_QRELS *qrels_ptr, *end_qrels;
long i;
/* Reserve space for output tr_tups, if needed */
if (trec_top->num_text_tr > max_tr_tup) {
if (max_tr_tup > 0)
(void) free ((char *) start_tr_tup);
max_tr_tup += trec_top->num_text_tr;
if (NULL == (start_tr_tup = Malloc (max_tr_tup, TR_TUP)))
return (UNDEF);
}
/* Sort trec_top by sim, breaking ties lexicographically using docno */
if (epi->time_flag) {
qsort ((char *) trec_top->text_tr,
(int) trec_top->num_text_tr,
sizeof (TEXT_TR),
comp_negsim_docno);
}
else {
qsort ((char *) trec_top->text_tr,
(int) trec_top->num_text_tr,
sizeof (TEXT_TR),
comp_sim_docno);
}
/* Add ranks to trec_top (starting at 1) */
for (i = 0; i < trec_top->num_text_tr; i++) {
trec_top->text_tr[i].rank = i+1;
}
/* Sort trec_top lexicographically */
qsort ((char *) trec_top->text_tr,
(int) trec_top->num_text_tr,
sizeof (TEXT_TR),
comp_tr_docno);
for (i = 1; i < trec_top->num_text_tr; i++) {
if (0 == strcmp (trec_top->text_tr[i].docno,
trec_top->text_tr[i-1].docno)) {
set_error (SM_ILLPA_ERR, "Duplicate top docs docno", "trec_eval");
return (UNDEF);
}
}
/* Sort trec_qrels lexicographically */
qsort ((char *) trec_qrels->text_qrels,
(int) trec_qrels->num_text_qrels,
sizeof (TEXT_QRELS),
comp_qrels_docno);
/* Find number of relevant docs, and check for duplicates */
*num_rel = 0;
for (i = 0; i < trec_qrels->num_text_qrels; i++) {
if (i > 0 && (0 == strcmp (trec_qrels->text_qrels[i].docno,
trec_qrels->text_qrels[i-1].docno))) {
set_error (SM_ILLPA_ERR, "Duplicate qrels docno", "trec_eval");
return (UNDEF);
}
if (trec_qrels->text_qrels[i].rel >= epi->relevance_level)
(*num_rel)++;
}
/* Go through trec_top, trec_qrels in parallel to determine which
docno's are in both (ie, which trec_top are relevant). Once relevance
is known, convert trec_top tuple into TR_TUP. */
tr_tup = start_tr_tup;
qrels_ptr = trec_qrels->text_qrels;
end_qrels = &trec_qrels->text_qrels[trec_qrels->num_text_qrels];
for (i = 0; i < trec_top->num_text_tr; i++) {
if (trec_top->text_tr[i].rank > epi->max_num_docs_per_topic)
/* Skip if evaluation desired over fewer docs than this rank */
continue;
while (qrels_ptr < end_qrels &&
strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) < 0)
qrels_ptr++;
if (qrels_ptr >= end_qrels ||
strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) > 0) {
/* Doc is non-judged */
tr_tup->rel = RELVALUE_NONPOOL;
/* Skip unjudged docs if desired */
if (epi->judged_docs_only_flag)
continue;
}
else {
/* Doc is in pool, assign relevance */
if (qrels_ptr->rel == -1)
/* In pool, but unjudged (eg, infAP uses a sample of pool) */
tr_tup->rel = RELVALUE_UNJUDGED;
else
tr_tup->rel = qrels_ptr->rel;
qrels_ptr++;
}
tr_tup->did = i;
tr_tup->rank = trec_top->text_tr[i].rank;
tr_tup->sim = trec_top->text_tr[i].sim;
tr_tup->action = 0;
tr_tup->iter = 0;
tr_tup++;
}
/* Form the full TR_VEC object for this qid */
tr_vec->qid = trec_top->qid;
tr_vec->num_tr = tr_tup - start_tr_tup;
tr_vec->tr = start_tr_tup;
/* If judged_docs_only_flag, then must fix up ranks to reflect unjudged
docs being thrown out. Note: done this way to preserve original
tie-breaking based on text docno */
if (epi->judged_docs_only_flag) {
/* Sort tuples by increasing rank */
qsort ((char *) tr_vec->tr,
(int) tr_vec->num_tr,
sizeof (TR_TUP),
comp_tr_tup_rank);
for (i = 0; i < tr_vec->num_tr; i++) {
tr_vec->tr[i].rank = i+1;
}
qsort ((char *) tr_vec->tr,
(int) tr_vec->num_tr,
sizeof (TR_TUP),
comp_tr_tup_did);
}
return (1);
}
static int
comp_sim_docno (ptr1, ptr2)
TEXT_TR *ptr1;
TEXT_TR *ptr2;
{
if (ptr1->sim > ptr2->sim)
return (-1);
if (ptr1->sim < ptr2->sim)
return (1);
return (strcmp (ptr2->docno, ptr1->docno));
}
static int
comp_negsim_docno (ptr1, ptr2)
TEXT_TR *ptr1;
TEXT_TR *ptr2;
{
if (ptr1->sim < ptr2->sim)
return (-1);
if (ptr1->sim > ptr2->sim)
return (1);
return (strcmp (ptr2->docno, ptr1->docno));
}
static int
comp_tr_docno (ptr1, ptr2)
TEXT_TR *ptr1;
TEXT_TR *ptr2;
{
return (strcmp (ptr1->docno, ptr2->docno));
}
static int
comp_qrels_docno (ptr1, ptr2)
TEXT_QRELS *ptr1;
TEXT_QRELS *ptr2;
{
return (strcmp (ptr1->docno, ptr2->docno));
}
static int
comp_tr_tup_rank (ptr1, ptr2)
TR_TUP *ptr1;
TR_TUP *ptr2;
{
return (ptr1->rank - ptr2->rank);
}
static int
comp_tr_tup_did (ptr1, ptr2)
TR_TUP *ptr1;
TR_TUP *ptr2;
{
return (ptr1->did - ptr2->did);
}
trec_eval.8.1/get_qrels.c
+169
View File
@@ -0,0 +1,169 @@
/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley. */
#include "common.h"
#include "sysfunc.h"
#include "smart_error.h"
#include "trec_eval.h"
#include <ctype.h>
/* Read all relevance information from text_qrels_file.
Relevance for each docno to qid is determined from text_qrels_file, which
consists of text tuples of the form
qid iter docno rel
giving TREC document numbers (docno, a string) and their relevance (rel,
an integer) to query qid (a string). iter string field is ignored.
Fields are separated by whitespace, string fields can contain no whitespace.
File may contain no NULL characters.
*/
int
get_qrels (text_qrels_file, all_trec_qrels)
char *text_qrels_file;
ALL_TREC_QRELS *all_trec_qrels;
{
int fd;
int size = 0;
char *trec_qrels_buf;
char *ptr;
char *current_qid;
char *qid_ptr, *docno_ptr, *rel_ptr;
long i;
long rel;
TREC_QRELS *current_qrels = NULL;
/* Read entire file into memory */
if (-1 == (fd = open (text_qrels_file, 0)) ||
-1 == (size = lseek (fd, 0L, 2)) ||
NULL == (trec_qrels_buf = malloc ((unsigned) size+2)) ||
-1 == lseek (fd, 0L, 0) ||
size != read (fd, trec_qrels_buf, size) ||
-1 == close (fd)) {
set_error (SM_ILLPA_ERR, "Cannot read qrels file", "trec_eval");
return (UNDEF);
}
current_qid = "";
/* Initialize all_trec_qrels */
all_trec_qrels->num_q_qrels = 0;
all_trec_qrels->max_num_q_qrels = INIT_NUM_QUERIES;
if (NULL == (all_trec_qrels->trec_qrels = Malloc (INIT_NUM_QUERIES,
TREC_QRELS)))
return (UNDEF);
if (size == 0)
return (0);
/* Append ending newline if not present, Append NULL terminator */
if (trec_qrels_buf[size-1] != '\n') {
trec_qrels_buf[size] = '\n';
size++;
}
trec_qrels_buf[size] = '\0';
ptr = trec_qrels_buf;
while (*ptr) {
/* Get current line */
/* Get qid */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
qid_ptr = ptr;
while (! isspace (*ptr)) ptr++;
if (*ptr == '\n') {
set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
return (UNDEF);
}
*ptr++ = '\0';
/* Skip iter */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
while (! isspace (*ptr)) ptr++;
if (*ptr++ == '\n') {
set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
return (UNDEF);
}
/* Get docno */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
docno_ptr = ptr;
while (! isspace (*ptr)) ptr++;
if (*ptr == '\n') {
set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
return (UNDEF);
}
*ptr++ = '\0';
/* Get relevance */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
if (*ptr == '\n') {
set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
return (UNDEF);
}
rel_ptr = ptr;
while (! isspace (*ptr)) ptr++;
if (*ptr != '\n') {
*ptr++ = '\0';
while (*ptr != '\n' && isspace (*ptr)) ptr++;
if (*ptr != '\n') {
set_error (SM_ILLPA_ERR, "malformed qrels line",
"trec_eval");
return (UNDEF);
}
}
*ptr++ = '\0';
if (0 != strcmp (qid_ptr, current_qid)) {
/* Query has changed. Must check if new query or this is more
judgements for an old query */
for (i = 0; i < all_trec_qrels->num_q_qrels; i++) {
if (0 == strcmp (qid_ptr, all_trec_qrels->trec_qrels[i].qid))
break;
}
if (i >= all_trec_qrels->num_q_qrels) {
/* New unseen query, add and initialize it */
if (all_trec_qrels->num_q_qrels >=
all_trec_qrels->max_num_q_qrels) {
all_trec_qrels->max_num_q_qrels *= 10;
if (NULL == (all_trec_qrels->trec_qrels =
Realloc (all_trec_qrels->trec_qrels,
all_trec_qrels->max_num_q_qrels,
TREC_QRELS)))
return (UNDEF);
}
current_qrels = &all_trec_qrels->trec_qrels[i];
current_qrels->qid = qid_ptr;
current_qrels->num_text_qrels = 0;
current_qrels->max_num_text_qrels = INIT_NUM_RELS;
if (NULL == (current_qrels->text_qrels =
Malloc (INIT_NUM_RELS, TEXT_QRELS)))
return (UNDEF);
all_trec_qrels->num_q_qrels++;
}
else {
/* Old query, just switch current_q_index */
current_qrels = &all_trec_qrels->trec_qrels[i];
}
current_qid = current_qrels->qid;
}
/* Add judgement to current query's list */
if (current_qrels->num_text_qrels >=
current_qrels->max_num_text_qrels) {
/* Need more space */
current_qrels->max_num_text_qrels *= 10;
if (NULL == (current_qrels->text_qrels =
Realloc (current_qrels->text_qrels,
current_qrels->max_num_text_qrels,
TEXT_QRELS)))
return (UNDEF);
}
current_qrels->text_qrels[current_qrels->num_text_qrels].docno =
docno_ptr;
rel = atol (rel_ptr);
current_qrels->text_qrels[current_qrels->num_text_qrels++].rel =
rel;
}
return (1);
}
trec_eval.8.1/get_top.c
+192
View File
@@ -0,0 +1,192 @@
/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley. */
#include "common.h"
#include "sysfunc.h"
#include "smart_error.h"
#include "trec_eval.h"
#include <ctype.h>
/* Read all retrieved results information from trec_top_file.
Read text tuples from trec_top_file of the form
030 Q0 ZF08-175-870 0 4238 prise1
qid iter docno rank sim run_id
giving TREC document numbers (a string) retrieved by query qid
(a string) with similarity sim (a float). The other fields are ignored,
with the exception that the run_id field of the last line is kept and
output. In particular, note that the rank field is ignored here;
internally ranks are assigned by sorting by the sim field with ties
broken determinstically (using docno).
Sim is assumed to be higher for the docs to be retrieved first.
File may contain no NULL characters.
Any field following run_id is ignored.
*/
int
get_top (trec_top_file, all_trec_top)
char *trec_top_file;
ALL_TREC_TOP *all_trec_top;
{
int fd;
int size = 0;
char *trec_top_buf;
char *ptr;
char *current_qid;
char *qid_ptr, *docno_ptr, *sim_ptr;
char *run_id_ptr = "";
long i;
TREC_TOP *current_top = NULL;
float sim;
/* Read entire file into memory */
if (-1 == (fd = open (trec_top_file, 0)) ||
-1 == (size = lseek (fd, 0L, 2)) ||
NULL == (trec_top_buf = malloc ((unsigned) size+2)) ||
-1 == lseek (fd, 0L, 0) ||
size != read (fd, trec_top_buf, size) ||
-1 == close (fd)) {
set_error (SM_ILLPA_ERR, "Cannot read results file", "trec_eval");
return (UNDEF);
}
current_qid = "";
/* Initialize all_trec_top */
all_trec_top->num_q_tr = 0;
all_trec_top->max_num_q_tr = INIT_NUM_QUERIES;
if (NULL == (all_trec_top->trec_top = Malloc (INIT_NUM_QUERIES,
TREC_TOP)))
return (UNDEF);
if (size == 0)
return (0);
/* Append ending newline if not present, Append NULL terminator */
if (trec_top_buf[size-1] != '\n') {
trec_top_buf[size] = '\n';
size++;
}
trec_top_buf[size] = '\0';
ptr = trec_top_buf;
while (*ptr) {
/* Get current line */
/* Get qid */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
if (*ptr == '\n') {
/* Ignore blank lines (people seem to insist on them!) */
ptr++;
continue;
}
qid_ptr = ptr;
while (! isspace (*ptr)) ptr++;
if (*ptr == '\n') {
set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
return (UNDEF);
}
*ptr++ = '\0';
/* Skip iter */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
while (! isspace (*ptr)) ptr++;
if (*ptr++ == '\n') {
set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
return (UNDEF);
}
/* Get docno */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
docno_ptr = ptr;
while (! isspace (*ptr)) ptr++;
if (*ptr == '\n') {
set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
return (UNDEF);
}
*ptr++ = '\0';
/* Skip rank */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
while (! isspace (*ptr)) ptr++;
if (*ptr++ == '\n') {
set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
return (UNDEF);
}
/* Get sim */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
sim_ptr = ptr;
while (! isspace (*ptr)) ptr++;
if (*ptr == '\n') {
set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
return (UNDEF);
}
*ptr++ = '\0';
/* Get run_id */
while (*ptr != '\n' && isspace (*ptr)) ptr++;
if (*ptr == '\n') {
set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
return (UNDEF);
}
run_id_ptr = ptr;
while (! isspace (*ptr)) ptr++;
if (*ptr != '\n') {
/* Skip over rest of line */
*ptr++ = '\0';
while (*ptr != '\n') ptr++;
}
*ptr++ = '\0';
if (0 != strcmp (qid_ptr, current_qid)) {
/* Query has changed. Must check if new query or this is more
judgements for an old query */
for (i = 0; i < all_trec_top->num_q_tr; i++) {
if (0 == strcmp (qid_ptr, all_trec_top->trec_top[i].qid))
break;
}
if (i >= all_trec_top->num_q_tr) {
/* New unseen query, add and initialize it */
if (all_trec_top->num_q_tr >=
all_trec_top->max_num_q_tr) {
all_trec_top->max_num_q_tr *= 10;
if (NULL == (all_trec_top->trec_top =
Realloc (all_trec_top->trec_top,
all_trec_top->max_num_q_tr,
TREC_TOP)))
return (UNDEF);
}
current_top = &all_trec_top->trec_top[i];
current_top->qid = qid_ptr;
current_top->num_text_tr = 0;
current_top->max_num_text_tr = INIT_NUM_RESULTS;
if (NULL == (current_top->text_tr =
Malloc (INIT_NUM_RESULTS, TEXT_TR)))
return (UNDEF);
all_trec_top->num_q_tr++;
}
else {
/* Old query, just switch current_q_index */
current_top = &all_trec_top->trec_top[i];
}
current_qid = current_top->qid;
}
/* Add retrieval docno/sim to current query's list */
if (current_top->num_text_tr >=
current_top->max_num_text_tr) {
/* Need more space */
current_top->max_num_text_tr *= 10;
if (NULL == (current_top->text_tr =
Realloc (current_top->text_tr,
current_top->max_num_text_tr,
TEXT_TR)))
return (UNDEF);
}
current_top->text_tr[current_top->num_text_tr].docno = docno_ptr;
sim = atof (sim_ptr);
current_top->text_tr[current_top->num_text_tr].sim = sim;
current_top->text_tr[current_top->num_text_tr++].rank = 0;
}
all_trec_top->run_id = run_id_ptr;
return (1);
}
trec_eval.8.1/measures.c
+286
View File
@@ -0,0 +1,286 @@
#ifdef RCSID
static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $";
#endif
/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley.
Permission is granted for use of this file in unmodified form for
research purposes. Please contact the SMART project to obtain
permission for other uses.
*/
#include "common.h"
#include "sysfunc.h"
#include "buf.h"
#include "trec_eval.h"
static long cutoff[] = CUTOFF_VALUES;
static char param_val[20];
static char *get_param_str_ircl_prn(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%4.2f", (float) index / (NUM_RP_PTS -1));
return (param_val);
}
static char *get_param_str_cutoff(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%ld", cutoff[index]);
return (param_val);
}
static char *get_param_str_Rcutoff(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%4.2f",
(float) MAX_RPREC * (index+1) /(float) (NUM_PREC_PTS - 1));
return (param_val);
}
static char *get_param_str_utility(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f",
epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d);
return (param_val);
}
static char *get_param_str_maxfallout(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%ld", (long) MAX_FALL_RET);
return (param_val);
}
static char *get_param_str_fall_recall(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%ld",
(long) (MAX_FALL_RET * index) / (NUM_FR_PTS - 1));
return (param_val);
}
static char *get_param_str_time_cutoff(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%ld",
(long) (index * MAX_TIME / NUM_TIME_PTS));
return (param_val);
}
static char *get_param_str_time_utility_cutoff(epi, index)
EVAL_PARAM_INFO *epi;
long index;
{
sprintf (param_val, "%3.1f_%3.1f_%3.1f_%3.1f-%ld",
epi->utility_a, epi->utility_b, epi->utility_c, epi->utility_d,
(long) (index * MAX_TIME / NUM_TIME_PTS));
return (param_val);
}
SINGLE_MEASURE sing_meas[] = {
{"num_ret", "Total number of documents retrieved over all queries",
1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_ret)},
{"num_rel", "Total number of relevant documents over all queries",
1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel)},
{"num_rel_ret", "Total number of relevant documents retrieved over all queries",
1, 1, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_rel_ret)},
{"map", "Mean Average Precision (MAP)",
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_recall_precis)},
{"gm_ap","Average Precision. Geometric Mean, q_score=log(MAX(map,.00001))",
0, 1, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_ap)},
{"R-prec", "R-Precision (Precision after R (= num-rel for topic) documents retrieved)",
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, R_recall_precis)},
{"bpref", "Binary Preference, top R judged nonrel",
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref)},
{"recip_rank", "Reciprical rank of top relevant document",
0, 1, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, recip_rank)},
/* end of short output measures (the major ones) */
{"num_nonrel_judged_ret", "Total number of judged non-relevant documents retrieved over all queries",
1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, num_nonrel_judged_ret)},
{"exact_prec", "Exact Precision over retrieved set",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_precis)},
{"exact_recall", "Exact Recall over retrieved set",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_recall)},
{"11-pt_avg", "Average over all 11 points of recall-precision graph",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av11_recall_precis)},
{"3-pt_avg", "Average over 3 points of recall-precision graph",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av3_recall_precis)},
{"avg_doc_prec", "Rel doc precision averaged over all relevant docs (NOT over topics)",
0, 0, 0, 0, 0, 0, 1, 0, offsetof(TREC_EVAL, avg_doc_prec)},
{"exact_relative_prec", "Exact relative precision",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_precis)},
{"avg_relative_prec", "Average relative precision",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_rel_precis)},
{"exact_unranked_avg_prec", "Exact Unranked Average Precision",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_uap)},
{"exact_relative_unranked_avg_prec", "Exact Relative Unranked Average Precision",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, exact_rel_uap)},
{"map_at_R", "Average Precision over first R docs retrieved",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_R_precis)},
{"int_map", "Interpolated Mean Average Precision",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_recall_precis)},
{"exact_int_R_rcl_prec", "Exact R-based-interpolated-Precision",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_R_recall_precis)},
{"int_map_at_R", "Average Interpolated Precision for first R docs retrieved",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, int_av_R_precis)},
{"time_integral_prec", "Time: Average Integral Precision",
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_precis)},
{"time_integral_relative_prec", "Time: Average Integral Relative Precision",
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_relprecis)},
{"time_integral_uap", "Time: Average Integral Unranked Precision",
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_uap)},
{"time_integral_relative_uap", "Time: Average Integral Unranked Relative Precision",
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_reluap)},
{"time_integral_cum_rel", "Time: Average (Integral) cumulative number relevant",
0, 0, 1, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, av_time_cum_rel)},
{"bpref_allnonrel", "Binary Preference, all judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_allnonrel)},
{"bpref_retnonrel", "Binary Preference, all retrieved judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retnonrel)},
{"bpref_topnonrel", "Binary Preference, top 100 judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_topnonrel)},
{"bpref_top5Rnonrel", "Binary Preference, top 5R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top5Rnonrel)},
{"bpref_top10Rnonrel", "Binary Preference, top 10R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10Rnonrel)},
{"bpref_top10pRnonrel", "Binary Preference, top 10 + R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top10pRnonrel)},
{"bpref_top25pRnonrel", "Binary Preference, top 25 + R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25pRnonrel)},
{"bpref_top50pRnonrel", "Binary Preference, top 50 + R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top50pRnonrel)},
{"bpref_top25p2Rnonrel", "Binary Preference, top 25 + 2*R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_top25p2Rnonrel)},
{"bpref_retall", "Binary Preference, Only retrieved judged rel and nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_retall)},
{"bpref_5", "Binary Preference, top 5 rel, top 5 nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_5)},
{"bpref_10", "Binary Preference, top 10 rel, top 10 nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_10)},
{"bpref_num_all", "Binary Preference, Number not retrieved before (all judged)",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_all)},
{"bpref_num_ret", "Binary Preference, Number retrieved after",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, bpref_num_ret)},
{"bpref_num_correct", "Binary Preference, Number correct preferences",
1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_correct)},
{"bpref_num_possible", "Binary Preference, Number possible correct_preferences",
1, 0, 0, 0, 0, 0, 0, 0, offsetof(TREC_EVAL, bpref_num_possible)},
{"old_bpref", "Buggy Version 7.3. Binary Preference, top R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref)},
{"old_bpref_top10pRnonrel", "Buggy Version 7.3. Binary Preference,top 10+R judged nonrel",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, old_bpref_top10pRnonrel)},
{"infAP", "Inferred AP. Calculate AP using only a judged random sample of the pool, averaging in unpooled documents as nonrel.",
0, 0, 0, 0, 0, 1, 0, 0, offsetof(TREC_EVAL, inf_ap)},
{"gm_bpref", "Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001))",
0, 0, 0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, gm_bpref)},
{"rank_first_rel", "Rank of top relevant document (0 if none)",
1, 0, 0, 1, 0, 0, 0, 0, offsetof(TREC_EVAL, rank_first_rel)},
};
int num_sing_meas = sizeof (sing_meas) / sizeof (sing_meas[0]);
PARAMETERIZED_MEASURE param_meas[] = {
{"Interpolated Recall - Precision Averages",
0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, int_recall_precis[0]), NUM_RP_PTS,
"ircl_prn.%s", " at %s recall",
get_param_str_ircl_prn},
{"Precision",
0, 1, 0, 0, 0, 1, offsetof(TREC_EVAL, precis_cut[0]), NUM_CUTOFF,
"P%s", " after %s docs retrieved",
get_param_str_cutoff},
/* end of short output measures (the major ones) */
{"Recall",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, recall_cut[0]), NUM_CUTOFF,
"recall%s", " after %s docs retrieved",
get_param_str_cutoff},
{"R-based precision",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, R_prec_cut[0]), NUM_PREC_PTS-1,
"%sR-prec", "- precision after %s * R docs retrieved",
get_param_str_Rcutoff},
{"Relative precision",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_precis_cut[0]), NUM_CUTOFF,
"relative_prec%s", " after %s docs retrieved",
get_param_str_cutoff},
{"Unranked Average Precision",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, uap_cut[0]), NUM_CUTOFF,
"unranked_avg_prec%s", " after %s docs retrieved",
get_param_str_cutoff},
{"Relative Unranked Average Precision",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, rel_uap_cut[0]), NUM_CUTOFF,
"relative_unranked_avg_prec%s", " after %s docs retrieved",
get_param_str_cutoff},
{"Utility (a,b,c,d)",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, exact_utility), 1,
"utility_%s", " Coefficients %s ",
get_param_str_utility},
{"Recall averaged at X nonrel docs",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, av_fall_recall), 1,
"rcl_at_%s_nonrel", " X= %s ",
get_param_str_maxfallout},
{"Fallout - Recall Averages",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, fall_recall[0]), NUM_FR_PTS,
"fallout_recall_%s", "- recall after %s nonrel docs retrieved",
get_param_str_fall_recall},
{"Interpolated R-based precision,",
0, 0, 0, 0, 0, 1, offsetof(TREC_EVAL, int_R_prec_cut[0]), NUM_PREC_PTS-1,
"int_%sR-prec", " after %s * R docs retrieved",
get_param_str_Rcutoff},
{"Time: Utility (a,b,c,d):",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, av_time_utility), 1,
"time_integral_utility_%s", " Coefficients %s ",
get_param_str_utility},
{"Time: num_rel at cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_rel[0]), NUM_TIME_PTS,
"time_num_rel_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: num_nonrel at cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_num_nrel[0]), NUM_TIME_PTS,
"time_num_nonrel_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: cumulative rel at cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_cum_rel[0]), NUM_TIME_PTS,
"time_cum_rel_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: precision at time cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS,
"time_precis_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: precision at time cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_precis[0]), NUM_TIME_PTS,
"time_precis_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: relative precision at time cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_relprecis[0]), NUM_TIME_PTS,
"time_relative_precis_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: unranked precision at time cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_uap[0]), NUM_TIME_PTS,
"time_uap_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: relative unranked precision at time cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_reluap[0]), NUM_TIME_PTS,
"time_relative_uap_%s", " after %s seconds",
get_param_str_time_cutoff},
{"Time: utility at time cutoff:",
0, 0, 1, 0, 0, 1, offsetof(TREC_EVAL, time_utility[0]), NUM_TIME_PTS,
"time_utility_%s", " after %s seconds",
get_param_str_time_utility_cutoff},
};
int num_param_meas = sizeof (param_meas) / sizeof (param_meas[0]);
MICRO_MEASURE micro_meas[] = {
{"micro_prec", "Total relevant retrieved documents / Total retrieved documents",
0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_ret)},
{"micro_recall", "Total relevant retrieved documents / Total relevant documents",
0, offsetof(TREC_EVAL, num_rel_ret), offsetof(TREC_EVAL, num_rel)},
{"micro_bpref", "Total correct preferences / Total possible preferences",
0, offsetof(TREC_EVAL, bpref_num_correct), offsetof(TREC_EVAL, bpref_num_possible)},
};
int num_micro_meas = sizeof (micro_meas) / sizeof (micro_meas[0]);
trec_eval.8.1/print_meas.c
+336
View File
@@ -0,0 +1,336 @@
#ifdef RCSID
static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/tr_eval.c,v 11.0 1992/07/21 18:20:33 chrisb Exp chrisb $";
#endif
/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley.
Permission is granted for use of this file in unmodified form for
research purposes. Please contact the SMART project to obtain
permission for other uses.
*/
#include "common.h"
#include "sysfunc.h"
#include "buf.h"
#include "trec_eval.h"
static SM_BUF internal_output = {0, 0, (char *) 0};
int add_buf_string();
extern SINGLE_MEASURE sing_meas[];
extern PARAMETERIZED_MEASURE param_meas[];
extern MICRO_MEASURE micro_meas[];
extern int num_param_meas, num_sing_meas, num_micro_meas;
int
accumulate_results (query_eval, accum_eval)
TREC_EVAL *query_eval;
TREC_EVAL *accum_eval;
{
long i,j;
float *float_query, *float_accum;
long *long_query, *long_accum;
if (query_eval->num_ret <= 0)
return (0);
accum_eval->num_queries++;
for (i = 0; i < num_sing_meas; i++) {
if (sing_meas[i].is_long_flag) {
long_query = (long *) (((char *) query_eval) +
sing_meas[i].byte_offset);
long_accum = (long *) (((char *) accum_eval) +
sing_meas[i].byte_offset);
*long_accum += *long_query;
}
else {
float_query = (float *) (((char *) query_eval) +
sing_meas[i].byte_offset);
float_accum = (float *) (((char *) accum_eval) +
sing_meas[i].byte_offset);
*float_accum += *float_query;
}
}
for (i = 0; i < num_param_meas; i++) {
for (j = 0; j < param_meas[i].num_values; j++) {
if (param_meas[i].is_long_flag) {
long_query = (long *) (((char *) query_eval) +
param_meas[i].byte_offset);
long_accum = (long *) (((char *) accum_eval) +
param_meas[i].byte_offset);
long_accum[j] += long_query[j];
}
else {
float_query = (float *) (((char *) query_eval) +
param_meas[i].byte_offset);
float_accum = (float *) (((char *) accum_eval) +
param_meas[i].byte_offset);
float_accum[j] += float_query[j];
}
}
}
return (0);
}
void
print_rel_trec_eval_list (is_single_query_flag, epi, eval, output)
long is_single_query_flag;
EVAL_PARAM_INFO *epi;
TREC_EVAL *eval;
SM_BUF *output;
{
long i,j;
char temp_buf[1024];
char q_buf[20];
char name_buf[80];
SM_BUF *out_p;
long long_eval;
float float_eval;
if (output == NULL) {
out_p = &internal_output;
out_p->end = 0;
}
else
out_p = output;
if (is_single_query_flag) {
(void) sprintf (q_buf, "%.20s", eval[0].qid);
}
else {
(void) sprintf (q_buf, "%s", "all");
(void) sprintf (temp_buf, "%-15s\t%s\t%ld\n",
"num_q", q_buf, eval->num_queries);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
for (i = 0; i < num_sing_meas; i++) {
if ((! sing_meas[i].print_short_flag) && (! epi->all_flag))
continue;
if (sing_meas[i].print_time_flag && (!epi->time_flag))
continue;
if (sing_meas[i].print_only_query_flag && (!is_single_query_flag))
continue;
if (sing_meas[i].print_only_average_flag && (is_single_query_flag))
continue;
if (sing_meas[i].is_long_flag) {
long_eval = *((long *) (((char *) eval) +
sing_meas[i].byte_offset));
if (sing_meas[i].avg_results_flag)
long_eval /= eval->num_queries;
(void) sprintf (temp_buf, "%-15s\t%s\t%ld\n",
sing_meas[i].name, q_buf, long_eval);
}
else {
float_eval = *((float *) (((char *) eval) +
sing_meas[i].byte_offset));
if (sing_meas[i].avg_results_flag)
float_eval /= eval->num_queries;
else if (sing_meas[i].avg_rel_results_flag && eval->num_rel > 0)
/* average over number of rel docs instead of number queries */
float_eval /= eval->num_rel;
else if (sing_meas[i].gm_results_flag) {
/* computing geometric mean instead of mean */
if (!is_single_query_flag && epi->average_complete_flag)
/* Must patch up averages for any missing queries, since */
/* value of 0 means perfection */
float_eval += (eval->num_queries - eval->num_orig_queries)*
log (MIN_GEO_MEAN);
float_eval = (float) exp ((double) (float_eval /
eval->num_queries));
}
(void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n",
sing_meas[i].name, q_buf, float_eval);
}
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
for (i = 0; i < num_param_meas; i++) {
if ((! param_meas[i].print_short_flag) && (! epi->all_flag))
continue;
if (param_meas[i].print_time_flag && (!epi->time_flag))
continue;
if (param_meas[i].print_only_query_flag && (!is_single_query_flag))
continue;
if (param_meas[i].print_only_average_flag && (is_single_query_flag))
continue;
for (j = 0; j < param_meas[i].num_values; j++) {
sprintf (name_buf, param_meas[i].format_string,
param_meas[i].get_param_str (epi, j));
if (param_meas[i].is_long_flag) {
long_eval = ((long *) (((char *) eval) +
param_meas[i].byte_offset))[j];
if (param_meas[i].avg_results_flag)
long_eval /= eval->num_queries;
(void) sprintf (temp_buf, "%-15s\t%s\t%ld\n",
name_buf, q_buf, long_eval);
}
else {
float_eval = ((float *) (((char *) eval) +
param_meas[i].byte_offset))[j];
if (param_meas[i].avg_results_flag)
float_eval /= eval->num_queries;
(void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n",
name_buf, q_buf, float_eval);
}
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
}
if (! is_single_query_flag) {
long denom_long_eval;
for (i = 0; i < num_micro_meas; i++) {
if ((! micro_meas[i].print_short_flag) && (! epi->all_flag))
continue;
long_eval = *((long *) (((char *) eval) +
micro_meas[i].numerator_byte_offset));
denom_long_eval = *((long *) (((char *) eval) +
micro_meas[i].denominator_byte_offset));
float_eval = (float) long_eval / (float) denom_long_eval;
(void) sprintf (temp_buf, "%-15s\t%s\t%6.4f\n",
micro_meas[i].name, q_buf, float_eval);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
}
if (output == NULL) {
(void) fwrite (out_p->buf, 1, out_p->end, stdout);
out_p->end = 0;
}
}
static long cutoff[] = CUTOFF_VALUES;
void
old_print_trec_eval_list (epi, eval, num_runs, output)
EVAL_PARAM_INFO *epi;
TREC_EVAL *eval;
int num_runs;
SM_BUF *output;
{
long i,j;
char temp_buf[1024];
SM_BUF *out_p;
if (output == NULL) {
out_p = &internal_output;
out_p->end = 0;
}
else
out_p = output;
/* Print total numbers retrieved/rel for all runs */
if (UNDEF == add_buf_string("\nQueryid (Num):\t", out_p))
return;
for (i = 0; i < num_runs; i++) {
if (UNDEF == add_buf_string (eval->qid, out_p))
return;
}
if (UNDEF == add_buf_string("\nTotal number of documents over all queries",
out_p))
return;
if (UNDEF == add_buf_string("\n Retrieved:", out_p))
return;
for (i = 0; i < num_runs; i++) {
(void) sprintf (temp_buf, " %5ld", eval[i].num_ret);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
if (UNDEF == add_buf_string("\n Relevant: ", out_p))
return;
for (i = 0; i < num_runs; i++) {
(void) sprintf (temp_buf, " %5ld", eval[i].num_rel);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
if (UNDEF == add_buf_string("\n Rel_ret: ", out_p))
return;
for (i = 0; i < num_runs; i++) {
(void) sprintf (temp_buf, " %5ld", eval[i].num_rel_ret);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
/* Print recall precision figures at NUM_RP_PTS recall levels */
if (UNDEF == add_buf_string
("\nInterpolated Recall - Precision Averages:", out_p))
return;
for (j = 0; j < NUM_RP_PTS; j++) {
(void) sprintf (temp_buf, "\n at %4.2f ",
(float) j / (NUM_RP_PTS - 1));
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
for (i = 0; i < num_runs; i++) {
(void) sprintf (temp_buf, " %6.4f ",
eval[i].int_recall_precis[j] /eval[i].num_queries);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
}
/* Print average recall precision and percentage improvement */
(void) sprintf (temp_buf,
"\nAverage precision (non-interpolated) for all rel docs(averaged over queries)\n ");
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
for (i = 0; i < num_runs; i++) {
(void) sprintf (temp_buf, " %6.4f ",
eval[i].av_recall_precis / eval[i].num_queries);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
if (num_runs > 1) {
(void) sprintf (temp_buf, "\n %% Change: ");
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
for (i = 1; i < num_runs; i++) {
(void) sprintf (temp_buf, " %6.1f ",
(((eval[i].av_recall_precis / eval[i].num_queries)/
(eval[0].av_recall_precis / eval[i].num_queries))
- 1.0) * 100.0);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
}
(void) sprintf (temp_buf, "\nPrecision:");
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
for (j = 0; j < NUM_CUTOFF; j++) {
(void) sprintf (temp_buf, "\n At %4ld docs:", cutoff[j]);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
for (i = 0; i < num_runs; i++) {
(void) sprintf (temp_buf, " %6.4f",
eval[i].precis_cut[j] / eval[i].num_queries);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
}
(void) sprintf (temp_buf, "\nR-Precision (precision after R (= num_rel for a query) docs retrieved):\n Exact: ");
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
for (i = 0; i < num_runs; i++) {
(void) sprintf (temp_buf, " %6.4f",
eval[i].R_recall_precis / eval[i].num_queries);
if (UNDEF == add_buf_string (temp_buf, out_p))
return;
}
if (UNDEF == add_buf_string ("\n", out_p))
return;
if (output == NULL) {
(void) fwrite (out_p->buf, 1, out_p->end, stdout);
out_p->end = 0;
}
return;
}
trec_eval.8.1/smart_error.h
+24
View File
@@ -0,0 +1,24 @@
#ifndef SMART_ERRORH
#define SMART_ERRORH
#include <errno.h>
#define SMART_MINERR 1000
#define SM_INCON_ERR 1000
#define SM_ILLSK_ERR 1001
#define SM_ILLMD_ERR 1002
#define SM_ILLPA_ERR 1003
#define SMART_NUMERR 4
extern int errno;
extern int smart_errno; /* If > 0 and <= sys_nerr then refers to */
/* sys_errlist, else if >= smart_errmin */
/* and <= smart_errmax, then smart_errlist */
extern char *smart_message; /* Message to be printed (often filename) */
extern char *smart_routine; /* Major routine issuing error message */
#define set_error(n,m,r) { if (n > 0) smart_errno = n;\
smart_message = m;\
smart_routine = r; }
#define clr_err() smart_errno = errno = 0
#endif /* SMART_ERRORH */
trec_eval.8.1/sysfunc.h
+39
View File
@@ -0,0 +1,39 @@
#ifndef SYSFUNCH
#define SYSFUNCH
/* Declarations of major functions within standard C libraries */
/* Once all of the major systems get their act together (and I follow
suit!), this file should just include system header files from
/usr/include. Until then... */
#include <unistd.h>
#include <limits.h>
#include <ctype.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <memory.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <sys/time.h>
#include <sys/mman.h>
/* For time being, define Berkeley constructs in terms of SVR4 constructs*/
#define bzero(dest,len) memset(dest,'\0',len)
#define bcopy(source,dest,len) memcpy(dest,source,len)
#define srandom(seed) srand(seed)
#define random() rand()
/* ANSI should give us an offsetof suitable for the implementation;
* otherwise, try a non-portable but commonly supported definition
*/
#ifdef __STDC__
#include <stddef.h>
#endif
#ifndef offsetof
#define offsetof(type, member) ((size_t) \
((char *)&((type*)0)->member - (char *)(type *)0))
#endif
#endif /* SYSFUNCH */
trec_eval.8.1/test/out.test
+29
View File
@@ -0,0 +1,29 @@
num_q all 3
num_ret all 1500
num_rel all 561
num_rel_ret all 131
map all 0.1785
gm_ap all 0.1051
R-prec all 0.2174
bpref all 0.1981
recip_rank all 0.4064
ircl_prn.0.00 all 0.4665
ircl_prn.0.10 all 0.3884
ircl_prn.0.20 all 0.3186
ircl_prn.0.30 all 0.2732
ircl_prn.0.40 all 0.2666
ircl_prn.0.50 all 0.2184
ircl_prn.0.60 all 0.0822
ircl_prn.0.70 all 0.0348
ircl_prn.0.80 all 0.0312
ircl_prn.0.90 all 0.0312
ircl_prn.1.00 all 0.0312
P5 all 0.2667
P10 all 0.3000
P15 all 0.3111
P20 all 0.3667
P30 all 0.3333
P100 all 0.2467
P200 all 0.1600
P500 all 0.0873
P1000 all 0.0437
trec_eval.8.1/test/out.test.a
+135
View File
@@ -0,0 +1,135 @@
num_q all 3
num_ret all 1500
num_rel all 561
num_rel_ret all 131
map all 0.1785
gm_ap all 0.1051
R-prec all 0.2174
bpref all 0.1981
recip_rank all 0.4064
num_nonrel_judged_ret all 607
exact_prec all 0.0873
exact_recall all 0.5997
11-pt_avg all 0.1947
3-pt_avg all 0.1894
avg_doc_prec all 0.0862
exact_relative_prec all 0.5997
avg_relative_prec all 0.5617
exact_unranked_avg_prec all 0.0354
exact_relative_unranked_avg_prec all 0.0102
map_at_R all 0.2948
int_map all 0.1895
exact_int_R_rcl_prec all 0.2552
int_map_at_R all 0.3393
bpref_allnonrel all 0.5684
bpref_retnonrel all 0.4531
bpref_topnonrel all 0.3268
bpref_top5Rnonrel all 0.2982
bpref_top10Rnonrel all 0.4472
bpref_top10pRnonrel all 0.2050
bpref_top25pRnonrel all 0.2222
bpref_top50pRnonrel all 0.2848
bpref_top25p2Rnonrel all 0.2640
bpref_retall all 0.7076
bpref_5 all 0.2933
bpref_10 all 0.3133
bpref_num_all all 45867.0000
bpref_num_ret all 5955.0000
bpref_num_correct all 30286
bpref_num_possible all 230705
old_bpref all 0.1845
old_bpref_top10pRnonrel all 0.1912
infAP all 0.1785
gm_bpref all 0.0083
ircl_prn.0.00 all 0.4665
ircl_prn.0.10 all 0.3884
ircl_prn.0.20 all 0.3186
ircl_prn.0.30 all 0.2732
ircl_prn.0.40 all 0.2666
ircl_prn.0.50 all 0.2184
ircl_prn.0.60 all 0.0822
ircl_prn.0.70 all 0.0348
ircl_prn.0.80 all 0.0312
ircl_prn.0.90 all 0.0312
ircl_prn.1.00 all 0.0312
P5 all 0.2667
P10 all 0.3000
P15 all 0.3111
P20 all 0.3667
P30 all 0.3333
P100 all 0.2467
P200 all 0.1600
P500 all 0.0873
P1000 all 0.0437
recall5 all 0.0173
recall10 all 0.0317
recall15 all 0.0534
recall20 all 0.1061
recall30 all 0.1335
recall100 all 0.4980
recall200 all 0.5533
recall500 all 0.5997
recall1000 all 0.5997
0.20R-prec all 0.3445
0.40R-prec all 0.3157
0.60R-prec all 0.2842
0.80R-prec all 0.2613
1.00R-prec all 0.2174
1.20R-prec all 0.1921
1.40R-prec all 0.1653
1.60R-prec all 0.1468
1.80R-prec all 0.1308
2.00R-prec all 0.1369
relative_prec5 all 0.2667
relative_prec10 all 0.3000
relative_prec15 all 0.3111
relative_prec20 all 0.3833
relative_prec30 all 0.3556
relative_prec100 all 0.5585
relative_prec200 all 0.5938
relative_prec500 all 0.5997
relative_prec1000 all 0.5997
unranked_avg_prec5 all 0.0139
unranked_avg_prec10 all 0.0215
unranked_avg_prec15 all 0.0417
unranked_avg_prec20 all 0.0580
unranked_avg_prec30 all 0.0721
unranked_avg_prec100 all 0.1071
unranked_avg_prec200 all 0.0648
unranked_avg_prec500 all 0.0354
unranked_avg_prec1000 all 0.0177
relative_unranked_avg_prec5 all 0.2133
relative_unranked_avg_prec10 all 0.1767
relative_unranked_avg_prec15 all 0.2193
relative_unranked_avg_prec20 all 0.2375
relative_unranked_avg_prec30 all 0.2007
relative_unranked_avg_prec100 all 0.3868
relative_unranked_avg_prec200 all 0.4569
relative_unranked_avg_prec500 all 0.4814
relative_unranked_avg_prec1000 all 0.0025
utility_1.0_-1.0_0.0_0.0 all -412.6667
rcl_at_142_nonrel all 0.3848
fallout_recall_0 all 0.0087
fallout_recall_14 all 0.1334
fallout_recall_28 all 0.2049
fallout_recall_42 all 0.3597
fallout_recall_56 all 0.3618
fallout_recall_71 all 0.4299
fallout_recall_85 all 0.4697
fallout_recall_99 all 0.5413
fallout_recall_113 all 0.5484
fallout_recall_127 all 0.5491
fallout_recall_142 all 0.5505
int_0.20R-prec all 0.3990
int_0.40R-prec all 0.3559
int_0.60R-prec all 0.3312
int_0.80R-prec all 0.2995
int_1.00R-prec all 0.2552
int_1.20R-prec all 0.2300
int_1.40R-prec all 0.2032
int_1.60R-prec all 0.1847
int_1.80R-prec all 0.1687
int_2.00R-prec all 0.1581
micro_prec all 0.0873
micro_recall all 0.2335
micro_bpref all 0.1313
trec_eval.8.1/test/out.test.aq
+525
View File
@@ -0,0 +1,525 @@
num_ret 301 500
num_rel 301 474
num_rel_ret 301 71
map 301 0.0324
R-prec 301 0.1456
bpref 301 0.1230
recip_rank 301 0.1667
num_nonrel_judged_ret 301 188
exact_prec 301 0.1420
exact_recall 301 0.1498
11-pt_avg 301 0.0450
3-pt_avg 301 0.0000
avg_doc_prec 301 0.0324
exact_relative_prec 301 0.1498
avg_relative_prec 301 0.1957
exact_unranked_avg_prec 301 0.0213
exact_relative_unranked_avg_prec 301 0.0202
map_at_R 301 0.1981
int_map 301 0.0339
exact_int_R_rcl_prec 301 0.1456
int_map_at_R 301 0.2095
bpref_allnonrel 301 0.1395
bpref_retnonrel 301 0.0824
bpref_topnonrel 301 0.0445
bpref_top5Rnonrel 301 0.1395
bpref_top10Rnonrel 301 0.1395
bpref_top10pRnonrel 301 0.1236
bpref_top25pRnonrel 301 0.1244
bpref_top50pRnonrel 301 0.1256
bpref_top25p2Rnonrel 301 0.1368
bpref_retall 301 0.5499
bpref_5 301 0.0000
bpref_10 301 0.1000
bpref_num_all 301 81606.0000
bpref_num_ret 301 7340.0000
bpref_num_correct 301 27646
bpref_num_possible 301 224676
old_bpref 301 0.0824
old_bpref_top10pRnonrel 301 0.0824
infAP 301 0.0324
rank_first_rel 301 6
ircl_prn.0.00 301 0.2857
ircl_prn.0.10 301 0.2096
ircl_prn.0.20 301 0.0000
ircl_prn.0.30 301 0.0000
ircl_prn.0.40 301 0.0000
ircl_prn.0.50 301 0.0000
ircl_prn.0.60 301 0.0000
ircl_prn.0.70 301 0.0000
ircl_prn.0.80 301 0.0000
ircl_prn.0.90 301 0.0000
ircl_prn.1.00 301 0.0000
P5 301 0.0000
P10 301 0.2000
P15 301 0.1333
P20 301 0.2500
P30 301 0.2333
P100 301 0.2300
P200 301 0.2100
P500 301 0.1420
P1000 301 0.0710
recall5 301 0.0000
recall10 301 0.0042
recall15 301 0.0042
recall20 301 0.0105
recall30 301 0.0148
recall100 301 0.0485
recall200 301 0.0886
recall500 301 0.1498
recall1000 301 0.1498
0.20R-prec 301 0.2211
0.40R-prec 301 0.2053
0.60R-prec 301 0.1930
0.80R-prec 301 0.1711
1.00R-prec 301 0.1456
1.20R-prec 301 0.1248
1.40R-prec 301 0.1069
1.60R-prec 301 0.0935
1.80R-prec 301 0.0831
2.00R-prec 301 0.0749
relative_prec5 301 0.0000
relative_prec10 301 0.2000
relative_prec15 301 0.1333
relative_prec20 301 0.2500
relative_prec30 301 0.2333
relative_prec100 301 0.2300
relative_prec200 301 0.2100
relative_prec500 301 0.1498
relative_prec1000 301 0.1498
unranked_avg_prec5 301 0.0000
unranked_avg_prec10 301 0.0008
unranked_avg_prec15 301 0.0006
unranked_avg_prec20 301 0.0026
unranked_avg_prec30 301 0.0034
unranked_avg_prec100 301 0.0112
unranked_avg_prec200 301 0.0186
unranked_avg_prec500 301 0.0213
unranked_avg_prec1000 301 0.0106
relative_unranked_avg_prec5 301 0.0000
relative_unranked_avg_prec10 301 0.0400
relative_unranked_avg_prec15 301 0.0178
relative_unranked_avg_prec20 301 0.0625
relative_unranked_avg_prec30 301 0.0544
relative_unranked_avg_prec100 301 0.0529
relative_unranked_avg_prec200 301 0.0441
relative_unranked_avg_prec500 301 0.0224
relative_unranked_avg_prec1000 301 0.0050
utility_1.0_-1.0_0.0_0.0 301 -358.0000
rcl_at_142_nonrel 301 0.0439
fallout_recall_0 301 0.0000
fallout_recall_14 301 0.0105
fallout_recall_28 301 0.0211
fallout_recall_42 301 0.0338
fallout_recall_56 301 0.0401
fallout_recall_71 301 0.0443
fallout_recall_85 301 0.0506
fallout_recall_99 301 0.0654
fallout_recall_113 301 0.0738
fallout_recall_127 301 0.0759
fallout_recall_142 301 0.0802
int_0.20R-prec 301 0.2414
int_0.40R-prec 301 0.2123
int_0.60R-prec 301 0.1937
int_0.80R-prec 301 0.1719
int_1.00R-prec 301 0.1456
int_1.20R-prec 301 0.1248
int_1.40R-prec 301 0.1069
int_1.60R-prec 301 0.0935
int_1.80R-prec 301 0.0831
int_2.00R-prec 301 0.0749
num_ret 302 500
num_rel 302 77
num_rel_ret 302 50
map 302 0.4175
R-prec 302 0.5065
bpref 302 0.4712
recip_rank 302 1.0000
num_nonrel_judged_ret 302 214
exact_prec 302 0.1000
exact_recall 302 0.6494
11-pt_avg 302 0.4327
3-pt_avg 302 0.4613
avg_doc_prec 302 0.4175
exact_relative_prec 302 0.6494
avg_relative_prec 302 0.6097
exact_unranked_avg_prec 302 0.0649
exact_relative_unranked_avg_prec 302 0.0100
map_at_R 302 0.6862
int_map 302 0.4288
exact_int_R_rcl_prec 302 0.5065
int_map_at_R 302 0.7061
bpref_allnonrel 302 0.6273
bpref_retnonrel 302 0.5481
bpref_topnonrel 302 0.4919
bpref_top5Rnonrel 302 0.5931
bpref_top10Rnonrel 302 0.6212
bpref_top10pRnonrel 302 0.4813
bpref_top25pRnonrel 302 0.4935
bpref_top50pRnonrel 302 0.5088
bpref_top25p2Rnonrel 302 0.5308
bpref_retall 302 0.8440
bpref_5 302 0.8800
bpref_10 302 0.8400
bpref_num_all 302 47531.0000
bpref_num_ret 302 9031.0000
bpref_num_correct 302 2640
bpref_num_possible 302 5929
old_bpref 302 0.4712
old_bpref_top10pRnonrel 302 0.4813
infAP 302 0.4175
rank_first_rel 302 1
ircl_prn.0.00 302 1.0000
ircl_prn.0.10 302 0.8421
ircl_prn.0.20 302 0.8421
ircl_prn.0.30 302 0.7059
ircl_prn.0.40 302 0.6863
ircl_prn.0.50 302 0.5417
ircl_prn.0.60 302 0.1420
ircl_prn.0.70 302 0.0000
ircl_prn.0.80 302 0.0000
ircl_prn.0.90 302 0.0000
ircl_prn.1.00 302 0.0000
P5 302 0.8000
P10 302 0.7000
P15 302 0.8000
P20 302 0.8000
P30 302 0.7333
P100 302 0.4200
P200 302 0.2200
P500 302 0.1000
P1000 302 0.0500
recall5 302 0.0519
recall10 302 0.0909
recall15 302 0.1558
recall20 302 0.2078
recall30 302 0.2857
recall100 302 0.5455
recall200 302 0.5714
recall500 302 0.6494
recall1000 302 0.6494
0.20R-prec 302 0.8125
0.40R-prec 302 0.7419
0.60R-prec 302 0.6596
0.80R-prec 302 0.6129
1.00R-prec 302 0.5065
1.20R-prec 302 0.4516
1.40R-prec 302 0.3889
1.60R-prec 302 0.3468
1.80R-prec 302 0.3094
2.00R-prec 302 0.2857
relative_prec5 302 0.8000
relative_prec10 302 0.7000
relative_prec15 302 0.8000
relative_prec20 302 0.8000
relative_prec30 302 0.7333
relative_prec100 302 0.5455
relative_prec200 302 0.5714
relative_prec500 302 0.6494
relative_prec1000 302 0.6494
unranked_avg_prec5 302 0.0416
unranked_avg_prec10 302 0.0636
unranked_avg_prec15 302 0.1247
unranked_avg_prec20 302 0.1662
unranked_avg_prec30 302 0.2095
unranked_avg_prec100 302 0.2291
unranked_avg_prec200 302 0.1257
unranked_avg_prec500 302 0.0649
unranked_avg_prec1000 302 0.0325
relative_unranked_avg_prec5 302 0.6400
relative_unranked_avg_prec10 302 0.4900
relative_unranked_avg_prec15 302 0.6400
relative_unranked_avg_prec20 302 0.6400
relative_unranked_avg_prec30 302 0.5378
relative_unranked_avg_prec100 302 0.2975
relative_unranked_avg_prec200 302 0.3265
relative_unranked_avg_prec500 302 0.4217
relative_unranked_avg_prec1000 302 0.0025
utility_1.0_-1.0_0.0_0.0 302 -400.0000
rcl_at_142_nonrel 302 0.5091
fallout_recall_0 302 0.0260
fallout_recall_14 302 0.3896
fallout_recall_28 302 0.4935
fallout_recall_42 302 0.5455
fallout_recall_56 302 0.5455
fallout_recall_71 302 0.5455
fallout_recall_85 302 0.5584
fallout_recall_99 302 0.5584
fallout_recall_113 302 0.5714
fallout_recall_127 302 0.5714
fallout_recall_142 302 0.5714
int_0.20R-prec 302 0.8421
int_0.40R-prec 302 0.7419
int_0.60R-prec 302 0.6863
int_0.80R-prec 302 0.6129
int_1.00R-prec 302 0.5065
int_1.20R-prec 302 0.4516
int_1.40R-prec 302 0.3889
int_1.60R-prec 302 0.3468
int_1.80R-prec 302 0.3094
int_2.00R-prec 302 0.2857
num_ret 303 500
num_rel 303 10
num_rel_ret 303 10
map 303 0.0858
R-prec 303 0.0000
bpref 303 0.0000
recip_rank 303 0.0526
num_nonrel_judged_ret 303 205
exact_prec 303 0.0200
exact_recall 303 1.0000
11-pt_avg 303 0.1065
3-pt_avg 303 0.1069
avg_doc_prec 303 0.0858
exact_relative_prec 303 1.0000
avg_relative_prec 303 0.8798
exact_unranked_avg_prec 303 0.0200
exact_relative_unranked_avg_prec 303 0.0004
map_at_R 303 0.0000
int_map 303 0.1058
exact_int_R_rcl_prec 303 0.1136
int_map_at_R 303 0.1023
bpref_allnonrel 303 0.9384
bpref_retnonrel 303 0.7288
bpref_topnonrel 303 0.4440
bpref_top5Rnonrel 303 0.1620
bpref_top10Rnonrel 303 0.5810
bpref_top10pRnonrel 303 0.0100
bpref_top25pRnonrel 303 0.0486
bpref_top50pRnonrel 303 0.2200
bpref_top25p2Rnonrel 303 0.1244
bpref_retall 303 0.7288
bpref_5 303 0.0000
bpref_10 303 0.0000
bpref_num_all 303 8464.0000
bpref_num_ret 303 1494.0000
bpref_num_correct 303 0
bpref_num_possible 303 100
old_bpref 303 0.0000
old_bpref_top10pRnonrel 303 0.0100
infAP 303 0.0858
rank_first_rel 303 19
ircl_prn.0.00 303 0.1136
ircl_prn.0.10 303 0.1136
ircl_prn.0.20 303 0.1136
ircl_prn.0.30 303 0.1136
ircl_prn.0.40 303 0.1136
ircl_prn.0.50 303 0.1136
ircl_prn.0.60 303 0.1045
ircl_prn.0.70 303 0.1045
ircl_prn.0.80 303 0.0935
ircl_prn.0.90 303 0.0935
ircl_prn.1.00 303 0.0935
P5 303 0.0000
P10 303 0.0000
P15 303 0.0000
P20 303 0.0500
P30 303 0.0333
P100 303 0.0900
P200 303 0.0500
P500 303 0.0200
P1000 303 0.0100
recall5 303 0.0000
recall10 303 0.0000
recall15 303 0.0000
recall20 303 0.1000
recall30 303 0.1000
recall100 303 0.9000
recall200 303 1.0000
recall500 303 1.0000
recall1000 303 1.0000
0.20R-prec 303 0.0000
0.40R-prec 303 0.0000
0.60R-prec 303 0.0000
0.80R-prec 303 0.0000
1.00R-prec 303 0.0000
1.20R-prec 303 0.0000
1.40R-prec 303 0.0000
1.60R-prec 303 0.0000
1.80R-prec 303 0.0000
2.00R-prec 303 0.0500
relative_prec5 303 0.0000
relative_prec10 303 0.0000
relative_prec15 303 0.0000
relative_prec20 303 0.1000
relative_prec30 303 0.1000
relative_prec100 303 0.9000
relative_prec200 303 1.0000
relative_prec500 303 1.0000
relative_prec1000 303 1.0000
unranked_avg_prec5 303 0.0000
unranked_avg_prec10 303 0.0000
unranked_avg_prec15 303 0.0000
unranked_avg_prec20 303 0.0050
unranked_avg_prec30 303 0.0033
unranked_avg_prec100 303 0.0810
unranked_avg_prec200 303 0.0500
unranked_avg_prec500 303 0.0200
unranked_avg_prec1000 303 0.0100
relative_unranked_avg_prec5 303 0.0000
relative_unranked_avg_prec10 303 0.0000
relative_unranked_avg_prec15 303 0.0000
relative_unranked_avg_prec20 303 0.0100
relative_unranked_avg_prec30 303 0.0100
relative_unranked_avg_prec100 303 0.8100
relative_unranked_avg_prec200 303 1.0000
relative_unranked_avg_prec500 303 1.0000
relative_unranked_avg_prec1000 303 0.0001
utility_1.0_-1.0_0.0_0.0 303 -480.0000
rcl_at_142_nonrel 303 0.6014
fallout_recall_0 303 0.0000
fallout_recall_14 303 0.0000
fallout_recall_28 303 0.1000
fallout_recall_42 303 0.5000
fallout_recall_56 303 0.5000
fallout_recall_71 303 0.7000
fallout_recall_85 303 0.8000
fallout_recall_99 303 1.0000
fallout_recall_113 303 1.0000
fallout_recall_127 303 1.0000
fallout_recall_142 303 1.0000
int_0.20R-prec 303 0.1136
int_0.40R-prec 303 0.1136
int_0.60R-prec 303 0.1136
int_0.80R-prec 303 0.1136
int_1.00R-prec 303 0.1136
int_1.20R-prec 303 0.1136
int_1.40R-prec 303 0.1136
int_1.60R-prec 303 0.1136
int_1.80R-prec 303 0.1136
int_2.00R-prec 303 0.1136
num_q all 3
num_ret all 1500
num_rel all 561
num_rel_ret all 131
map all 0.1785
gm_ap all 0.1051
R-prec all 0.2174
bpref all 0.1981
recip_rank all 0.4064
num_nonrel_judged_ret all 607
exact_prec all 0.0873
exact_recall all 0.5997
11-pt_avg all 0.1947
3-pt_avg all 0.1894
avg_doc_prec all 0.0862
exact_relative_prec all 0.5997
avg_relative_prec all 0.5617
exact_unranked_avg_prec all 0.0354
exact_relative_unranked_avg_prec all 0.0102
map_at_R all 0.2948
int_map all 0.1895
exact_int_R_rcl_prec all 0.2552
int_map_at_R all 0.3393
bpref_allnonrel all 0.5684
bpref_retnonrel all 0.4531
bpref_topnonrel all 0.3268
bpref_top5Rnonrel all 0.2982
bpref_top10Rnonrel all 0.4472
bpref_top10pRnonrel all 0.2050
bpref_top25pRnonrel all 0.2222
bpref_top50pRnonrel all 0.2848
bpref_top25p2Rnonrel all 0.2640
bpref_retall all 0.7076
bpref_5 all 0.2933
bpref_10 all 0.3133
bpref_num_all all 45867.0000
bpref_num_ret all 5955.0000
bpref_num_correct all 30286
bpref_num_possible all 230705
old_bpref all 0.1845
old_bpref_top10pRnonrel all 0.1912
infAP all 0.1785
gm_bpref all 0.0083
ircl_prn.0.00 all 0.4665
ircl_prn.0.10 all 0.3884
ircl_prn.0.20 all 0.3186
ircl_prn.0.30 all 0.2732
ircl_prn.0.40 all 0.2666
ircl_prn.0.50 all 0.2184
ircl_prn.0.60 all 0.0822
ircl_prn.0.70 all 0.0348
ircl_prn.0.80 all 0.0312
ircl_prn.0.90 all 0.0312
ircl_prn.1.00 all 0.0312
P5 all 0.2667
P10 all 0.3000
P15 all 0.3111
P20 all 0.3667
P30 all 0.3333
P100 all 0.2467
P200 all 0.1600
P500 all 0.0873
P1000 all 0.0437
recall5 all 0.0173
recall10 all 0.0317
recall15 all 0.0534
recall20 all 0.1061
recall30 all 0.1335
recall100 all 0.4980
recall200 all 0.5533
recall500 all 0.5997
recall1000 all 0.5997
0.20R-prec all 0.3445
0.40R-prec all 0.3157
0.60R-prec all 0.2842
0.80R-prec all 0.2613
1.00R-prec all 0.2174
1.20R-prec all 0.1921
1.40R-prec all 0.1653
1.60R-prec all 0.1468
1.80R-prec all 0.1308
2.00R-prec all 0.1369
relative_prec5 all 0.2667
relative_prec10 all 0.3000
relative_prec15 all 0.3111
relative_prec20 all 0.3833
relative_prec30 all 0.3556
relative_prec100 all 0.5585
relative_prec200 all 0.5938
relative_prec500 all 0.5997
relative_prec1000 all 0.5997
unranked_avg_prec5 all 0.0139
unranked_avg_prec10 all 0.0215
unranked_avg_prec15 all 0.0417
unranked_avg_prec20 all 0.0580
unranked_avg_prec30 all 0.0721
unranked_avg_prec100 all 0.1071
unranked_avg_prec200 all 0.0648
unranked_avg_prec500 all 0.0354
unranked_avg_prec1000 all 0.0177
relative_unranked_avg_prec5 all 0.2133
relative_unranked_avg_prec10 all 0.1767
relative_unranked_avg_prec15 all 0.2193
relative_unranked_avg_prec20 all 0.2375
relative_unranked_avg_prec30 all 0.2007
relative_unranked_avg_prec100 all 0.3868
relative_unranked_avg_prec200 all 0.4569
relative_unranked_avg_prec500 all 0.4814
relative_unranked_avg_prec1000 all 0.0025
utility_1.0_-1.0_0.0_0.0 all -412.6667
rcl_at_142_nonrel all 0.3848
fallout_recall_0 all 0.0087
fallout_recall_14 all 0.1334
fallout_recall_28 all 0.2049
fallout_recall_42 all 0.3597
fallout_recall_56 all 0.3618
fallout_recall_71 all 0.4299
fallout_recall_85 all 0.4697
fallout_recall_99 all 0.5413
fallout_recall_113 all 0.5484
fallout_recall_127 all 0.5491
fallout_recall_142 all 0.5505
int_0.20R-prec all 0.3990
int_0.40R-prec all 0.3559
int_0.60R-prec all 0.3312
int_0.80R-prec all 0.2995
int_1.00R-prec all 0.2552
int_1.20R-prec all 0.2300
int_1.40R-prec all 0.2032
int_1.60R-prec all 0.1847
int_1.80R-prec all 0.1687
int_2.00R-prec all 0.1581
micro_prec all 0.0873
micro_recall all 0.2335
micro_bpref all 0.1313
trec_eval.8.1/test/out.test.aqc
+395
View File
@@ -0,0 +1,395 @@
num_ret 301 500
num_rel 301 474
num_rel_ret 301 71
map 301 0.0324
R-prec 301 0.1456
bpref 301 0.1230
recip_rank 301 0.1667
num_nonrel_judged_ret 301 188
exact_prec 301 0.1420
exact_recall 301 0.1498
11-pt_avg 301 0.0450
3-pt_avg 301 0.0000
avg_doc_prec 301 0.0324
exact_relative_prec 301 0.1498
avg_relative_prec 301 0.1957
exact_unranked_avg_prec 301 0.0213
exact_relative_unranked_avg_prec 301 0.0202
map_at_R 301 0.1981
int_map 301 0.0339
exact_int_R_rcl_prec 301 0.1456
int_map_at_R 301 0.2095
bpref_allnonrel 301 0.1395
bpref_retnonrel 301 0.0824
bpref_topnonrel 301 0.0445
bpref_top5Rnonrel 301 0.1395
bpref_top10Rnonrel 301 0.1395
bpref_top10pRnonrel 301 0.1236
bpref_top25pRnonrel 301 0.1244
bpref_top50pRnonrel 301 0.1256
bpref_top25p2Rnonrel 301 0.1368
bpref_retall 301 0.5499
bpref_5 301 0.0000
bpref_10 301 0.1000
bpref_num_all 301 81606.0000
bpref_num_ret 301 7340.0000
bpref_num_correct 301 27646
bpref_num_possible 301 224676
old_bpref 301 0.0824
old_bpref_top10pRnonrel 301 0.0824
infAP 301 0.0324
rank_first_rel 301 6
ircl_prn.0.00 301 0.2857
ircl_prn.0.10 301 0.2096
ircl_prn.0.20 301 0.0000
ircl_prn.0.30 301 0.0000
ircl_prn.0.40 301 0.0000
ircl_prn.0.50 301 0.0000
ircl_prn.0.60 301 0.0000
ircl_prn.0.70 301 0.0000
ircl_prn.0.80 301 0.0000
ircl_prn.0.90 301 0.0000
ircl_prn.1.00 301 0.0000
P5 301 0.0000
P10 301 0.2000
P15 301 0.1333
P20 301 0.2500
P30 301 0.2333
P100 301 0.2300
P200 301 0.2100
P500 301 0.1420
P1000 301 0.0710
recall5 301 0.0000
recall10 301 0.0042
recall15 301 0.0042
recall20 301 0.0105
recall30 301 0.0148
recall100 301 0.0485
recall200 301 0.0886
recall500 301 0.1498
recall1000 301 0.1498
0.20R-prec 301 0.2211
0.40R-prec 301 0.2053
0.60R-prec 301 0.1930
0.80R-prec 301 0.1711
1.00R-prec 301 0.1456
1.20R-prec 301 0.1248
1.40R-prec 301 0.1069
1.60R-prec 301 0.0935
1.80R-prec 301 0.0831
2.00R-prec 301 0.0749
relative_prec5 301 0.0000
relative_prec10 301 0.2000
relative_prec15 301 0.1333
relative_prec20 301 0.2500
relative_prec30 301 0.2333
relative_prec100 301 0.2300
relative_prec200 301 0.2100
relative_prec500 301 0.1498
relative_prec1000 301 0.1498
unranked_avg_prec5 301 0.0000
unranked_avg_prec10 301 0.0008
unranked_avg_prec15 301 0.0006
unranked_avg_prec20 301 0.0026
unranked_avg_prec30 301 0.0034
unranked_avg_prec100 301 0.0112
unranked_avg_prec200 301 0.0186
unranked_avg_prec500 301 0.0213
unranked_avg_prec1000 301 0.0106
relative_unranked_avg_prec5 301 0.0000
relative_unranked_avg_prec10 301 0.0400
relative_unranked_avg_prec15 301 0.0178
relative_unranked_avg_prec20 301 0.0625
relative_unranked_avg_prec30 301 0.0544
relative_unranked_avg_prec100 301 0.0529
relative_unranked_avg_prec200 301 0.0441
relative_unranked_avg_prec500 301 0.0224
relative_unranked_avg_prec1000 301 0.0050
utility_1.0_-1.0_0.0_0.0 301 -358.0000
rcl_at_142_nonrel 301 0.0439
fallout_recall_0 301 0.0000
fallout_recall_14 301 0.0105
fallout_recall_28 301 0.0211
fallout_recall_42 301 0.0338
fallout_recall_56 301 0.0401
fallout_recall_71 301 0.0443
fallout_recall_85 301 0.0506
fallout_recall_99 301 0.0654
fallout_recall_113 301 0.0738
fallout_recall_127 301 0.0759
fallout_recall_142 301 0.0802
int_0.20R-prec 301 0.2414
int_0.40R-prec 301 0.2123
int_0.60R-prec 301 0.1937
int_0.80R-prec 301 0.1719
int_1.00R-prec 301 0.1456
int_1.20R-prec 301 0.1248
int_1.40R-prec 301 0.1069
int_1.60R-prec 301 0.0935
int_1.80R-prec 301 0.0831
int_2.00R-prec 301 0.0749
num_ret 303 84
num_rel 303 10
num_rel_ret 303 6
map 303 0.2723
R-prec 303 0.4000
bpref 303 0.3300
recip_rank 303 0.3333
num_nonrel_judged_ret 303 61
exact_prec 303 0.0714
exact_recall 303 0.6000
11-pt_avg 303 0.3354
3-pt_avg 303 0.3282
avg_doc_prec 303 0.2723
exact_relative_prec 303 0.6000
avg_relative_prec 303 0.5637
exact_unranked_avg_prec 303 0.0429
exact_relative_unranked_avg_prec 303 0.0051
map_at_R 303 0.3449
int_map 303 0.3089
exact_int_R_rcl_prec 303 0.4000
int_map_at_R 303 0.5087
bpref_allnonrel 303 0.5968
bpref_retnonrel 303 0.5525
bpref_topnonrel 303 0.5710
bpref_top5Rnonrel 303 0.5420
bpref_top10Rnonrel 303 0.5710
bpref_top10pRnonrel 303 0.4550
bpref_top25pRnonrel 303 0.5171
bpref_top50pRnonrel 303 0.5517
bpref_top25p2Rnonrel 303 0.5356
bpref_retall 303 0.9208
bpref_5 303 0.4400
bpref_10 303 0.5500
bpref_num_all 303 5383.0000
bpref_num_ret 303 337.0000
bpref_num_correct 303 33
bpref_num_possible 303 100
old_bpref 303 0.3300
old_bpref_top10pRnonrel 303 0.4550
infAP 303 0.2723
rank_first_rel 303 3
ircl_prn.0.00 303 0.6000
ircl_prn.0.10 303 0.6000
ircl_prn.0.20 303 0.6000
ircl_prn.0.30 303 0.6000
ircl_prn.0.40 303 0.5714
ircl_prn.0.50 303 0.3846
ircl_prn.0.60 303 0.3333
ircl_prn.0.70 303 0.0000
ircl_prn.0.80 303 0.0000
ircl_prn.0.90 303 0.0000
ircl_prn.1.00 303 0.0000
P5 303 0.6000
P10 303 0.4000
P15 303 0.3333
P20 303 0.3000
P30 303 0.2000
P100 303 0.0600
P200 303 0.0300
P500 303 0.0120
P1000 303 0.0060
recall5 303 0.3000
recall10 303 0.4000
recall15 303 0.5000
recall20 303 0.6000
recall30 303 0.6000
recall100 303 0.6000
recall200 303 0.6000
recall500 303 0.6000
recall1000 303 0.6000
0.20R-prec 303 0.0000
0.40R-prec 303 0.5000
0.60R-prec 303 0.5000
0.80R-prec 303 0.5000
1.00R-prec 303 0.4000
1.20R-prec 303 0.3333
1.40R-prec 303 0.3571
1.60R-prec 303 0.3125
1.80R-prec 303 0.3333
2.00R-prec 303 0.3000
relative_prec5 303 0.6000
relative_prec10 303 0.4000
relative_prec15 303 0.5000
relative_prec20 303 0.6000
relative_prec30 303 0.6000
relative_prec100 303 0.6000
relative_prec200 303 0.6000
relative_prec500 303 0.6000
relative_prec1000 303 0.6000
unranked_avg_prec5 303 0.1800
unranked_avg_prec10 303 0.1600
unranked_avg_prec15 303 0.1667
unranked_avg_prec20 303 0.1800
unranked_avg_prec30 303 0.1200
unranked_avg_prec100 303 0.0360
unranked_avg_prec200 303 0.0180
unranked_avg_prec500 303 0.0072
unranked_avg_prec1000 303 0.0036
relative_unranked_avg_prec5 303 0.3600
relative_unranked_avg_prec10 303 0.1600
relative_unranked_avg_prec15 303 0.2500
relative_unranked_avg_prec20 303 0.3600
relative_unranked_avg_prec30 303 0.3600
relative_unranked_avg_prec100 303 0.0036
relative_unranked_avg_prec200 303 0.0009
relative_unranked_avg_prec500 303 0.0001
relative_unranked_avg_prec1000 303 0.0000
utility_1.0_-1.0_0.0_0.0 303 -72.0000
rcl_at_142_nonrel 303 0.5796
fallout_recall_0 303 0.0000
fallout_recall_14 303 0.6000
fallout_recall_28 303 0.6000
fallout_recall_42 303 0.6000
fallout_recall_56 303 0.6000
fallout_recall_71 303 0.6000
fallout_recall_85 303 0.6000
fallout_recall_99 303 0.6000
fallout_recall_113 303 0.6000
fallout_recall_127 303 0.6000
fallout_recall_142 303 0.6000
int_0.20R-prec 303 0.6000
int_0.40R-prec 303 0.6000
int_0.60R-prec 303 0.5714
int_0.80R-prec 303 0.5000
int_1.00R-prec 303 0.4000
int_1.20R-prec 303 0.3846
int_1.40R-prec 303 0.3571
int_1.60R-prec 303 0.3333
int_1.80R-prec 303 0.3333
int_2.00R-prec 303 0.3000
num_q all 3
num_ret all 584
num_rel all 484
num_rel_ret all 77
map all 0.1016
gm_ap all 0.0045
R-prec all 0.1819
bpref all 0.1510
recip_rank all 0.1667
num_nonrel_judged_ret all 249
exact_prec all 0.0711
exact_recall all 0.2499
11-pt_avg all 0.1268
3-pt_avg all 0.1094
avg_doc_prec all 0.0374
exact_relative_prec all 0.2499
avg_relative_prec all 0.2531
exact_unranked_avg_prec all 0.0214
exact_relative_unranked_avg_prec all 0.0084
map_at_R all 0.1810
int_map all 0.1143
exact_int_R_rcl_prec all 0.1819
int_map_at_R all 0.2394
bpref_allnonrel all 0.2454
bpref_retnonrel all 0.2116
bpref_topnonrel all 0.2052
bpref_top5Rnonrel all 0.2272
bpref_top10Rnonrel all 0.2368
bpref_top10pRnonrel all 0.1929
bpref_top25pRnonrel all 0.2138
bpref_top50pRnonrel all 0.2258
bpref_top25p2Rnonrel all 0.2241
bpref_retall all 0.4902
bpref_5 all 0.1467
bpref_10 all 0.2167
bpref_num_all all 28996.3340
bpref_num_ret all 2559.0000
bpref_num_correct all 27679
bpref_num_possible all 224776
old_bpref all 0.1375
old_bpref_top10pRnonrel all 0.1791
infAP all 0.1016
gm_bpref all 0.0074
ircl_prn.0.00 all 0.2952
ircl_prn.0.10 all 0.2699
ircl_prn.0.20 all 0.2000
ircl_prn.0.30 all 0.2000
ircl_prn.0.40 all 0.1905
ircl_prn.0.50 all 0.1282
ircl_prn.0.60 all 0.1111
ircl_prn.0.70 all 0.0000
ircl_prn.0.80 all 0.0000
ircl_prn.0.90 all 0.0000
ircl_prn.1.00 all 0.0000
P5 all 0.2000
P10 all 0.2000
P15 all 0.1556
P20 all 0.1833
P30 all 0.1444
P100 all 0.0967
P200 all 0.0800
P500 all 0.0513
P1000 all 0.0257
recall5 all 0.1000
recall10 all 0.1347
recall15 all 0.1681
recall20 all 0.2035
recall30 all 0.2049
recall100 all 0.2162
recall200 all 0.2295
recall500 all 0.2499
recall1000 all 0.2499
0.20R-prec all 0.0737
0.40R-prec all 0.2351
0.60R-prec all 0.2310
0.80R-prec all 0.2237
1.00R-prec all 0.1819
1.20R-prec all 0.1527
1.40R-prec all 0.1547
1.60R-prec all 0.1353
1.80R-prec all 0.1388
2.00R-prec all 0.1250
relative_prec5 all 0.2000
relative_prec10 all 0.2000
relative_prec15 all 0.2111
relative_prec20 all 0.2833
relative_prec30 all 0.2778
relative_prec100 all 0.2767
relative_prec200 all 0.2700
relative_prec500 all 0.2499
relative_prec1000 all 0.2499
unranked_avg_prec5 all 0.0600
unranked_avg_prec10 all 0.0536
unranked_avg_prec15 all 0.0557
unranked_avg_prec20 all 0.0609
unranked_avg_prec30 all 0.0411
unranked_avg_prec100 all 0.0157
unranked_avg_prec200 all 0.0122
unranked_avg_prec500 all 0.0095
unranked_avg_prec1000 all 0.0047
relative_unranked_avg_prec5 all 0.1200
relative_unranked_avg_prec10 all 0.0667
relative_unranked_avg_prec15 all 0.0893
relative_unranked_avg_prec20 all 0.1408
relative_unranked_avg_prec30 all 0.1381
relative_unranked_avg_prec100 all 0.0188
relative_unranked_avg_prec200 all 0.0150
relative_unranked_avg_prec500 all 0.0075
relative_unranked_avg_prec1000 all 0.0017
utility_1.0_-1.0_0.0_0.0 all -143.3333
rcl_at_142_nonrel all 0.2078
fallout_recall_0 all 0.0000
fallout_recall_14 all 0.2035
fallout_recall_28 all 0.2070
fallout_recall_42 all 0.2113
fallout_recall_56 all 0.2134
fallout_recall_71 all 0.2148
fallout_recall_85 all 0.2169
fallout_recall_99 all 0.2218
fallout_recall_113 all 0.2246
fallout_recall_127 all 0.2253
fallout_recall_142 all 0.2267
int_0.20R-prec all 0.2805
int_0.40R-prec all 0.2708
int_0.60R-prec all 0.2551
int_0.80R-prec all 0.2240
int_1.00R-prec all 0.1819
int_1.20R-prec all 0.1698
int_1.40R-prec all 0.1547
int_1.60R-prec all 0.1423
int_1.80R-prec all 0.1388
int_2.00R-prec all 0.1250
micro_prec all 0.1318
micro_recall all 0.1591
micro_bpref all 0.1231
trec_eval.8.1/test/out.test.aqcM
+395
View File
@@ -0,0 +1,395 @@
num_ret 301 100
num_rel 301 474
num_rel_ret 301 23
map 301 0.0118
R-prec 301 0.0485
bpref 301 0.0456
recip_rank 301 0.1667
num_nonrel_judged_ret 301 50
exact_prec 301 0.2300
exact_recall 301 0.0485
11-pt_avg 301 0.0260
3-pt_avg 301 0.0000
avg_doc_prec 301 0.0118
exact_relative_prec 301 0.2300
avg_relative_prec 301 0.2217
exact_unranked_avg_prec 301 0.0112
exact_relative_unranked_avg_prec 301 0.0529
map_at_R 301 0.1225
int_map 301 0.0129
exact_int_R_rcl_prec 301 0.0485
int_map_at_R 301 0.1311
bpref_allnonrel 301 0.0474
bpref_retnonrel 301 0.0208
bpref_topnonrel 301 0.0347
bpref_top5Rnonrel 301 0.0474
bpref_top10Rnonrel 301 0.0474
bpref_top10pRnonrel 301 0.0457
bpref_top25pRnonrel 301 0.0457
bpref_top50pRnonrel 301 0.0459
bpref_top25p2Rnonrel 301 0.0471
bpref_retall 301 0.4296
bpref_5 301 0.0000
bpref_10 301 0.1000
bpref_num_all 301 27726.0000
bpref_num_ret 301 494.0000
bpref_num_correct 301 10246
bpref_num_possible 301 224676
old_bpref 301 0.0208
old_bpref_top10pRnonrel 301 0.0208
infAP 301 0.0118
rank_first_rel 301 6
ircl_prn.0.00 301 0.2857
ircl_prn.0.10 301 0.0000
ircl_prn.0.20 301 0.0000
ircl_prn.0.30 301 0.0000
ircl_prn.0.40 301 0.0000
ircl_prn.0.50 301 0.0000
ircl_prn.0.60 301 0.0000
ircl_prn.0.70 301 0.0000
ircl_prn.0.80 301 0.0000
ircl_prn.0.90 301 0.0000
ircl_prn.1.00 301 0.0000
P5 301 0.0000
P10 301 0.2000
P15 301 0.1333
P20 301 0.2500
P30 301 0.2333
P100 301 0.2300
P200 301 0.1150
P500 301 0.0460
P1000 301 0.0230
recall5 301 0.0000
recall10 301 0.0042
recall15 301 0.0042
recall20 301 0.0105
recall30 301 0.0148
recall100 301 0.0485
recall200 301 0.0485
recall500 301 0.0485
recall1000 301 0.0485
0.20R-prec 301 0.2211
0.40R-prec 301 0.1211
0.60R-prec 301 0.0807
0.80R-prec 301 0.0605
1.00R-prec 301 0.0485
1.20R-prec 301 0.0404
1.40R-prec 301 0.0346
1.60R-prec 301 0.0303
1.80R-prec 301 0.0269
2.00R-prec 301 0.0243
relative_prec5 301 0.0000
relative_prec10 301 0.2000
relative_prec15 301 0.1333
relative_prec20 301 0.2500
relative_prec30 301 0.2333
relative_prec100 301 0.2300
relative_prec200 301 0.1150
relative_prec500 301 0.0485
relative_prec1000 301 0.0485
unranked_avg_prec5 301 0.0000
unranked_avg_prec10 301 0.0008
unranked_avg_prec15 301 0.0006
unranked_avg_prec20 301 0.0026
unranked_avg_prec30 301 0.0034
unranked_avg_prec100 301 0.0112
unranked_avg_prec200 301 0.0056
unranked_avg_prec500 301 0.0022
unranked_avg_prec1000 301 0.0011
relative_unranked_avg_prec5 301 0.0000
relative_unranked_avg_prec10 301 0.0400
relative_unranked_avg_prec15 301 0.0178
relative_unranked_avg_prec20 301 0.0625
relative_unranked_avg_prec30 301 0.0544
relative_unranked_avg_prec100 301 0.0529
relative_unranked_avg_prec200 301 0.0132
relative_unranked_avg_prec500 301 0.0021
relative_unranked_avg_prec1000 301 0.0005
utility_1.0_-1.0_0.0_0.0 301 -54.0000
rcl_at_142_nonrel 301 0.0360
fallout_recall_0 301 0.0000
fallout_recall_14 301 0.0105
fallout_recall_28 301 0.0211
fallout_recall_42 301 0.0338
fallout_recall_56 301 0.0401
fallout_recall_71 301 0.0443
fallout_recall_85 301 0.0485
fallout_recall_99 301 0.0485
fallout_recall_113 301 0.0485
fallout_recall_127 301 0.0485
fallout_recall_142 301 0.0485
int_0.20R-prec 301 0.2300
int_0.40R-prec 301 0.1211
int_0.60R-prec 301 0.0807
int_0.80R-prec 301 0.0605
int_1.00R-prec 301 0.0485
int_1.20R-prec 301 0.0404
int_1.40R-prec 301 0.0346
int_1.60R-prec 301 0.0303
int_1.80R-prec 301 0.0269
int_2.00R-prec 301 0.0243
num_ret 303 84
num_rel 303 10
num_rel_ret 303 6
map 303 0.2723
R-prec 303 0.4000
bpref 303 0.3300
recip_rank 303 0.3333
num_nonrel_judged_ret 303 61
exact_prec 303 0.0714
exact_recall 303 0.6000
11-pt_avg 303 0.3354
3-pt_avg 303 0.3282
avg_doc_prec 303 0.2723
exact_relative_prec 303 0.6000
avg_relative_prec 303 0.5637
exact_unranked_avg_prec 303 0.0429
exact_relative_unranked_avg_prec 303 0.0051
map_at_R 303 0.3449
int_map 303 0.3089
exact_int_R_rcl_prec 303 0.4000
int_map_at_R 303 0.5087
bpref_allnonrel 303 0.5968
bpref_retnonrel 303 0.5525
bpref_topnonrel 303 0.5710
bpref_top5Rnonrel 303 0.5420
bpref_top10Rnonrel 303 0.5710
bpref_top10pRnonrel 303 0.4550
bpref_top25pRnonrel 303 0.5171
bpref_top50pRnonrel 303 0.5517
bpref_top25p2Rnonrel 303 0.5356
bpref_retall 303 0.9208
bpref_5 303 0.4400
bpref_10 303 0.5500
bpref_num_all 303 5383.0000
bpref_num_ret 303 337.0000
bpref_num_correct 303 33
bpref_num_possible 303 100
old_bpref 303 0.3300
old_bpref_top10pRnonrel 303 0.4550
infAP 303 0.2723
rank_first_rel 303 3
ircl_prn.0.00 303 0.6000
ircl_prn.0.10 303 0.6000
ircl_prn.0.20 303 0.6000
ircl_prn.0.30 303 0.6000
ircl_prn.0.40 303 0.5714
ircl_prn.0.50 303 0.3846
ircl_prn.0.60 303 0.3333
ircl_prn.0.70 303 0.0000
ircl_prn.0.80 303 0.0000
ircl_prn.0.90 303 0.0000
ircl_prn.1.00 303 0.0000
P5 303 0.6000
P10 303 0.4000
P15 303 0.3333
P20 303 0.3000
P30 303 0.2000
P100 303 0.0600
P200 303 0.0300
P500 303 0.0120
P1000 303 0.0060
recall5 303 0.3000
recall10 303 0.4000
recall15 303 0.5000
recall20 303 0.6000
recall30 303 0.6000
recall100 303 0.6000
recall200 303 0.6000
recall500 303 0.6000
recall1000 303 0.6000
0.20R-prec 303 0.0000
0.40R-prec 303 0.5000
0.60R-prec 303 0.5000
0.80R-prec 303 0.5000
1.00R-prec 303 0.4000
1.20R-prec 303 0.3333
1.40R-prec 303 0.3571
1.60R-prec 303 0.3125
1.80R-prec 303 0.3333
2.00R-prec 303 0.3000
relative_prec5 303 0.6000
relative_prec10 303 0.4000
relative_prec15 303 0.5000
relative_prec20 303 0.6000
relative_prec30 303 0.6000
relative_prec100 303 0.6000
relative_prec200 303 0.6000
relative_prec500 303 0.6000
relative_prec1000 303 0.6000
unranked_avg_prec5 303 0.1800
unranked_avg_prec10 303 0.1600
unranked_avg_prec15 303 0.1667
unranked_avg_prec20 303 0.1800
unranked_avg_prec30 303 0.1200
unranked_avg_prec100 303 0.0360
unranked_avg_prec200 303 0.0180
unranked_avg_prec500 303 0.0072
unranked_avg_prec1000 303 0.0036
relative_unranked_avg_prec5 303 0.3600
relative_unranked_avg_prec10 303 0.1600
relative_unranked_avg_prec15 303 0.2500
relative_unranked_avg_prec20 303 0.3600
relative_unranked_avg_prec30 303 0.3600
relative_unranked_avg_prec100 303 0.0036
relative_unranked_avg_prec200 303 0.0009
relative_unranked_avg_prec500 303 0.0001
relative_unranked_avg_prec1000 303 0.0000
utility_1.0_-1.0_0.0_0.0 303 -72.0000
rcl_at_142_nonrel 303 0.5796
fallout_recall_0 303 0.0000
fallout_recall_14 303 0.6000
fallout_recall_28 303 0.6000
fallout_recall_42 303 0.6000
fallout_recall_56 303 0.6000
fallout_recall_71 303 0.6000
fallout_recall_85 303 0.6000
fallout_recall_99 303 0.6000
fallout_recall_113 303 0.6000
fallout_recall_127 303 0.6000
fallout_recall_142 303 0.6000
int_0.20R-prec 303 0.6000
int_0.40R-prec 303 0.6000
int_0.60R-prec 303 0.5714
int_0.80R-prec 303 0.5000
int_1.00R-prec 303 0.4000
int_1.20R-prec 303 0.3846
int_1.40R-prec 303 0.3571
int_1.60R-prec 303 0.3333
int_1.80R-prec 303 0.3333
int_2.00R-prec 303 0.3000
num_q all 3
num_ret all 184
num_rel all 484
num_rel_ret all 29
map all 0.0947
gm_ap all 0.0032
R-prec all 0.1495
bpref all 0.1252
recip_rank all 0.1667
num_nonrel_judged_ret all 111
exact_prec all 0.1005
exact_recall all 0.2162
11-pt_avg all 0.1205
3-pt_avg all 0.1094
avg_doc_prec all 0.0172
exact_relative_prec all 0.2767
avg_relative_prec all 0.2618
exact_unranked_avg_prec all 0.0180
exact_relative_unranked_avg_prec all 0.0193
map_at_R all 0.1558
int_map all 0.1073
exact_int_R_rcl_prec all 0.1495
int_map_at_R all 0.2133
bpref_allnonrel all 0.2147
bpref_retnonrel all 0.1911
bpref_topnonrel all 0.2019
bpref_top5Rnonrel all 0.1965
bpref_top10Rnonrel all 0.2061
bpref_top10pRnonrel all 0.1669
bpref_top25pRnonrel all 0.1876
bpref_top50pRnonrel all 0.1992
bpref_top25p2Rnonrel all 0.1942
bpref_retall all 0.4501
bpref_5 all 0.1467
bpref_10 all 0.2167
bpref_num_all all 11036.3330
bpref_num_ret all 277.0000
bpref_num_correct all 10279
bpref_num_possible all 224776
old_bpref all 0.1169
old_bpref_top10pRnonrel all 0.1586
infAP all 0.0947
gm_bpref all 0.0053
ircl_prn.0.00 all 0.2952
ircl_prn.0.10 all 0.2000
ircl_prn.0.20 all 0.2000
ircl_prn.0.30 all 0.2000
ircl_prn.0.40 all 0.1905
ircl_prn.0.50 all 0.1282
ircl_prn.0.60 all 0.1111
ircl_prn.0.70 all 0.0000
ircl_prn.0.80 all 0.0000
ircl_prn.0.90 all 0.0000
ircl_prn.1.00 all 0.0000
P5 all 0.2000
P10 all 0.2000
P15 all 0.1556
P20 all 0.1833
P30 all 0.1444
P100 all 0.0967
P200 all 0.0483
P500 all 0.0193
P1000 all 0.0097
recall5 all 0.1000
recall10 all 0.1347
recall15 all 0.1681
recall20 all 0.2035
recall30 all 0.2049
recall100 all 0.2162
recall200 all 0.2162
recall500 all 0.2162
recall1000 all 0.2162
0.20R-prec all 0.0737
0.40R-prec all 0.2070
0.60R-prec all 0.1936
0.80R-prec all 0.1868
1.00R-prec all 0.1495
1.20R-prec all 0.1246
1.40R-prec all 0.1306
1.60R-prec all 0.1143
1.80R-prec all 0.1201
2.00R-prec all 0.1081
relative_prec5 all 0.2000
relative_prec10 all 0.2000
relative_prec15 all 0.2111
relative_prec20 all 0.2833
relative_prec30 all 0.2778
relative_prec100 all 0.2767
relative_prec200 all 0.2383
relative_prec500 all 0.2162
relative_prec1000 all 0.2162
unranked_avg_prec5 all 0.0600
unranked_avg_prec10 all 0.0536
unranked_avg_prec15 all 0.0557
unranked_avg_prec20 all 0.0609
unranked_avg_prec30 all 0.0411
unranked_avg_prec100 all 0.0157
unranked_avg_prec200 all 0.0079
unranked_avg_prec500 all 0.0031
unranked_avg_prec1000 all 0.0016
relative_unranked_avg_prec5 all 0.1200
relative_unranked_avg_prec10 all 0.0667
relative_unranked_avg_prec15 all 0.0893
relative_unranked_avg_prec20 all 0.1408
relative_unranked_avg_prec30 all 0.1381
relative_unranked_avg_prec100 all 0.0188
relative_unranked_avg_prec200 all 0.0047
relative_unranked_avg_prec500 all 0.0008
relative_unranked_avg_prec1000 all 0.0002
utility_1.0_-1.0_0.0_0.0 all -42.0000
rcl_at_142_nonrel all 0.2052
fallout_recall_0 all 0.0000
fallout_recall_14 all 0.2035
fallout_recall_28 all 0.2070
fallout_recall_42 all 0.2113
fallout_recall_56 all 0.2134
fallout_recall_71 all 0.2148
fallout_recall_85 all 0.2162
fallout_recall_99 all 0.2162
fallout_recall_113 all 0.2162
fallout_recall_127 all 0.2162
fallout_recall_142 all 0.2162
int_0.20R-prec all 0.2767
int_0.40R-prec all 0.2404
int_0.60R-prec all 0.2174
int_0.80R-prec all 0.1868
int_1.00R-prec all 0.1495
int_1.20R-prec all 0.1417
int_1.40R-prec all 0.1306
int_1.60R-prec all 0.1212
int_1.80R-prec all 0.1201
int_2.00R-prec all 0.1081
micro_prec all 0.1576
micro_recall all 0.0599
micro_bpref all 0.0457
trec_eval.8.1/test/out.test.aql
+525
View File
@@ -0,0 +1,525 @@
num_ret 301 500
num_rel 301 12
num_rel_ret 301 1
map 301 0.0003
R-prec 301 0.0000
bpref 301 0.0000
recip_rank 301 0.0033
num_nonrel_judged_ret 301 258
exact_prec 301 0.0020
exact_recall 301 0.0833
11-pt_avg 301 0.0003
3-pt_avg 301 0.0000
avg_doc_prec 301 0.0003
exact_relative_prec 301 0.0833
avg_relative_prec 301 0.0323
exact_unranked_avg_prec 301 0.0002
exact_relative_unranked_avg_prec 301 0.0000
map_at_R 301 0.0000
int_map 301 0.0003
exact_int_R_rcl_prec 301 0.0033
int_map_at_R 301 0.0030
bpref_allnonrel 301 0.0739
bpref_retnonrel 301 0.0213
bpref_topnonrel 301 0.0000
bpref_top5Rnonrel 301 0.0000
bpref_top10Rnonrel 301 0.0417
bpref_top10pRnonrel 301 0.0000
bpref_top25pRnonrel 301 0.0000
bpref_top50pRnonrel 301 0.0000
bpref_top25p2Rnonrel 301 0.0000
bpref_retall 301 0.2558
bpref_5 301 0.0000
bpref_10 301 0.0000
bpref_num_all 301 1504.0000
bpref_num_ret 301 66.0000
bpref_num_correct 301 0
bpref_num_possible 301 144
old_bpref 301 0.0000
old_bpref_top10pRnonrel 301 0.0000
infAP 301 0.0003
rank_first_rel 301 307
ircl_prn.0.00 301 0.0033
ircl_prn.0.10 301 0.0000
ircl_prn.0.20 301 0.0000
ircl_prn.0.30 301 0.0000
ircl_prn.0.40 301 0.0000
ircl_prn.0.50 301 0.0000
ircl_prn.0.60 301 0.0000
ircl_prn.0.70 301 0.0000
ircl_prn.0.80 301 0.0000
ircl_prn.0.90 301 0.0000
ircl_prn.1.00 301 0.0000
P5 301 0.0000
P10 301 0.0000
P15 301 0.0000
P20 301 0.0000
P30 301 0.0000
P100 301 0.0000
P200 301 0.0000
P500 301 0.0020
P1000 301 0.0010
recall5 301 0.0000
recall10 301 0.0000
recall15 301 0.0000
recall20 301 0.0000
recall30 301 0.0000
recall100 301 0.0000
recall200 301 0.0000
recall500 301 0.0833
recall1000 301 0.0833
0.20R-prec 301 0.0000
0.40R-prec 301 0.0000
0.60R-prec 301 0.0000
0.80R-prec 301 0.0000
1.00R-prec 301 0.0000
1.20R-prec 301 0.0000
1.40R-prec 301 0.0000
1.60R-prec 301 0.0000
1.80R-prec 301 0.0000
2.00R-prec 301 0.0000
relative_prec5 301 0.0000
relative_prec10 301 0.0000
relative_prec15 301 0.0000
relative_prec20 301 0.0000
relative_prec30 301 0.0000
relative_prec100 301 0.0000
relative_prec200 301 0.0000
relative_prec500 301 0.0833
relative_prec1000 301 0.0833
unranked_avg_prec5 301 0.0000
unranked_avg_prec10 301 0.0000
unranked_avg_prec15 301 0.0000
unranked_avg_prec20 301 0.0000
unranked_avg_prec30 301 0.0000
unranked_avg_prec100 301 0.0000
unranked_avg_prec200 301 0.0000
unranked_avg_prec500 301 0.0002
unranked_avg_prec1000 301 0.0001
relative_unranked_avg_prec5 301 0.0000
relative_unranked_avg_prec10 301 0.0000
relative_unranked_avg_prec15 301 0.0000
relative_unranked_avg_prec20 301 0.0000
relative_unranked_avg_prec30 301 0.0000
relative_unranked_avg_prec100 301 0.0000
relative_unranked_avg_prec200 301 0.0000
relative_unranked_avg_prec500 301 0.0069
relative_unranked_avg_prec1000 301 0.0000
utility_1.0_-1.0_0.0_0.0 301 -498.0000
rcl_at_142_nonrel 301 0.0000
fallout_recall_0 301 0.0000
fallout_recall_14 301 0.0000
fallout_recall_28 301 0.0000
fallout_recall_42 301 0.0000
fallout_recall_56 301 0.0000
fallout_recall_71 301 0.0000
fallout_recall_85 301 0.0000
fallout_recall_99 301 0.0000
fallout_recall_113 301 0.0000
fallout_recall_127 301 0.0000
fallout_recall_142 301 0.0000
int_0.20R-prec 301 0.0033
int_0.40R-prec 301 0.0033
int_0.60R-prec 301 0.0033
int_0.80R-prec 301 0.0033
int_1.00R-prec 301 0.0033
int_1.20R-prec 301 0.0033
int_1.40R-prec 301 0.0033
int_1.60R-prec 301 0.0033
int_1.80R-prec 301 0.0033
int_2.00R-prec 301 0.0033
num_ret 302 500
num_rel 302 77
num_rel_ret 302 50
map 302 0.4175
R-prec 302 0.5065
bpref 302 0.4712
recip_rank 302 1.0000
num_nonrel_judged_ret 302 214
exact_prec 302 0.1000
exact_recall 302 0.6494
11-pt_avg 302 0.4327
3-pt_avg 302 0.4613
avg_doc_prec 302 0.4175
exact_relative_prec 302 0.6494
avg_relative_prec 302 0.6097
exact_unranked_avg_prec 302 0.0649
exact_relative_unranked_avg_prec 302 0.0100
map_at_R 302 0.6862
int_map 302 0.4288
exact_int_R_rcl_prec 302 0.5065
int_map_at_R 302 0.7061
bpref_allnonrel 302 0.6273
bpref_retnonrel 302 0.5481
bpref_topnonrel 302 0.4919
bpref_top5Rnonrel 302 0.5931
bpref_top10Rnonrel 302 0.6212
bpref_top10pRnonrel 302 0.4813
bpref_top25pRnonrel 302 0.4935
bpref_top50pRnonrel 302 0.5088
bpref_top25p2Rnonrel 302 0.5308
bpref_retall 302 0.8440
bpref_5 302 0.8800
bpref_10 302 0.8400
bpref_num_all 302 47531.0000
bpref_num_ret 302 9031.0000
bpref_num_correct 302 2640
bpref_num_possible 302 5929
old_bpref 302 0.4712
old_bpref_top10pRnonrel 302 0.4813
infAP 302 0.4175
rank_first_rel 302 1
ircl_prn.0.00 302 1.0000
ircl_prn.0.10 302 0.8421
ircl_prn.0.20 302 0.8421
ircl_prn.0.30 302 0.7059
ircl_prn.0.40 302 0.6863
ircl_prn.0.50 302 0.5417
ircl_prn.0.60 302 0.1420
ircl_prn.0.70 302 0.0000
ircl_prn.0.80 302 0.0000
ircl_prn.0.90 302 0.0000
ircl_prn.1.00 302 0.0000
P5 302 0.8000
P10 302 0.7000
P15 302 0.8000
P20 302 0.8000
P30 302 0.7333
P100 302 0.4200
P200 302 0.2200
P500 302 0.1000
P1000 302 0.0500
recall5 302 0.0519
recall10 302 0.0909
recall15 302 0.1558
recall20 302 0.2078
recall30 302 0.2857
recall100 302 0.5455
recall200 302 0.5714
recall500 302 0.6494
recall1000 302 0.6494
0.20R-prec 302 0.8125
0.40R-prec 302 0.7419
0.60R-prec 302 0.6596
0.80R-prec 302 0.6129
1.00R-prec 302 0.5065
1.20R-prec 302 0.4516
1.40R-prec 302 0.3889
1.60R-prec 302 0.3468
1.80R-prec 302 0.3094
2.00R-prec 302 0.2857
relative_prec5 302 0.8000
relative_prec10 302 0.7000
relative_prec15 302 0.8000
relative_prec20 302 0.8000
relative_prec30 302 0.7333
relative_prec100 302 0.5455
relative_prec200 302 0.5714
relative_prec500 302 0.6494
relative_prec1000 302 0.6494
unranked_avg_prec5 302 0.0416
unranked_avg_prec10 302 0.0636
unranked_avg_prec15 302 0.1247
unranked_avg_prec20 302 0.1662
unranked_avg_prec30 302 0.2095
unranked_avg_prec100 302 0.2291
unranked_avg_prec200 302 0.1257
unranked_avg_prec500 302 0.0649
unranked_avg_prec1000 302 0.0325
relative_unranked_avg_prec5 302 0.6400
relative_unranked_avg_prec10 302 0.4900
relative_unranked_avg_prec15 302 0.6400
relative_unranked_avg_prec20 302 0.6400
relative_unranked_avg_prec30 302 0.5378
relative_unranked_avg_prec100 302 0.2975
relative_unranked_avg_prec200 302 0.3265
relative_unranked_avg_prec500 302 0.4217
relative_unranked_avg_prec1000 302 0.0025
utility_1.0_-1.0_0.0_0.0 302 -400.0000
rcl_at_142_nonrel 302 0.5091
fallout_recall_0 302 0.0260
fallout_recall_14 302 0.3896
fallout_recall_28 302 0.4935
fallout_recall_42 302 0.5455
fallout_recall_56 302 0.5455
fallout_recall_71 302 0.5455
fallout_recall_85 302 0.5584
fallout_recall_99 302 0.5584
fallout_recall_113 302 0.5714
fallout_recall_127 302 0.5714
fallout_recall_142 302 0.5714
int_0.20R-prec 302 0.8421
int_0.40R-prec 302 0.7419
int_0.60R-prec 302 0.6863
int_0.80R-prec 302 0.6129
int_1.00R-prec 302 0.5065
int_1.20R-prec 302 0.4516
int_1.40R-prec 302 0.3889
int_1.60R-prec 302 0.3468
int_1.80R-prec 302 0.3094
int_2.00R-prec 302 0.2857
num_ret 303 500
num_rel 303 8
num_rel_ret 303 8
map 303 0.0823
R-prec 303 0.0000
bpref 303 0.0000
recip_rank 303 0.0526
num_nonrel_judged_ret 303 138
exact_prec 303 0.0160
exact_recall 303 1.0000
11-pt_avg 303 0.1049
3-pt_avg 303 0.1106
avg_doc_prec 303 0.0823
exact_relative_prec 303 1.0000
avg_relative_prec 303 0.8963
exact_unranked_avg_prec 303 0.0160
exact_relative_unranked_avg_prec 303 0.0003
map_at_R 303 0.0000
int_map 303 0.1065
exact_int_R_rcl_prec 303 0.1136
int_map_at_R 303 0.0994
bpref_allnonrel 303 0.9689
bpref_retnonrel 303 0.7962
bpref_topnonrel 303 0.7188
bpref_top5Rnonrel 303 0.3688
bpref_top10Rnonrel 303 0.6844
bpref_top10pRnonrel 303 0.0625
bpref_top25pRnonrel 303 0.2727
bpref_top50pRnonrel 303 0.5259
bpref_top25p2Rnonrel 303 0.3811
bpref_retall 303 0.7962
bpref_5 303 0.0000
bpref_10 303 0.0125
bpref_num_all 303 7007.0000
bpref_num_ret 303 879.0000
bpref_num_correct 303 0
bpref_num_possible 303 64
old_bpref 303 0.0000
old_bpref_top10pRnonrel 303 0.0625
infAP 303 0.1200
rank_first_rel 303 19
ircl_prn.0.00 303 0.1136
ircl_prn.0.10 303 0.1136
ircl_prn.0.20 303 0.1136
ircl_prn.0.30 303 0.1136
ircl_prn.0.40 303 0.1136
ircl_prn.0.50 303 0.1136
ircl_prn.0.60 303 0.1136
ircl_prn.0.70 303 0.1045
ircl_prn.0.80 303 0.1045
ircl_prn.0.90 303 0.0748
ircl_prn.1.00 303 0.0748
P5 303 0.0000
P10 303 0.0000
P15 303 0.0000
P20 303 0.0500
P30 303 0.0333
P100 303 0.0700
P200 303 0.0400
P500 303 0.0160
P1000 303 0.0080
recall5 303 0.0000
recall10 303 0.0000
recall15 303 0.0000
recall20 303 0.1250
recall30 303 0.1250
recall100 303 0.8750
recall200 303 1.0000
recall500 303 1.0000
recall1000 303 1.0000
0.20R-prec 303 0.0000
0.40R-prec 303 0.0000
0.60R-prec 303 0.0000
0.80R-prec 303 0.0000
1.00R-prec 303 0.0000
1.20R-prec 303 0.0000
1.40R-prec 303 0.0000
1.60R-prec 303 0.0000
1.80R-prec 303 0.0000
2.00R-prec 303 0.0000
relative_prec5 303 0.0000
relative_prec10 303 0.0000
relative_prec15 303 0.0000
relative_prec20 303 0.1250
relative_prec30 303 0.1250
relative_prec100 303 0.8750
relative_prec200 303 1.0000
relative_prec500 303 1.0000
relative_prec1000 303 1.0000
unranked_avg_prec5 303 0.0000
unranked_avg_prec10 303 0.0000
unranked_avg_prec15 303 0.0000
unranked_avg_prec20 303 0.0063
unranked_avg_prec30 303 0.0042
unranked_avg_prec100 303 0.0613
unranked_avg_prec200 303 0.0400
unranked_avg_prec500 303 0.0160
unranked_avg_prec1000 303 0.0080
relative_unranked_avg_prec5 303 0.0000
relative_unranked_avg_prec10 303 0.0000
relative_unranked_avg_prec15 303 0.0000
relative_unranked_avg_prec20 303 0.0156
relative_unranked_avg_prec30 303 0.0156
relative_unranked_avg_prec100 303 0.7656
relative_unranked_avg_prec200 303 1.0000
relative_unranked_avg_prec500 303 1.0000
relative_unranked_avg_prec1000 303 0.0001
utility_1.0_-1.0_0.0_0.0 303 -484.0000
rcl_at_142_nonrel 303 0.6523
fallout_recall_0 303 0.0000
fallout_recall_14 303 0.0000
fallout_recall_28 303 0.1250
fallout_recall_42 303 0.6250
fallout_recall_56 303 0.6250
fallout_recall_71 303 0.8750
fallout_recall_85 303 0.8750
fallout_recall_99 303 1.0000
fallout_recall_113 303 1.0000
fallout_recall_127 303 1.0000
fallout_recall_142 303 1.0000
int_0.20R-prec 303 0.1136
int_0.40R-prec 303 0.1136
int_0.60R-prec 303 0.1136
int_0.80R-prec 303 0.1136
int_1.00R-prec 303 0.1136
int_1.20R-prec 303 0.1136
int_1.40R-prec 303 0.1136
int_1.60R-prec 303 0.1136
int_1.80R-prec 303 0.1136
int_2.00R-prec 303 0.1136
num_q all 3
num_ret all 1500
num_rel all 97
num_rel_ret all 59
map all 0.1667
gm_ap all 0.0210
R-prec all 0.1688
bpref all 0.1571
recip_rank all 0.3520
num_nonrel_judged_ret all 610
exact_prec all 0.0393
exact_recall all 0.5776
11-pt_avg all 0.1793
3-pt_avg all 0.1906
avg_doc_prec all 0.3382
exact_relative_prec all 0.5776
avg_relative_prec all 0.5128
exact_unranked_avg_prec all 0.0270
exact_relative_unranked_avg_prec all 0.0034
map_at_R all 0.2287
int_map all 0.1785
exact_int_R_rcl_prec all 0.2078
int_map_at_R all 0.2695
bpref_allnonrel all 0.5567
bpref_retnonrel all 0.4552
bpref_topnonrel all 0.4036
bpref_top5Rnonrel all 0.3206
bpref_top10Rnonrel all 0.4491
bpref_top10pRnonrel all 0.1813
bpref_top25pRnonrel all 0.2554
bpref_top50pRnonrel all 0.3449
bpref_top25p2Rnonrel all 0.3040
bpref_retall all 0.6320
bpref_5 all 0.2933
bpref_10 all 0.2842
bpref_num_all all 18680.6660
bpref_num_ret all 3325.3333
bpref_num_correct all 2640
bpref_num_possible all 6137
old_bpref all 0.1571
old_bpref_top10pRnonrel all 0.1813
infAP all 0.1792
gm_bpref all 0.0004
ircl_prn.0.00 all 0.3723
ircl_prn.0.10 all 0.3186
ircl_prn.0.20 all 0.3186
ircl_prn.0.30 all 0.2732
ircl_prn.0.40 all 0.2666
ircl_prn.0.50 all 0.2184
ircl_prn.0.60 all 0.0852
ircl_prn.0.70 all 0.0348
ircl_prn.0.80 all 0.0348
ircl_prn.0.90 all 0.0249
ircl_prn.1.00 all 0.0249
P5 all 0.2667
P10 all 0.2333
P15 all 0.2667
P20 all 0.2833
P30 all 0.2556
P100 all 0.1633
P200 all 0.0867
P500 all 0.0393
P1000 all 0.0197
recall5 all 0.0173
recall10 all 0.0303
recall15 all 0.0519
recall20 all 0.1109
recall30 all 0.1369
recall100 all 0.4735
recall200 all 0.5238
recall500 all 0.5776
recall1000 all 0.5776
0.20R-prec all 0.2708
0.40R-prec all 0.2473
0.60R-prec all 0.2199
0.80R-prec all 0.2043
1.00R-prec all 0.1688
1.20R-prec all 0.1505
1.40R-prec all 0.1296
1.60R-prec all 0.1156
1.80R-prec all 0.1031
2.00R-prec all 0.0952
relative_prec5 all 0.2667
relative_prec10 all 0.2333
relative_prec15 all 0.2667
relative_prec20 all 0.3083
relative_prec30 all 0.2861
relative_prec100 all 0.4735
relative_prec200 all 0.5238
relative_prec500 all 0.5776
relative_prec1000 all 0.5776
unranked_avg_prec5 all 0.0139
unranked_avg_prec10 all 0.0212
unranked_avg_prec15 all 0.0416
unranked_avg_prec20 all 0.0575
unranked_avg_prec30 all 0.0712
unranked_avg_prec100 all 0.0968
unranked_avg_prec200 all 0.0552
unranked_avg_prec500 all 0.0270
unranked_avg_prec1000 all 0.0135
relative_unranked_avg_prec5 all 0.2133
relative_unranked_avg_prec10 all 0.1633
relative_unranked_avg_prec15 all 0.2133
relative_unranked_avg_prec20 all 0.2185
relative_unranked_avg_prec30 all 0.1845
relative_unranked_avg_prec100 all 0.3544
relative_unranked_avg_prec200 all 0.4422
relative_unranked_avg_prec500 all 0.4762
relative_unranked_avg_prec1000 all 0.0009
utility_1.0_-1.0_0.0_0.0 all -460.6667
rcl_at_142_nonrel all 0.3871
fallout_recall_0 all 0.0087
fallout_recall_14 all 0.1299
fallout_recall_28 all 0.2062
fallout_recall_42 all 0.3902
fallout_recall_56 all 0.3902
fallout_recall_71 all 0.4735
fallout_recall_85 all 0.4778
fallout_recall_99 all 0.5195
fallout_recall_113 all 0.5238
fallout_recall_127 all 0.5238
fallout_recall_142 all 0.5238
int_0.20R-prec all 0.3197
int_0.40R-prec all 0.2863
int_0.60R-prec all 0.2677
int_0.80R-prec all 0.2433
int_1.00R-prec all 0.2078
int_1.20R-prec all 0.1895
int_1.40R-prec all 0.1686
int_1.60R-prec all 0.1546
int_1.80R-prec all 0.1421
int_2.00R-prec all 0.1342
micro_prec all 0.0393
micro_recall all 0.6082
micro_bpref all 0.4302
trec_eval.8.1/test/qrels.rel_level
+3681
View File
File diff suppressed because it is too large Load Diff
trec_eval.8.1/test/qrels.test
+3681
View File
File diff suppressed because it is too large Load Diff
trec_eval.8.1/test/results.test
+1500
View File
File diff suppressed because it is too large Load Diff
trec_eval.8.1/test/results.trunc
+584
View File
@@ -0,0 +1,584 @@
301 Q0 FBIS3-27619 99 2.138276 STANDARD more junk at
303 Q0 FT941-17652 99 1.857372 STANDARD end of lines.
301 Q0 FBIS3-2934 98 2.139655 STANDARD reserved for
301 Q0 FBIS3-25796 97 2.143418 STANDARD future
303 Q0 FT934-4015 97 1.875063 STANDARD expansion
301 Q0 FBIS4-46846 96 2.144021 STANDARD
303 Q0 FT941-793 96 1.901362 STANDARD
301 Q0 FT943-2588 95 2.150375 STANDARD
303 Q0 FT933-3699 95 1.910402 STANDARD
301 Q0 LA042189-0086 94 2.150424 STANDARD
303 Q0 FT933-6323 94 1.919063 STANDARD
301 Q0 FBIS4-40452 93 2.153299 STANDARD
301 Q0 FBIS4-67072 92 2.157889 STANDARD
301 Q0 FBIS3-45072 91 2.160266 STANDARD
301 Q0 FBIS4-45296 90 2.160571 STANDARD
301 Q0 FR940620-1-00007 9 2.971818 STANDARD
301 Q0 FBIS4-55845 89 2.162241 STANDARD
303 Q0 FT924-286 89 2.025797 STANDARD
301 Q0 FBIS4-44401 88 2.162794 STANDARD
303 Q0 FT934-4583 88 2.035755 STANDARD
301 Q0 FBIS3-25940 87 2.164362 STANDARD
301 Q0 FBIS4-41839 86 2.170714 STANDARD
301 Q0 FBIS3-3729 85 2.172867 STANDARD
301 Q0 LA062189-0010 84 2.173631 STANDARD
301 Q0 FBIS3-41348 83 2.175046 STANDARD
301 Q0 FBIS4-4063 82 2.176446 STANDARD
301 Q0 FBIS4-26072 81 2.183235 STANDARD
301 Q0 FR940620-1-00008 80 2.183501 STANDARD
303 Q0 FT934-5418 8 3.747635 STANDARD
301 Q0 FR940620-1-00009 8 3.023369 STANDARD
301 Q0 FBIS3-46420 79 2.188316 STANDARD
301 Q0 FBIS3-25902 78 2.191656 STANDARD
301 Q0 FBIS3-26112 77 2.202529 STANDARD
301 Q0 FBIS3-24190 76 2.211735 STANDARD
301 Q0 FBIS3-23432 75 2.227293 STANDARD
303 Q0 FT934-3191 75 2.122274 STANDARD
301 Q0 FT943-13315 74 2.232947 STANDARD
301 Q0 FBIS3-17394 73 2.233619 STANDARD
301 Q0 FBIS3-26720 72 2.236461 STANDARD
301 Q0 FBIS4-1796 71 2.236936 STANDARD
301 Q0 FBIS4-54904 70 2.237512 STANDARD
303 Q0 FT934-4132 70 2.207861 STANDARD
303 Q0 FT934-2516 7 3.768137 STANDARD
301 Q0 FBIS3-20552 7 3.133914 STANDARD
301 Q0 FBIS4-62079 69 2.239440 STANDARD
303 Q0 FT921-3432 69 2.237047 STANDARD
303 Q0 FT933-6946 68 2.295164 STANDARD
301 Q0 FBIS3-58025 68 2.243509 STANDARD
303 Q0 FT941-15661 67 2.307403 STANDARD
301 Q0 FBIS3-58055 67 2.243509 STANDARD
301 Q0 FBIS4-40930 66 2.243923 STANDARD
301 Q0 FBIS4-2725 65 2.245463 STANDARD
301 Q0 FBIS4-44512 64 2.251601 STANDARD
301 Q0 FBIS3-60984 63 2.254571 STANDARD
301 Q0 FBIS4-62372 62 2.262802 STANDARD
301 Q0 FBIS4-41215 61 2.266717 STANDARD
303 Q0 FT933-6678 60 2.373152 STANDARD
301 Q0 FBIS3-59285 60 2.277126 STANDARD
301 Q0 FBIS3-20551 6 3.137958 STANDARD
301 Q0 FBIS4-45333 59 2.277421 STANDARD
301 Q0 FBIS4-50083 58 2.284391 STANDARD
301 Q0 FBIS3-26914 57 2.314254 STANDARD
301 Q0 FBIS3-59284 56 2.317995 STANDARD
301 Q0 FBIS4-41840 55 2.335550 STANDARD
301 Q0 FBIS3-26415 54 2.336936 STANDARD
301 Q0 FBIS4-51202 53 2.339266 STANDARD
301 Q0 FBIS3-18129 52 2.342345 STANDARD
301 Q0 FBIS4-45453 51 2.346620 STANDARD
301 Q0 FBIS3-20713 500 1.655716 STANDARD
301 Q0 FBIS3-45676 50 2.349077 STANDARD
301 Q0 FBIS4-24388 5 3.176254 STANDARD
301 Q0 FBIS3-27288 499 1.655729 STANDARD
301 Q0 FBIS4-47008 498 1.655917 STANDARD
301 Q0 FR940303-1-00021 497 1.655963 STANDARD
301 Q0 FBIS3-58540 496 1.658499 STANDARD
301 Q0 FBIS3-12094 495 1.658651 STANDARD
303 Q0 FT943-7096 495 0.803135 STANDARD
301 Q0 FBIS3-60007 494 1.659024 STANDARD
303 Q0 FT942-11262 494 0.803779 STANDARD
301 Q0 FR941006-0-00045 493 1.659331 STANDARD
301 Q0 FBIS4-57133 492 1.659471 STANDARD
301 Q0 FT942-17001 491 1.660437 STANDARD
301 Q0 FBIS4-22346 490 1.660598 STANDARD
301 Q0 FBIS3-41247 49 2.350864 STANDARD
301 Q0 FR941230-2-00138 489 1.662662 STANDARD
303 Q0 FR940602-2-00109 489 0.806242 STANDARD
301 Q0 FBIS4-20961 488 1.663079 STANDARD
301 Q0 FBIS3-10979 487 1.663086 STANDARD
301 Q0 LA101990-0076 486 1.665950 STANDARD
301 Q0 FBIS4-49380 485 1.666289 STANDARD
303 Q0 FR941205-2-00054 485 0.807621 STANDARD
301 Q0 FBIS3-3728 484 1.667296 STANDARD
303 Q0 FT931-15900 484 0.808374 STANDARD
301 Q0 FBIS3-37418 483 1.668495 STANDARD
301 Q0 FR940721-2-00054 482 1.668723 STANDARD
301 Q0 FBIS4-67231 481 1.669646 STANDARD
301 Q0 FBIS4-47254 480 1.672893 STANDARD
301 Q0 FBIS4-49754 48 2.351483 STANDARD
301 Q0 LA041689-0147 479 1.673101 STANDARD
301 Q0 FT943-12601 478 1.673670 STANDARD
301 Q0 FBIS4-53139 477 1.674022 STANDARD
301 Q0 FBIS3-59322 476 1.674562 STANDARD
301 Q0 FR940404-0-00087 475 1.674784 STANDARD
301 Q0 LA120389-0125 474 1.674817 STANDARD
303 Q0 FT931-10187 474 0.817357 STANDARD
301 Q0 FR940804-0-00125 473 1.674901 STANDARD
301 Q0 FBIS4-31652 472 1.675422 STANDARD
301 Q0 FBIS4-30283 471 1.676491 STANDARD
301 Q0 FR940216-1-00014 470 1.677013 STANDARD
301 Q0 FBIS4-1553 47 2.362436 STANDARD
301 Q0 FT944-12673 469 1.677549 STANDARD
301 Q0 FBIS3-42341 468 1.677783 STANDARD
301 Q0 FBIS4-67160 467 1.678897 STANDARD
303 Q0 FT943-15250 467 0.822134 STANDARD
301 Q0 FBIS4-45235 466 1.679272 STANDARD
303 Q0 FT921-832 466 0.822476 STANDARD
301 Q0 FT923-13103 465 1.679853 STANDARD
301 Q0 LA011190-0084 464 1.680453 STANDARD
301 Q0 LA041789-0010 463 1.680776 STANDARD
301 Q0 FBIS4-50167 462 1.681279 STANDARD
301 Q0 FBIS4-66978 461 1.681566 STANDARD
301 Q0 FBIS4-4127 460 1.682513 STANDARD
301 Q0 FBIS4-1764 46 2.364943 STANDARD
301 Q0 FBIS3-21844 459 1.682594 STANDARD
301 Q0 FBIS4-47199 458 1.682735 STANDARD
301 Q0 FBIS4-40720 457 1.683077 STANDARD
301 Q0 FBIS4-45450 456 1.683376 STANDARD
301 Q0 FBIS4-34666 455 1.684233 STANDARD
301 Q0 FBIS4-44181 454 1.685975 STANDARD
303 Q0 FR940926-2-00073 454 0.832528 STANDARD
301 Q0 FBIS4-23427 453 1.686031 STANDARD
301 Q0 FBIS4-45346 452 1.687554 STANDARD
301 Q0 FBIS4-64345 451 1.687968 STANDARD
303 Q0 FR941221-0-00051 451 0.834705 STANDARD
301 Q0 FBIS3-10609 450 1.688209 STANDARD
301 Q0 FBIS4-2105 45 2.368016 STANDARD
301 Q0 FBIS4-21294 449 1.688366 STANDARD
301 Q0 FBIS4-40359 448 1.690737 STANDARD
301 Q0 FBIS3-31354 447 1.691054 STANDARD
301 Q0 FBIS3-51005 446 1.691843 STANDARD
303 Q0 FR941006-2-00076 446 0.836143 STANDARD
301 Q0 FBIS4-33615 445 1.693773 STANDARD
301 Q0 FBIS3-26651 444 1.693784 STANDARD
301 Q0 FBIS4-52929 443 1.696057 STANDARD
301 Q0 FBIS4-41803 442 1.696270 STANDARD
301 Q0 FBIS3-1108 441 1.699095 STANDARD
301 Q0 FBIS4-2039 440 1.701881 STANDARD
301 Q0 FR940620-1-00006 44 2.375205 STANDARD
301 Q0 FR940727-0-00060 439 1.702908 STANDARD
303 Q0 FT924-4358 439 0.843043 STANDARD
301 Q0 FT942-7830 438 1.703799 STANDARD
303 Q0 FR940622-2-00073 438 0.843079 STANDARD
301 Q0 FT942-2423 437 1.704746 STANDARD
301 Q0 FBIS4-41538 436 1.705109 STANDARD
301 Q0 FBIS3-21770 435 1.708522 STANDARD
301 Q0 FBIS4-26786 434 1.708904 STANDARD
301 Q0 FBIS3-60076 433 1.708965 STANDARD
303 Q0 FT923-9781 433 0.847790 STANDARD
301 Q0 FBIS4-2511 432 1.709049 STANDARD
301 Q0 FBIS3-24246 431 1.712145 STANDARD
301 Q0 FBIS4-62077 430 1.713174 STANDARD
303 Q0 FT931-6554 43 2.681227 STANDARD
301 Q0 FBIS4-43801 43 2.389463 STANDARD
301 Q0 FBIS3-50136 429 1.713585 STANDARD
303 Q0 FT934-4525 429 0.848461 STANDARD
301 Q0 FBIS3-27374 428 1.715253 STANDARD
301 Q0 FT931-1053 427 1.716405 STANDARD
301 Q0 FBIS4-67291 426 1.718286 STANDARD
301 Q0 FT944-18651 425 1.718812 STANDARD
301 Q0 FBIS3-5774 424 1.719125 STANDARD
301 Q0 FBIS4-54900 423 1.719201 STANDARD
301 Q0 FT911-2671 422 1.719461 STANDARD
303 Q0 FT932-4616 422 0.853034 STANDARD
301 Q0 FBIS4-65501 421 1.721008 STANDARD
301 Q0 FBIS3-39430 420 1.721082 STANDARD
303 Q0 FR941221-0-00048 420 0.853809 STANDARD
301 Q0 FBIS3-3190 42 2.392541 STANDARD
301 Q0 FBIS4-67336 419 1.721329 STANDARD
301 Q0 FBIS3-8781 418 1.722128 STANDARD
301 Q0 FBIS4-57722 417 1.722171 STANDARD
301 Q0 FBIS4-25262 416 1.722552 STANDARD
301 Q0 FBIS3-60022 415 1.724387 STANDARD
301 Q0 FR940202-2-00151 414 1.724760 STANDARD
301 Q0 FBIS3-24181 413 1.726911 STANDARD
301 Q0 LA061090-0040 412 1.729130 STANDARD
303 Q0 FT943-14510 412 0.859534 STANDARD
301 Q0 LA082690-0090 411 1.729742 STANDARD
301 Q0 FBIS4-22596 410 1.729780 STANDARD
303 Q0 FT944-128 41 2.760373 STANDARD
301 Q0 FBIS4-25028 41 2.394200 STANDARD
301 Q0 FBIS3-41143 409 1.729987 STANDARD
303 Q0 FT921-3809 409 0.862736 STANDARD
301 Q0 FR940830-2-00003 408 1.731386 STANDARD
301 Q0 FR940303-1-00014 407 1.732111 STANDARD
303 Q0 FR941130-2-00086 407 0.863714 STANDARD
301 Q0 FBIS3-54773 406 1.733266 STANDARD
301 Q0 FT934-6443 405 1.733755 STANDARD
301 Q0 FBIS3-52075 404 1.734758 STANDARD
301 Q0 FBIS3-59073 403 1.734932 STANDARD
301 Q0 FBIS4-49547 402 1.735723 STANDARD
301 Q0 FBIS4-3077 401 1.735977 STANDARD
301 Q0 LA082990-0130 400 1.737108 STANDARD
301 Q0 FBIS3-41105 40 2.399732 STANDARD
301 Q0 FBIS3-9399 4 3.215889 STANDARD
301 Q0 FBIS4-26038 399 1.737455 STANDARD
301 Q0 FBIS4-64135 398 1.738539 STANDARD
301 Q0 FBIS3-24145 397 1.738575 STANDARD
301 Q0 LA052089-0047 396 1.738631 STANDARD
301 Q0 FBIS4-25476 395 1.738840 STANDARD
301 Q0 FBIS3-38466 394 1.738962 STANDARD
301 Q0 FBIS3-61345 393 1.740630 STANDARD
301 Q0 FBIS4-66179 392 1.741075 STANDARD
301 Q0 LA062390-0102 391 1.742990 STANDARD
303 Q0 FT932-15782 391 0.874581 STANDARD
301 Q0 FBIS4-49928 390 1.743994 STANDARD
303 Q0 FR941221-0-00052 390 0.874838 STANDARD
301 Q0 FBIS4-45477 39 2.404399 STANDARD
301 Q0 FBIS4-50806 389 1.744591 STANDARD
301 Q0 FBIS4-41958 388 1.745697 STANDARD
303 Q0 FR940906-2-00139 388 0.876992 STANDARD
301 Q0 FBIS3-4313 387 1.746026 STANDARD
303 Q0 FR940513-2-00145 387 0.877140 STANDARD
301 Q0 FBIS4-24694 386 1.747539 STANDARD
301 Q0 FBIS3-60144 385 1.748958 STANDARD
301 Q0 FBIS3-19646 384 1.749999 STANDARD
301 Q0 LA010790-0228 383 1.750647 STANDARD
301 Q0 FBIS4-68720 382 1.750805 STANDARD
301 Q0 FBIS4-24788 381 1.751424 STANDARD
303 Q0 FT942-786 381 0.882208 STANDARD
301 Q0 LA021989-0204 380 1.752505 STANDARD
301 Q0 FBIS4-50898 38 2.419756 STANDARD
301 Q0 FBIS4-49483 379 1.754356 STANDARD
301 Q0 FR940804-0-00119 378 1.755399 STANDARD
301 Q0 FBIS4-46851 377 1.755611 STANDARD
301 Q0 FBIS3-14961 376 1.756365 STANDARD
301 Q0 FBIS4-39570 375 1.757526 STANDARD
301 Q0 FBIS4-20988 374 1.757675 STANDARD
301 Q0 FR940727-0-00079 373 1.759109 STANDARD
301 Q0 LA041789-0055 372 1.760346 STANDARD
301 Q0 FBIS4-26727 371 1.760759 STANDARD
301 Q0 FBIS4-45189 370 1.761385 STANDARD
301 Q0 FBIS4-31645 37 2.425397 STANDARD
301 Q0 FBIS4-16950 369 1.761678 STANDARD
301 Q0 FBIS4-68847 368 1.762087 STANDARD
301 Q0 FR940804-0-00103 367 1.762352 STANDARD
301 Q0 FBIS4-65896 366 1.762550 STANDARD
301 Q0 FBIS4-7717 365 1.763895 STANDARD
301 Q0 FR940804-0-00112 364 1.764905 STANDARD
301 Q0 LA060189-0150 363 1.766305 STANDARD
301 Q0 FBIS4-62049 362 1.767618 STANDARD
303 Q0 FT943-5596 362 0.897657 STANDARD
301 Q0 FR940727-0-00077 361 1.768786 STANDARD
301 Q0 FBIS4-57959 360 1.770075 STANDARD
301 Q0 FBIS3-25359 36 2.432513 STANDARD
301 Q0 FBIS3-46076 359 1.770914 STANDARD
303 Q0 FT943-13317 359 0.903902 STANDARD
301 Q0 FBIS3-2549 358 1.773210 STANDARD
301 Q0 FBIS4-46757 357 1.773279 STANDARD
301 Q0 FBIS4-40181 356 1.775154 STANDARD
301 Q0 FBIS4-24419 355 1.777109 STANDARD
301 Q0 FBIS3-57406 354 1.777276 STANDARD
301 Q0 FBIS3-45602 353 1.777504 STANDARD
301 Q0 FBIS3-51349 352 1.777761 STANDARD
301 Q0 FBIS4-43791 351 1.777947 STANDARD
301 Q0 FBIS4-68498 350 1.779944 STANDARD
301 Q0 FBIS3-3020 35 2.436040 STANDARD
301 Q0 FBIS4-2546 349 1.781359 STANDARD
301 Q0 FBIS4-56992 348 1.782381 STANDARD
301 Q0 FBIS4-25332 347 1.783756 STANDARD
301 Q0 FBIS4-68426 346 1.785070 STANDARD
301 Q0 FBIS4-40360 345 1.785083 STANDARD
303 Q0 FT932-4803 345 0.915780 STANDARD
301 Q0 LA042990-0044 344 1.785390 STANDARD
301 Q0 LA080790-0035 343 1.787194 STANDARD
301 Q0 FBIS4-8957 342 1.788093 STANDARD
301 Q0 FBIS4-38410 341 1.790607 STANDARD
303 Q0 FT931-2231 341 0.921274 STANDARD
301 Q0 FT942-8808 340 1.791096 STANDARD
301 Q0 FBIS4-26351 34 2.436919 STANDARD
301 Q0 FR940721-2-00075 339 1.791411 STANDARD
301 Q0 FBIS3-36565 338 1.792058 STANDARD
301 Q0 FBIS3-61238 337 1.792962 STANDARD
301 Q0 FT923-2348 336 1.794087 STANDARD
303 Q0 FT942-5468 336 0.924340 STANDARD
301 Q0 FBIS3-26742 335 1.795461 STANDARD
301 Q0 FT942-852 334 1.796588 STANDARD
301 Q0 LA100590-0029 333 1.797265 STANDARD
301 Q0 FR940503-2-00146 332 1.799753 STANDARD
301 Q0 FBIS3-3019 331 1.799830 STANDARD
301 Q0 FBIS4-41394 330 1.800183 STANDARD
301 Q0 FBIS4-1967 33 2.446289 STANDARD
301 Q0 FBIS4-26643 329 1.800264 STANDARD
301 Q0 FBIS3-55966 328 1.800644 STANDARD
301 Q0 FBIS3-21961 327 1.800842 STANDARD
301 Q0 FR940203-1-00038 326 1.800881 STANDARD
301 Q0 FBIS4-65446 325 1.801062 STANDARD
301 Q0 FBIS4-16502 324 1.802201 STANDARD
301 Q0 FBIS4-2318 323 1.802893 STANDARD
301 Q0 FBIS4-41395 322 1.803218 STANDARD
301 Q0 FR940727-0-00078 321 1.806117 STANDARD
301 Q0 FBIS4-68801 320 1.808038 STANDARD
301 Q0 FBIS3-19420 32 2.448735 STANDARD
301 Q0 FBIS4-2498 319 1.809141 STANDARD
303 Q0 FT942-795 319 0.951376 STANDARD
301 Q0 FBIS3-15586 318 1.809177 STANDARD
301 Q0 FBIS3-41090 317 1.809227 STANDARD
301 Q0 FBIS4-20367 316 1.810602 STANDARD
301 Q0 FBIS4-67183 315 1.811862 STANDARD
301 Q0 FT944-14183 314 1.812210 STANDARD
303 Q0 FT924-12943 314 0.956181 STANDARD
301 Q0 FBIS4-51255 313 1.812859 STANDARD
303 Q0 FR941221-0-00049 313 0.956703 STANDARD
301 Q0 FBIS3-32620 312 1.814571 STANDARD
301 Q0 FBIS4-2049 311 1.815679 STANDARD
301 Q0 FBIS4-45157 310 1.817723 STANDARD
301 Q0 FBIS4-1667 31 2.452689 STANDARD
301 Q0 FBIS4-21321 309 1.817834 STANDARD
301 Q0 FR940303-1-00019 308 1.820319 STANDARD
301 Q0 FT943-16238 307 1.821578 STANDARD
301 Q0 FBIS3-26006 306 1.822304 STANDARD
301 Q0 FR940804-0-00116 305 1.822332 STANDARD
301 Q0 FBIS4-21249 304 1.822357 STANDARD
301 Q0 FBIS3-27051 303 1.826845 STANDARD
301 Q0 FR940622-2-00053 302 1.830749 STANDARD
301 Q0 FBIS4-41399 301 1.831922 STANDARD
301 Q0 LA032490-0049 300 1.835254 STANDARD
301 Q0 FBIS4-45469 30 2.470949 STANDARD
301 Q0 FBIS3-22085 3 3.228945 STANDARD
301 Q0 FBIS4-4077 299 1.836900 STANDARD
301 Q0 LA100390-0069 298 1.838421 STANDARD
303 Q0 FT922-11472 298 0.978346 STANDARD
301 Q0 FR940429-0-00132 297 1.839996 STANDARD
301 Q0 LA071389-0002 296 1.840566 STANDARD
301 Q0 FR940303-1-00012 295 1.841413 STANDARD
301 Q0 FBIS3-1975 294 1.844307 STANDARD
301 Q0 FBIS4-10739 293 1.845839 STANDARD
301 Q0 FBIS4-21188 292 1.847648 STANDARD
301 Q0 FBIS3-41288 291 1.847715 STANDARD
301 Q0 FBIS3-22049 290 1.849501 STANDARD
301 Q0 FBIS4-1865 29 2.473547 STANDARD
301 Q0 FBIS4-67075 289 1.849748 STANDARD
301 Q0 FBIS3-26805 288 1.850634 STANDARD
301 Q0 FR940727-0-00093 287 1.850940 STANDARD
301 Q0 FBIS4-41144 286 1.853102 STANDARD
301 Q0 FBIS4-2512 285 1.857639 STANDARD
301 Q0 FBIS4-66264 284 1.860067 STANDARD
301 Q0 FBIS4-1794 283 1.861069 STANDARD
301 Q0 FR940429-0-00128 282 1.862317 STANDARD
301 Q0 FBIS4-46775 281 1.863126 STANDARD
301 Q0 FBIS4-2931 280 1.863184 STANDARD
301 Q0 FBIS4-25032 28 2.480994 STANDARD
301 Q0 FBIS4-68669 279 1.864410 STANDARD
301 Q0 FBIS4-56776 278 1.864500 STANDARD
303 Q0 FT923-10876 278 1.011683 STANDARD
301 Q0 FBIS3-41244 277 1.865750 STANDARD
301 Q0 FBIS4-25706 276 1.867508 STANDARD
301 Q0 FBIS3-46228 275 1.868788 STANDARD
303 Q0 FR940119-2-00100 275 1.015211 STANDARD
301 Q0 FBIS3-24197 274 1.870808 STANDARD
303 Q0 FT923-5257 274 1.017270 STANDARD
301 Q0 FR940203-1-00039 273 1.870957 STANDARD
303 Q0 FT943-3693 273 1.019071 STANDARD
301 Q0 FBIS3-30458 272 1.871451 STANDARD
301 Q0 FBIS3-30686 271 1.871451 STANDARD
301 Q0 FBIS3-40077 270 1.872271 STANDARD
303 Q0 FT943-11292 270 1.020416 STANDARD
301 Q0 FBIS4-24387 27 2.481159 STANDARD
301 Q0 FBIS3-27474 269 1.873794 STANDARD
301 Q0 LA070890-0129 268 1.876154 STANDARD
303 Q0 FT932-15788 268 1.023917 STANDARD
301 Q0 FBIS4-45239 267 1.877867 STANDARD
301 Q0 FBIS3-24247 266 1.878037 STANDARD
301 Q0 FBIS4-62543 265 1.880077 STANDARD
301 Q0 FBIS3-21779 264 1.880893 STANDARD
301 Q0 FBIS3-38878 263 1.882401 STANDARD
301 Q0 FBIS4-25065 262 1.883741 STANDARD
301 Q0 FBIS3-10204 261 1.885856 STANDARD
301 Q0 FR940503-2-00147 260 1.886508 STANDARD
301 Q0 FBIS3-2393 26 2.494489 STANDARD
301 Q0 FBIS4-24386 259 1.886517 STANDARD
301 Q0 FBIS4-1549 258 1.887025 STANDARD
301 Q0 FBIS4-39881 257 1.887065 STANDARD
301 Q0 FBIS4-25161 256 1.887178 STANDARD
303 Q0 FT923-7711 256 1.055355 STANDARD
301 Q0 FBIS3-2115 255 1.887945 STANDARD
301 Q0 FBIS4-66178 254 1.889605 STANDARD
303 Q0 FR941221-0-00047 254 1.057115 STANDARD
301 Q0 FT931-3563 253 1.889801 STANDARD
301 Q0 FBIS4-14080 252 1.890979 STANDARD
301 Q0 FBIS4-50842 251 1.892349 STANDARD
301 Q0 FBIS4-46806 250 1.893980 STANDARD
301 Q0 FBIS4-40260 25 2.528623 STANDARD
301 Q0 FBIS3-26913 249 1.896585 STANDARD
301 Q0 FBIS3-18507 248 1.896854 STANDARD
301 Q0 LA071889-0026 247 1.897763 STANDARD
301 Q0 FBIS4-40935 246 1.898004 STANDARD
301 Q0 FBIS3-26645 245 1.898311 STANDARD
301 Q0 FT924-227 244 1.898500 STANDARD
301 Q0 FBIS3-8746 243 1.899552 STANDARD
301 Q0 FBIS3-3223 242 1.899980 STANDARD
301 Q0 FT944-15444 241 1.901744 STANDARD
301 Q0 FBIS3-11210 240 1.903990 STANDARD
301 Q0 FBIS4-41541 24 2.550802 STANDARD
301 Q0 FBIS3-45003 239 1.906086 STANDARD
301 Q0 FT944-4555 238 1.906426 STANDARD
301 Q0 FR940728-2-00151 237 1.907816 STANDARD
301 Q0 FBIS3-26367 236 1.908945 STANDARD
301 Q0 LA071990-0150 235 1.909468 STANDARD
301 Q0 FBIS4-41991 234 1.911603 STANDARD
301 Q0 FR940203-1-00036 233 1.912871 STANDARD
301 Q0 FBIS4-40481 232 1.914181 STANDARD
301 Q0 FBIS4-1668 231 1.915419 STANDARD
301 Q0 FBIS3-24277 230 1.918759 STANDARD
301 Q0 FR940620-1-00005 23 2.569489 STANDARD
301 Q0 FBIS4-43797 229 1.919818 STANDARD
301 Q0 FBIS4-46734 228 1.920978 STANDARD
303 Q0 FT943-10128 228 1.109917 STANDARD
301 Q0 FBIS3-24284 227 1.921283 STANDARD
303 Q0 FT941-5396 227 1.110329 STANDARD
301 Q0 FBIS4-39330 226 1.923398 STANDARD
303 Q0 FT922-7904 226 1.118909 STANDARD
301 Q0 FBIS4-40934 225 1.923446 STANDARD
301 Q0 FBIS4-24523 224 1.926240 STANDARD
301 Q0 FBIS4-51332 223 1.930347 STANDARD
301 Q0 FBIS3-21908 222 1.930625 STANDARD
301 Q0 FBIS3-2605 221 1.932684 STANDARD
301 Q0 FBIS3-3303 220 1.933041 STANDARD
301 Q0 FBIS4-16951 22 2.586972 STANDARD
301 Q0 FBIS3-11095 219 1.939206 STANDARD
301 Q0 FBIS4-22345 218 1.943247 STANDARD
301 Q0 FBIS4-46584 217 1.944825 STANDARD
301 Q0 FBIS3-15636 216 1.945223 STANDARD
301 Q0 FBIS4-38481 215 1.945278 STANDARD
301 Q0 FBIS4-41667 214 1.946881 STANDARD
301 Q0 FBIS4-2514 213 1.947558 STANDARD
301 Q0 FT941-3237 212 1.948220 STANDARD
301 Q0 FBIS4-49845 211 1.954449 STANDARD
301 Q0 FBIS3-42315 210 1.956193 STANDARD
303 Q0 FT932-16246 210 1.161086 STANDARD
301 Q0 FBIS4-41952 21 2.622548 STANDARD
301 Q0 FT933-12037 209 1.957280 STANDARD
301 Q0 FBIS3-45756 208 1.957361 STANDARD
303 Q0 FR941020-2-00110 208 1.170328 STANDARD
301 Q0 FBIS3-22088 207 1.960365 STANDARD
301 Q0 FBIS4-50056 206 1.961327 STANDARD
303 Q0 FT921-15863 206 1.175726 STANDARD
301 Q0 FBIS4-68746 205 1.962427 STANDARD
301 Q0 FBIS3-44612 204 1.963338 STANDARD
301 Q0 LA080989-0129 203 1.964162 STANDARD
303 Q0 FT934-2685 203 1.182910 STANDARD
301 Q0 FBIS3-11212 202 1.971704 STANDARD
301 Q0 FBIS4-24633 201 1.975092 STANDARD
303 Q0 FT931-1868 201 1.188802 STANDARD
301 Q0 FBIS3-41285 200 1.978161 STANDARD
301 Q0 FBIS4-21302 20 2.632815 STANDARD
301 Q0 FBIS3-21938 2 3.280215 STANDARD
301 Q0 FBIS3-21930 199 1.980598 STANDARD
301 Q0 FBIS3-23901 198 1.982240 STANDARD
301 Q0 FBIS3-41158 197 1.982367 STANDARD
301 Q0 FBIS4-19949 196 1.988816 STANDARD
301 Q0 FBIS3-24453 195 1.989023 STANDARD
301 Q0 FBIS4-49289 194 1.990929 STANDARD
301 Q0 FBIS3-21905 193 1.991868 STANDARD
301 Q0 FBIS4-2721 192 1.993366 STANDARD
303 Q0 FT944-9936 192 1.217489 STANDARD
301 Q0 FBIS4-51118 191 1.996027 STANDARD
303 Q0 FT943-5598 191 1.218840 STANDARD
301 Q0 FBIS4-2510 190 1.996562 STANDARD
303 Q0 FT921-7107 19 3.363091 STANDARD
301 Q0 FBIS4-7688 19 2.669399 STANDARD
301 Q0 FBIS3-45601 189 1.996655 STANDARD
301 Q0 FBIS3-39566 188 1.997787 STANDARD
301 Q0 FBIS4-6448 187 1.998577 STANDARD
301 Q0 FR940303-1-00006 186 1.999081 STANDARD
303 Q0 FT932-12850 186 1.249146 STANDARD
301 Q0 FBIS3-38787 185 1.999308 STANDARD
301 Q0 FBIS3-26005 184 2.001203 STANDARD
301 Q0 FBIS3-21937 183 2.001406 STANDARD
303 Q0 FR940304-2-00134 183 1.254448 STANDARD
301 Q0 FBIS4-41863 182 2.002898 STANDARD
301 Q0 FT942-13766 181 2.003752 STANDARD
301 Q0 LA041789-0046 180 2.004429 STANDARD
301 Q0 FBIS3-21750 18 2.670143 STANDARD
301 Q0 FBIS3-26218 179 2.004786 STANDARD
301 Q0 FBIS3-41163 178 2.006615 STANDARD
301 Q0 FBIS4-2048 177 2.011086 STANDARD
303 Q0 FT941-3758 177 1.278597 STANDARD
301 Q0 FBIS3-45789 176 2.012594 STANDARD
301 Q0 FBIS4-21330 175 2.013537 STANDARD
301 Q0 FR940727-0-00092 174 2.014195 STANDARD
301 Q0 FBIS4-55395 173 2.014687 STANDARD
301 Q0 FBIS3-21765 172 2.016918 STANDARD
301 Q0 FBIS3-17175 171 2.017663 STANDARD
301 Q0 FBIS3-44864 170 2.017798 STANDARD
301 Q0 FBIS4-3044 17 2.693945 STANDARD
301 Q0 FBIS4-3370 169 2.018858 STANDARD
301 Q0 FBIS3-24037 168 2.022115 STANDARD
301 Q0 FBIS4-22471 167 2.022665 STANDARD
301 Q0 FBIS3-1849 166 2.024102 STANDARD
301 Q0 FBIS3-41385 165 2.026598 STANDARD
303 Q0 FT922-13455 165 1.346386 STANDARD
301 Q0 FBIS4-51335 164 2.027163 STANDARD
303 Q0 FT933-2180 164 1.349473 STANDARD
301 Q0 FBIS3-2327 163 2.034978 STANDARD
301 Q0 FBIS3-35272 162 2.038591 STANDARD
301 Q0 FBIS4-49431 161 2.038758 STANDARD
301 Q0 FBIS3-51766 160 2.041041 STANDARD
303 Q0 FT921-8919 160 1.360642 STANDARD
301 Q0 FBIS3-23986 16 2.712466 STANDARD
301 Q0 FBIS4-7390 159 2.043206 STANDARD
301 Q0 FBIS4-43965 158 2.044602 STANDARD
301 Q0 FBIS3-33020 157 2.047786 STANDARD
301 Q0 FBIS4-20985 156 2.048596 STANDARD
301 Q0 FBIS4-50209 155 2.049279 STANDARD
301 Q0 FBIS4-1863 154 2.053106 STANDARD
301 Q0 FBIS4-14483 153 2.053272 STANDARD
301 Q0 FBIS4-66291 152 2.057615 STANDARD
301 Q0 FBIS4-66307 151 2.057615 STANDARD
301 Q0 FR940727-0-00091 150 2.059793 STANDARD
301 Q0 FBIS3-3586 15 2.785274 STANDARD
301 Q0 FBIS4-3230 149 2.064102 STANDARD
301 Q0 FBIS4-3367 148 2.064102 STANDARD
303 Q0 FT923-3530 148 1.409420 STANDARD
301 Q0 FR940804-0-00102 147 2.064112 STANDARD
301 Q0 FT921-15491 146 2.065663 STANDARD
301 Q0 FT923-14709 145 2.066168 STANDARD
301 Q0 FBIS3-58028 144 2.069669 STANDARD
301 Q0 FBIS3-58058 143 2.069669 STANDARD
301 Q0 FBIS4-1628 142 2.070523 STANDARD
301 Q0 FT941-10546 141 2.078446 STANDARD
301 Q0 FBIS3-25901 140 2.078458 STANDARD
301 Q0 FBIS3-3622 14 2.785274 STANDARD
301 Q0 FBIS3-3412 139 2.080240 STANDARD
303 Q0 FT924-14355 139 1.478320 STANDARD
301 Q0 FBIS4-26335 138 2.080463 STANDARD
301 Q0 FBIS3-24143 137 2.081314 STANDARD
301 Q0 LA101590-0071 136 2.082767 STANDARD
301 Q0 FBIS3-46116 135 2.083308 STANDARD
301 Q0 FT944-15443 134 2.084795 STANDARD
301 Q0 FBIS4-43552 133 2.087620 STANDARD
301 Q0 FBIS3-25894 132 2.091815 STANDARD
303 Q0 FT943-11617 132 1.503932 STANDARD
301 Q0 FBIS3-41349 131 2.092097 STANDARD
301 Q0 FBIS4-46425 130 2.092915 STANDARD
301 Q0 FBIS3-45599 13 2.795794 STANDARD
301 Q0 FBIS4-44396 129 2.095230 STANDARD
301 Q0 FBIS3-54944 128 2.095287 STANDARD
301 Q0 FBIS4-45482 127 2.096435 STANDARD
301 Q0 FBIS3-52858 126 2.097152 STANDARD
301 Q0 FT944-8297 125 2.100060 STANDARD
301 Q0 FR940202-2-00154 124 2.104024 STANDARD
301 Q0 FBIS4-50993 123 2.106927 STANDARD
301 Q0 FBIS4-7811 122 2.108214 STANDARD
301 Q0 FBIS4-50513 121 2.108639 STANDARD
301 Q0 FBIS3-17255 120 2.110716 STANDARD
301 Q0 FBIS3-3189 12 2.826465 STANDARD
301 Q0 FBIS4-49075 119 2.112576 STANDARD
303 Q0 FT933-10324 119 1.642809 STANDARD
301 Q0 FBIS4-34879 118 2.112775 STANDARD
301 Q0 FBIS3-24362 117 2.114117 STANDARD
303 Q0 FR940304-2-00135 117 1.645932 STANDARD
301 Q0 FBIS4-42757 116 2.114265 STANDARD
301 Q0 FBIS4-38364 115 2.114667 STANDARD
301 Q0 FBIS4-26192 114 2.117829 STANDARD
301 Q0 FBIS4-62078 113 2.118467 STANDARD
301 Q0 FBIS4-45419 112 2.120282 STANDARD
303 Q0 FT933-678 112 1.695116 STANDARD
301 Q0 FBIS4-34515 111 2.122778 STANDARD
301 Q0 FBIS4-47045 110 2.123794 STANDARD
301 Q0 FR940303-1-00022 11 2.918622 STANDARD
301 Q0 FR940620-1-00004 109 2.125475 STANDARD
301 Q0 FBIS4-25845 108 2.126431 STANDARD
301 Q0 FBIS4-10817 107 2.127389 STANDARD
301 Q0 FBIS4-1842 106 2.127805 STANDARD
301 Q0 FBIS4-45552 105 2.127882 STANDARD
301 Q0 FR940202-2-00150 104 2.129133 STANDARD
301 Q0 FBIS3-55219 103 2.129514 STANDARD
301 Q0 FBIS4-63597 102 2.132991 STANDARD
303 Q0 FT934-3766 102 1.797155 STANDARD
301 Q0 FBIS4-2356 101 2.136141 STANDARD
301 Q0 FBIS4-56982 100 2.137468 STANDARD
303 Q0 FT934-3325 100 1.846668 STANDARD
301 Q0 FR940804-0-00127 10 2.920190 STANDARD
301 Q0 FBIS4-50478 1 3.340779 STANDARD
trec_eval.8.1/tr_vec.h
+21
View File
@@ -0,0 +1,21 @@
#ifndef TR_VECH
#define TR_VECH
/* $Header: /home/smart/release/./src/h/tr_vec.h,v 10.1 91/11/05 23:47:35 smart Exp Locker: smart $*/
typedef struct {
long did; /* document id */
long rank; /* Rank of this document */
char action; /* what action a user has taken with doc */
char rel; /* whether doc judged relevant(1) or not(0) */
char iter; /* Number of feedback runs for this query */
char trtup_unused; /* Presently unused field */
float sim; /* similarity of did to qid */
} TR_TUP;
typedef struct {
char *qid; /* query id */
long num_tr; /* Number of tuples for tr_vec */
TR_TUP *tr; /* tuples. Invariant: tr sorted increasing did */
} TR_VEC;
#endif /* TR_VECH */
trec_eval.8.1/trec_eval
Executable
BIN
View File
Binary file not shown.
trec_eval.8.1/trec_eval.c
+257
View File
@@ -0,0 +1,257 @@
static char *VersionID = VERSIONID;
/* "Version 7.3 trec_eval Dec 15, 2004"; */
/* Copyright (c) 2004, 2003, 1991, 1990, 1984 - Chris Buckley. */
/******************** PROCEDURE DESCRIPTION ************************
*0 Take TREC results text file, TREC qrels file, and evaluate
*1 local.convert.obj.trec_eval
*2 trec_eval [-q] [-a] [-t] [-o] [-v] [-n num] trec_rel_file trec_top_file
*7 Read text tuples from trec_top_file of the form
*7 030 Q0 ZF08-175-870 0 4238 prise1
*7 qid iter docno rank sim run_id
*7 giving TREC document numbers (a string) retrieved by query qid
*7 (an integer) with similarity sim (a float). The other fields are ignored.
*7 Input is asssumed to be sorted numerically by qid.
*7 Sim is assumed to be higher for the docs to be retrieved first.
*7 Relevance for each docno to qid is determined from text_qrels_file, which
*7 consists of text tuples of the form
*7 qid iter docno rel
*7 giving TREC document numbers (a string) and their relevance to query qid
*7 (a non-negative integer less than 128, or -1 to indicate unjudged).
*7 Tuples are asssumed to be sorted numerically by qid.
*7 The text tuples with relevence judgements are converted to TR_VEC form
*7 and then submitted to the evaluation routines.
*7
*7 -q: In addition to summary evaluation, give evaluation for each query
*7 -a: Print all evaluation measures calculated, instead of just the
*7 official measures for TREC 2.
*7 -o: Print everything out in old, non-relational format
*7 -v: Print version number and exit
*7 -h: Print full help message and exit
*7 -t: Treat similarity as time that document retrieved. Compute
*7 several time-based measures after ranking docs by time retrieved
*7 (first doc (lowest sim) retrieved ranked highest).
*7 Only done if -a selected.
*7 -J: Calculate all measures only over judged documents that appear
*7 in qrels. (DO NOT USE)
*7 -n<num>: following integer is the number of queries to average over.
*7 -ua<num>: Value to use for 'a' coefficient of utility computation.
*7 -ub<num>: Value to use for 'b' coefficient of utility computation.
*7 -uc<num>: Value to use for 'c' coefficient of utility computation.
*7 -ud<num>: Value to use for 'd' coefficient of utility computation.
*7 -N<num>: Number of docs in collection
*7 -M<num>:Max number of results to evaluate per topic
*8 Procedure is to read all the docs retrieved for a query, and all the
*8 relevant docs for that query,
*8 sort and rank the retrieved docs by sim/docno,
*8 and look up docno in the relevant docs to determine relevance.
*8 The qid,did,rank,sim,rel fields of of TR_VEC are filled in;
*8 action,iter fields are set to 0.
*8 Queries for which there is no relevance information are ignored completely.
***********************************************************************/
#include "common.h"
#include "sysfunc.h"
#include "smart_error.h"
#include "tr_vec.h"
#include "trec_eval.h"
#include "buf.h"
void print_error();
void old_print_trec_eval_list();
void print_rel_trec_eval_list();
int trec_eval_help(EVAL_PARAM_INFO *epi);
int accumulate_results (TREC_EVAL *query_eval, TREC_EVAL *accum_eval);
int get_top (char *trec_top_file, ALL_TREC_TOP *all_trec_top);
int get_qrels (char *text_qrels_file, ALL_TREC_QRELS *all_trec_qrels);
int form_trvec (EVAL_PARAM_INFO *ep, TREC_TOP *trec_top,
TREC_QRELS *trec_qrels, TR_VEC *tr_vec, long *num_rel);
int trvec_trec_eval (EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
TREC_EVAL *eval, long num_rel, long num_nonrel);
static char *usage = "Usage: trec_eval [-h] [-q] [-a] [-o] [-v] trec_rel_file trec_top_file\n\
-h: Give full help information, including other options\n\
-q: In addition to summary evaluation, give evaluation for each query\n\
-a: Print all evaluation measures, instead of just official measures\n\
-o: Print requested measures in old non-relational format\n";
int
main (argc, argv)
int argc;
char *argv[];
{
char *trec_rel_file, *trec_top_file;
ALL_TREC_TOP all_trec_top;
ALL_TREC_QRELS all_trec_qrels;
TREC_EVAL accum_eval, query_eval;
TR_VEC tr_vec;
long num_rel;
long num_eval_q;
long i,j;
EVAL_PARAM_INFO epi;
/* Initialize static info before getting program optional args */
epi.query_flag = epi.all_flag = epi.time_flag = epi.average_complete_flag = 0;
epi.judged_docs_only_flag = 0;
epi.relation_flag = 1;
epi.utility_a = UTILITY_A; epi.utility_b = UTILITY_B;
epi.utility_c = UTILITY_C; epi.utility_d = UTILITY_D;
epi.num_docs_in_coll = 0;
epi.relevance_level = 1;
epi.max_num_docs_per_topic = MAXLONG;
/* Should use getopts, but some people may not have it. */
/* This keeps growing over the years. Should redo */
while (argc > 1 && argv[1][0] == '-') {
if (argv[1][1] == 'q')
epi.query_flag++;
else if (argv[1][1] == 'v') {
fprintf (stderr, "trec_eval version %s\n", VersionID);
exit (0);
}
else if (argv[1][1] == 'h') {
(void) trec_eval_help(&epi);
exit (0);
}
else if (argv[1][1] == 'a')
epi.all_flag++;
else if (argv[1][1] == 'o')
epi.relation_flag = 0;
else if (argv[1][1] == 'c') {
epi.average_complete_flag++;
}
else if (argv[1][1] == 'l') {
epi.relevance_level = atol (&argv[1][2]);
}
else if (argv[1][1] == 'J') {
epi.judged_docs_only_flag++;
}
else if (argv[1][1] == 'N')
epi.num_docs_in_coll = atol (&argv[1][2]);
else if (argv[1][1] == 'M')
epi.max_num_docs_per_topic = atol (&argv[1][2]);
else if (argv[1][1] == 'U') {
if (argv[1][2] == 'a')
epi.utility_a = atof (&argv[1][3]);
else if (argv[1][2] == 'b')
epi.utility_b = atof (&argv[1][3]);
else if (argv[1][2] == 'c')
epi.utility_c = atof (&argv[1][3]);
else if (argv[1][2] == 'd')
epi.utility_d = atof (&argv[1][3]);
else {
(void) fputs (usage,stderr);
exit (1);
}
}
else if (argv[1][1] == 'T')
epi.time_flag++;
else {
(void) fputs (usage,stderr);
exit (1);
}
argc--; argv++;
}
if (argc != 3) {
(void) fputs (usage,stderr);
exit (1);
}
trec_rel_file = argv[1];
trec_top_file = argv[2];
/* Get qrels and top results information for all queries from the
input text files */
if (UNDEF == get_qrels (trec_rel_file, &all_trec_qrels) ||
UNDEF == get_top (trec_top_file, &all_trec_top)) {
print_error ("trec_eval: input error", "Quit");
exit (2);
}
/* For each topic which has both qrels and top results information,
calculate, possibly print (if query_flag), and accumulate
evaluation measures. */
num_eval_q = 0;
(void) memset ((void *) &accum_eval, 0, sizeof (TREC_EVAL));
accum_eval.qid = "All";
for (i = 0; i < all_trec_top.num_q_tr; i++) {
/* Find rel info for this query (skip if no rel info) */
for (j = 0; j < all_trec_qrels.num_q_qrels; j++) {
if (0 == strcmp (all_trec_top.trec_top[i].qid,
all_trec_qrels.trec_qrels[j].qid))
break;
}
if (j >= all_trec_qrels.num_q_qrels)
continue;
/* Form results/rel into SMART TR_VEC form */
if (UNDEF == form_trvec (&epi,
&all_trec_top.trec_top[i],
&all_trec_qrels.trec_qrels[j],
&tr_vec,
&num_rel)) {
print_error ("trec_eval: form_tr_vec error", "Quit");
exit (3);
}
/* Evaluate results/rel for this query */
if (UNDEF == trvec_trec_eval (&epi,
&tr_vec,
&query_eval,
num_rel,
all_trec_qrels.trec_qrels[j].num_text_qrels - num_rel)) {
print_error ("trec_eval: evaluation error", "Quit");
exit (4);
}
/* Print results for this query, if desired */
if (epi.query_flag) {
if (epi.relation_flag)
print_rel_trec_eval_list (1, &epi, &query_eval, (SM_BUF *) NULL);
else
old_print_trec_eval_list (&epi, &query_eval, 1, (SM_BUF *) NULL);
}
/* Accumulate results for later averaging */
if (UNDEF == accumulate_results (&query_eval, &accum_eval)) {
print_error ("trec_eval: accumulation error", "Quit");
exit (5);
}
num_eval_q++;
}
/******** REMOVE THIS ONCE WARNING FLAG ADDED */
/* Warn if numq_flag_num < num_eval_q */
if (num_eval_q == 0) {
set_error (SM_INCON_ERR,
"No queries with both results and relevance info",
"trec_eval");
return (UNDEF);
print_error ("trec_eval", "Quit");
exit (6);
}
if (epi.average_complete_flag) {
/* Want to average over possibly missing queries. Pass in actual
* number of queries in num_orig_queries */
accum_eval.num_orig_queries = accum_eval.num_queries;
accum_eval.num_queries = all_trec_qrels.num_q_qrels;
}
/* Print final evaluation results */
if (epi.relation_flag)
print_rel_trec_eval_list (0, &epi, &accum_eval, (SM_BUF *) NULL);
else
old_print_trec_eval_list (&epi, &accum_eval, 1, (SM_BUF *) NULL);
exit (0);
}
trec_eval.8.1/trec_eval.h
+373
View File
@@ -0,0 +1,373 @@
#ifndef TRECEVALH
#define TRECEVALH
/* Static state info; set at beginning, possibly from program options, */
/* but then remains constant throughout. */
typedef struct {
long query_flag; /* 0. If set, evaluation output will be
printed for each query, in addition
to summary at end. */
long all_flag; /* 0. If set, all evaluation measures will
be printed instead of just the
final TREC 2 measures. */
long time_flag; /* 0. If set, calculate time-based measures*/
long relation_flag; /* 1. If set, print in relational form */
long average_complete_flag; /* 0. If set, average over the complete set
of relevance judgements (qrels), instead
of the number of queries
in the intersection of qrels and result */
long judged_docs_only_flag; /* 0. If set, throw out all unjudged docs
for the retrieved set before calculating
any measures. */
double utility_a; /* UTILITY_A. Default utility values */
double utility_b; /* UTILITY_B. Default utility values */
double utility_c; /* UTILITY_C. Default utility values */
double utility_d; /* UTILITY_D. Default utility values */
long num_docs_in_coll; /* 0. number of docs in collection */
long relevance_level; /* 1. In relevance judgements, the level at
which a doc is considered relevant for
this evaluation */
long max_num_docs_per_topic; /* MAXLONG. evaluate only this many docs */
} EVAL_PARAM_INFO;
/* Measure characteristics (how to print them, average them). */
/* List of measures is in measures.c */
/* Three types of measures:
single measures - single measure and name
parameterized measures - arrays of a measure, whose measure name
depends on parameter (eg P5, P10)
micro measures - measures defined as the micro average over all
docs retrieved independent of topic. Only calculated
and printed for the "all" pseudo-query.
Eg micro_prec = num_rel_ret / num_ret
*/
typedef struct {
char *name;
char *long_name;
unsigned char is_long_flag; /* otherwise float */
unsigned char print_short_flag; /* if set, measure is always printed
(not just if all_flag set) */
unsigned char print_time_flag; /* if set, measure is printed only
if time_flag is set */
unsigned char print_only_query_flag; /* if set, measure is printed only
when printing individual query output*/
unsigned char print_only_average_flag; /* if set, measure is printed only
when printing overall average output*/
unsigned char avg_results_flag; /* if set, average results over queries */
unsigned char avg_rel_results_flag;/* if set,average results over num_rel*/
unsigned char gm_results_flag; /* if set, measure uses geometric mean. ie
exponentiate the average before
printing */
long byte_offset;
} SINGLE_MEASURE;
typedef struct {
char *long_name;
unsigned char is_long_flag; /* otherwise float */
unsigned char print_short_flag; /* if set, print in short output */
unsigned char print_time_flag; /* if set, measure is printed only
if time_flag is set */
unsigned char print_only_query_flag; /* if set, measure is printed only
when printing individual query output*/
unsigned char print_only_average_flag; /* if set, measure is printed only
when printing overall average output*/
unsigned char avg_results_flag; /* if set, average results over queries */
long byte_offset;
long num_values;
char *format_string;
char *long_format_string;
char *(*get_param_str) (EVAL_PARAM_INFO *ip, long index);
} PARAMETERIZED_MEASURE;
typedef struct {
char *name;
char *long_name;
unsigned char print_short_flag; /* if set, measure is always printed
(not just if all_flag set) */
long numerator_byte_offset;
long denominator_byte_offset;
} MICRO_MEASURE;
typedef struct { /* For each retrieved document result */
char *docno; /* document id */
float sim; /* score */
long rank; /* rank assigned after breaking ties */
} TEXT_TR;
typedef struct { /* For each query in retrieved results */
char *qid; /* query id */
long num_text_tr; /* number of TEXT_TR results for query*/
long max_num_text_tr; /* number results space reserved for */
TEXT_TR *text_tr; /* Array of TEXT_TR results */
} TREC_TOP;
typedef struct { /* Overall retrieved results */
char *run_id; /* run id */
long num_q_tr; /* Number of TREC_TOP queries */
long max_num_q_tr; /* Num queries space reserved for*/
TREC_TOP *trec_top; /* Array of TREC_TOP query results */
} ALL_TREC_TOP;
typedef struct { /* For each relevance judgement */
char *docno; /* document id */
long rel; /* document judgement */
} TEXT_QRELS;
typedef struct { /* For each query in rel judgements */
char *qid; /* query id */
long num_text_qrels; /* number of judged documents */
long max_num_text_qrels; /* Num docs space reserved for */
TEXT_QRELS *text_qrels; /* Array of judged TEXT_QRELS */
} TREC_QRELS;
typedef struct { /* Overall relevance judgements */
long num_q_qrels; /* Number of TREC_QRELS queries */
long max_num_q_qrels; /* Num queries space reserved for */
TREC_QRELS *trec_qrels; /* Array of TREC_QRELS queries */
} ALL_TREC_QRELS;
#define INIT_NUM_QUERIES 50
#define INIT_NUM_RESULTS 1000
#define INIT_NUM_RELS 2000
/* Non standard values for tr_vec->rel field */
#define RELVALUE_NONPOOL -1
#define RELVALUE_UNJUDGED -2
/* Set retrieval is based on contingency table:
relevant nonrelevant
retrieved a b
nonretrieved c d
Often you see r == num_rel_ret == a
R == num_rel == a+c
n == num_ret == a+b
N == num_docs == a+b+c+d
Some of these definitions are used in comments below
*/
/* ----------------------------------------------- */
/* Defined constants that are collection/purpose dependent */
/* Number of cutoffs for recall,precision, and rel_precis measures. */
/* CUTOFF_VALUES gives the number of retrieved docs that these */
/* evaluation mesures are applied at. */
#define NUM_CUTOFF 9
#define CUTOFF_VALUES {5, 10, 15, 20, 30, 100, 200, 500, 1000}
/* Maximum fallout value, expressed in number of non-rel docs retrieved. */
/* (Make the approximation that number of non-rel docs in collection */
/* is equal to the number of number of docs in collection) */
#define MAX_FALL_RET 142
/* Maximum multiple of R (number of rel docs for this query) to calculate */
/* R-based precision at */
#define MAX_RPREC 2.0
#define MAX_TIME 300.0
#define NUM_TIME_PTS 60
/* Set a maximum number of nonrel docs to be used for preference measures */
#define PREF_TOP_NONREL_NUM 100
/* ----------------------------------------------- */
/* Defined constants that are collection/purpose independent. If you
change these, you probably need to change comments and documentation,
and some variable names may not be appropriate any more! */
#define NUM_RP_PTS 11
#define THREE_PTS {2, 5, 8}
#define NUM_FR_PTS 11
#define NUM_PREC_PTS 11
#define UTILITY_A 1.0
#define UTILITY_B -1.0
#define UTILITY_C 0.0
#define UTILITY_D 0.0
#define MIN_GEO_MEAN .00001
#define INFAP_EPSILON .00001
typedef struct {
char *qid; /* query id */
long num_queries; /* Number of queries for this eval */
long num_orig_queries; /* Number of queries for this eval without
missing values, if using trec_eval -c */
/* Summary Numbers over all queries */
long num_rel; /* Number of relevant docs */
long num_ret; /* Number of retrieved docs */
long num_rel_ret; /* Number of relevant retrieved docs */
long num_nonrel_judged_ret; /* Number of non-relevant retrieved
judged docs */
float avg_doc_prec; /* Average of precision over all
relevant documents (query independent)*/
/* Measures after num_ret docs */
float exact_recall; /* Recall after num_ret docs */
float exact_precis; /* Precision after num_ret docs */
float exact_rel_precis; /* Relative Precision (or recall) */
/* Defined to be precision / max possible
precision */
float exact_uap; /* Unranked Average Precision */
/* Every rel doc in retrieved set gets
precision, every nonret rel doc gets 0.
Average over all rel docs */
/* Note this = exact_recall *
exact_precision for a query */
/* Preferred measure for evaluation of
unranked sets of arbitrary size. */
float exact_rel_uap; /* Relative Unranked Average Precision */
/* Above, but relativized given size of
retrieved set */
/* If (n<R) set num_rel to n
If (n>R) set num_ret to R
Then use uap formula */
/* exact_rel_precis ** 2 */
float exact_utility; /* From contingency table, by default:
UTILITY_A * a + UTILITY_B * b +
UTILITY_C * c + UTILITY_D * d.
By default, a-b (or r - (n-r)) */
float recip_rank; /* reciprical rank of top retrieved
relevant document */
long rank_first_rel; /* Rank of top retrieved rel doc. Set to
0 if none. Unaveraged */
/* Measures after each document */
float recall_cut[NUM_CUTOFF]; /* Recall after cutoff[i] docs */
float precis_cut[NUM_CUTOFF]; /* precision after cutoff[i] docs. If
less than cutoff[i] docs retrieved,
then assume an additional
cutoff[i]-num_ret non-relevant docs
are retrieved. */
float rel_precis_cut[NUM_CUTOFF];/* Relative precision after cutoff[i]
docs. (Note relative precision is
identical to relative recall) */
float uap_cut[NUM_CUTOFF]; /* uap (is recall * precision) after
cutoff[i] docs. Not recommended */
float rel_uap_cut[NUM_CUTOFF]; /* rel_uap at cutoff[i] docs */
float av_rel_precis; /* average (integral) of rel_precis
after each doc. Do not use if
number of docs retrieved varies */
float av_rel_uap; /* average (integral) of rel_uap
after each doc. Do not use if
number of docs retrieved varies */
/* Measures after each rel doc */
float av_recall_precis; /* MAP! average(integral) of precision at
all rel doc ranks. THE MAJOR
EVALUATION MEASURE FOR RANKED DOCS */
float int_av_recall_precis; /* Same as above, but the precision values
have been interpolated, so that prec(X)
is actually MAX prec(Y) for all
Y >= X */
float int_recall_precis[NUM_RP_PTS];/* interpolated precision at
0.1 increments of recall */
float int_av3_recall_precis; /* interpolated average at 3 intermediate
points */
float int_av11_recall_precis; /* interpolated average at NUM_RP_PTS
intermediate points (recall_level) */
/* Measures after each non-rel doc */
float fall_recall[NUM_FR_PTS]; /* max recall after each non-rel doc,
at 11 points starting at 0.0 and
ending at MAX_FALL_RET /num_docs */
float av_fall_recall; /* Average of fallout-recall, after each
non-rel doc until fallout of
MAX_FALL_RET / num_docs achieved */
/* Measures after R-related cutoffs. R is the number of relevant
docs for a particular query, but note that these cutoffs are after
R docs, whether relevant or non-relevant, have been retrieved.
R-related cutoffs are really only applicable to a situtation where
there are many relevant docs per query (or lots of queries). */
float R_recall_precis; /* Recall or precision after R docs
(note they are equal at this point) */
float av_R_precis; /* Average (or integral) of precision at
each doc until R docs have been
retrieved */
float R_prec_cut[NUM_PREC_PTS]; /* Precision measured after multiples of
R docs have been retrieved. 10
equal points, with max multiple
having value MAX_RPREC */
float int_R_recall_precis; /* Interpolated precision after R docs
Prec(X) = MAX(prec(Y)) for all Y>=X */
float int_av_R_precis; /* Interpolated */
float int_R_prec_cut[NUM_PREC_PTS]; /* Interpolated */
/* Measures after particular time relative to size of eventual retrieved
set. Eg, precision is num_rel_so_far/num_ret
relprecision is num_rel_so_far/MIN(num_ret,num_rel)
uap is num_rel_so_far**2/(num_ret*MIN(num_ret,num_rel))
reluap is relprecision * relprecision */
float time_num_rel[NUM_TIME_PTS]; /* Number of rel docs in time bucket*/
float time_num_nrel[NUM_TIME_PTS];/* Number of nrel docs in each bucket*/
float time_cum_rel[NUM_TIME_PTS]; /* Cumulative time_num_rel */
float time_precis[NUM_TIME_PTS]; /* First Precision in each bucket */
float time_relprecis[NUM_TIME_PTS];/* First rel-Precision in each bucket */
float time_uap[NUM_TIME_PTS]; /* First uap in bucket*/
float time_reluap[NUM_TIME_PTS]; /* First relative uap in bucket*/
float time_utility[NUM_TIME_PTS]; /* First Utility (default 1,-1,0,0)
in bucket */
float av_time_precis; /* Sum (integral) of time_precis */
float av_time_relprecis; /* Sum (integral) of time_relprecis */
float av_time_uap; /* Sum (integral) of time_uap */
float av_time_reluap; /* Sum (integral) of time_reluap */
float av_time_utility; /* Sum (integral) of time_utility */
float av_time_cum_rel; /* Sum (integral) of time_cum_rel */
/* Measures dependent on only judged documents */
/* Binary Pref relations: fraction of nonrel documents retrieved after
each rel doc */
float bpref; /* real BPREF. Top num_rel nonrel docs */
float bpref_top5Rnonrel; /* Top 5 * num_rel nonrel docs */
float bpref_top10Rnonrel; /* Top 10 * num_rel nonrel docs */
/* float bpref_topRnonrel; * renamed as bpref */
float bpref_allnonrel; /* all judged nonrel docs */
float bpref_retnonrel; /* Only retrieved nonrel docs */
float bpref_topnonrel; /* Top PREF_TOPNREL_NUM nonrel docs */
float bpref_top50pRnonrel; /* Top 50 + num_rel nonrel docs */
float bpref_top25pRnonrel; /* Top 25 + num_rel nonrel docs */
float bpref_top10pRnonrel; /* Top 10 + num_rel nonrel docs.
Bad version used in SIGIR 2004 paper */
float old_bpref_top10pRnonrel; /* bad old version. Top 10 + num_rel
nonrel docs. Used in SIGIR 2004 paper*/
float bpref_top25p2Rnonrel; /* Top 25 + 2 * num_rel nonrel docs */
float bpref_retall; /* Only retrieved rel,nonrel docs */
float bpref_5; /* Only top 5 rel, top 5 nonrel */
float bpref_10; /* Only top 10 rel, top 10 nonrel */
float old_bpref; /* Bad old bpref. Top num_rel nonrel docs.
Only used retrieved nonrel docs.
Used in TREC 12,13, mention in
SIGIR 2004 paper */
float bpref_num_all; /* num not retrieved before (all judged)*/
float bpref_num_ret; /* num retrieved after */
long bpref_num_correct; /* num correct preferences */
long bpref_num_possible; /* num possible correct preferences */
/* Measures that allow sampling of judgement pool: Qrels/results divided
into unpooled, pooled_but_unjudged, pooled_judged_rel,
pooled_judged_nonrel. */
/* Inf_ap: "Estimating Average Precision with Incomplete and Imperfect
Judgments", Emine Yilmaz and Javed A. Aslam.
My intuition of it: Calculate P at rel doc using higher retrieved judged
docs, then average in 0's from higher pooled docs. */
float inf_ap; /* Inferred AP. see Aslam et al,
Estimating Average Precision with
Incomplete Information */
/* Measures that use Geometric Mean
avg_Score = exp (SUM (log (MAX (query_score, .00001))) / N)
WARNING: Geometric Mean measures special cased for "trec_eval -c".
Works, but be careful when implementing new measure */
float gm_ap; /* Geometric Mean version of MAP */
float gm_bpref; /* Geometric Mean version of bpref. Note
bpref has lots of 0.0 values */
} TREC_EVAL;
#endif /* TRECEVALH */
trec_eval.8.1/trec_eval_help.c
+208
View File
@@ -0,0 +1,208 @@
/* Copyright (c) 2003, 1991, 1990, 1984 - Chris Buckley. */
#include "common.h"
#include "trec_eval.h"
static char *help_message =
"trec_eval [-h] [-q] [-a] [-o] [-c] [-l<num> [-N<num>] [-M<num>] [-Ua<num>] [-Ub<num>] [-Uc<num>] [-Ud<num>] [-T] trec_rel_file trec_top_file \n\
\n\
Calculate and print various evaluation measures, evaluating the results \n\
in trec_top_file against the relevance judgements in trec_rel_file. \n\
\n\
There are a fair number of options, of which only the lower case options are \n\
normally ever used. \n\
-h: Print full help message and exit \n\
-q: In addition to summary evaluation, give evaluation for each query \n\
-a: Print all evaluation measures calculated, instead of just the \n\
main official measures for TREC. \n\
-o: Print everything out in old, nonrelational format (default is relational) \n\
-c: Average over the complete set of queries in the relevance judgements \n\
instead of the queries in the intersection of relevance judgements \n\
and results. Missing queries will contribute a value of 0 to all \n\
evaluation measures (which may or may not be reasonable for a \n\
particular evaluation measure, but is reasonable for standard TREC \n\
measures.) \n\
-l<num>: Num indicates the minimum relevance judgement value needed for \n\
a document to be called relevant. (All measures used by TREC eval are \n\
based on binary relevance). Used if trec_rel_file contains relevance \n\
judged on a multi-relevance scale. Default is 1. \n\
-N<num>: Number of docs in collection \n\
-M<num>: Max number of docs per topic to use in evaluation (discard rest). \n\
-Ua<num>: Value to use for 'a' coefficient of utility computation. \n\
relevant nonrelevant \n\
retrieved a b \n\
nonretrieved c d \n\
-Ub<num>: Value to use for 'b' coefficient of utility computation. \n\
-Uc<num>: Value to use for 'c' coefficient of utility computation. \n\
-Ud<num>: Value to use for 'd' coefficient of utility computation. \n\
-J: Calculate all values only over the judged (either relevant or \n\
nonrelevant) documents. All unjudged documents are removed from the \n\
retrieved set before any calculations (possibly leaving an empty set). \n\
DO NOT USE, unless you really know what you're doing - very easy to get \n\
reasonable looking, but invalid, numbers. \n\
-T: Treat similarity as time that document retrieved. Compute \n\
several time-based measures after ranking docs by time retrieved \n\
(first doc (lowest sim) retrieved ranked highest). \n\
Only done if -a selected. \n\
\n\
\n\
Read text tuples from trec_top_file of the form \n\
030 Q0 ZF08-175-870 0 4238 prise1 \n\
qid iter docno rank sim run_id \n\
giving TREC document numbers (a string) retrieved by query qid \n\
(a string) with similarity sim (a float). The other fields are ignored, \n\
with the exception that the run_id field of the last line is kept and \n\
output. In particular, note that the rank field is ignored here; \n\
internally ranks are assigned by sorting by the sim field with ties \n\
broken deterministicly (using docno). \n\
Sim is assumed to be higher for the docs to be retrieved first. \n\
File may contain no NULL characters. \n\
Lines may contain fields after the run_id; they are ignored. \n\
\n\
Relevance for each docno to qid is determined from text_qrels_file, which \n\
consists of text tuples of the form \n\
qid iter docno rel \n\
giving TREC document numbers (docno, a string) and their relevance (rel, \n\
a non-negative integer less than 128, or -1 (unjudged)) \n\
to query qid (a string). iter string field is ignored. \n\
Fields are separated by whitespace, string fields can contain no whitespace. \n\
File may contain no NULL characters. \n\
\n\
The text tuples with relevance judgements are converted to TR_VEC form \n\
and then submitted to the SMART evaluation routines. \n\
The did,rank,sim fields of TR_VEC are filled in from trec_top_file; \n\
action,iter fields are set to 0. \n\
The rel field is set to -1 if the document was not in the pool (not in \n\
text_qrels_file) or -2 if the document was in the pool but unjudged (some \n\
measures (infAP) allow the pool to be sampled instead of judged fully). \n\
Otherwise it is set to the value in text_qrels_file. \n\
Most measures, but not all, will treat -1 or -2 the same as 0, \n\
namely nonrelevant. Note that relevance_level is used to \n\
determine if the document is relevant during score calculations. \n\
Queries for which there is no relevance information are ignored. \n\
Warning: queries for which there are relevant docs but no retrieved docs \n\
are also ignored by default. This allows systems to evaluate over subsets \n\
of the relevant docs, but means if a system improperly retrieves no docs, \n\
it will not be detected. Use the -c flag to avoid this behavior. \n\
\n\
EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. \n\
Relational Format prints the same values, but all lines are of the form \n\
measure_name query value \n\
\n\
1. Total number of documents over all queries \n\
Retrieved: \n\
Relevant: \n\
Rel_ret: (relevant and retrieved) \n\
These should be self-explanatory. All values are totals over all \n\
queries being evaluated. \n\
2. Interpolated Recall - Precision Averages: \n\
at 0.00 \n\
at 0.10 \n\
... \n\
at 1.00 \n\
See any standard IR text (especially by Salton) for more details of \n\
recall-precision evaluation. Measures precision (percent of retrieved \n\
docs that are relevant) at various recall levels (after a certain \n\
percentage of all the relevant docs for that query have been retrieved). \n\
'Interpolated' means that, for example, precision at recall \n\
0.10 (ie, after 10% of rel docs for a query have been retrieved) is \n\
taken to be MAXIMUM of precision at all recall points >= 0.10. \n\
Values are averaged over all queries (for each of the 11 recall levels). \n\
These values are used for Recall-Precision graphs. \n\
3. Average precision (non-interpolated) over all rel docs \n\
The precision is calculated after each relevant doc is retrieved. \n\
If a relevant doc is not retrieved, its precision is 0.0. \n\
All precision values are then averaged together to get a single number \n\
for the performance of a query. Conceptually this is the area \n\
underneath the recall-precision graph for the query. \n\
The values are then averaged over all queries. \n\
4. Precision: \n\
at 5 docs \n\
at 10 docs \n\
... \n\
at 1000 docs \n\
The precision (percent of retrieved docs that are relevant) after X \n\
documents (whether relevant or nonrelevant) have been retrieved. \n\
Values averaged over all queries. If X docs were not retrieved \n\
for a query, then all missing docs are assumed to be non-relevant. \n\
5. R-Precision (precision after R (= num_rel for a query) docs retrieved): \n\
Measures precision (or recall, they're the same) after R docs \n\
have been retrieved, where R is the total number of relevant docs \n\
for a query. Thus if a query has 40 relevant docs, then precision \n\
is measured after 40 docs, while if it has 600 relevant docs, precision \n\
is measured after 600 docs. This avoids some of the averaging \n\
problems of the 'precision at X docs' values in (4) above. \n\
If R is greater than the number of docs retrieved for a query, then \n\
the nonretrieved docs are all assumed to be nonrelevant. \n\
";
extern SINGLE_MEASURE sing_meas[];
extern PARAMETERIZED_MEASURE param_meas[];
extern MICRO_MEASURE micro_meas[];
extern int num_param_meas, num_sing_meas, num_micro_meas;
int
trec_eval_help(epi)
EVAL_PARAM_INFO *epi;
{
long i, j;
/* Note this trusts the format_strings in measures.c will not overflow */
char temp_buf1[200];
char temp_buf2[200];
printf ("%s\n", help_message);
printf ("Major measures (again) with their relational names:\n");
for (i = 0; i < num_sing_meas; i++) {
if (sing_meas[i].print_short_flag)
printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name);
}
for (i = 0; i < num_param_meas; i++) {
if (param_meas[i].print_short_flag) {
for (j = 0; j < param_meas[i].num_values; j++) {
sprintf (temp_buf1, param_meas[i].format_string,
param_meas[i].get_param_str (epi, j));
sprintf (temp_buf2, param_meas[i].long_format_string,
param_meas[i].get_param_str (epi, j));
printf ("%-15s\t%s%s\n", temp_buf1,
param_meas[i].long_name, temp_buf2);
}
}
}
for (i = 0; i < num_micro_meas; i++) {
if (micro_meas[i].print_short_flag)
printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name);
}
printf ("\n\nMinor measures with their relational names:\n");
for (i = 0; i < num_sing_meas; i++) {
if (sing_meas[i].print_short_flag)
continue;
if (sing_meas[i].print_time_flag && (! epi->time_flag))
continue;
if (! sing_meas[i].print_short_flag)
printf ("%-15s\t%s\n", sing_meas[i].name, sing_meas[i].long_name);
}
for (i = 0; i < num_param_meas; i++) {
if (param_meas[i].print_short_flag)
continue;
if (param_meas[i].print_time_flag && (! epi->time_flag))
continue;
for (j = 0; j < param_meas[i].num_values; j++) {
sprintf (temp_buf1, param_meas[i].format_string,
param_meas[i].get_param_str (epi, j));
sprintf (temp_buf2, param_meas[i].long_format_string,
param_meas[i].get_param_str (epi, j));
printf ("%-15s\t%s%s\n", temp_buf1,
param_meas[i].long_name, temp_buf2);
}
}
for (i = 0; i < num_micro_meas; i++) {
if (! micro_meas[i].print_short_flag)
printf ("%-15s\t%s\n", micro_meas[i].name, micro_meas[i].long_name);
}
return (1);
}
trec_eval.8.1/trvec_teval.c
+700
View File
@@ -0,0 +1,700 @@
#ifdef RCSID
static char rcsid[] = "$Header: /home/smart/release/src/libevaluate/trvec_trec_eval.c,v 11.0 1992/07/21 18:20:35 chrisb Exp chrisb $";
#endif
/* Copyright (c) 2005
*/
#include "common.h"
#include "sysfunc.h"
#include "smart_error.h"
#include "tr_vec.h"
#include "trec_eval.h"
static int compare_iter_rank();
static void calc_cutoff_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
TREC_EVAL *eval, long num_rel,
long num_nonrel);
static void calc_bpref_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
TREC_EVAL *eval, long num_rel,
long num_nonrel);
static void calc_average_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
TREC_EVAL *eval, long num_rel,
long num_nonrel);
static void calc_exact_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
TREC_EVAL *eval, long num_rel,
long num_nonrel);
static void calc_time_measures(EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
TREC_EVAL *eval, long num_rel,
long num_nonrel);
int
trvec_trec_eval (epi, tr_vec, eval, num_rel, num_nonrel)
EVAL_PARAM_INFO *epi;
TR_VEC *tr_vec;
TREC_EVAL *eval;
long num_rel; /* Number relevant judged */
long num_nonrel; /* Number nonrelevant judged */
{
long j;
long max_iter;
if (tr_vec == (TR_VEC *) NULL)
return (UNDEF);
/* Initialize everything to 0 */
bzero ((char *) eval, sizeof (TREC_EVAL));
eval->qid = tr_vec->qid;
eval->num_queries = 1;
/* If no retrieved docs, then just return */
if (tr_vec->num_tr == 0) {
return (0);
}
eval->num_rel = num_rel;
/* Evaluate only the docs on the last iteration of new_tr_vec */
/* Sort the tr tuples for this query by decreasing iter and
increasing rank */
qsort ((char *) tr_vec->tr,
(int) tr_vec->num_tr,
sizeof (TR_TUP),
compare_iter_rank);
max_iter = tr_vec->tr[0].iter;
for (j = 0; j < tr_vec->num_tr; j++) {
if (tr_vec->tr[j].iter == max_iter) {
eval->num_ret++;
if (tr_vec->tr[j].rel >= epi->relevance_level)
eval->num_rel_ret++;
}
else {
if (tr_vec->tr[j].rel >= epi->relevance_level)
eval->num_rel--;
}
}
/* Calculate cutoff measures, and those measures dependant on them */
/* Also includes recip_rank and rank_first_rel */
calc_cutoff_measures (epi, tr_vec, eval, num_rel, num_nonrel);
/* Calculate bpref and related measures (judged docs only) */
calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel);
/* Calculate measures that average over ret or rel docs */
calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel);
/* Calculate exact measures over entire retrieved sets */
calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel);
/* Calculate time measures, if wanted */
if (epi->time_flag)
calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel);
return (1);
}
static int
compare_iter_rank (tr1, tr2)
TR_TUP *tr1;
TR_TUP *tr2;
{
if (tr1->iter > tr2->iter)
return (-1);
if (tr1->iter < tr2->iter)
return (1);
if (tr1->rank < tr2->rank)
return (-1);
if (tr1->rank > tr2->rank)
return (1);
return (0);
}
/* ********************************************************************* */
/* calculate cutoff measures */
/* cutoff values for recall precision output */
static int cutoff[NUM_CUTOFF] = CUTOFF_VALUES;
static int three_pts[3] = THREE_PTS;
static void
calc_cutoff_measures(epi, tr_vec, eval, num_rel, num_nonrel)
EVAL_PARAM_INFO *epi;
TR_VEC *tr_vec;
TREC_EVAL *eval;
long num_rel; /* Number relevant judged */
long num_nonrel; /* Number nonrelevant judged */
{
double recall, precis; /* current recall, precision values */
double rel_precis, rel_uap;/* relative precision, uap values */
double int_precis; /* current interpolated precision values */
long i,j;
long cut_rp[NUM_RP_PTS]; /* number of rel docs needed to be retrieved
for each recall-prec cutoff */
long cut_fr[NUM_FR_PTS]; /* number of non-rel docs needed to be
retrieved for each fall-recall cutoff */
long cut_rprec[NUM_PREC_PTS]; /* Number of docs needed to be retrieved
for each R-based prec cutoff */
long current_cutoff, current_cut_rp, current_cut_fr, current_cut_rprec;
long rel_so_far = eval->num_rel_ret;
/* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all
Y >= X) */
int_precis = (float) rel_so_far / (float) eval->num_ret;
/* Discover cutoff values for this query */
current_cutoff = NUM_CUTOFF - 1;
while (current_cutoff > 0 && cutoff[current_cutoff] > eval->num_ret)
current_cutoff--;
for (i = 0; i < NUM_RP_PTS; i++)
cut_rp[i] = ((eval->num_rel * i) + NUM_RP_PTS - 2) / (NUM_RP_PTS - 1);
current_cut_rp = NUM_RP_PTS - 1;
while (current_cut_rp > 0 && cut_rp[current_cut_rp] > eval->num_rel_ret)
current_cut_rp--;
for (i = 0; i < NUM_FR_PTS; i++)
cut_fr[i] = ((MAX_FALL_RET * i) + NUM_FR_PTS - 2) / (NUM_FR_PTS - 1);
current_cut_fr = NUM_FR_PTS - 1;
while (current_cut_fr > 0 && cut_fr[current_cut_fr] > eval->num_ret - eval->num_rel_ret)
current_cut_fr--;
for (i = 1; i < NUM_PREC_PTS+1; i++)
cut_rprec[i-1] = ((MAX_RPREC * eval->num_rel * i) + NUM_PREC_PTS - 2)
/ (NUM_PREC_PTS - 1);
current_cut_rprec = NUM_PREC_PTS - 1;
while (current_cut_rprec > 0 && cut_rprec[current_cut_rprec]>eval->num_ret)
current_cut_rprec--;
/* Loop over all retrieved docs in reverse order */
for (j = eval->num_ret; j > 0; j--) {
if (rel_so_far > 0) {
recall = (float) rel_so_far / (float) eval->num_rel;
precis = (float) rel_so_far / (float) j;
if (j > eval->num_rel) {
rel_precis = (float) rel_so_far / (float) eval->num_rel;
}
else {
rel_precis = (float) rel_so_far / (float) j;
}
}
else {
recall = 0.0;
precis = 0.0;
rel_precis = 0.0;
}
rel_uap = rel_precis * rel_precis;
if (int_precis < precis)
int_precis = precis;
while (j == cutoff[current_cutoff]) {
eval->recall_cut[current_cutoff] = recall;
eval->precis_cut[current_cutoff] = precis;
eval->rel_precis_cut[current_cutoff] = rel_precis;
eval->uap_cut[current_cutoff] = precis * recall;
eval->rel_uap_cut[current_cutoff] = rel_uap;
current_cutoff--;
}
while (j == cut_rprec[current_cut_rprec]) {
eval->R_prec_cut[current_cut_rprec] = precis;
eval->int_R_prec_cut[current_cut_rprec] = int_precis;
current_cut_rprec--;
}
if (j == eval->num_rel) {
eval->R_recall_precis = precis;
eval->int_R_recall_precis = int_precis;
}
if (tr_vec->tr[j-1].rel >= epi->relevance_level) {
while (rel_so_far == cut_rp[current_cut_rp]) {
eval->int_recall_precis[current_cut_rp] = int_precis;
current_cut_rp--;
}
eval->recip_rank = 1.0 / (float) j;
eval->rank_first_rel = j;
rel_so_far--;
}
else {
/* Note: for fallout-recall, the recall at X non-rel docs
is used for the recall 'after' (X-1) non-rel docs.
Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for
Y retrieved docs where X-1 non-rel retrieved */
while (current_cut_fr >= 0 &&
j - rel_so_far == cut_fr[current_cut_fr] + 1) {
eval->fall_recall[current_cut_fr] = recall;
current_cut_fr--;
}
}
}
/* Fill in the 0.0 value for recall-precision (== max precision
at any point in the retrieval ranking) */
eval->int_recall_precis[0] = int_precis;
/* Fill in those cutoff values and averages that were not achieved
because insufficient docs were retrieved. */
for (i = 0; i < NUM_CUTOFF; i++) {
if (eval->num_ret < cutoff[i]) {
if (eval->num_rel_ret > 0) {
eval->recall_cut[i] = ((float) eval->num_rel_ret /
(float) eval->num_rel);
eval->precis_cut[i] = ((float) eval->num_rel_ret /
(float) cutoff[i]);
}
eval->rel_precis_cut[i] = (cutoff[i] < eval->num_rel) ?
eval->precis_cut[i] :
eval->recall_cut[i];
eval->uap_cut[i] = eval->precis_cut[i] *
eval->recall_cut[i];
eval->rel_uap_cut[i] = eval->precis_cut[i] *
eval->precis_cut[i];
}
}
for (i = 0; i < NUM_FR_PTS; i++) {
if (eval->num_ret - eval->num_rel_ret < cut_fr[i]) {
if (eval->num_rel_ret > 0)
eval->fall_recall[i] = (float) eval->num_rel_ret /
(float) eval->num_rel;
}
}
for (i = 0; i < NUM_PREC_PTS; i++) {
if (eval->num_ret < cut_rprec[i]) {
eval->R_prec_cut[i] = (float) eval->num_rel_ret /
(float) cut_rprec[i];
eval->int_R_prec_cut[i] = (float) eval->num_rel_ret /
(float) cut_rprec[i];
}
}
if (eval->num_rel > eval->num_ret) {
eval->R_recall_precis = (float) eval->num_rel_ret /
(float)eval->num_rel;
eval->int_R_recall_precis = (float) eval->num_rel_ret /
(float)eval->num_rel;
}
/* Calculate other indirect evaluation measure averages. */
/* average recall-precis of 3 and 11 intermediate points */
eval->int_av3_recall_precis =
(eval->int_recall_precis[three_pts[0]] +
eval->int_recall_precis[three_pts[1]] +
eval->int_recall_precis[three_pts[2]]) / 3.0;
for (i = 0; i < NUM_RP_PTS; i++) {
eval->int_av11_recall_precis += eval->int_recall_precis[i];
}
eval->int_av11_recall_precis /= NUM_RP_PTS;
}
static void
calc_bpref_measures (epi, tr_vec, eval, num_rel, num_nonrel)
EVAL_PARAM_INFO *epi;
TR_VEC *tr_vec;
TREC_EVAL *eval;
long num_rel; /* Number relevant judged */
long num_nonrel; /* Number nonrelevant judged */
{
long j;
long nonrel_ret;
long nonrel_so_far, rel_so_far, pool_unjudged_so_far;
long bounded_5R_nonrel_so_far, bounded_10R_nonrel_so_far;
long pref_top_nonrel_num = PREF_TOP_NONREL_NUM;
long pref_top_50pRnonrel_num;
long pref_top_25pRnonrel_num;
long pref_top_25p2Rnonrel_num;
long pref_top_10pRnonrel_num;
long pref_top_Rnonrel_num;
/* Calculate judgement based measures (dependent on only
judged docs; no assumption of non-relevance if not judged) */
/* Binary Preference measures; here expressed as all docs with a higher
value of rel are to be preferred. Optimize by keeping track of nonrel
seen so far */
pref_top_nonrel_num = PREF_TOP_NONREL_NUM;
pref_top_50pRnonrel_num = 50 + eval->num_rel;
pref_top_25pRnonrel_num = 25 + eval->num_rel;
pref_top_10pRnonrel_num = 10 + eval->num_rel;
pref_top_Rnonrel_num = eval->num_rel;
pref_top_25p2Rnonrel_num = 25 + (2 * eval->num_rel);
nonrel_ret = 0;
for (j = 0; j < tr_vec->num_tr; j++) {
if (tr_vec->tr[j].rel >= 0 && tr_vec->tr[j].rel < epi->relevance_level)
nonrel_ret++;
}
nonrel_so_far = 0;
rel_so_far = 0;
pool_unjudged_so_far = 0;
bounded_5R_nonrel_so_far = 0;
bounded_10R_nonrel_so_far = 0;
for (j = 0; j < tr_vec->num_tr; j++) {
if (tr_vec->tr[j].rel == RELVALUE_NONPOOL)
/* document not in pool. Skip */
continue;
if (tr_vec->tr[j].rel == RELVALUE_UNJUDGED) {
/* document in pool but unjudged. */
pool_unjudged_so_far++;
continue;
}
if (tr_vec->tr[j].rel >= 0 && tr_vec->tr[j].rel < epi->relevance_level) {
/* Judged Nonrel document */
if (nonrel_so_far < 5 * eval->num_rel) {
bounded_5R_nonrel_so_far++;
if (nonrel_so_far < 10 * eval->num_rel) {
bounded_10R_nonrel_so_far++;
}
}
nonrel_so_far++;
}
else {
/* Judged Rel doc */
rel_so_far++;
/* Add fraction of correct preferences. */
/* Special case nonrel_so_far == 0 to avoid division by 0 */
if (nonrel_so_far > 0) {
eval->bpref_allnonrel += 1.0 - (((float) nonrel_so_far) /
(float) num_nonrel);
eval->bpref_retnonrel += 1.0 - (((float) nonrel_so_far) /
(float) nonrel_ret);
eval->bpref_retall += 1.0 - (((float) nonrel_so_far) /
(float) nonrel_ret);
eval->bpref_num_correct +=
MIN (num_nonrel, pref_top_Rnonrel_num) -
MIN (nonrel_so_far, pref_top_Rnonrel_num);
eval->bpref += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) /
(float) MIN (num_nonrel, pref_top_Rnonrel_num));
eval->old_bpref += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_Rnonrel_num)) /
(float) MIN (nonrel_ret, pref_top_Rnonrel_num));
eval->bpref_topnonrel += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_nonrel_num)) /
(float) MIN (num_nonrel, pref_top_nonrel_num));
eval->bpref_top50pRnonrel += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_50pRnonrel_num)) /
(float) MIN (num_nonrel, pref_top_50pRnonrel_num));
eval->bpref_top25pRnonrel += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_25pRnonrel_num)) /
(float) MIN (num_nonrel, pref_top_25pRnonrel_num));
eval->bpref_top10pRnonrel += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) /
(float) MIN (num_nonrel, pref_top_10pRnonrel_num));
eval->old_bpref_top10pRnonrel += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_10pRnonrel_num)) /
(float) MIN (nonrel_ret, pref_top_10pRnonrel_num));
eval->bpref_top25p2Rnonrel += 1.0 -
(((float) MIN (nonrel_so_far, pref_top_25p2Rnonrel_num)) /
(float) MIN (num_nonrel, pref_top_25p2Rnonrel_num));
if (rel_so_far <= 5 && nonrel_so_far < 5)
eval->bpref_5 += 1.0 - (float) nonrel_so_far /
(float) MIN (num_nonrel, 5);
if (rel_so_far <= 10 && nonrel_so_far < 10)
eval->bpref_10 += 1.0 - (float) nonrel_so_far /
(float) MIN (num_nonrel, 10);
}
else {
eval->bpref += 1.0;
eval->old_bpref += 1.0;
eval->bpref_allnonrel += 1.0;
eval->bpref_retnonrel += 1.0;
eval->bpref_retall += 1.0;
eval->bpref_topnonrel += 1.0;
eval->bpref_top50pRnonrel += 1.0;
eval->bpref_top25pRnonrel += 1.0;
eval->bpref_top10pRnonrel += 1.0;
eval->old_bpref_top10pRnonrel += 1.0;
eval->bpref_top25p2Rnonrel += 1.0;
if (rel_so_far <= 5)
eval->bpref_5 += 1.0;
if (rel_so_far <= 10)
eval->bpref_10 += 1.0;
}
eval->bpref_top5Rnonrel += 1.0 -
(((float) bounded_5R_nonrel_so_far) /
(float) MIN (num_nonrel, eval->num_rel * 5));
eval->bpref_top10Rnonrel += 1.0 -
(((float) bounded_10R_nonrel_so_far) /
(float) MIN (num_nonrel, eval->num_rel * 10));
eval->bpref_num_all += num_nonrel - nonrel_so_far;
eval->bpref_num_ret += nonrel_ret - nonrel_so_far;
/* inf_ap */
if (0 == j)
eval->inf_ap += 1.0;
else {
float fj = (float) j;
eval->inf_ap += 1.0 / (fj+1.0) +
(fj / (fj+1.0)) *
((rel_so_far-1+nonrel_so_far+pool_unjudged_so_far) / fj) *
((rel_so_far-1 + INFAP_EPSILON) /
(rel_so_far-1 + nonrel_so_far + 2 * INFAP_EPSILON));
}
}
}
if (eval->num_rel) {
eval->bpref /= eval->num_rel;
eval->old_bpref /= eval->num_rel;
eval->bpref_allnonrel /= eval->num_rel;
eval->bpref_retnonrel /= eval->num_rel;
eval->bpref_topnonrel /= eval->num_rel;
eval->bpref_top5Rnonrel /= eval->num_rel;
eval->bpref_top10Rnonrel /= eval->num_rel;
eval->bpref_top50pRnonrel /= eval->num_rel;
eval->bpref_top25pRnonrel /= eval->num_rel;
eval->bpref_top10pRnonrel /= eval->num_rel;
eval->old_bpref_top10pRnonrel /= eval->num_rel;
eval->bpref_top25p2Rnonrel /= eval->num_rel;
if (eval->num_rel_ret) {
eval->bpref_retall /= eval->num_rel_ret;
eval->bpref_5 /= MIN (rel_so_far, 5);
eval->bpref_10 /= MIN (rel_so_far, 10);
}
eval->bpref_num_possible = eval->num_rel *
MIN (num_nonrel, pref_top_Rnonrel_num);
eval->inf_ap /= eval->num_rel;
}
eval->num_nonrel_judged_ret = nonrel_ret;
/* For those bpref measure variants which use the geometric mean instead
of straight averages, compute them here. Original measure value
is constrained to be greater than MIN_GEO_MEAN (for time being .00001,
since trec_eval prints to four significant digits) */
eval->gm_bpref = (float) log ((double)(MAX (eval->bpref,
MIN_GEO_MEAN)));
}
static void
calc_average_measures (epi, tr_vec, eval, num_rel, num_nonrel)
EVAL_PARAM_INFO *epi;
TR_VEC *tr_vec;
TREC_EVAL *eval;
long num_rel; /* Number relevant judged */
long num_nonrel; /* Number nonrelevant judged */
{
double recall, precis; /* current recall, precision values */
double rel_precis, rel_uap;/* relative precision, uap values */
double int_precis; /* current interpolated precision values */
long i,j;
long rel_so_far;
/* Note for interpolated precision values (Prec(X) = MAX (PREC(Y)) for all
Y >= X) */
rel_so_far = eval->num_rel_ret;
int_precis = (float) rel_so_far / (float) eval->num_ret;
/* Loop over all retrieved docs in reverse order */
for (j = eval->num_ret; j > 0; j--) {
if (rel_so_far > 0) {
recall = (float) rel_so_far / (float) eval->num_rel;
precis = (float) rel_so_far / (float) j;
if (j > eval->num_rel) {
rel_precis = (float) rel_so_far / (float) eval->num_rel;
}
else {
rel_precis = (float) rel_so_far / (float) j;
}
}
else {
recall = 0.0;
precis = 0.0;
rel_precis = 0.0;
}
rel_uap = rel_precis * rel_precis;
if (int_precis < precis)
int_precis = precis;
eval->av_rel_precis += rel_precis;
eval->av_rel_uap += rel_uap;
if (j < eval->num_rel) {
eval->av_R_precis += precis;
eval->int_av_R_precis += int_precis;
}
if (tr_vec->tr[j-1].rel >= epi->relevance_level) {
eval->int_av_recall_precis += int_precis;
eval->av_recall_precis += precis;
eval->avg_doc_prec += precis;
rel_so_far--;
}
else {
/* Note: for fallout-recall, the recall at X non-rel docs
is used for the recall 'after' (X-1) non-rel docs.
Ie. recall_used(X-1 non-rel docs) = MAX (recall(Y)) for
Y retrieved docs where X-1 non-rel retrieved */
if (j - rel_so_far < MAX_FALL_RET) {
eval->av_fall_recall += recall;
}
}
}
if (eval->num_ret - eval->num_rel_ret < MAX_FALL_RET) {
if (eval->num_rel_ret > 0)
eval->av_fall_recall += ((MAX_FALL_RET -
(eval->num_ret - eval->num_rel_ret))
* ((float)eval->num_rel_ret /
(float)eval->num_rel));
}
if (eval->num_rel > eval->num_ret) {
for (i = eval->num_ret; i < eval->num_rel; i++) {
eval->av_R_precis += (float) eval->num_rel_ret /
(float) i;
eval->int_av_R_precis += (float) eval->num_rel_ret /
(float) i;
}
}
/* Calculate all the other averages */
if (eval->num_rel_ret > 0) {
eval->av_recall_precis /= eval->num_rel;
eval->int_av_recall_precis /= eval->num_rel;
}
eval->av_fall_recall /= MAX_FALL_RET;
eval->av_rel_precis /= eval->num_ret;
eval->av_rel_uap /= eval->num_ret;
if (eval->num_rel) {
eval->av_R_precis /= eval->num_rel;
eval->int_av_R_precis /= eval->num_rel;
}
/* For those measure variants which use the geometric mean instead
of straight averages, compute them here. Original measure value
is constrained to be greater than MIN_GEO_MEAN (for time being .00001,
since trec_eval prints to four significant digits) */
eval->gm_ap = (float) log ((double)(MAX (eval->av_recall_precis,
MIN_GEO_MEAN)));
}
static void
calc_exact_measures (epi, tr_vec, eval, num_rel, num_nonrel)
EVAL_PARAM_INFO *epi;
TR_VEC *tr_vec;
TREC_EVAL *eval;
long num_rel; /* Number relevant judged */
long num_nonrel; /* Number nonrelevant judged */
{
if (eval->num_rel) {
eval->exact_recall = (double) eval->num_rel_ret / eval->num_rel;
eval->exact_precis = (double) eval->num_rel_ret / eval->num_ret;
eval->exact_uap = eval->exact_recall * eval->exact_precis;
if (eval->num_rel > eval->num_ret) {
eval->exact_rel_precis = eval->exact_precis;
}
else {
eval->exact_rel_precis = eval->exact_recall;
}
eval->exact_rel_uap = eval->exact_precis * eval->exact_precis;
eval->exact_utility =
epi->utility_a * eval->num_rel_ret +
epi->utility_b * (eval->num_ret - eval->num_rel_ret) +
epi->utility_c * (eval->num_rel - eval->num_rel_ret) +
epi->utility_d * (epi->num_docs_in_coll + eval->num_rel_ret
- eval->num_ret - eval->num_rel);
}
}
static void
calc_time_measures (epi, tr_vec, eval, num_rel, num_nonrel)
EVAL_PARAM_INFO *epi;
TR_VEC *tr_vec;
TREC_EVAL *eval;
long num_rel; /* Number relevant judged */
long num_nonrel; /* Number nonrelevant judged */
{
double recall, precis; /* current recall, precision values */
double rel_precis, rel_uap;/* relative precision, uap values */
double int_precis = 0.0; /* current interpolated precision values */
long i,j;
long bucket;
long last_time_bucket = NUM_TIME_PTS; /* Last time bucket filled in */
long rel_so_far = eval->num_rel_ret;
long min_ret_rel = MIN(eval->num_rel, eval->num_ret);
/* Loop over all retrieved docs in reverse order */
for (j = eval->num_ret; j > 0; j--) {
if (rel_so_far > 0) {
recall = (float) rel_so_far / (float) eval->num_rel;
precis = (float) rel_so_far / (float) j;
if (j > eval->num_rel) {
rel_precis = (float) rel_so_far / (float) eval->num_rel;
}
else {
rel_precis = (float) rel_so_far / (float) j;
}
}
else {
recall = 0.0;
precis = 0.0;
rel_precis = 0.0;
}
rel_uap = rel_precis * rel_precis;
if (int_precis < precis)
int_precis = precis;
bucket = tr_vec->tr[j-1].sim *
((double) NUM_TIME_PTS / (double) MAX_TIME);
if (bucket < 0) bucket = 0;
if (bucket >= NUM_TIME_PTS) bucket = NUM_TIME_PTS-1;
if (tr_vec->tr[j-1].rel >= epi->relevance_level)
eval->time_num_rel[bucket]++;
else
eval->time_num_nrel[bucket]++;
eval->time_precis[bucket] = (float)rel_so_far /
(float) eval->num_ret;
eval->time_relprecis[bucket] = ((float)rel_so_far) /
(float) min_ret_rel;
eval->time_uap[bucket] = (float) rel_so_far * rel_so_far /
((float) eval->num_ret * (float) min_ret_rel);
eval->time_reluap[bucket] = (float) rel_so_far * rel_so_far /
((float) min_ret_rel * (float) min_ret_rel);
eval->time_utility[bucket] =
epi->utility_a * rel_so_far +
epi->utility_b * (j - rel_so_far) +
epi->utility_c * (eval->num_rel - rel_so_far) +
epi->utility_d * (epi->num_docs_in_coll +
rel_so_far - j - eval->num_rel);
/* Need to fill in buckets up to last bucket */
/* note assumes buckets are decreasing */
/* Must do here since utility can be negative and zero
cannot be used as flag later */
for (i = bucket+1; i < last_time_bucket; i++) {
eval->time_precis[i] = eval->time_precis[bucket];
eval->time_relprecis[i] = eval->time_relprecis[bucket];
eval->time_uap[i] = eval->time_uap[bucket];
eval->time_reluap[i] = eval->time_reluap[bucket];
eval->time_utility[i] = eval->time_utility[bucket];
}
last_time_bucket = bucket;
}
eval->time_cum_rel[0] = eval->time_num_rel[0];
eval->av_time_cum_rel = eval->time_num_rel[0];
for (i=1; i< NUM_TIME_PTS; i++) {
eval->time_cum_rel[i] = eval->time_cum_rel[i-1] + eval->time_num_rel[i];
eval->av_time_cum_rel += eval->time_cum_rel[i];
eval->av_time_precis += eval->time_precis[i];
eval->av_time_relprecis += eval->time_relprecis[i];
eval->av_time_uap += eval->time_uap[i];
eval->av_time_reluap += eval->time_reluap[i];
eval->av_time_utility += eval->time_utility[i];
}
eval->av_time_cum_rel /= NUM_TIME_PTS;
eval->av_time_precis /= NUM_TIME_PTS;
eval->av_time_relprecis /= NUM_TIME_PTS;
eval->av_time_uap /= NUM_TIME_PTS;
eval->av_time_reluap /= NUM_TIME_PTS;
eval->av_time_utility /= NUM_TIME_PTS;
}
trecqa.py
+177
View File
@@ -0,0 +1,177 @@
from __future__ import print_function
import os
import re
import random
import pickle
import itertools
from keras_attention_model import make_model
from utils.dictionary import Dictionary
random.seed(42)
data_path = '/media/moloch/HHD/MachineLearning/data/trecqa/jacana-qa-naacl2013-data-results'
def gen_pairs(fname, gen=False, pair_file=os.path.join('models', 'trec.pairs')):
if os.path.exists(pair_file) and not gen:
return pickle.load(open(pair_file, 'rb'))
else:
with open(os.path.join(data_path, fname), 'r') as f:
lines = f.read()
qa_pairs = list()
for pair in re.finditer("<QApairs id='[\d\.]+'>(.+?)</QApairs>", lines, flags=re.DOTALL):
text = pair.group(1)
q = re.findall('<question>.+?</question>', text, flags=re.DOTALL)[0].split('\n')[1].split('\t')
pos, neg = list(), list()
for a in re.finditer("<(positive|negative)>(.+?)</.+?>", text, flags=re.DOTALL):
cl = a.group(1)
text = ' '.join(a.group(2).split('\n')[1].split('\t'))
if cl[0] == 'p':
pos.append(text)
else:
neg.append(text)
qa_pairs.append({'question': ' '.join(q), 'positive': pos, 'negative': neg})
pickle.dump(qa_pairs, open(pair_file, 'wb'))
return qa_pairs
def gen_dict(pairs, gen=False, dic_file=os.path.join('models', 'trecqa.dict'), dic=None):
if os.path.exists(dic_file) and not gen:
dic = Dictionary.load(dic_file)
else:
if dic is None:
dic = Dictionary()
for ps in pairs:
for q in ps:
dic.add(q['question'])
for ans in itertools.chain(q['positive'], q['negative']):
dic.add(ans)
dic.save(dic_file)
return dic
def gen_eval(dic, pairs, q_maxlen, a_maxlen):
from keras.preprocessing.sequence import pad_sequences
q_data = list()
a_data = list()
n_good = list()
for pair in pairs:
pos = dic.convert(pair['positive'])
neg = dic.convert(pair['negative'])
q = dic.convert(pair['question'])
q_data.append(pad_sequences([q], maxlen=q_maxlen, padding='post', truncating='post', value=len(dic)))
a_data.append(pad_sequences(pos + neg, maxlen=a_maxlen, padding='post', truncating='post', value=len(dic)))
n_good.append(len(pos))
return q_data, a_data, n_good
def gen_data(dic, pairs, q_maxlen, a_maxlen, gen=False, data_file=os.path.join('models', 'trecqa.data'), even=False):
if os.path.exists(data_file) and not gen:
from numpy import load as npload
f = npload(open(data_file, 'rb'))
return f['q'], f['p'], f['n']
else:
from keras.preprocessing.sequence import pad_sequences
questions = list()
pos_answers = list()
neg_answers = list()
for pair in pairs:
pos = dic.convert(pair['positive'])
neg = dic.convert(pair['negative'])
q = dic.convert(pair['question'])
questions += q * len(pos)
pos_answers += pos
neg_answers += neg
questions = pad_sequences(questions, maxlen=q_maxlen, padding='post', truncating='post', dtype='int32', value=len(dic))
pos_answers = pad_sequences(pos_answers, maxlen=a_maxlen, padding='post', truncating='post', dtype='int32', value=len(dic))
neg_answers = pad_sequences(neg_answers, maxlen=a_maxlen, padding='post', truncating='post', dtype='int32', value=len(dic))
if even:
m = min(len(pos_answers), len(neg_answers))
pos_answers = pos_answers[:m]
neg_answers = neg_answers[:m]
from numpy import savez as npsavez
npsavez(open(data_file, 'wb'), q=questions, p=pos_answers, n=neg_answers)
return questions, pos_answers, neg_answers
def get_mrr(model, questions, all_answers, n_good, n_eval=-1):
import numpy as np
from scipy.stats import rankdata
if n_eval != -1:
questions = questions[-n_eval:]
all_answers = all_answers[-n_eval:]
n_good = n_good[-n_eval:]
c = 0
for i in range(len(questions)):
question = questions[i]
ans = all_answers[i]
qs = np.repeat(question, len(ans), 0)
sims = model.predict([qs, ans]).flatten()
r = rankdata(sims)
max_r = np.argmax(r)
max_n = np.argmax(r[:n_good[i]])
x = 1 / float(r[max_r] - r[max_n] + 1)
c += x
return c / len(questions)
gen = True
qa_pairs = gen_pairs('train2393.cleanup.xml', gen=gen, pair_file=os.path.join('models', 'trecqa.pairs'))
qa_pairs_dev = gen_pairs('dev-less-than-40.manual-edit.xml', gen=gen, pair_file=os.path.join('models', 'trec_dev.pairs'))
qa_pairs_test = gen_pairs('test-less-than-40.manual-edit.xml', gen=gen, pair_file=os.path.join('models', 'trec_test.pairs'))
dic = gen_dict([qa_pairs, qa_pairs_test, qa_pairs_dev], gen=gen)
q_maxlen = 10
a_maxlen = 40
dic.top(20000)
n_words = len(dic) + 1
questions, pos_answers, neg_answers = gen_data(dic, qa_pairs, q_maxlen, a_maxlen, data_file=os.path.join('models', 'trecqa.data'), gen=gen)
questions_dev, pos_answers_dev, neg_answers_dev = gen_data(dic, qa_pairs_dev, q_maxlen, a_maxlen, even=True, data_file=os.path.join('models', 'trecqa_dev.data'), gen=gen)
questions_test, answers_test, n_good_test = gen_eval(dic, qa_pairs_test, q_maxlen, a_maxlen)
from numpy import asarray
targets = asarray([0] * len(questions))
targets_dev = asarray([0] * len(questions_dev))
print('Generating model')
train_model, test_model = make_model(q_maxlen, a_maxlen, n_words, n_embed_dims=400, n_lstm_dims=64)
for i in range(1000):
print('----- %d -----' % i)
import numpy.random as nprandom
nprandom.shuffle(neg_answers)
neg_answers_train = neg_answers[:len(pos_answers)]
train_model.fit([questions, pos_answers, neg_answers_train], targets, nb_epoch=1, batch_size=128, validation_data=[[questions_dev, pos_answers_dev, neg_answers_dev], targets_dev])
if i % 20 == 0:
train_model.save_weights(os.path.join('models', 'trecqa_model_for_training_iter_%d.h5' % (i+1)), overwrite=True)
test_model.save_weights(os.path.join('models', 'trecqa_model_for_testing_iter_%d.h5' % (i+1)), overwrite=True)
print('MRR: {}'.format(get_mrr(test_model, questions_test, answers_test, n_good_test)))
utils/dictionary.py
+19 -11
View File
@@ -5,15 +5,11 @@ try:
except ImportError:
import pickle
from numpy import asarray
import numpy as np
from gensim.utils import tokenize
class Dictionary:
def __init__(self, min_len=1):
self._token_counts = dict()
self._id = 1
self._id = 0
self._min_len = min_len
self.token2id = dict()
@@ -43,18 +39,21 @@ class Dictionary:
return self.token2id.get(item, self._id)
def __getitem__(self, item):
return self.id2token[item] if 0 <= item < len(self.token2id) else 'UNKNOWN'
return self.id2token[item] if item < self._id else 'X'
def __len__(self):
return self._id + 1
def convert(self, text):
if isinstance(text, str):
docs = [tokenize(text, to_lower=True)]
else:
docs = [tokenize(t, to_lower=True) for t in text]
from gensim.utils import tokenize
from numpy import asarray
return [asarray([self(t) for t in doc], dtype=np.int32) for doc in docs]
if isinstance(text, str):
docs = [tokenize(text, to_lower=True, deacc=True)]
else:
docs = [tokenize(t, to_lower=True, deacc=True) for t in text]
return [asarray([self(t) for t in doc], dtype='int32') for doc in docs]
def revert(self, tokens):
texts = list()
@@ -64,6 +63,15 @@ class Dictionary:
return texts
def top(self, n):
import operator
sorted_tokens = sorted(self._token_counts.items(), reverse=True, key=operator.itemgetter(1))[:n]
self._token_counts = dict((k, v) for k, v in sorted_tokens)
self.id2token = [k for k in self._token_counts.keys()]
self.token2id = dict((v, k) for k, v in enumerate(self.id2token))
self._id = len(self.id2token)
def strip(self, n):
self._token_counts = dict((k, v) for k, v in self._token_counts.items() if v > n)
self.id2token = [k for k in self._token_counts.keys()]