Castor sm-model first commit

sm-model/README.md

@@ -0,0 +1,30 @@
+## Similarity Measure model (SM model)
+
+#### References:
+1. Aliaksei Severyn and Alessandro Moschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738
+
+
+#### TODOs:
+1. figure out if the L2 regularization is correct
+2. Batch size of 50 (current batch_size = 1)
+
+
+#### Running it
+
+``1.`` Make TrecEval:
+```
+$ cd trec_eval-8.0
+$ make clean
+$ make
+```
+
+``2.`` Get the Overlapping features for Q and A:
+```
+$ python overlap_features.py TrecQA
+```
+
+``3.`` To run the S&M model on TrecQA, please follow the same parameter setting:
+```
+$ python main.py ../../data/aquaint+wiki.txt.gz.ndim\=50.bin ../../data/TrecQA/ sm --batch_size 1
+```
+

sm-model/main.py

@@ -0,0 +1,129 @@
+import os 
+import sys
+import time
+import glob
+import argparse
+import numpy as np
+
+import pandas as pd
+import subprocess
+
+import torch
+import torch.optim as optim
+import torch.nn as nn
+from torch.autograd import Variable
+
+
+from model import QAModel
+import utils
+from train import Trainer
+
+# logging setup
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+def logargs(func):
+    def inner(*args, **kwargs):
+        logger.info('%s : %s %s' % (func.__name__, args, kwargs))
+        return func(*args, **kwargs)
+    return inner
+
+
+def compute_map_mrr(dataset_folder, set_folder, test_scores):
+    logger.info( "Running trec_eval script..." )
+    N = len(test_scores)
+
+    qids_test, y_test = utils.get_test_qids_labels(dataset_folder, set_folder)
+
+    # Call TrecEval code to calc MAP and MRR
+    df_submission = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rank', 'sim', 'run_id'])
+    df_submission['qid'] = qids_test
+    df_submission['iter'] = 0
+    df_submission['docno'] = np.arange(N)
+    df_submission['rank'] = 0
+    df_submission['sim'] = test_scores
+    df_submission['run_id'] = 'smmodel'
+    df_submission.to_csv(os.path.join(args.dataset_folder, 'submission.txt'), header=False, index=False, sep=' ')
+
+    df_gold = pd.DataFrame(index=np.arange(N), columns=['qid', 'iter', 'docno', 'rel'])
+    df_gold['qid'] = qids_test
+    df_gold['iter'] = 0
+    df_gold['docno'] = np.arange(N)
+    df_gold['rel'] = y_test
+    df_gold.to_csv(os.path.join(args.dataset_folder, 'gold.txt'), header=False, index=False, sep=' ')
+
+    subprocess.call("/bin/sh run_eval.sh '{}'".format(args.dataset_folder), shell=True)
+
+
+if __name__ == "__main__":
+    ap = argparse.ArgumentParser(description='pytorch port of the SM model')
+    ap.add_argument('word_vectors_file', help='NOTE: a cache will be created for faster loading for word vectors')
+    ap.add_argument('dataset_folder', help='directory containing train, dev, test sets')
+    ap.add_argument('model_fname', help='model will be saved in args.dataset_folder/<model_fname>')
+    ap.add_argument('--batch_size', type=int, default=1)
+    ap.add_argument('--filter_width', type=int, default=5)
+    ap.add_argument('--epochs', type=int, default=25)
+    ap.add_argument('--eta', help='Initial learning rate', default=0.01, type=float)
+    ap.add_argument('--mom', help='SGD Momentum', default=0.9, type=float)    
+    ap.add_argument('--classes', type=int, default=2)
+    ap.add_argument('--patience', type=int, default=5, help="if there is no appreciable change in model after <patience> epochs, then stop")
+    
+    args = ap.parse_args()
+
+    torch.manual_seed(1234)
+    np.random.seed(1234)
+
+    # cache word embeddings
+    cache_file = os.path.splitext(args.word_vectors_file)[0] + '.cache'    
+    utils.cache_word_embeddings(args.word_vectors_file, cache_file)
+
+    vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file)
+    
+    # instantiate model
+    net = QAModel(vec_dim, args.filter_width) #filter width is 5
+    QAModel.save(net, args.dataset_folder, args.model_fname)
+    
+    trainer = Trainer(net)
+
+    best_accuracy = 0.0
+    best_model = 0
+    
+    for i in range(args.epochs):
+        logger.info('Training epoch {} -------------'.format(i+1))        
+        train_accuracy = trainer.train(args.dataset_folder, 'train', args.batch_size, cache_file)        
+        # sys.exit(0)
+        dev_accuracy, dev_scores = trainer.test(args.dataset_folder, 'clean-dev', args.batch_size, cache_file)
+        if dev_accuracy > best_accuracy:            
+            best_model = i
+            best_accuracy = dev_accuracy
+            QAModel.save(net, args.dataset_folder, args.model_fname)
+            logger.info('Achieved better dev_accuracy ... saved model')
+        
+        compute_map_mrr(args.dataset_folder, 'clean-dev', dev_scores)
+
+        if (i - best_model) >= args.patience:
+            logger.warning('No improvement since the last {} epochs. Stopping training'.format(i - best_model))
+            break
+    
+    logger.info('Training epochs completed ------------')        
+    logger.info('Best accuracy in training phase = {:.4f}'.format(best_accuracy))
+
+    logger.info('Evaluating over test set...')
+    model = QAModel.load(args.dataset_folder, args.model_fname)
+
+    evaluator = Trainer(model)
+    test_accuracy, test_scores = evaluator.test(args.dataset_folder, 'clean-test', args.batch_size, cache_file)
+
+    logger.info('Test set accuracy = {:.4f}'.format(test_accuracy))
+
+    compute_map_mrr(args.dataset_folder, 'clean-test', test_scores)
+    
+
+    

sm-model/model.py

@@ -0,0 +1,81 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+import numpy as np
+
+import os
+
+# logging setup
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.INFO)
+formatter = logging.Formatter('%(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+
+class QAModel(nn.Module):
+    
+    @staticmethod
+    def save(model, out_folder, model_fname):
+        torch.save(model, os.path.join(out_folder, model_fname))
+
+    @staticmethod
+    def load(in_folder, model_fname):
+        return torch.load(os.path.join(in_folder, model_fname))
+
+    def __init__(self, input_n_dim, filter_width, ext_feats_size=4, n_classes=2):
+        super(QAModel, self).__init__()
+
+        self.conv_channels = 100
+        n_hidden = 2*self.conv_channels + 1
+
+        self.conv_q = nn.Sequential(
+            nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),
+            nn.Tanh()            
+        )
+
+        self.conv_a = nn.Sequential(
+            nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),
+            nn.Tanh()            
+        )
+        
+        self.combined_feature_vector = nn.Linear(2*self.conv_channels+ext_feats_size, n_hidden)
+        #TODO: add +1 to Linear layer^. Will need change in forward function
+        self.combined_features_activation = nn.Tanh()
+        self.dropout = nn.Dropout(0.5)
+        self.hidden = nn.Linear(n_hidden, n_classes)
+        self.logsoftmax = nn.LogSoftmax()
+                                
+
+    def forward(self, question, answer, ext_feats):                        
+        
+        q = self.conv_q.forward(question)        
+        q = F.max_pool1d(q, q.size()[2])
+        q = q.view(-1, self.conv_channels)
+        logger.debug('forward q: {}'.format(q))
+
+        a = self.conv_a.forward(answer)
+        a = F.max_pool1d(a, a.size()[2])
+        a = a.view(-1, self.conv_channels)
+
+        x = torch.cat([q, a, ext_feats], 1)        
+        # logger.debug('featvec x: {}'.format(x))
+        # logger.debug(x.creator)
+
+        x = self.combined_feature_vector.forward(x)
+        x = self.combined_features_activation.forward(x)
+        x = self.dropout(x)
+        x = self.hidden(x)
+        x = self.logsoftmax(x)
+
+        logger.debug('x data {}'.format(x.data))
+        logger.debug('x grad {}'.format(x.grad))
+
+        return x
+
+

sm-model/overlap_features.py

@@ -0,0 +1,186 @@
+import sys
+import re
+import os
+import numpy as np
+
+import argparse
+
+#from nltk.corpus import stopwords
+from nltk.stem.porter import *
+from collections import defaultdict
+
+def load_data(dname):
+  stemmer = PorterStemmer()
+  qids, questions, answers, labels = [], [], [], []
+  print dname
+  with open(dname+'a.toks') as f:
+    for line in f:
+      line = unicode(line, errors='ignore')
+      question = line.strip().split()
+      question = [stemmer.stem(word) for word in question]
+      questions.append(question)
+  with open(dname+'b.toks') as f:
+    for line in f:
+      line = unicode(line, errors='ignore')
+      answer = line.decode('utf-8').strip().split()
+      answer = [stemmer.stem(word) for word in answer]
+      answers.append(answer)
+  with open(dname+'id.txt') as f:
+    for line in f:
+      qids.append(line.strip())
+  with open(dname+'sim.txt') as f:
+    for line in f:
+      labels.append(int(line.strip())) 
+  return qids, questions, answers, labels
+
+def compute_overlap_features(questions, answers, word2df=None, stoplist=None):
+  word2df = word2df if word2df else {}
+  stoplist = stoplist if stoplist else set()
+  feats_overlap = []
+  for question, answer in zip(questions, answers):
+    # q_set = set(question)
+    # a_set = set(answer)
+    q_set = set([q for q in question if q not in stoplist])
+    a_set = set([a for a in answer if a not in stoplist])
+    word_overlap = q_set.intersection(a_set)
+    # overlap = float(len(word_overlap)) / (len(q_set) * len(a_set) + 1e-8)
+    if len(q_set) == 0 and len(a_set) == 0:
+      overlap = 0
+    else:
+      overlap = float(len(word_overlap)) / (len(q_set) + len(a_set))
+
+    # q_set = set([q for q in question if q not in stoplist])
+    # a_set = set([a for a in answer if a not in stoplist])
+    word_overlap = q_set.intersection(a_set)
+    df_overlap = 0.0
+    for w in word_overlap:
+      df_overlap += word2df[w]
+    if len(q_set) == 0 and len(a_set) == 0:
+      df_overlap = 0
+    else:
+      df_overlap /= (len(q_set) + len(a_set))
+
+    feats_overlap.append(np.array([
+                         overlap,
+                         df_overlap,
+                         ]))
+  return np.array(feats_overlap)
+
+def compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length):
+  stoplist = stoplist if stoplist else []
+  feats_overlap = []
+  q_indices, a_indices = [], []
+  for question, answer in zip(questions, answers):
+    q_set = set([q for q in question if q not in stoplist])
+    a_set = set([a for a in answer if a not in stoplist])
+    word_overlap = q_set.intersection(a_set)
+
+    q_idx = np.ones(q_max_sent_length) * 2
+    for i, q in enumerate(question):
+      value = 0
+      if q in word_overlap:
+        value = 1
+      q_idx[i] = value
+    q_indices.append(q_idx)
+
+    #### ERROR
+    # a_idx = np.ones(a_max_sent_length) * 2
+    # for i, q in enumerate(question):
+    #   value = 0
+    #   if q in word_overlap:
+
+    a_idx = np.ones(a_max_sent_length) * 2
+    for i, a in enumerate(answer):
+      value = 0
+      if a in word_overlap:
+        value = 1
+      a_idx[i] = value
+    a_indices.append(a_idx)
+
+  q_indices = np.vstack(q_indices).astype('int32')
+  a_indices = np.vstack(a_indices).astype('int32')
+
+  return q_indices, a_indices
+
+def compute_dfs(docs):
+  word2df = defaultdict(float)
+  for doc in docs:
+    for w in set(doc):
+      word2df[w] += 1.0
+  num_docs = len(docs)
+  for w, value in word2df.iteritems():
+    word2df[w] /= np.math.log(num_docs / value)
+  return word2df
+
+if __name__ == '__main__':
+  ap = argparse.ArgumentParser(description="compute overlap features for SM model")
+  ap.add_argument("dataset", help="path/to/dataset-directory", default="../../data/TrecQA")
+  args = ap.parse_args()
+
+  stoplist = set([line.strip() for line in open('stopwords.txt')])
+  import string
+  punct = set(string.punctuation)
+  stoplist.update(punct) 
+  #stoplist = None 
+ 
+  all_questions, all_answers, all_qids = [], [], []
+  base_dir = args.dataset
+  # base_dir = '../../data/' + sys.argv[1] + '/'
+  # sub_dirs = ['train/', 'raw-dev/', 'test.minimal/','test.complete/']
+  # sub_dirs = ['train-all/', 'raw-dev/', 'raw-test/']
+  sub_dirs = ['train/', 'clean-dev/', 'clean-test/']
+  for sub in sub_dirs:
+    qids, questions, answers, labels = load_data(base_dir+sub)
+    all_questions.extend(questions)
+    all_answers.extend(answers)
+    all_qids.extend(qids)
+  
+  seen = set()
+  unique_questions = []
+  for q, qid in zip(all_questions, all_qids):
+    if qid not in seen:
+      seen.add(qid)
+      unique_questions.append(q)
+  
+  docs = all_answers + unique_questions
+  word2dfs = compute_dfs(docs)
+  print word2dfs.items()[:10]
+
+  q_max_sent_length = max(map(lambda x: len(x), all_questions))
+  a_max_sent_length = max(map(lambda x: len(x), all_answers))
+  print 'q_max_sent_length', q_max_sent_length
+  print 'a_max_sent_length', a_max_sent_length
+
+  for sub in sub_dirs:
+    print sub
+    qids, questions, answers, labels = load_data(base_dir+sub)
+
+    overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs)
+    overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs)
+    overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist])
+    print overlap_feats[:3]
+    print 'overlap_feats', overlap_feats.shape
+    
+    from sklearn.preprocessing import StandardScaler
+    scaler = StandardScaler()
+    print "Scaling overlap features"
+    overlap_feats = scaler.fit_transform(overlap_feats)
+    print overlap_feats[:3]    
+
+    '''q_overlap_indices, a_overlap_indices = compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length)
+    print q_overlap_indices[:3], a_overlap_indices[:3]
+    print 'q_overlap_indices', q_overlap_indices.shape
+    print 'a_overlap_indices', a_overlap_indices.shape'''
+
+    with open(base_dir+sub+'overlap_feats.txt', 'w') as f:
+      for i in range(overlap_feats.shape[0]):
+        for j in range(4):
+          f.write(str(overlap_feats[i][j]) + ' ')
+        f.write('\n')
+    '''with open(base_dir+sub+'overlap_indices.txt', 'w') as f:
+      for i in range(q_overlap_indices.shape[0]):
+        for j in range(q_max_sent_length):
+          f.write(str(q_overlap_indices[i][j]) + ' ')
+        for j in range(a_max_sent_length):
+          f.write(str(a_overlap_indices[i][j]) + ' ')
+        f.write('\n')'''

sm-model/run_eval.sh

@@ -0,0 +1,9 @@
+#!/bin/csh -f
+
+exp_dir=$1
+judgement=${exp_dir}/gold.txt
+output=${exp_dir}/submission.txt
+
+./trec_eval-8.0/trec_eval -q -c ${judgement} ${output} > ${output}.treceval
+tail -29 ${output}.treceval | grep -e 'map' -e 'recip_rank'
+exit 0

sm-model/stopwords.txt

@@ -0,0 +1,238 @@
+'s
+I
+a
+aboard
+about
+above
+across
+after
+afterwards
+against
+agin
+ago
+agreed-upon
+ah
+alas
+albeit
+all
+all-over
+almost
+along
+alongside
+altho
+although
+amid
+amidst
+among
+amongst
+an
+and
+another
+any
+anyone
+anything
+around
+as
+aside
+astride
+at
+atop
+avec
+away
+back
+be
+because
+before
+beforehand
+behind
+behynde
+below
+beneath
+beside
+besides
+between
+bewteen
+beyond
+bi
+both
+but
+by
+ca.
+de
+des
+despite
+do
+down
+due
+durin
+during
+each
+eh
+either
+en
+every
+ever
+everyone
+everything
+except
+far
+fer
+for
+from
+go
+goddamn
+goody
+gosh
+half
+have
+he
+hell
+her
+herself
+hey
+him
+himself
+his
+ho
+how
+however
+i
+if
+in
+inside
+insofar
+instead
+into
+it
+its
+itself
+la
+le
+les
+lest
+lieu
+like
+me
+minus
+moreover
+my
+myself
+near
+near-by
+nearer
+nearest
+neither
+nevertheless
+next
+no
+nor
+not
+nothing
+notwithstanding
+o
+o'er
+of
+off
+on
+once
+one
+oneself
+only
+onto
+or
+other
+others
+otherwise
+our
+ours
+ourselves
+out
+outside
+outta
+over
+per
+rather
+regardless
+round
+se
+she
+should
+since
+so
+some
+someone
+something
+than
+that
+the
+their
+them
+themselves
+then
+there
+therefore
+these
+they
+thine
+this
+those
+thou
+though
+through
+throughout
+thru
+till
+to
+together
+toward
+towardes
+towards
+uh
+under
+underneath
+unless
+unlike
+until
+unto
+up
+upon
+uppon
+us
+via
+vis-a-vis
+vis-�-vis
+we
+well
+what
+whatever
+whatsoever
+when
+whenever
+where
+whereas
+wherefore
+whereupon
+whether
+which
+whichever
+while
+who
+whoever
+whom
+whose
+why
+with
+withal
+within
+without
+ye
+yea
+yeah
+yes
+yet
+yonder
+you
+your
+yours
+yourself
+yourselves

sm-model/train.py

@@ -0,0 +1,240 @@
+import os 
+import sys
+
+import time
+import glob
+import argparse
+import numpy as np
+
+import torch
+import torch.optim as optim
+import torch.nn as nn
+from torch.autograd import Variable
+
+import utils
+
+# logging setup
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+
+class Trainer(object):
+    
+    def __init__(self, model):
+        self.reg = 1e-5
+        self.model = model
+        self.criterion = nn.CrossEntropyLoss()
+        self.optimizer = optim.SGD(self.model.parameters(), lr=0.001, weight_decay=self.reg)
+
+
+    def regularize_loss(self, loss):       
+        
+        flattened_params = []
+
+        for p in self.model.parameters():
+            f = p.data.clone()
+            flattened_params.append(f.view(-1))
+        
+        fp = torch.cat(flattened_params)
+
+        loss = loss + 0.5 * self.reg * fp.norm() * fp.norm()
+
+        # for p in self.model.parameters():
+        #     loss = loss + 0.5 * self.reg * p.norm() * p.norm()
+            
+        return loss    
+
+    
+    def _train(self, xq, xa, ext_feats, ys):
+                
+        self.optimizer.zero_grad()        
+        output = self.model(xq, xa, ext_feats)                    
+        loss = self.criterion(output, ys)        
+        logger.debug('loss after criterion {}'.format(loss))
+
+        # NOTE: regularizing location 1
+        # loss = self.regularize_loss(loss)
+        # logger.debug('loss after regularizing {}'.format(loss))
+        
+        loss.backward()
+        
+        logger.debug('AFTER backward')
+        #logger.debug('params {}'.format([p for p in self.model.parameters()]))
+        logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()]))
+       
+
+        # NOTE: regularizing location 2. It would seem that location 1 is correct?
+        loss = self.regularize_loss(loss)
+        logger.debug('loss after regularizing {}'.format(loss))
+
+        self.optimizer.step()
+
+        logger.debug('AFTER step')
+        #logger.debug('params {}'.format([p for p in self.model.parameters()]))
+        logger.debug('params grads {}'.format([p.grad for p in self.model.parameters()]))
+
+        return loss.data[0], self.pred_equals_y(output, ys)
+
+
+    def pred_equals_y(self, pred, y):
+        # logger.debug('pred_equals_y:')
+        # logger.debug(pred)
+        # logger.debug(y)
+        _, best = pred.max(1)
+        # logger.debug('{} {}'.format(_, best))
+        best = best.data.long().squeeze()
+        # logger.debug(best)
+        return torch.sum(y.data.long() == best)
+
+
+    def test(self, dataset_folder, set_folder, batch_size, word_vectors_cache_file):
+        logger.info('Predictions on {} -----'.format(set_folder))
+                
+        questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \
+                utils.read_in_dataset(dataset_folder, set_folder)
+
+        # load word embeddings for training set vocab
+        word_vectors, vec_dim = utils.load_cached_embeddings(word_vectors_cache_file, vocab)
+
+        self.model.eval()
+
+        batch_size = 1
+
+        total_loss = 0.0
+        total_correct = 0.0
+        num_batches = np.ceil(len(questions)/batch_size )
+        y_pred = np.zeros(len(questions))
+        ypc = 0
+            
+        for k in xrange(int(num_batches)):
+            batch_start = k * batch_size
+            batch_end = (k+1) * batch_size
+            # convert raw questions and sentences to tensors
+            batch_inputs, batch_labels = self.get_tensorized_inputs(
+                    questions[batch_start:batch_end], 
+                    sentences[batch_start:batch_end], 
+                    labels[batch_start:batch_end],
+                    ext_feats[batch_start:batch_end], 
+                    word_vectors, vocab, vec_dim
+                )
+            
+            xq, xa, x_ext_feats = batch_inputs[0]
+            y = batch_labels[0]
+            
+            pred = self.model(xq, xa, x_ext_feats)
+            loss = self.criterion(pred, y)        
+            total_loss += loss
+            total_correct += self.pred_equals_y(pred, y)
+
+            p_score, p_class = pred.max(1)
+            logger.debug('pred {}'.format(pred))
+            logger.debug('pred.max {} {}'.format(p_score, p_class))
+            logger.debug('score {}'.format(p_score.squeeze()))
+            logger.debug('score {}'.format(p_score.squeeze()[0]))            
+            y_pred[ypc] = p_score.data.squeeze()[0]
+            ypc += 1         
+               
+
+        logger.info('{}_correct {}'.format(set_folder, total_correct))
+        logger.info('{}_loss {}'.format(set_folder, total_loss.data[0]))
+        logger.info('{} total {}'.format(set_folder, len(labels)))
+        logger.info('{}_loss = {:.4f}, acc = {:.4f}'.format( set_folder, total_loss.data[0]/len(labels), float(total_correct)/len(labels) ))
+
+        return float(total_correct)/len(labels), y_pred
+
+
+    def train(self, dataset_folder, set_folder, batch_size, word_vectors_cache_file):
+        # read in training data
+        questions, sentences, labels, vocab, maxlen_q, maxlen_s, ext_feats = \
+                utils.read_in_dataset(dataset_folder, set_folder)
+
+        # load word embeddings for training set vocab
+        word_vectors, vec_dim = utils.load_cached_embeddings(word_vectors_cache_file, vocab)
+
+        # set model for training modep
+        self.model.train()
+
+        train_loss, train_correct = 0., 0.
+        num_batches = np.ceil(len(questions)/float(batch_size) )
+
+        for k in xrange(int(num_batches)):
+            batch_start = k * batch_size
+            batch_end = (k+1) * batch_size
+
+            # convert raw questions and sentences to tensors
+            batch_inputs, batch_labels = self.get_tensorized_inputs(
+                    questions[batch_start:batch_end], 
+                    sentences[batch_start:batch_end], 
+                    labels[batch_start:batch_end],
+                    ext_feats[batch_start:batch_end], 
+                    word_vectors, vocab, vec_dim
+                )
+            
+            xq, xa, x_ext_feats = batch_inputs[0]
+
+            ys = batch_labels[0]
+
+            batch_loss, batch_correct = self._train(xq, xa, x_ext_feats, ys)                                        
+            
+            # logger.debug('batch_loss {}, batch_correct {}'.format(batch_loss, batch_correct))
+            train_loss += batch_loss
+            train_correct += batch_correct
+            # break
+
+        logger.info('train_correct {}'.format(train_correct))
+        logger.info('train_loss {}'.format(train_loss))
+        logger.info('total training batches = {}'.format(num_batches))
+        logger.info('train_loss = {:.4f}, acc = {:.4f}'.format(
+            train_loss/num_batches, train_correct/num_batches
+        ))
+        return train_correct/num_batches
+        
+
+    def make_input_matrix(self,  sentence, word_vectors, vec_dim):
+        terms = sentence.strip().split()        
+        # word_embeddings = torch.zeros(max_len, vec_dim).type(torch.DoubleTensor)
+        word_embeddings = torch.zeros(len(terms), vec_dim).type(torch.DoubleTensor)
+        for i in xrange(len(terms)):
+            word = terms[i]        
+            emb = torch.from_numpy(word_vectors[word])                        
+            word_embeddings[i] = emb            
+
+        input_tensor = torch.zeros(1, vec_dim, len(terms))
+        input_tensor[0] = torch.transpose(word_embeddings, 0 , 1)
+        return input_tensor
+
+
+    def get_tensorized_inputs(self, batch_ques, batch_sents, batch_labels, batch_ext_feats, word_vectors, vocab, vec_dim):
+        batch_size = len(batch_ques)
+        # NOTE: ideal batch size is one, because sentences are all of different length.
+        # In other words, we have no option but to feed in sentences one by one into the model
+        # and compute loss at the end.
+
+        # TODO: what if the sentences in a batch are all of different lengths?
+        # - should be have the longest sentence as 2nd dim?
+        #   - would zero endings work for other smaller sentences?
+
+        y = torch.LongTensor(batch_size).type(torch.LongTensor)
+
+        tensorized_inputs = []
+        for i in xrange(len(batch_ques)):
+            xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim) ) #, requires_grad=False)
+            xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim) ) #, requires_grad=False)
+            # ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
+            ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i]))
+            ext_feats =torch.unsqueeze(ext_feats, 0)
+            y[i] = batch_labels[i]
+            tensorized_inputs.append((xq, xs, ext_feats))
+
+        return tensorized_inputs, Variable(y)
+
+
+
+

sm-model/trec_eval-8.0/Makefile

@@ -0,0 +1,141 @@
+BIN = /home/smart/bin
+H   = .
+
+VERSIONID = 8.0
+
+# gcc
+CC       = gcc
+CFLAGS   = -g -I$H -O3 -Wall -DVERSIONID=\"$(VERSIONID)\"
+CFLAGS   = -g -I$H  -Wall -DVERSIONID=\"$(VERSIONID)\"
+
+# cc
+###CC       = cc
+###CFLAGS   = -I$H -g -DVERSIONID=\"$(VERSIONID)\"
+
+# Other macros used in some or all makefiles
+INSTALL = /bin/mv
+
+
+OBJS = trec_eval.o get_qrels.o get_top.o form_trvec.o measures.o print_meas.o\
+	trvec_teval.o buf_util.o error_msgs.o \
+       trec_eval_help.o
+
+
+SRCS = trec_eval.c get_qrels.c get_top.c form_trvec.c measures.c print_meas.c\
+       trvec_teval.c buf_util.c error_msgs.c \
+       trec_eval_help.c
+
+SRCH = common.h trec_eval.h smart_error.h sysfunc.h tr_vec.h buf.h
+
+SRCOTHER = README Makefile test bpref_bug
+
+trec_eval: $(SRCS) Makefile $(SRCH)
+	$(CC) $(CFLAGS)  -o trec_eval $(SRCS) -lm
+
+install: $(BIN)/trec_eval
+
+quicktest: trec_eval
+	./trec_eval test/qrels.test test/results.test | diff - test/out.test
+	./trec_eval -a test/qrels.test test/results.test | diff - test/out.test.a
+	./trec_eval -a -q test/qrels.test test/results.test | diff - test/out.test.aq
+	./trec_eval -a -q -c test/qrels.test test/results.trunc | diff - test/out.test.aqc
+	./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc | diff - test/out.test.aqcM
+	/bin/echo "Test succeeeded"
+
+longtest: trec_eval
+	/bin/rm -rf test.long; mkdir test.long
+	./trec_eval test/qrels.test test/results.test > test.long/out.test
+	./trec_eval -a test/qrels.test test/results.test > test.long/out.test.a
+	./trec_eval -a -q test/qrels.test test/results.test > test.long/out.test.aq
+	./trec_eval -a -q -c test/qrels.test test/results.trunc > test.long/out.test.aqc
+	./trec_eval -a -q -c -M100 test/qrels.test test/results.trunc > test.long/out.test.aqcM
+	diff test.long test
+
+$(BIN)/trec_eval: trec_eval
+	if [ -f $@ ]; then $(INSTALL) $@ $@.old; fi;
+	$(INSTALL) trec_eval $@
+
+##4##########################################################################
+##5##########################################################################
+#  All code below this line (except for automatically created dependencies)
+#  is independent of this particular makefile, and should not be changed!
+#############################################################################
+
+#########################################################################
+# Odds and ends                                                         #
+#########################################################################
+clean semiclean:
+	/bin/rm -f *.o *.BAK *~ trec_eval trec_eval.*.shar out.trec_eval Makefile.bak
+
+shar:
+	shar -X $(SRCOTHER) $(SRCS) $(SRCH) > trec_eval.$(VERSIONID).shar
+
+lint:
+	lint $(SRCS)
+
+#########################################################################
+# Determining program dependencies                                      #
+#########################################################################
+depend:
+	grep '^#[ ]*include' *.c \
+		| sed -e 's?:[^"]*"\([^"]*\)".*?: \$H/\1?' \
+			-e '/</d' \
+			-e '/functions.h/d' \
+		        -e 's/\.c/.o/' \
+		        -e 's/\.y/.o/' \
+		        -e 's/\.l/.o/' \
+		> makedep
+	echo '/^# DO NOT DELETE THIS LINE/+2,$$d' >eddep
+	echo '$$r makedep' >>eddep
+	echo 'w' >>eddep
+	cp Makefile Makefile.bak
+	ed - Makefile < eddep
+	/bin/rm eddep makedep
+	echo '# DEPENDENCIES MUST END AT END OF FILE' >> Makefile
+	echo '# IF YOU PUT STUFF HERE IT WILL GO AWAY' >> Makefile
+	echo '# see make depend above' >> Makefile
+
+# DO NOT DELETE THIS LINE -- make depend uses it
+
+buf_util.o: ./common.h
+buf_util.o: ./sysfunc.h
+buf_util.o: ./buf.h
+error_msgs.o: ./smart_error.h
+error_msgs.o: ./sysfunc.h
+form_trvec.o: ./common.h
+form_trvec.o: ./sysfunc.h
+form_trvec.o: ./smart_error.h
+form_trvec.o: ./tr_vec.h
+form_trvec.o: ./trec_eval.h
+form_trvec.o: ./buf.h
+get_qrels.o: ./common.h
+get_qrels.o: ./sysfunc.h
+get_qrels.o: ./smart_error.h
+get_qrels.o: ./trec_eval.h
+get_top.o: ./common.h
+get_top.o: ./sysfunc.h
+get_top.o: ./smart_error.h
+get_top.o: ./trec_eval.h
+measures.o: ./common.h
+measures.o: ./sysfunc.h
+measures.o: ./buf.h
+measures.o: ./trec_eval.h
+print_meas.o: ./common.h
+print_meas.o: ./sysfunc.h
+print_meas.o: ./buf.h
+print_meas.o: ./trec_eval.h
+trec_eval.o: ./common.h
+trec_eval.o: ./sysfunc.h
+trec_eval.o: ./smart_error.h
+trec_eval.o: ./tr_vec.h
+trec_eval.o: ./trec_eval.h
+trec_eval.o: ./buf.h
+trec_eval_help.o: ./common.h
+trvec_teval.o: ./common.h
+trvec_teval.o: ./sysfunc.h
+trvec_teval.o: ./smart_error.h
+trvec_teval.o: ./tr_vec.h
+trvec_teval.o: ./trec_eval.h
+# DEPENDENCIES MUST END AT END OF FILE
+# IF YOU PUT STUFF HERE IT WILL GO AWAY
+# see make depend above

sm-model/trec_eval-8.0/README

@@ -0,0 +1,385 @@
+trec_eval is the standard tool used by the TREC community for
+evaluating an ad hoc retrieval run, given the results file and a
+standard set of judged results.  
+
+------------------------------------------------------------------------------
+Installation: Should be as easy as typing "make" in the source directory,
+if gcc is available.  Otherwise, comment out the gcc lines (lines 5-6) and
+uncomment out the cc lines (lines 9-10)
+If you wish the trec_eval binary to be placed in a standard location, alter
+the first line of Makefile appropriately.
+
+------------------------------------------------------------------------------
+Testing: sample input and output files are included in the directory test.
+"make quicktest" will perform some sample simple evaluations and compare
+the results.
+
+------------------------------------------------------------------------------
+Usage:  Most options can be ignored.  The only one most folks will need
+is the "-q" flag, to indicate whether to output results for individual 
+queries as well as the averages over all queries.  Official TREC usage
+might be something like 
+	trec_eval -q -c -M1000 official_qrels submitted_results 
+to ensure correct evaluation if submitted_results doesn't have results
+for all queries, or returns more than 1000 documents per query.
+
+
+------------------------------------------------------------------------------
+Change Log
+------------------------------------------------------------------------------
+Version 8.0, full bpref bug fix, see file bpref_bug.  I decided to up the 
+        version number since bpref results are incompatible with previous 
+        results (though the changes are small).
+
+11/8/05: Bpref_bug: New file explaining bug and impact (conclusions after
+	rerunning all of SIGIR 2004 bpref paper experiments).
+11/5/05: Added new measures: micro_prec, micro_recall, micro_bpref. I thought
+	I had an application for micro_bpref averaging (summing components of 
+	measure over all docs (ignoring topics) and then computing measure), 
+	but micro_bpref still proved a rotten measure.  Left code in case
+	someone ever actually finds an application for valid micro averaging.
+11/5/05: Added new measures: old_bpref, old_bpref_top10pRnonrel.  These are
+	the old buggy measures included only for backward comparisons.
+11/5/05: trvec_teval.c: Broke apart old trvec_trec_eval to calculate
+	different types of measures separately.  Very hard to decipher
+	old code (though still difficult with new code) since parts of
+	the calculations for a measure were so far apart.
+
+------------------------------------------------------------------------------
+Version 7.4, minor changes from 7.3
+
+11/4/05: trvec_teval.c: fixed bpref bug if very low (< R) numbers of non-rel
+          judgements available (divided by num_nonrel_ret instead of 
+	  num_nonrel). (pointed out by Ian Soboroff).
+11/3/05: trvec_teval.c: bpref_10, bpref_5 had zero division problems if
+          no rel docs were retrieved. (pointed out by Ian Soboroff).
+10/23/05: form_trvec.c: Added check for duplicate docno's in results and qrels.
+          (pointed out by Shlomo Geva. Default behavior used to be that
+	   duplicate result docno's were always non-rel, but that changed in
+	   later versions, so had better test explicitly for it and complain).
+10/23/05: README: sample invocation of trec_eval had arguments reversed.
+          (pointed out by Carol Peters).
+10/23/05: moved gm_ap to be a major measure (always printed). changed 
+          measures.c, test/out*, README
+
+------------------------------------------------------------------------------
+
+Version 7.3, a reasonably major rewrite from earlier versions in terms
+of internal structure and default output format (now relational), but 
+the input format and measures calculated remain the same (or at least 
+upward compatible).
+
+------------------------------------------------------------------------------
+end of ChangeLog
+------------------------------------------------------------------------------
+
+Adding measures: To add a new measure:
+	1. Add space for the measure in TREC_EVAL structure of "trec_eval.h"
+	2. Add description of measure in "measures.c".  See "trec_eval.h" for
+	   definition of the fields.  This description is used for
+	   printing, accumulating, and averaging the measure values.
+	3. Calculate the measure in "trvec_teval.c".  Unfortunately, this has
+	   gotten very long over the years as more measures are added, but
+	   most of it can be ignored.  I should write a simple version of
+	   just the "short" measures so the structure can be seen better.
+
+------------------------------------------------------------------------------
+
+Files:
+Makefile     Compile and test trec_eval
+README       This file
+test         Collection of sample input and output for trec_eval
+trec_eval.c  Main procedure
+get_qrels.c  Called by main to read the standard judged documents (qrels)
+get_top.c    Called by main to read the results file to be evaluated
+form_trvec.c Called by main to put the results and qrels for an individual
+	     query in the proper format to be evaluated.
+trvec_teval.c Called by main to evaluate an individual query
+print_meas.c Called by main to print an evaluated query, and to accumulate
+	     the results for later averaging over the queries.
+measures.c   Description of the measures used by printing.
+trec_eval_help.c Descriptions of trec_eval, the output, and the measures.
+
+trec_eval.h  Basic evaluation structures.
+bpref_bug:   Description of bug in bpref that existed in trec_eval versions 6
+             through 7.3.
+
+The rest of the files are small utility portions from SMART.
+tr_vec.h
+smart_error.h
+sysfunc.h
+buf.h
+common.h
+buf_util.c
+error_msgs.c
+
+------------------------------------------------------------------------------
+
+The rest of this file consists of information printed by "trec_eval -h":
+(If you REALLY want a complete list of measures calculated, you can add the
+time based measures and run "trec_eval -T -h".)
+
+trec_eval [-h] [-q] [-a] [-o] [-c] [-l<num>  [-N<num>] [-M<num>] [-Ua<num>] [-Ub<num>] [-Uc<num>] [-Ud<num>] [-T] trec_rel_file trec_top_file 
+ 
+Calculate and print various evaluation measures, evaluating the results  
+in trec_top_file against the relevance judgements in trec_rel_file. 
+ 
+There are a fair number of options, of which only the lower case options are 
+normally ever used.   
+ -h: Print full help message and exit 
+ -q: In addition to summary evaluation, give evaluation for each query 
+ -a: Print all evaluation measures calculated, instead of just the 
+     main official measures for TREC. 
+ -o: Print everything out in old, nonrelational format (default is relational) 
+ -c: Average over the complete set of queries in the relevance judgements  
+      instead of the queries in the intersection of relevance judgements 
+      and results.  Missing queries will contribute a value of 0 to all 
+      evaluation measures (which may or may not be reasonable for a  
+      particular evaluation measure, but is reasonable for standard TREC 
+      measures.) 
+ -l<num>: Num indicates the minimum relevance judgement value needed for 
+      a document to be called relevant. (All measures used by TREC eval are 
+      based on binary relevance).  Used if trec_rel_file contains relevance 
+      judged on a multi-relevance scale.  Default is 1. 
+ -N<num>: Number of docs in collection 
+ -M<num>: Max number of docs per topic to use in evaluation (discard rest). 
+ -Ua<num>: Value to use for 'a' coefficient of utility computation. 
+                        relevant  nonrelevant 
+      retrieved            a          b 
+      nonretrieved         c          d 
+ -Ub<num>: Value to use for 'b' coefficient of utility computation. 
+ -Uc<num>: Value to use for 'c' coefficient of utility computation. 
+ -Ud<num>: Value to use for 'd' coefficient of utility computation. 
+ -J: Calculate all values only over the judged (either relevant or  
+     nonrelevant) documents.  All unjudged documents are removed from the 
+     retrieved set before any calculations (possibly leaving an empty set). 
+     DO NOT USE, unless you really know what you're doing - very easy to get 
+     reasonable looking, but invalid, numbers.  
+ -T: Treat similarity as time that document retrieved.  Compute 
+      several time-based measures after ranking docs by time retrieved 
+      (first doc (lowest sim) retrieved ranked highest).  
+      Only done if -a selected. 
+ 
+ 
+Read text tuples from trec_top_file of the form 
+     030  Q0  ZF08-175-870  0   4238   prise1 
+     qid iter   docno      rank  sim   run_id 
+giving TREC document numbers (a string) retrieved by query qid  
+(a string) with similarity sim (a float).  The other fields are ignored, 
+with the exception that the run_id field of the last line is kept and 
+output.  In particular, note that the rank field is ignored here; 
+internally ranks are assigned by sorting by the sim field with ties  
+broken deterministicly (using docno). 
+Sim is assumed to be higher for the docs to be retrieved first. 
+File may contain no NULL characters. 
+Lines may contain fields after the run_id; they are ignored. 
+ 
+Relevance for each docno to qid is determined from text_qrels_file, which 
+consists of text tuples of the form 
+   qid  iter  docno  rel 
+giving TREC document numbers (docno, a string) and their relevance (rel,  
+an integer) to query qid (a string).  iter string field is ignored.   
+Fields are separated by whitespace, string fields can contain no whitespace. 
+File may contain no NULL characters. 
+ 
+The text tuples with relevance judgements are converted to TR_VEC form 
+and then submitted to the SMART evaluation routines. 
+The qid,did,rank,sim,rel fields of TR_VEC are filled in; 
+action,iter fields are set to 0. 
+The rel field is set to -1 if the document was not judged (not in  
+text_qrels_file).  Most measures, but not all, will treat -1 the same as 0, 
+namely nonrelevant.  Note that relevance_level is used to determine if the 
+document is relevant during score calculations. 
+Queries for which there are no relevant docs are ignored. 
+Warning: queries for which there are relevant docs but no retrieved docs 
+are also ignored by default.  This allows systems to evaluate over subsets  
+of the relevant docs, but means if a system improperly retrieves no docs,  
+it will not be detected.  Use the -c flag to avoid this behavior. 
+ 
+EXPLANATION OF OFFICIAL VALUES PRINTED OF OLD NON-RELATIONAL FORMAT. 
+Relational Format prints the same values, but all lines are of the form 
+    measure_name   query   value 
+ 
+1. Total number of documents over all queries 
+        Retrieved: 
+        Relevant: 
+        Rel_ret:     (relevant and retrieved) 
+   These should be self-explanatory.  All values are totals over all 
+   queries being evaluated. 
+2. Interpolated Recall - Precision Averages: 
+        at 0.00 
+        at 0.10 
+        ... 
+        at 1.00 
+   See any standard IR text (especially by Salton) for more details of  
+   recall-precision evaluation.  Measures precision (percent of retrieved 
+   docs that are relevant) at various recall levels (after a certain 
+   percentage of all the relevant docs for that query have been retrieved). 
+   'Interpolated' means that, for example, precision at recall 
+   0.10 (ie, after 10% of rel docs for a query have been retrieved) is 
+   taken to be MAXIMUM of precision at all recall points >= 0.10. 
+   Values are averaged over all queries (for each of the 11 recall levels). 
+   These values are used for Recall-Precision graphs. 
+3. Average precision (non-interpolated) over all rel docs 
+   The precision is calculated after each relevant doc is retrieved. 
+   If a relevant doc is not retrieved, its precision is 0.0. 
+   All precision values are then averaged together to get a single number 
+   for the performance of a query.  Conceptually this is the area 
+   underneath the recall-precision graph for the query. 
+   The values are then averaged over all queries. 
+4. Precision: 
+       at 5    docs 
+       at 10   docs 
+       ... 
+       at 1000 docs    
+   The precision (percent of retrieved docs that are relevant) after X 
+   documents (whether relevant or nonrelevant) have been retrieved. 
+   Values averaged over all queries.  If X docs were not retrieved 
+   for a query, then all missing docs are assumed to be non-relevant. 
+5. R-Precision (precision after R (= num_rel for a query) docs retrieved): 
+   Measures precision (or recall, they're the same) after R docs 
+   have been retrieved, where R is the total number of relevant docs 
+   for a query.  Thus if a query has 40 relevant docs, then precision 
+   is measured after 40 docs, while if it has 600 relevant docs, precision 
+   is measured after 600 docs.  This avoids some of the averaging 
+   problems of the 'precision at X docs' values in (4) above. 
+   If R is greater than the number of docs retrieved for a query, then 
+   the nonretrieved docs are all assumed to be nonrelevant. 
+
+Major measures (again) with their relational names:
+num_ret        	Total number of documents retrieved over all queries
+num_rel        	Total number of relevant documents over all queries
+num_rel_ret    	Total number of relevant documents retrieved over all queries
+map            	Mean Average Precision (MAP)
+gm_ap          	Average Precision. Geometric Mean, q_score=log(MAX(map,.00001))
+R-prec         	R-Precision (Precision after R (= num-rel for topic) documents retrieved)
+bpref          	Binary Preference, top R judged nonrel
+recip_rank     	Reciprical rank of top relevant document
+ircl_prn.0.00  	Interpolated Recall - Precision Averages at 0.00 recall
+ircl_prn.0.10  	Interpolated Recall - Precision Averages at 0.10 recall
+ircl_prn.0.20  	Interpolated Recall - Precision Averages at 0.20 recall
+ircl_prn.0.30  	Interpolated Recall - Precision Averages at 0.30 recall
+ircl_prn.0.40  	Interpolated Recall - Precision Averages at 0.40 recall
+ircl_prn.0.50  	Interpolated Recall - Precision Averages at 0.50 recall
+ircl_prn.0.60  	Interpolated Recall - Precision Averages at 0.60 recall
+ircl_prn.0.70  	Interpolated Recall - Precision Averages at 0.70 recall
+ircl_prn.0.80  	Interpolated Recall - Precision Averages at 0.80 recall
+ircl_prn.0.90  	Interpolated Recall - Precision Averages at 0.90 recall
+ircl_prn.1.00  	Interpolated Recall - Precision Averages at 1.00 recall
+P5             	Precision after 5 docs retrieved
+P10            	Precision after 10 docs retrieved
+P15            	Precision after 15 docs retrieved
+P20            	Precision after 20 docs retrieved
+P30            	Precision after 30 docs retrieved
+P100           	Precision after 100 docs retrieved
+P200           	Precision after 200 docs retrieved
+P500           	Precision after 500 docs retrieved
+P1000          	Precision after 1000 docs retrieved
+
+
+Minor measures with their relational names:
+exact_prec     	Exact Precision over retrieved set
+exact_recall   	Exact Recall over retrieved set
+11-pt_avg      	Average over all 11 points of recall-precision graph
+3-pt_avg       	Average over 3 points of recall-precision graph
+avg_doc_prec   	Rel doc precision averaged over all relevant docs (NOT over topics)
+exact_relative_prec	Exact relative precision
+avg_relative_prec	Average relative precision
+exact_unranked_avg_prec	Exact Unranked Average Precision
+exact_relative_unranked_avg_prec	Exact Relative Unranked Average Precision
+map_at_R       	Average Precision over first R docs retrieved
+int_map        	Interpolated Mean Average Precision
+exact_int_R_rcl_prec	Exact R-based-interpolated-Precision
+int_map_at_R   	Average Interpolated Precision for first R docs retrieved
+bpref_allnonrel	Binary Preference, all judged nonrel
+bpref_retnonrel	Binary Preference, all retrieved judged nonrel
+bpref_topnonrel	Binary Preference, top 100 judged nonrel
+bpref_top5Rnonrel	Binary Preference, top 5R judged nonrel
+bpref_top10Rnonrel	Binary Preference, top 10R judged nonrel
+bpref_top10pRnonrel	Binary Preference, top 10 + R judged nonrel
+bpref_top25pRnonrel	Binary Preference, top 25 + R judged nonrel
+bpref_top50pRnonrel	Binary Preference, top 50 + R judged nonrel
+bpref_top25p2Rnonrel	Binary Preference, top 25 + 2*R judged nonrel
+bpref_retall   	Binary Preference, Only retrieved judged rel and nonrel
+bpref_5        	Binary Preference, top 5 rel, top 5 nonrel
+bpref_10       	Binary Preference, top 10 rel, top 10 nonrel
+bpref_num_all  	Binary Preference, Number not retrieved before (all judged)
+bpref_num_ret  	Binary Preference, Number retrieved after
+bpref_num_correct	Binary Preference, Number correct preferences
+bpref_num_possible	Binary Preference, Number possible correct_preferences
+old_bpref      	Buggy Version 7.3. Binary Preference, top R judged nonrel
+old_bpref_top10pRnonrel	Buggy Version 7.3. Binary Preference,top 10+R judged nonrel
+gm_bpref       	Binary Preference, top R judged nonrel, Geometric Mean, q_score=log(MAX(bpref,.00001))
+rank_first_rel 	Rank of top relevant document (0 if none)
+recall5        	Recall after 5 docs retrieved
+recall10       	Recall after 10 docs retrieved
+recall15       	Recall after 15 docs retrieved
+recall20       	Recall after 20 docs retrieved
+recall30       	Recall after 30 docs retrieved
+recall100      	Recall after 100 docs retrieved
+recall200      	Recall after 200 docs retrieved
+recall500      	Recall after 500 docs retrieved
+recall1000     	Recall after 1000 docs retrieved
+0.20R-prec     	R-based precision- precision after 0.20 * R docs retrieved
+0.40R-prec     	R-based precision- precision after 0.40 * R docs retrieved
+0.60R-prec     	R-based precision- precision after 0.60 * R docs retrieved
+0.80R-prec     	R-based precision- precision after 0.80 * R docs retrieved
+1.00R-prec     	R-based precision- precision after 1.00 * R docs retrieved
+1.20R-prec     	R-based precision- precision after 1.20 * R docs retrieved
+1.40R-prec     	R-based precision- precision after 1.40 * R docs retrieved
+1.60R-prec     	R-based precision- precision after 1.60 * R docs retrieved
+1.80R-prec     	R-based precision- precision after 1.80 * R docs retrieved
+2.00R-prec     	R-based precision- precision after 2.00 * R docs retrieved
+relative_prec5 	Relative precision after 5 docs retrieved
+relative_prec10	Relative precision after 10 docs retrieved
+relative_prec15	Relative precision after 15 docs retrieved
+relative_prec20	Relative precision after 20 docs retrieved
+relative_prec30	Relative precision after 30 docs retrieved
+relative_prec100	Relative precision after 100 docs retrieved
+relative_prec200	Relative precision after 200 docs retrieved
+relative_prec500	Relative precision after 500 docs retrieved
+relative_prec1000	Relative precision after 1000 docs retrieved
+unranked_avg_prec5	Unranked Average Precision after 5 docs retrieved
+unranked_avg_prec10	Unranked Average Precision after 10 docs retrieved
+unranked_avg_prec15	Unranked Average Precision after 15 docs retrieved
+unranked_avg_prec20	Unranked Average Precision after 20 docs retrieved
+unranked_avg_prec30	Unranked Average Precision after 30 docs retrieved
+unranked_avg_prec100	Unranked Average Precision after 100 docs retrieved
+unranked_avg_prec200	Unranked Average Precision after 200 docs retrieved
+unranked_avg_prec500	Unranked Average Precision after 500 docs retrieved
+unranked_avg_prec1000	Unranked Average Precision after 1000 docs retrieved
+relative_unranked_avg_prec5	Relative Unranked Average Precision after 5 docs retrieved
+relative_unranked_avg_prec10	Relative Unranked Average Precision after 10 docs retrieved
+relative_unranked_avg_prec15	Relative Unranked Average Precision after 15 docs retrieved
+relative_unranked_avg_prec20	Relative Unranked Average Precision after 20 docs retrieved
+relative_unranked_avg_prec30	Relative Unranked Average Precision after 30 docs retrieved
+relative_unranked_avg_prec100	Relative Unranked Average Precision after 100 docs retrieved
+relative_unranked_avg_prec200	Relative Unranked Average Precision after 200 docs retrieved
+relative_unranked_avg_prec500	Relative Unranked Average Precision after 500 docs retrieved
+relative_unranked_avg_prec1000	Relative Unranked Average Precision after 1000 docs retrieved
+utility_1.0_-1.0_0.0_0.0	Utility (a,b,c,d) Coefficients 1.0_-1.0_0.0_0.0 
+rcl_at_142_nonrel	Recall averaged at X nonrel docs    X= 142 
+fallout_recall_0	Fallout - Recall Averages- recall after 0 nonrel docs retrieved
+fallout_recall_14	Fallout - Recall Averages- recall after 14 nonrel docs retrieved
+fallout_recall_28	Fallout - Recall Averages- recall after 28 nonrel docs retrieved
+fallout_recall_42	Fallout - Recall Averages- recall after 42 nonrel docs retrieved
+fallout_recall_56	Fallout - Recall Averages- recall after 56 nonrel docs retrieved
+fallout_recall_71	Fallout - Recall Averages- recall after 71 nonrel docs retrieved
+fallout_recall_85	Fallout - Recall Averages- recall after 85 nonrel docs retrieved
+fallout_recall_99	Fallout - Recall Averages- recall after 99 nonrel docs retrieved
+fallout_recall_113	Fallout - Recall Averages- recall after 113 nonrel docs retrieved
+fallout_recall_127	Fallout - Recall Averages- recall after 127 nonrel docs retrieved
+fallout_recall_142	Fallout - Recall Averages- recall after 142 nonrel docs retrieved
+int_0.20R-prec 	Interpolated R-based precision, after 0.20 * R docs retrieved
+int_0.40R-prec 	Interpolated R-based precision, after 0.40 * R docs retrieved
+int_0.60R-prec 	Interpolated R-based precision, after 0.60 * R docs retrieved
+int_0.80R-prec 	Interpolated R-based precision, after 0.80 * R docs retrieved
+int_1.00R-prec 	Interpolated R-based precision, after 1.00 * R docs retrieved
+int_1.20R-prec 	Interpolated R-based precision, after 1.20 * R docs retrieved
+int_1.40R-prec 	Interpolated R-based precision, after 1.40 * R docs retrieved
+int_1.60R-prec 	Interpolated R-based precision, after 1.60 * R docs retrieved
+int_1.80R-prec 	Interpolated R-based precision, after 1.80 * R docs retrieved
+int_2.00R-prec 	Interpolated R-based precision, after 2.00 * R docs retrieved
+micro_prec     	Total relevant retrieved documents / Total retrieved documents
+micro_recall   	Total relevant retrieved documents / Total relevant documents
+micro_bpref    	Total correct preferences / Total possible preferences

sm-model/trec_eval-8.0/buf.h

@@ -0,0 +1,13 @@
+#ifndef BUFH
+#define BUFH
+/*        $Header: /home/smart/release/src/h/buf.h,v 11.0 1992/07/21 18:18:32 chrisb Exp $*/
+
+/* structure used for passing around text (buf) which possibly includes
+   NULLs.  see buf_util.c for add_buf(). */
+typedef struct {
+    int size;                      
+    int end;
+    char *buf;
+} SM_BUF;
+
+#endif /* BUFH */

sm-model/trec_eval-8.0/buf_util.c

@@ -0,0 +1,86 @@
+#ifdef RCSID
+static char rcsid[] = "$Header: /home/smart/release/src/libgeneral/buf_util.c,v 11.0 1992/07/21 18:21:04 chrisb Exp $";
+#endif
+
+/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. 
+
+   Permission is granted for use of this file in unmodified form for
+   research purposes. Please contact the SMART project to obtain 
+   permission for other uses.
+*/
+
+/********************   PROCEDURE DESCRIPTION   ************************
+ *0 Utility procedure to add the memory contents of new.buf to result.buf
+ *2 add_buf (new, result)
+ *3  SM_BUF *new;
+ *3  SM_BUF *result;
+ *7 Both new and result are of type
+ *7 typedef struct {
+ *7     int size;                      * allocated space for buf *
+ *7     int end;                       * end of valid data in buf *
+ *7     char *buf;                     * buffer of arbitrary data *
+ *7 } SM_BUF;
+ *7     
+ *7 Append the data in new to the end of the data in result.  The data can
+ *7 be arbitrary data, eg, include '\0's.
+ *7 Return UNDEF if can't allocate enough space for the result, 0 otherwise.
+***********************************************************************/
+
+#include "common.h"
+#include "sysfunc.h"
+#include "buf.h"
+
+int 
+add_buf (new, result)
+SM_BUF *new, *result;
+{
+    if (result->size == 0) {
+        if (NULL == (result->buf = malloc ((unsigned) new->end * 2 + 1)))
+            return (UNDEF);
+        result->size = 2 * new->end + 1;
+        result->end = 0;
+    }
+    else if (new->end >= result->size - result->end) {
+        if (NULL == (result->buf =
+                     realloc (result->buf,
+                              (unsigned) result->size * 2 + new->end)))
+            return (UNDEF);
+        result->size += result->size + new->end;
+    }
+
+    bcopy (new->buf, &result->buf[result->end], new->end);
+    result->end += new->end;
+    
+    return (0);
+}
+
+/********************   PROCEDURE DESCRIPTION   ************************
+ *0 Utility procedure to add the string new to result.buf
+ *2 add_buf_string (new, result)
+ *3  char *new;
+ *3  SM_BUF *result;
+
+ *7 Result is of type
+ *7 typedef struct {
+ *7     int size;                      * allocated space for buf *
+ *7     int end;                       * end of valid data in buf *
+ *7     char *buf;                     * buffer of arbitrary data *
+ *7 } SM_BUF;
+ *7     
+ *7 Append the data in new to the end of the data in result.
+ *7 Return UNDEF if can't allocate enough space for the result, 0 otherwise.
+***********************************************************************/
+
+int 
+add_buf_string (new, result)
+char *new;
+SM_BUF *result;
+{
+    SM_BUF temp_buf;
+    temp_buf.end = strlen (new);
+    temp_buf.buf = new;
+
+    return (add_buf (&temp_buf, result));
+
+}
+

sm-model/trec_eval-8.0/common.h

@@ -0,0 +1,31 @@
+#ifndef COMMONH
+#define COMMONH
+
+#include <stdio.h>
+
+#ifndef FALSE
+#define FALSE   0
+#endif
+#ifndef TRUE
+#define TRUE    1
+#endif
+
+#define UNDEF   -1
+
+#define MAX(A,B)  ((A) > (B) ? (A) : (B))
+#define MIN(A,B)  ((A) > (B) ? (B) : (A))
+
+#ifndef MAXLONG
+#define MAXLONG 2147483647L             /* largest long int. no. */
+#endif
+
+/*
+ * Some useful macros for making malloc et al easier to use.
+ * Macros handle the casting and the like that's needed.
+ */
+#define Malloc(n,type) (type *) malloc( (unsigned) ((n)*sizeof(type)))
+#define Realloc(loc,n,type) (type *) realloc( (char *)(loc), \
+                                              (unsigned) ((n)*sizeof(type)))
+#define Free(loc) (void) free( (char *)(loc) )
+
+#endif /* COMMONH */

sm-model/trec_eval-8.0/error_msgs.c

@@ -0,0 +1,93 @@
+#ifdef RCSID
+static char rcsid[] = "$Header: /home/smart/release/./src/libgeneral/error_msgs.c,v 10.1 91/11/05 23:49:06 smart Exp Locker: smart $";
+#endif
+
+/* Copyright (c) 1991, 1990, 1984 - Gerard Salton, Chris Buckley. 
+
+   Permission is granted for use of this file in unmodified form for
+   research purposes. Please contact the SMART project to obtain 
+   permission for other uses.
+*/
+
+/********************   PROCEDURE DESCRIPTION   ************************
+ *0 print a SMART error message
+ *2 print_error (new_routine, new_message)
+ *3  char *new_routine;
+ *3  char *new_message;
+ *6 Global UNIX variables errno, sys_nerr, sys_errlist are used, as well
+ *6 as SMART global variables smart_errlist and smart_errno;
+
+ *7 Print an error message to stderr.  At point of error determination,
+ *7 either smart_errno should be set, or (if UNIX library error) errno will
+ *7 be set.  If smart_errno is set, then the routine name that detected the
+ *7 error and a message are printed.  In addition, the routine name that prints
+ *7 the error and it's message (eg action to be taken) are printed.
+ *9 smart_errno should be more widely used, in particular to locate the
+ *9 procedure the error occurs in.  Many errors can only get "located"
+ *9 by setting trace.
+***********************************************************************/
+
+#include <stdio.h>
+#include "smart_error.h"
+#include "sysfunc.h"
+
+/* Declarations of external variables defined in "smart_error.h" */
+
+int smart_errno;               /* If > 0 and <= sys_nerr then refers to  */
+                                /* sys_errlist, else if >= smart_errmin */
+                                /* and <= smart_errmax, then smart_errlist */
+char *smart_message;            /* Message to be printed (often filename) */
+char *smart_routine;            /* Major routine issuing error message */
+
+
+extern int  errno;
+
+char *smart_errlist[] = {
+    "Inconsistency check",
+    "Illegal value for seek",
+    "Illegal mode for object",
+    "Illegal parameter value"
+};
+
+void
+print_error (new_routine, new_message)
+char *new_routine;
+char *new_message;
+{
+    if (smart_errno > 0 && smart_errno < SMART_MINERR) {
+        (void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n",
+                 new_routine,
+                 smart_routine,
+                 smart_message,
+                 strerror(smart_errno),
+                 new_message);
+    }
+    else if (smart_errno >= SMART_MINERR && 
+             smart_errno < SMART_MINERR + SMART_NUMERR) {
+        (void) fprintf (stderr, "%s: in %s: '%s' %s - %s\n",
+                 new_routine,
+                 smart_routine,
+                 smart_message,
+                 smart_errlist[smart_errno - SMART_MINERR],
+                 new_message);
+    }
+    else if (smart_errno == 0 && errno != 0) {
+        /* Presumably error detected directly by new_routine */
+        /* after system call */
+        (void) fprintf (stderr, "%s: '%s' - %s\n",
+                 new_routine,
+                 strerror(errno),
+                 new_message);
+    }
+    else {
+        (void) fprintf (stderr, "%s: Undetermined error detected - %s\n",
+                 new_routine,
+                 new_message);
+    }
+
+    /* Reset the global error indicators */
+    errno = 0;
+    smart_errno = 0;
+    smart_message = NULL;
+    smart_routine = NULL;
+}

sm-model/trec_eval-8.0/form_trvec.c

@@ -0,0 +1,201 @@
+#include "common.h"
+#include "sysfunc.h"
+#include "smart_error.h"
+#include "tr_vec.h"
+#include "trec_eval.h"
+#include "buf.h"
+
+static int comp_tr_tup_rank(), comp_tr_tup_did(), comp_tr_docno(), 
+    comp_qrels_docno(), comp_sim_docno(), comp_negsim_docno();
+
+/* Space reserved for output TR_TUP tuples */
+static TR_TUP *start_tr_tup;
+static long max_tr_tup = 0;
+
+int
+form_trvec (epi, trec_top, trec_qrels, tr_vec, num_rel)
+EVAL_PARAM_INFO *epi;
+TREC_TOP *trec_top;
+TREC_QRELS *trec_qrels;
+TR_VEC *tr_vec;
+long *num_rel;
+{
+    TR_TUP *tr_tup;
+    TEXT_QRELS *qrels_ptr, *end_qrels;
+    long i;
+
+    /* Reserve space for output tr_tups, if needed */
+    if (trec_top->num_text_tr > max_tr_tup) {
+        if (max_tr_tup > 0) 
+            (void) free ((char *) start_tr_tup);
+        max_tr_tup += trec_top->num_text_tr;
+        if (NULL == (start_tr_tup = Malloc (max_tr_tup, TR_TUP)))
+            return (UNDEF);
+    }
+
+    /* Sort trec_top by sim, breaking ties lexicographically using docno */
+    if (epi->time_flag) {
+	qsort ((char *) trec_top->text_tr,
+	       (int) trec_top->num_text_tr,
+	       sizeof (TEXT_TR),
+	       comp_negsim_docno);
+    }
+    else {
+	qsort ((char *) trec_top->text_tr,
+	       (int) trec_top->num_text_tr,
+	       sizeof (TEXT_TR),
+	       comp_sim_docno);
+    }
+
+    /* Add ranks to trec_top (starting at 1) */
+    for (i = 0; i < trec_top->num_text_tr; i++) {
+        trec_top->text_tr[i].rank = i+1;
+    }
+
+    /* Sort trec_top lexicographically */
+    qsort ((char *) trec_top->text_tr,
+           (int) trec_top->num_text_tr,
+           sizeof (TEXT_TR),
+           comp_tr_docno);
+
+    for (i = 1; i < trec_top->num_text_tr; i++) {
+	if (0 == strcmp (trec_top->text_tr[i].docno,
+			 trec_top->text_tr[i-1].docno)) {
+	    set_error (SM_ILLPA_ERR, "Duplicate top docs docno", trec_top->text_tr[i].docno);
+	    return (UNDEF);
+	}
+    }
+
+    /* Sort trec_qrels lexicographically */
+    qsort ((char *) trec_qrels->text_qrels,
+           (int) trec_qrels->num_text_qrels,
+           sizeof (TEXT_QRELS),
+           comp_qrels_docno);
+
+    /* Find number of relevant docs, and check for duplicates */
+    *num_rel = 0;
+    for (i = 0; i < trec_qrels->num_text_qrels; i++) {
+	//if (i > 0 && (0 == strcmp (trec_qrels->text_qrels[i].docno,
+	//			   trec_qrels->text_qrels[i-1].docno))) {
+	//    set_error (SM_ILLPA_ERR, "Duplicate qrels docno", trec_qrels->text_qrels[i].docno);
+	//    return (UNDEF);
+	//}
+	if (trec_qrels->text_qrels[i].rel >= epi->relevance_level)
+	    (*num_rel)++;
+    }
+
+    /* Go through trec_top, trec_qrels in parallel to determine which
+       docno's are in both (ie, which trec_top are relevant).  Once relevance
+       is known, convert trec_top tuple into TR_TUP. */
+    tr_tup = start_tr_tup;
+    qrels_ptr = trec_qrels->text_qrels;
+    end_qrels = &trec_qrels->text_qrels[trec_qrels->num_text_qrels];
+    for (i = 0; i < trec_top->num_text_tr; i++) {
+	if (trec_top->text_tr[i].rank > epi->max_num_docs_per_topic)
+	    /* Skip if evaluation desired over fewer docs than this rank */
+	    continue;
+        while (qrels_ptr < end_qrels &&
+               strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) < 0)
+            qrels_ptr++;
+        if (qrels_ptr >= end_qrels ||
+            strcmp (qrels_ptr->docno, trec_top->text_tr[i].docno) > 0) {
+            /* Doc is non-judged */
+            tr_tup->rel = -1;
+	    /* Skip unjudged docs if desired */
+	    if (epi->judged_docs_only_flag) 
+		continue;
+	}
+        else {
+            /* Doc is judged; assign relevance */
+	    tr_tup->rel = qrels_ptr->rel;
+	    qrels_ptr++;
+        }
+        tr_tup->did = i;
+        tr_tup->rank = trec_top->text_tr[i].rank;
+        tr_tup->sim = trec_top->text_tr[i].sim;
+        tr_tup->action = 0;
+        tr_tup->iter = 0;
+        tr_tup++;
+    }
+
+    /* Form the full TR_VEC object for this qid */
+    tr_vec->qid = trec_top->qid;
+    tr_vec->num_tr = tr_tup - start_tr_tup;
+    tr_vec->tr = start_tr_tup;
+
+    /* If judged_docs_only_flag, then must fix up ranks to reflect unjudged
+       docs being thrown out. Note: done this way to preserve original
+       tie-breaking based on text docno */
+    if (epi->judged_docs_only_flag) {
+	/* Sort tuples by increasing rank */
+	qsort ((char *) tr_vec->tr,
+	       (int) tr_vec->num_tr,
+	       sizeof (TR_TUP),
+	       comp_tr_tup_rank);
+	for (i = 0; i < tr_vec->num_tr; i++) {
+	    tr_vec->tr[i].rank = i+1;
+	}
+	qsort ((char *) tr_vec->tr,
+	       (int) tr_vec->num_tr,
+	       sizeof (TR_TUP),
+	       comp_tr_tup_did);
+    }
+
+    return (1);
+}
+
+static int 
+comp_sim_docno (ptr1, ptr2)
+TEXT_TR *ptr1;
+TEXT_TR *ptr2;
+{
+    if (ptr1->sim > ptr2->sim)
+        return (-1);
+    if (ptr1->sim < ptr2->sim)
+        return (1);
+    return (strcmp (ptr2->docno, ptr1->docno));
+}
+
+static int 
+comp_negsim_docno (ptr1, ptr2)
+TEXT_TR *ptr1;
+TEXT_TR *ptr2;
+{
+    if (ptr1->sim < ptr2->sim)
+        return (-1);
+    if (ptr1->sim > ptr2->sim)
+        return (1);
+    return (strcmp (ptr2->docno, ptr1->docno));
+}
+
+static int 
+comp_tr_docno (ptr1, ptr2)
+TEXT_TR *ptr1;
+TEXT_TR *ptr2;
+{
+    return (strcmp (ptr1->docno, ptr2->docno));
+}
+
+static int 
+comp_qrels_docno (ptr1, ptr2)
+TEXT_QRELS *ptr1;
+TEXT_QRELS *ptr2;
+{
+    return (strcmp (ptr1->docno, ptr2->docno));
+}
+
+static int 
+comp_tr_tup_rank (ptr1, ptr2)
+TR_TUP *ptr1;
+TR_TUP *ptr2;
+{
+    return (ptr1->rank - ptr2->rank);
+}
+
+static int 
+comp_tr_tup_did (ptr1, ptr2)
+TR_TUP *ptr1;
+TR_TUP *ptr2;
+{
+    return (ptr1->did - ptr2->did);
+}

sm-model/trec_eval-8.0/get_qrels.c

@@ -0,0 +1,169 @@
+/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley.  */
+
+
+#include "common.h"
+#include "sysfunc.h"
+#include "smart_error.h"
+#include "trec_eval.h"
+#include <ctype.h>
+
+
+/* Read all relevance information from text_qrels_file.
+Relevance for each docno to qid is determined from text_qrels_file, which
+consists of text tuples of the form
+   qid  iter  docno  rel
+giving TREC document numbers (docno, a string) and their relevance (rel, 
+an integer) to query qid (a string).  iter string field is ignored.  
+Fields are separated by whitespace, string fields can contain no whitespace.
+File may contain no NULL characters.
+*/
+
+int 
+get_qrels (text_qrels_file, all_trec_qrels)
+char *text_qrels_file;
+ALL_TREC_QRELS *all_trec_qrels;
+{
+    int fd;
+    int size = 0;
+    char *trec_qrels_buf;
+    char *ptr;
+    char *current_qid;
+    char *qid_ptr, *docno_ptr, *rel_ptr;
+    long i;
+    long rel;
+    TREC_QRELS *current_qrels = NULL;
+
+    /* Read entire file into memory */
+    if (-1 == (fd = open (text_qrels_file, 0)) ||
+        -1 == (size = lseek (fd, 0L, 2)) ||
+        NULL == (trec_qrels_buf = malloc ((unsigned) size+2)) ||
+        -1 == lseek (fd, 0L, 0) ||
+        size != read (fd, trec_qrels_buf, size) ||
+	-1 == close (fd)) {
+
+        set_error (SM_ILLPA_ERR, "Cannot read qrels file", "trec_eval");
+        return (UNDEF);
+    }
+
+    current_qid = "";
+
+    /* Initialize all_trec_qrels */
+    all_trec_qrels->num_q_qrels = 0;
+    all_trec_qrels->max_num_q_qrels = INIT_NUM_QUERIES;
+    if (NULL == (all_trec_qrels->trec_qrels = Malloc (INIT_NUM_QUERIES,
+						     TREC_QRELS)))
+	return (UNDEF);
+    
+    if (size == 0)
+	return (0);
+
+    /* Append ending newline if not present, Append NULL terminator */
+    if (trec_qrels_buf[size-1] != '\n') {
+	trec_qrels_buf[size] = '\n';
+	size++;
+    }
+    trec_qrels_buf[size] = '\0';
+
+    ptr = trec_qrels_buf;
+
+    while (*ptr) {
+	/* Get current line */
+	/* Get qid */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	qid_ptr = ptr;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
+	    return (UNDEF);
+	}
+	*ptr++ = '\0';
+	/* Skip iter */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr++ == '\n') {
+	    set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
+	    return (UNDEF);
+	}
+	/* Get docno */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	docno_ptr = ptr;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
+	    return (UNDEF);
+	}
+	*ptr++ = '\0';
+	/* Get relevance */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    set_error (SM_ILLPA_ERR, "Malformed qrels line", "trec_eval");
+	    return (UNDEF);
+	}
+	rel_ptr = ptr;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr != '\n') {
+	    *ptr++ = '\0';
+	    while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	    if (*ptr != '\n') {
+		set_error (SM_ILLPA_ERR, "malformed qrels line",
+			   "trec_eval");
+		return (UNDEF);
+	    }
+	}
+	*ptr++ = '\0';
+
+	if (0 != strcmp (qid_ptr, current_qid)) {
+	    /* Query has changed. Must check if new query or this is more
+	       judgements for an old query */
+	    for (i = 0; i < all_trec_qrels->num_q_qrels; i++) {
+		if (0 == strcmp (qid_ptr, all_trec_qrels->trec_qrels[i].qid))
+		    break;
+	    }
+	    if (i >= all_trec_qrels->num_q_qrels) {
+		/* New unseen query, add and initialize it */
+		if (all_trec_qrels->num_q_qrels >=
+		    all_trec_qrels->max_num_q_qrels) {
+		    all_trec_qrels->max_num_q_qrels *= 10;
+		    if (NULL == (all_trec_qrels->trec_qrels = 
+				 Realloc (all_trec_qrels->trec_qrels,
+					  all_trec_qrels->max_num_q_qrels,
+					  TREC_QRELS)))
+			return (UNDEF);
+		}
+		current_qrels = &all_trec_qrels->trec_qrels[i];
+		current_qrels->qid = qid_ptr;
+		current_qrels->num_text_qrels = 0;
+		current_qrels->max_num_text_qrels = INIT_NUM_RELS;
+		if (NULL == (current_qrels->text_qrels =
+			     Malloc (INIT_NUM_RELS, TEXT_QRELS)))
+		    return (UNDEF);
+		all_trec_qrels->num_q_qrels++;
+	    }
+	    else {
+		/* Old query, just switch current_q_index */
+		current_qrels = &all_trec_qrels->trec_qrels[i];
+	    }
+	    current_qid = current_qrels->qid;
+	}
+	
+	/* Add judgement to current query's list */
+	if (current_qrels->num_text_qrels >= 
+	    current_qrels->max_num_text_qrels) {
+	    /* Need more space */
+	    current_qrels->max_num_text_qrels *= 10;
+	    if (NULL == (current_qrels->text_qrels = 
+			 Realloc (current_qrels->text_qrels,
+				  current_qrels->max_num_text_qrels,
+				  TEXT_QRELS)))
+		return (UNDEF);
+	}
+	current_qrels->text_qrels[current_qrels->num_text_qrels].docno =
+		docno_ptr;
+	rel = atol (rel_ptr);
+	current_qrels->text_qrels[current_qrels->num_text_qrels++].rel =
+	    rel;
+    }
+
+    return (1);
+}
+

sm-model/trec_eval-8.0/get_top.c

@@ -0,0 +1,192 @@
+/* Copyright (c) 2003, 1991, 1990, 1984 Chris Buckley.  */
+
+
+#include "common.h"
+#include "sysfunc.h"
+#include "smart_error.h"
+#include "trec_eval.h"
+#include <ctype.h>
+
+/* Read all retrieved results information from trec_top_file.
+Read text tuples from trec_top_file of the form
+     030  Q0  ZF08-175-870  0   4238   prise1
+     qid iter   docno      rank  sim   run_id
+giving TREC document numbers (a string) retrieved by query qid 
+(a string) with similarity sim (a float).  The other fields are ignored,
+with the exception that the run_id field of the last line is kept and
+output.  In particular, note that the rank field is ignored here;
+internally ranks are assigned by sorting by the sim field with ties 
+broken determinstically (using docno).
+Sim is assumed to be higher for the docs to be retrieved first.
+File may contain no NULL characters.
+Any field following run_id is ignored.
+*/
+
+int 
+get_top (trec_top_file, all_trec_top)
+char *trec_top_file;
+ALL_TREC_TOP *all_trec_top;
+{
+    int fd;
+    int size = 0;
+    char *trec_top_buf;
+    char *ptr;
+    char *current_qid;
+    char *qid_ptr, *docno_ptr, *sim_ptr;
+    char *run_id_ptr = "";
+    long i;
+    TREC_TOP *current_top = NULL;
+    float sim;
+
+    /* Read entire file into memory */
+    if (-1 == (fd = open (trec_top_file, 0)) ||
+        -1 == (size = lseek (fd, 0L, 2)) ||
+        NULL == (trec_top_buf = malloc ((unsigned) size+2)) ||
+        -1 == lseek (fd, 0L, 0) ||
+        size != read (fd, trec_top_buf, size) ||
+	-1 == close (fd)) {
+
+        set_error (SM_ILLPA_ERR, "Cannot read qrels file", "trec_eval");
+        return (UNDEF);
+    }
+
+    current_qid = "";
+
+    /* Initialize all_trec_top */
+    all_trec_top->num_q_tr = 0;
+    all_trec_top->max_num_q_tr = INIT_NUM_QUERIES;
+    if (NULL == (all_trec_top->trec_top = Malloc (INIT_NUM_QUERIES,
+						     TREC_TOP)))
+	return (UNDEF);
+    
+    if (size == 0)
+	return (0);
+
+    /* Append ending newline if not present, Append NULL terminator */
+    if (trec_top_buf[size-1] != '\n') {
+	trec_top_buf[size] = '\n';
+	size++;
+    }
+    trec_top_buf[size] = '\0';
+
+    ptr = trec_top_buf;
+
+    while (*ptr) {
+	/* Get current line */
+	/* Get qid */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    /* Ignore blank lines (people seem to insist on them!) */
+	    ptr++;
+	    continue;
+	}
+	qid_ptr = ptr;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
+	    return (UNDEF);
+	}
+	*ptr++ = '\0';
+	/* Skip iter */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr++ == '\n') {
+	    set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
+	    return (UNDEF);
+	}
+	/* Get docno */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	docno_ptr = ptr;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
+	    return (UNDEF);
+	}
+	*ptr++ = '\0';
+	/* Skip rank */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr++ == '\n') {
+	    set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
+	    return (UNDEF);
+	}
+	/* Get sim */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	sim_ptr = ptr;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
+	    return (UNDEF);
+	}
+	*ptr++ = '\0';
+	/* Get run_id */
+	while (*ptr != '\n' && isspace (*ptr)) ptr++;
+	if (*ptr == '\n') {
+	    set_error (SM_ILLPA_ERR,"malformed top results line", "trec_eval");
+	    return (UNDEF);
+	}
+	run_id_ptr = ptr;
+	while (! isspace (*ptr)) ptr++;
+	if (*ptr != '\n') {
+	    /* Skip over rest of line */
+	    *ptr++ = '\0';
+	    while (*ptr != '\n') ptr++;
+	}
+	*ptr++ = '\0';
+
+	if (0 != strcmp (qid_ptr, current_qid)) {
+	    /* Query has changed. Must check if new query or this is more
+	       judgements for an old query */
+	    for (i = 0; i < all_trec_top->num_q_tr; i++) {
+		if (0 == strcmp (qid_ptr, all_trec_top->trec_top[i].qid))
+		    break;
+	    }
+	    if (i >= all_trec_top->num_q_tr) {
+		/* New unseen query, add and initialize it */
+		if (all_trec_top->num_q_tr >=
+		    all_trec_top->max_num_q_tr) {
+		    all_trec_top->max_num_q_tr *= 10;
+		    if (NULL == (all_trec_top->trec_top = 
+				 Realloc (all_trec_top->trec_top,
+					  all_trec_top->max_num_q_tr,
+					  TREC_TOP)))
+			return (UNDEF);
+		}
+		current_top = &all_trec_top->trec_top[i];
+		current_top->qid = qid_ptr;
+		current_top->num_text_tr = 0;
+		current_top->max_num_text_tr = INIT_NUM_RESULTS;
+		if (NULL == (current_top->text_tr =
+			     Malloc (INIT_NUM_RESULTS, TEXT_TR)))
+		    return (UNDEF);
+		all_trec_top->num_q_tr++;
+	    }
+	    else {
+		/* Old query, just switch current_q_index */
+		current_top = &all_trec_top->trec_top[i];
+	    }
+	    current_qid = current_top->qid;
+	}
+	
+	/* Add retrieval docno/sim to current query's list */
+	if (current_top->num_text_tr >= 
+	    current_top->max_num_text_tr) {
+	    /* Need more space */
+	    current_top->max_num_text_tr *= 10;
+	    if (NULL == (current_top->text_tr = 
+			 Realloc (current_top->text_tr,
+				  current_top->max_num_text_tr,
+				  TEXT_TR)))
+		return (UNDEF);
+	}
+	current_top->text_tr[current_top->num_text_tr].docno = docno_ptr;
+	sim = atof (sim_ptr);
+	current_top->text_tr[current_top->num_text_tr].sim = sim;
+	current_top->text_tr[current_top->num_text_tr++].rank = 0;
+    }
+
+    all_trec_top->run_id = run_id_ptr;
+
+    return (1);
+}
+

sm-model/trec_eval-8.0/smart_error.h

@@ -0,0 +1,24 @@
+#ifndef SMART_ERRORH
+#define SMART_ERRORH
+#include <errno.h>
+#define SMART_MINERR 1000
+#define SM_INCON_ERR 1000
+#define SM_ILLSK_ERR 1001
+#define SM_ILLMD_ERR 1002
+#define SM_ILLPA_ERR 1003
+#define SMART_NUMERR 4
+
+
+extern int  errno;
+extern int  smart_errno;        /* If > 0 and <= sys_nerr then refers to  */
+                                /* sys_errlist, else if >= smart_errmin */
+                                /* and <= smart_errmax, then smart_errlist */
+extern char *smart_message;     /* Message to be printed (often filename) */
+extern char *smart_routine;     /* Major routine issuing error message */
+
+
+#define set_error(n,m,r) {  if (n > 0) smart_errno = n;\
+                            smart_message = m;\
+                            smart_routine = r; }
+#define clr_err() smart_errno = errno = 0
+#endif /* SMART_ERRORH */

sm-model/trec_eval-8.0/sysfunc.h

@@ -0,0 +1,39 @@
+#ifndef SYSFUNCH
+#define SYSFUNCH
+/* Declarations of major functions within standard C libraries */
+/* Once all of the major systems get their act together (and I follow
+   suit!), this file should just include system header files from 
+   /usr/include.  Until then... */
+
+#include <unistd.h>
+#include <limits.h>
+#include <ctype.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+#include <memory.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <sys/time.h>
+#include <sys/mman.h>
+
+/* For time being, define Berkeley constructs in terms of SVR4 constructs*/
+#define bzero(dest,len)      memset(dest,'\0',len)
+#define bcopy(source,dest,len)   memcpy(dest,source,len)
+#define srandom(seed)        srand(seed)
+#define random()             rand()
+
+/* ANSI should give us an offsetof suitable for the implementation;
+ * otherwise, try a non-portable but commonly supported definition
+ */
+#ifdef __STDC__	
+#include <stddef.h>
+#endif
+#ifndef offsetof
+#define offsetof(type, member) ((size_t) \
+	((char *)&((type*)0)->member - (char *)(type *)0))
+#endif
+
+#endif /* SYSFUNCH */

sm-model/trec_eval-8.0/tr_vec.h

@@ -0,0 +1,21 @@
+#ifndef TR_VECH
+#define TR_VECH
+/*        $Header: /home/smart/release/./src/h/tr_vec.h,v 10.1 91/11/05 23:47:35 smart Exp Locker: smart $*/
+
+typedef struct {
+    long  did;          /* document id */
+    long  rank;         /* Rank of this document */
+    char  action;       /* what action a user has taken with doc */
+    char  rel;          /* whether doc judged relevant(1) or not(0) */
+    char  iter;         /* Number of feedback runs for this query */
+    char trtup_unused;  /* Presently unused field */
+    float sim;          /* similarity of did to qid */
+} TR_TUP;
+
+typedef struct {
+    char *qid;          /* query id */
+    long  num_tr;       /* Number of tuples for tr_vec */
+    TR_TUP *tr;         /* tuples.  Invariant: tr sorted increasing did */
+} TR_VEC;
+
+#endif /* TR_VECH */

sm-model/trec_eval-8.0/trec_eval.c

@@ -0,0 +1,256 @@
+static char *VersionID = VERSIONID;
+/* "Version 7.3  trec_eval Dec 15, 2004"; */
+
+/* Copyright (c) 2004, 2003, 1991, 1990, 1984 - Chris Buckley. */
+
+/********************   PROCEDURE DESCRIPTION   ************************
+ *0 Take TREC results text file, TREC qrels file, and evaluate
+ *1 local.convert.obj.trec_eval
+ *2 trec_eval [-q] [-a] [-t] [-o] [-v] [-n num] trec_rel_file trec_top_file
+
+ *7 Read text tuples from trec_top_file of the form
+ *7     030  Q0  ZF08-175-870  0   4238   prise1
+ *7     qid iter   docno      rank  sim   run_id
+ *7 giving TREC document numbers (a string) retrieved by query qid 
+ *7 (an integer) with similarity sim (a float).  The other fields are ignored.
+ *7 Input is asssumed to be sorted numerically by qid.
+ *7 Sim is assumed to be higher for the docs to be retrieved first.
+ *7 Relevance for each docno to qid is determined from text_qrels_file, which
+ *7 consists of text tuples of the form
+ *7    qid  iter  docno  rel
+ *7 giving TREC document numbers (a string) and their relevance to query qid
+ *7 (an integer). Tuples are asssumed to be sorted numerically by qid.
+ *7 The text tuples with relevence judgements are converted to TR_VEC form
+ *7 and then submitted to the evaluation routines.
+ *7
+ *7 -q: In addition to summary evaluation, give evaluation for each query
+ *7 -a: Print all evaluation measures calculated, instead of just the
+ *7      official measures for TREC 2.
+ *7 -o: Print everything out in old, non-relational format
+ *7 -v: Print version number and exit
+ *7 -h: Print full help message and exit
+ *7 -t: Treat similarity as time that document retrieved.  Compute
+ *7      several time-based measures after ranking docs by time retrieved
+ *7      (first doc (lowest sim) retrieved ranked highest). 
+ *7      Only done if -a selected.
+ *7 -J: Calculate all measures only over judged documents that appear 
+ *7     in qrels. (DO NOT USE)
+ *7 -n<num>: following integer is the number of queries to average over.
+ *7 -ua<num>: Value to use for 'a' coefficient of utility computation.
+ *7 -ub<num>: Value to use for 'b' coefficient of utility computation.
+ *7 -uc<num>: Value to use for 'c' coefficient of utility computation.
+ *7 -ud<num>: Value to use for 'd' coefficient of utility computation.
+ *7 -N<num>: Number of docs in collection
+ *7 -M<num>:Max number of results to evaluate per topic
+
+ *8 Procedure is to read all the docs retrieved for a query, and all the
+ *8 relevant docs for that query,
+ *8 sort and rank the retrieved docs by sim/docno, 
+ *8 and look up docno in the relevant docs to determine relevance.
+ *8 The qid,did,rank,sim,rel fields of of TR_VEC are filled in; 
+ *8 action,iter fields are set to 0.
+ *8 Queries for which there are no relevant docs are ignored completely.
+
+***********************************************************************/
+
+#include "common.h"
+#include "sysfunc.h"
+#include "smart_error.h"
+#include "tr_vec.h"
+#include "trec_eval.h"
+#include "buf.h"
+
+void print_error();
+void old_print_trec_eval_list();
+void print_rel_trec_eval_list();
+
+int trec_eval_help(EVAL_PARAM_INFO *epi);
+int accumulate_results (TREC_EVAL *query_eval, TREC_EVAL *accum_eval);
+int get_top (char *trec_top_file, ALL_TREC_TOP *all_trec_top);
+int get_qrels (char *text_qrels_file, ALL_TREC_QRELS *all_trec_qrels);
+int form_trvec (EVAL_PARAM_INFO *ep, TREC_TOP *trec_top,
+		TREC_QRELS *trec_qrels, TR_VEC *tr_vec, long *num_rel);
+int trvec_trec_eval (EVAL_PARAM_INFO *epi, TR_VEC *tr_vec,
+		     TREC_EVAL *eval, long num_rel, long num_nonrel);
+
+static char *usage = "Usage: trec_eval [-h] [-q] [-a] [-o] [-v] trec_rel_file trec_top_file\n\
+   -h: Give full help information, including other options\n\
+   -q: In addition to summary evaluation, give evaluation for each query\n\
+   -a: Print all evaluation measures, instead of just official measures\n\
+   -o: Print requested measures in old non-relational format\n";
+
+
+int
+main (argc, argv)
+int argc;
+char *argv[];
+{
+    char *trec_rel_file, *trec_top_file;
+    ALL_TREC_TOP all_trec_top;
+    ALL_TREC_QRELS all_trec_qrels;
+    TREC_EVAL accum_eval, query_eval;
+    TR_VEC tr_vec;
+    long num_rel;
+    long num_eval_q;
+    long i,j;
+    EVAL_PARAM_INFO epi;
+
+    /* Initialize static info before getting program optional args */
+    epi.query_flag = epi.all_flag = epi.time_flag = epi.average_complete_flag = 0;
+    epi.judged_docs_only_flag = 0;
+    epi.relation_flag = 1;
+    epi.utility_a = UTILITY_A; epi.utility_b = UTILITY_B;
+    epi.utility_c = UTILITY_C; epi.utility_d = UTILITY_D;
+    epi.num_docs_in_coll = 0;
+    epi.relevance_level = 1;
+    epi.max_num_docs_per_topic = MAXLONG;
+
+    /* Should use getopts, but some people may not have it. */
+    /* This keeps growing over the years.  Should redo */
+    while (argc > 1 && argv[1][0] == '-') {
+        if (argv[1][1] == 'q')
+            epi.query_flag++;
+	else if (argv[1][1] == 'v') {
+	    fprintf (stderr, "trec_eval version %s\n", VersionID);
+	    exit (0);
+	}
+	else if (argv[1][1] == 'h') {
+	    (void) trec_eval_help(&epi);
+	    exit (0);
+	}
+        else if (argv[1][1] == 'a')
+            epi.all_flag++;
+        else if (argv[1][1] == 'o')
+            epi.relation_flag = 0;
+        else if (argv[1][1] == 'c') {
+	    epi.average_complete_flag++;
+	}
+        else if (argv[1][1] == 'l') {
+	    epi.relevance_level = atol (&argv[1][2]);
+	}
+        else if (argv[1][1] == 'J') {
+	    epi.judged_docs_only_flag++;
+	}
+        else if (argv[1][1] == 'N')
+            epi.num_docs_in_coll = atol (&argv[1][2]);
+        else if (argv[1][1] == 'M')
+            epi.max_num_docs_per_topic = atol (&argv[1][2]);
+        else if (argv[1][1] == 'U') {
+	    if (argv[1][2] == 'a')
+		epi.utility_a = atof (&argv[1][3]);
+	    else if (argv[1][2] == 'b')
+		epi.utility_b = atof (&argv[1][3]);
+	    else if (argv[1][2] == 'c')
+		epi.utility_c = atof (&argv[1][3]);
+	    else if (argv[1][2] == 'd')
+		epi.utility_d = atof (&argv[1][3]);
+	    else {
+		(void) fputs (usage,stderr);
+		exit (1);
+	    }
+	}
+        else if (argv[1][1] == 'T')
+            epi.time_flag++;
+        else {
+            (void) fputs (usage,stderr);
+            exit (1);
+        }
+        argc--; argv++;
+    }
+
+    if (argc != 3) {
+        (void) fputs (usage,stderr);
+        exit (1);
+    }
+
+    trec_rel_file = argv[1];
+    trec_top_file = argv[2];
+
+    /* Get qrels and top results information for all queries from the
+       input text files */
+    if (UNDEF == get_qrels (trec_rel_file, &all_trec_qrels) ||
+	UNDEF == get_top (trec_top_file, &all_trec_top)) {
+        print_error ("trec_eval: input error", "Quit");
+        exit (2);
+    }
+
+    /* For each topic which has both qrels and top results information,
+       calculate, possibly print (if query_flag), and accumulate
+       evaluation measures. */
+    num_eval_q = 0;
+    (void) memset ((void *) &accum_eval, 0, sizeof (TREC_EVAL));
+    accum_eval.qid = "All";
+
+    for (i = 0; i < all_trec_top.num_q_tr; i++) {
+	/* Find rel info for this query (skip if no rel info) */
+	for (j = 0; j < all_trec_qrels.num_q_qrels; j++) {
+	    if (0 == strcmp (all_trec_top.trec_top[i].qid,
+			     all_trec_qrels.trec_qrels[j].qid))
+		break;
+	}
+	if (j >= all_trec_qrels.num_q_qrels)
+	    continue;
+
+	/* Form results/rel into SMART TR_VEC form */
+	if (UNDEF == form_trvec (&epi,
+				 &all_trec_top.trec_top[i],
+				 &all_trec_qrels.trec_qrels[j],
+				 &tr_vec,
+				 &num_rel)) {
+	    print_error ("trec_eval: form_tr_vec error", "Quit");
+	    exit (3);
+	}
+
+	/* Evaluate results/rel for this query */
+	if (UNDEF == trvec_trec_eval (&epi,
+				      &tr_vec,
+				      &query_eval,
+				      num_rel,
+				      all_trec_qrels.trec_qrels[j].num_text_qrels - num_rel)) {
+	    print_error ("trec_eval: evaluation error", "Quit");
+	    exit (4);
+	}
+
+	/* Print results for this query, if desired */
+	if (epi.query_flag) {
+	    if (epi.relation_flag)
+		print_rel_trec_eval_list (1, &epi, &query_eval, (SM_BUF *) NULL);
+	    else
+		old_print_trec_eval_list (&epi, &query_eval, 1, (SM_BUF *) NULL);
+        }
+
+	/* Accumulate results for later averaging */
+	if (UNDEF == accumulate_results (&query_eval, &accum_eval)) {
+	    print_error ("trec_eval: accumulation error", "Quit");
+	    exit (5);
+	}
+
+	num_eval_q++;
+    }
+
+    /******** REMOVE THIS ONCE WARNING FLAG ADDED */
+    /* Warn if numq_flag_num < num_eval_q */
+    if (num_eval_q == 0) {
+	    set_error (SM_INCON_ERR,
+		       "No queries with both results and relevance info",
+		       "trec_eval");
+	    return (UNDEF);
+	print_error ("trec_eval", "Quit");
+	exit (6);
+    }
+
+    if (epi.average_complete_flag) {
+	/* Want to average over possibly missing queries.  Pass in actual
+	 *  number of queries in num_orig_queries */
+	accum_eval.num_orig_queries = accum_eval.num_queries;
+	accum_eval.num_queries = all_trec_qrels.num_q_qrels;
+    }
+
+    /* Print final evaluation results */
+    if (epi.relation_flag)
+	print_rel_trec_eval_list (0, &epi, &accum_eval, (SM_BUF *) NULL);
+    else
+	old_print_trec_eval_list (&epi, &accum_eval, 1, (SM_BUF *) NULL);
+
+    exit (0);
+}

sm-model/utils.py

@@ -0,0 +1,156 @@
+# file input output 
+# LATER model input output as well
+
+import os
+import sys
+import numpy as np
+from gensim.models.keyedvectors import KeyedVectors
+
+import torch
+
+import cPickle
+
+
+
+# logging setup
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+
+
+def load_bin_vec(fname, words):
+  """
+  Loads 300x1 word vecs from Google (Mikolov) word2vec
+  """
+  print fname
+  vocab = set(words)
+  word_vecs = {}
+  with open(fname, "rb") as f:
+    header = f.readline()
+    vocab_size, layer1_size = map(int, header.split())
+    binary_len = numpy.dtype('float32').itemsize * layer1_size
+    print 'vocab_size, layer1_size', vocab_size, layer1_size
+    count = 0
+    for i, line in enumerate(xrange(vocab_size)):
+      if i % 100000 == 0:
+        print '.',
+      word = []
+      while True:
+        ch = f.read(1)
+        if ch == ' ':
+            word = ''.join(word)
+            break
+        if ch != '\n':
+            word.append(ch)
+      if word in vocab:
+        count += 1
+        word_vecs[word] = numpy.fromstring(f.read(binary_len), dtype='float32')
+      else:
+          f.read(binary_len)
+    print "done"
+    print "Words found in wor2vec embeddings", count
+    return word_vecs
+
+
+def logargs(func):
+    def inner(*args, **kwargs):
+        logger.info('%s : %s %s' % (func.__name__, args, kwargs))
+        return func(*args, **kwargs)
+    return inner
+
+
+def cache_word_embeddings(word_embeddings_file, cache_file):
+    if not word_embeddings_file.endswith('.gz'):
+        logger.warning( 'WARNING: expecting a .gz file. Is the {} in the correct format?'.format(word_embeddings_file))
+
+    vocab_size, vec_dim = 0, 0
+
+    if not os.path.exists(cache_file):
+        # cache does not exist
+        if not os.path.exists(os.path.dirname(cache_file)):
+            # make cache folder if needed
+            os.mkdir(os.path.dirname(cache_file))
+        
+        logger.info( 'caching the word embeddings in np.memmap format'     )   
+        wv = KeyedVectors.load_word2vec_format(word_embeddings_file, binary=True)
+        # print len(wv.syn0), wv.syn0.shape
+        # print len(wv.syn0norm) if wv.syn0norm else None
+        fp = np.memmap(cache_file, dtype=np.double, mode='w+', shape=wv.syn0.shape)
+        fp[:] = wv.syn0[:]
+        with open(cache_file + '.vocab', 'w') as f:
+            logger.info( 'writing out vocab for {}'.format(word_embeddings_file))
+            for _, w in sorted( (voc.index, word) for word, voc in wv.vocab.items()):
+                print >> f, w.encode('utf-8')
+        with open(cache_file + '.dimensions', 'w') as f:
+            logger.info( 'writing out dimensions for {}'.format(word_embeddings_file))
+            print >> f, wv.syn0.shape[0], wv.syn0.shape[1]
+        vocab_size, vec_dim  =  wv.syn0.shape
+        del fp, wv
+        print 'cached {} into {}'.format(word_embeddings_file, cache_file)
+
+    return vocab_size, vec_dim
+
+
+def load_embedding_dimensions(cache_file):
+    vocab_size, vec_dim = 0, 0
+    with open(cache_file + '.dimensions') as d:
+        vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
+    return vocab_size, vec_dim
+
+    
+def load_cached_embeddings(cache_file, vocab_list):
+    logger.debug( 'loading cached embeddings ')
+    w2v_dict = {}
+    with open(cache_file + '.dimensions') as d:
+        vocab_size, vec_dim = [int(e) for e in d.read().strip().split()]
+
+    W = np.memmap(cache_file, dtype=np.double, shape=(vocab_size, vec_dim))
+
+    with open(cache_file + '.vocab') as f:
+        logger.debug( 'loading vocab')
+        w2v_vocab_list = map(str.strip, f.readlines())
+
+    vocab_dict = {w:k for k,w in enumerate(w2v_vocab_list)}
+    
+    # Read w2v for vocab appears in Q and A
+    for word in vocab_list:
+        if word in vocab_dict:
+            w2v_dict[word] = W[vocab_dict[word]]
+        else:
+            w2v_dict[word] = np.random.uniform(-0.25, 0.25, vec_dim) 
+    return w2v_dict, vec_dim
+
+
+def read_in_dataset(dataset_folder, set_folder):
+    """
+    read in the data to return (question, sentence, label)
+    set_folder = {train|dev|test}
+    """
+    max_q = 0
+    max_s = 0
+    set_path = os.path.join(dataset_folder, set_folder)
+    len_q_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'a.toks')).readlines() ]
+    questions = [ line.strip() for line in open(os.path.join(set_path, 'a.toks')).readlines() ]
+    len_s_list =[ len(line.strip().split()) for line in open(os.path.join(set_path, 'b.toks')).readlines() ]
+    sentences = [ line.strip() for line in open(os.path.join(set_path, 'b.toks')).readlines() ]
+    labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ])
+    ext_feats = np.array([ map(float, line.strip().split(' ')) for line in open(os.path.join(set_path, 'overlap_feats.txt')).readlines() ])
+
+    #y = torch.from_numpy(labels)
+    #return questions, sentences, y
+
+    vocab = [ line.strip() for line in open(os.path.join(dataset_folder, 'vocab.txt')).readlines() ]
+    return questions, sentences, labels, vocab, max(len_q_list), max(len_s_list), ext_feats
+
+def get_test_qids_labels(dataset_folder, set_folder):
+    set_path = os.path.join(dataset_folder, set_folder)
+    qids = [ line.strip() for line in open(os.path.join(set_path, 'id.txt')).readlines() ]
+    labels = np.array([ int(line.strip()) for line in open(os.path.join(set_path, 'sim.txt')).readlines() ])
+    return qids, labels