TrecQA for MP-CNN (#77)

* Add TrecQA dataset and modularize MP-CNN infra

* Stylistic improvements

* Fix and warn about trec_eval path issue

* Update README for MP-CNN

* Update incorrect map/mrr

* MP-CNN: address code review comments

* Create common Castor pair Dataset class

* Move map and mrr computation to Castor utils

* Make map mrr utility trec_eval path more general

.gitignore

@@ -6,5 +6,6 @@ __pycache__
 *idfsim
 *.swp
 trec_eval.9.0/
+trec_eval-9.0.5
 *.pt
 text/

datasets/castor_dataset.py

@@ -0,0 +1,43 @@
+from abc import ABCMeta, abstractmethod
+import os
+
+from torchtext.data.dataset import Dataset
+from torchtext.data.example import Example
+from torchtext.data.field import Field
+
+from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features
+
+
+class CastorPairDataset(Dataset, metaclass=ABCMeta):
+
+    # Child classes must define
+    NAME = None
+    NUM_CLASSES = None
+    ID_FIELD = None
+    TEXT_FIELD = None
+    EXT_FEATS_FIELD = None
+    LABEL_FIELD = None
+
+    @abstractmethod
+    def __init__(self, path):
+        """
+        Create a Castor dataset involving pairs of texts
+        """
+        fields = [('id', self.ID_FIELD), ('sentence_1', self.TEXT_FIELD), ('sentence_2', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)]
+
+        examples = []
+        with open(os.path.join(path, 'a.toks'), 'r') as f1, open(os.path.join(path, 'b.toks'), 'r') as f2:
+            sent_list_1 = [l.rstrip('.\n').split(' ') for l in f1]
+            sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2]
+
+        word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2)
+        overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt)
+
+        with open(os.path.join(path, 'id.txt'), 'r') as id_file, open(os.path.join(path, 'sim.txt'), 'r') as label_file:
+            for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file):
+                pair_id = pair_id.rstrip('.\n')
+                label = label.rstrip('.\n')
+                example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields)
+                examples.append(example)
+
+        super(CastorPairDataset, self).__init__(examples, fields)

datasets/msrvid.py

@@ -3,13 +3,13 @@ import os
 
 import numpy as np
 import torch
-from torchtext.data.dataset import Dataset
 from torchtext.data.example import Example
 from torchtext.data.field import Field
 from torchtext.data.iterator import BucketIterator
 from torchtext.data.pipeline import Pipeline
 from torchtext.vocab import Vectors
 
+from datasets.castor_dataset import CastorPairDataset
 from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features
 
 
@@ -28,7 +28,7 @@ def get_class_probs(sim, *args):
     return class_probs
 
 
-class MSRVID(Dataset):
+class MSRVID(CastorPairDataset):
     NAME = 'msrvid'
     NUM_CLASSES = 6
     ID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
@@ -38,35 +38,13 @@ class MSRVID(Dataset):
 
     @staticmethod
     def sort_key(ex):
-        return len(ex.a)
+        return len(ex.sentence_1)
 
     def __init__(self, path):
         """
         Create a MSRVID dataset instance
         """
-        fields = [('id', self.ID_FIELD), ('a', self.TEXT_FIELD), ('b', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)]
-
-        examples = []
-        f1 = open(os.path.join(path, 'a.txt'), 'r')
-        f2 = open(os.path.join(path, 'b.txt'), 'r')
-        id_file = open(os.path.join(path, 'id.txt'), 'r')
-        label_file = open(os.path.join(path, 'sim.txt'), 'r')
-
-        sent_list_1 = [l.rstrip('.\n').split(' ') for l in f1]
-        sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2]
-
-        word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2)
-        overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt)
-
-        for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file):
-            pair_id = pair_id.rstrip('.\n')
-            label = label.rstrip('.\n')
-            example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields)
-            examples.append(example)
-
-        map(lambda f: f.close(), [f1, f2, label_file])
-
-        super(MSRVID, self).__init__(examples, fields)
+        super(MSRVID, self).__init__(path)
 
     @classmethod
     def splits(cls, path, train='train', test='test', **kwargs):

datasets/sick.py

@@ -3,13 +3,13 @@ import os
 
 import numpy as np
 import torch
-from torchtext.data.dataset import Dataset
 from torchtext.data.example import Example
 from torchtext.data.field import Field
 from torchtext.data.iterator import BucketIterator
 from torchtext.data.pipeline import Pipeline
 from torchtext.vocab import Vectors
 
+from datasets.castor_dataset import CastorPairDataset
 from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features
 
 
@@ -28,7 +28,7 @@ def get_class_probs(sim, *args):
     return class_probs
 
 
-class SICK(Dataset):
+class SICK(CastorPairDataset):
     NAME = 'sick'
     NUM_CLASSES = 5
     ID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
@@ -38,35 +38,13 @@ class SICK(Dataset):
 
     @staticmethod
     def sort_key(ex):
-        return len(ex.a)
+        return len(ex.sentence_1)
 
     def __init__(self, path):
         """
         Create a SICK dataset instance
         """
-        fields = [('id', self.ID_FIELD), ('a', self.TEXT_FIELD), ('b', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)]
-
-        examples = []
-        f1 = open(os.path.join(path, 'a.txt'), 'r')
-        f2 = open(os.path.join(path, 'b.txt'), 'r')
-        id_file = open(os.path.join(path, 'id.txt'), 'r')
-        label_file = open(os.path.join(path, 'sim.txt'), 'r')
-
-        sent_list_1 = [l.rstrip('.\n').split(' ') for l in f1]
-        sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2]
-
-        word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2)
-        overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt)
-
-        for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file):
-            pair_id = pair_id.rstrip('.\n')
-            label = label.rstrip('.\n')
-            example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields)
-            examples.append(example)
-
-        map(lambda f: f.close(), [f1, f2, label_file])
-
-        super(SICK, self).__init__(examples, fields)
+        super(SICK, self).__init__(path)
 
     @classmethod
     def splits(cls, path, train='train', validation='dev', test='test', **kwargs):

datasets/trecqa.py

@@ -0,0 +1,54 @@
+import os
+
+import torch
+from torchtext.data.example import Example
+from torchtext.data.field import Field
+from torchtext.data.iterator import BucketIterator
+from torchtext.vocab import Vectors
+
+from datasets.castor_dataset import CastorPairDataset
+from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features
+
+
+class TRECQA(CastorPairDataset):
+    NAME = 'trecqa'
+    NUM_CLASSES = 2
+    ID_FIELD = Field(sequential=False, tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True)
+    TEXT_FIELD = Field(batch_first=True, tokenize=lambda x: x)  # tokenizer is identity since we already tokenized it to compute external features
+    EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x)
+    LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
+
+    @staticmethod
+    def sort_key(ex):
+        return len(ex.sentence_1)
+
+    def __init__(self, path):
+        """
+        Create a TRECQA dataset instance
+        """
+        super(TRECQA, self).__init__(path)
+
+    @classmethod
+    def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs):
+        return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs)
+
+    @classmethod
+    def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, unk_init=torch.Tensor.zero_):
+        """
+        :param path: directory containing train, test, dev files
+        :param vectors_name: name of word vectors file
+        :param vectors_cache: directory containing word vectors file
+        :param batch_size: batch size
+        :param device: GPU device
+        :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes
+        :param unk_init: function used to generate vector for OOV words
+        :return:
+        """
+        if vectors is None:
+            vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init)
+
+        train, validation, test = cls.splits(path)
+
+        cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors)
+
+        return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device)

mp_cnn/README.md

@@ -20,6 +20,8 @@ Directory layout should be like this:
 │   └── GloVe/
 ```
 
+## SICK Dataset
+
 To run MP-CNN on the SICK dataset, use the following command. `--dropout 0` is for mimicking the original paper, although adding dropout can improve performance. If you have any problems running it check the Troubleshooting section below.
 
 ```
@@ -29,7 +31,9 @@ python main.py mpcnn.sick.model.castor --dataset sick --epochs 19 --epsilon 1e-7
 | Implementation and config        | Pearson's r   | Spearman's p  |
 | -------------------------------- |:-------------:|:-------------:|
 | Paper                            | 0.8686        |   0.8047      |
-| PyTorch using above config       | 0.8763        |   0.8215      |
+| PyTorch using above config       | 0.8684        |   0.8083      |
+
+## MSRVID Dataset
 
 To run MP-CNN on the MSRVID dataset, use the following command:
 ```
@@ -39,7 +43,23 @@ python main.py mpcnn.msrvid.model.castor --dataset msrvid --batch-size 16 --epsi
 | Implementation and config        | Pearson's r   |
 | -------------------------------- |:-------------:|
 | Paper                            | 0.9090        |
-| PyTorch using above config       | 0.9050        |
+| PyTorch using above config       | 0.8911        |
+
+## TrecQA Dataset
+
+To run MP-CNN on (Raw) TrecQA, you first need to run `./get_trec_eval.sh` in `utils` under the repo root while inside the `utils` directory. This will download and compile the official `trec_eval` tool used for evaluation.
+
+Then, you can run:
+```
+python main.py mpcnn.trecqa.model --dataset trecqa --epochs 5 --regularization 0.0005 --dropout 0.5 --eps 0.1
+```
+
+| Implementation and config        | map    | mrr    |
+| -------------------------------- |:------:|:------:|
+| Paper                            | 0.762  | 0.830  |
+| PyTorch using above config       | 0.7904 | 0.8223 |
+
+The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872).
 
 These are not the optimal hyperparameters but they are decent. This README will be updated with more optimal hyperparameters and results in the future.
 

mp_cnn/dataset.py

@@ -1,30 +1,14 @@
-from collections import defaultdict
-from enum import Enum
-import math
 import os
 
-import numpy as np
 import torch
-from torch.autograd import Variable
 import torch.nn as nn
-import torch.utils.data as data
 
 from datasets.sick import SICK
 from datasets.msrvid import MSRVID
-
-# logging setup
-import logging
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-ch = logging.StreamHandler()
-ch.setLevel(logging.DEBUG)
-formatter = logging.Formatter('%(levelname)s - %(message)s')
-ch.setFormatter(formatter)
-logger.addHandler(ch)
+from datasets.trecqa import TRECQA
 
 
-class UnknownWorcVecCache(object):
+class UnknownWordVecCache(object):
     """
     Caches the first randomly generated word vector for a certain size to make it is reused.
     """
@@ -47,7 +31,7 @@ class MPCNNDatasetFactory(object):
     def get_dataset(dataset_name, word_vectors_dir, word_vectors_file, batch_size, device):
         if dataset_name == 'sick':
             dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'sick/')
-            train_loader, dev_loader, test_loader = SICK.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWorcVecCache.unk)
+            train_loader, dev_loader, test_loader = SICK.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
             embedding_dim = SICK.TEXT_FIELD.vocab.vectors.size()
             embedding = nn.Embedding(embedding_dim[0], embedding_dim[1])
             embedding.weight = nn.Parameter(SICK.TEXT_FIELD.vocab.vectors)
@@ -55,11 +39,20 @@ class MPCNNDatasetFactory(object):
         elif dataset_name == 'msrvid':
             dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'msrvid/')
             dev_loader = None
-            train_loader, test_loader = MSRVID.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWorcVecCache.unk)
+            train_loader, test_loader = MSRVID.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
             embedding_dim = MSRVID.TEXT_FIELD.vocab.vectors.size()
             embedding = nn.Embedding(embedding_dim[0], embedding_dim[1])
             embedding.weight = nn.Parameter(MSRVID.TEXT_FIELD.vocab.vectors)
             return MSRVID, embedding, train_loader, test_loader, dev_loader
+        elif dataset_name == 'trecqa':
+            if not os.path.exists('../utils/trec_eval-9.0.5/trec_eval'):
+                raise FileNotFoundError('TrecQA requires the trec_eval tool to run. Please run get_trec_eval.sh inside Castor/utils (as working directory) before continuing.')
+            dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'TrecQA/')
+            train_loader, dev_loader, test_loader = TRECQA.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
+            embedding_dim = TRECQA.TEXT_FIELD.vocab.vectors.size()
+            embedding = nn.Embedding(embedding_dim[0], embedding_dim[1])
+            embedding.weight = nn.Parameter(TRECQA.TEXT_FIELD.vocab.vectors)
+            return TRECQA, embedding, train_loader, test_loader, dev_loader
         else:
             raise ValueError('{} is not a valid dataset.'.format(dataset_name))
 

mp_cnn/evaluation.py

@@ -1,117 +1,29 @@
-from scipy.stats import pearsonr, spearmanr
-import torch
-import torch.nn.functional as F
-from torch.autograd import Variable
+from mp_cnn.evaluators.sick_evaluator import SICKEvaluator
+from mp_cnn.evaluators.msrvid_evaluator import MSRVIDEvaluator
+from mp_cnn.evaluators.trecqa_evaluator import TRECQAEvaluator
 
 
 class MPCNNEvaluatorFactory(object):
     """
     Get the corresponding Evaluator class for a particular dataset.
     """
+    evaluator_map = {
+        'sick': SICKEvaluator,
+        'msrvid': MSRVIDEvaluator,
+        'trecqa': TRECQAEvaluator
+    }
+
     @staticmethod
     def get_evaluator(dataset_cls, model, data_loader, batch_size, device):
         if data_loader is None:
             return None
 
-        if hasattr(dataset_cls, 'NAME') and dataset_cls.NAME == 'sick':
-            return SICKEvaluator(dataset_cls, model, data_loader, batch_size, device)
-        elif hasattr(dataset_cls, 'NAME') and dataset_cls.NAME == 'msrvid':
-            return MSRVIDEvaluator(dataset_cls, model, data_loader, batch_size, device)
-        else:
-            raise ValueError('{} is not a valid dataset.'.format(dataset_cls))
+        if not hasattr(dataset_cls, 'NAME'):
+            raise ValueError('Invalid dataset. Dataset should have NAME attribute.')
 
+        if dataset_cls.NAME not in MPCNNEvaluatorFactory.evaluator_map:
+            raise ValueError('{} is not implemented.'.format(dataset_cls))
 
-class Evaluator(object):
-    """
-    Evaluates performance of model on a Dataset, using metrics specific to the Dataset.
-    """
-
-    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
-        self.dataset_cls = dataset_cls
-        self.model = model
-        self.data_loader = data_loader
-        self.batch_size = batch_size
-        self.device = device
-
-    def get_scores(self):
-        """
-        Get the scores used to evaluate the model.
-        Should return ([score1, score2, ..], [score1_name, score2_name, ...]).
-        The first score is the primary score used to determine if the model has improved.
-        """
-        raise NotImplementedError('Evaluator subclass needs to implement get_score')
-
-
-class SICKEvaluator(Evaluator):
-
-    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
-        super(SICKEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device)
-
-    def get_scores(self):
-        self.model.eval()
-        num_classes = self.dataset_cls.NUM_CLASSES
-        predict_classes = torch.arange(1, num_classes + 1).expand(self.batch_size, num_classes)
-        test_kl_div_loss = 0
-        predictions = []
-        true_labels = []
-
-        for batch in self.data_loader:
-            output = self.model(batch.a, batch.b, batch.ext_feats)
-            test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0]
-            # handle last batch which might have smaller size
-            if len(predict_classes) != len(batch.a):
-                predict_classes = torch.arange(1, num_classes + 1).expand(len(batch.a), num_classes)
-
-            if self.data_loader.device != -1:
-                with torch.cuda.device(self.device):
-                    predict_classes = predict_classes.cuda()
-
-            true_labels.append((predict_classes * batch.label.data).sum(dim=1))
-            predictions.append((predict_classes * output.data.exp()).sum(dim=1))
-
-            del output
-
-        predictions = torch.cat(predictions).cpu().numpy()
-        true_labels = torch.cat(true_labels).cpu().numpy()
-        test_kl_div_loss /= len(batch.dataset.examples)
-        pearson_r = pearsonr(predictions, true_labels)[0]
-        spearman_r = spearmanr(predictions, true_labels)[0]
-
-        return [pearson_r, spearman_r, test_kl_div_loss], ['pearson_r', 'spearman_r', 'KL-divergence loss']
-
-
-class MSRVIDEvaluator(Evaluator):
-
-    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
-        super(MSRVIDEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device)
-
-    def get_scores(self):
-        self.model.eval()
-        num_classes = self.dataset_cls.NUM_CLASSES
-        predict_classes = torch.arange(0, num_classes).expand(self.batch_size, num_classes)
-        test_kl_div_loss = 0
-        predictions = []
-        true_labels = []
-
-        for batch in self.data_loader:
-            output = self.model(batch.a, batch.b, batch.ext_feats)
-            test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0]
-            # handle last batch which might have smaller size
-            if len(predict_classes) != len(batch.a):
-                predict_classes = torch.arange(0, num_classes).expand(len(batch.a), num_classes)
-
-            if self.data_loader.device != -1:
-                with torch.cuda.device(self.device):
-                    predict_classes = predict_classes.cuda()
-
-            true_labels.append((predict_classes * batch.label.data).sum(dim=1))
-            predictions.append((predict_classes * output.data.exp()).sum(dim=1))
-
-            del output
-
-        predictions = torch.cat(predictions).cpu().numpy()
-        true_labels = torch.cat(true_labels).cpu().numpy()
-        test_kl_div_loss /= len(batch.dataset.examples)
-        pearson_r = pearsonr(predictions, true_labels)[0]
-
-        return [pearson_r, test_kl_div_loss], ['pearson_r', 'KL-divergence loss']
+        return MPCNNEvaluatorFactory.evaluator_map[dataset_cls.NAME](
+            dataset_cls, model, data_loader, batch_size, device
+        )

mp_cnn/evaluators/evaluator.py

@@ -0,0 +1,19 @@
+class Evaluator(object):
+    """
+    Evaluates performance of model on a Dataset, using metrics specific to the Dataset.
+    """
+
+    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
+        self.dataset_cls = dataset_cls
+        self.model = model
+        self.data_loader = data_loader
+        self.batch_size = batch_size
+        self.device = device
+
+    def get_scores(self):
+        """
+        Get the scores used to evaluate the model.
+        Should return ([score1, score2, ..], [score1_name, score2_name, ...]).
+        The first score is the primary score used to determine if the model has improved.
+        """
+        raise NotImplementedError('Evaluator subclass needs to implement get_score')

mp_cnn/evaluators/msrvid_evaluator.py

@@ -0,0 +1,42 @@
+from scipy.stats import pearsonr
+import torch
+import torch.nn.functional as F
+
+from mp_cnn.evaluators.evaluator import Evaluator
+
+
+class MSRVIDEvaluator(Evaluator):
+
+    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
+        super(MSRVIDEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device)
+
+    def get_scores(self):
+        self.model.eval()
+        num_classes = self.dataset_cls.NUM_CLASSES
+        predict_classes = torch.arange(0, num_classes).expand(self.batch_size, num_classes)
+        test_kl_div_loss = 0
+        predictions = []
+        true_labels = []
+
+        for batch in self.data_loader:
+            output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0]
+            # handle last batch which might have smaller size
+            if len(predict_classes) != len(batch.sentence_1):
+                predict_classes = torch.arange(0, num_classes).expand(len(batch.sentence_1), num_classes)
+
+            if self.data_loader.device != -1:
+                with torch.cuda.device(self.device):
+                    predict_classes = predict_classes.cuda()
+
+            true_labels.append((predict_classes * batch.label.data).sum(dim=1))
+            predictions.append((predict_classes * output.data.exp()).sum(dim=1))
+
+            del output
+
+        predictions = torch.cat(predictions).cpu().numpy()
+        true_labels = torch.cat(true_labels).cpu().numpy()
+        test_kl_div_loss /= len(batch.dataset.examples)
+        pearson_r = pearsonr(predictions, true_labels)[0]
+
+        return [pearson_r, test_kl_div_loss], ['pearson_r', 'KL-divergence loss']

mp_cnn/evaluators/sick_evaluator.py

@@ -0,0 +1,43 @@
+from scipy.stats import pearsonr, spearmanr
+import torch
+import torch.nn.functional as F
+
+from mp_cnn.evaluators.evaluator import Evaluator
+
+
+class SICKEvaluator(Evaluator):
+
+    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
+        super(SICKEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device)
+
+    def get_scores(self):
+        self.model.eval()
+        num_classes = self.dataset_cls.NUM_CLASSES
+        predict_classes = torch.arange(1, num_classes + 1).expand(self.batch_size, num_classes)
+        test_kl_div_loss = 0
+        predictions = []
+        true_labels = []
+
+        for batch in self.data_loader:
+            output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0]
+            # handle last batch which might have smaller size
+            if len(predict_classes) != len(batch.sentence_1):
+                predict_classes = torch.arange(1, num_classes + 1).expand(len(batch.sentence_1), num_classes)
+
+            if self.data_loader.device != -1:
+                with torch.cuda.device(self.device):
+                    predict_classes = predict_classes.cuda()
+
+            true_labels.append((predict_classes * batch.label.data).sum(dim=1))
+            predictions.append((predict_classes * output.data.exp()).sum(dim=1))
+
+            del output
+
+        predictions = torch.cat(predictions).cpu().numpy()
+        true_labels = torch.cat(true_labels).cpu().numpy()
+        test_kl_div_loss /= len(batch.dataset.examples)
+        pearson_r = pearsonr(predictions, true_labels)[0]
+        spearman_r = spearmanr(predictions, true_labels)[0]
+
+        return [pearson_r, spearman_r, test_kl_div_loss], ['pearson_r', 'spearman_r', 'KL-divergence loss']

mp_cnn/evaluators/trecqa_evaluator.py

@@ -0,0 +1,34 @@
+import torch.nn.functional as F
+
+from mp_cnn.evaluators.evaluator import Evaluator
+from utils.relevancy_metrics import get_map_mrr
+
+
+class TRECQAEvaluator(Evaluator):
+
+    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
+        super(TRECQAEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device)
+
+    def get_scores(self):
+        self.model.eval()
+        test_cross_entropy_loss = 0
+        qids = []
+        true_labels = []
+        predictions = []
+
+        for batch in self.data_loader:
+            qids.extend(batch.id.data.cpu().numpy())
+            output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            test_cross_entropy_loss += F.cross_entropy(output, batch.label, size_average=False).data[0]
+
+            true_labels.extend(batch.label.data.cpu().numpy())
+            predictions.extend(output.data.exp()[:, 1].cpu().numpy())
+
+            del output
+
+        qids = list(map(lambda n: int(round(n * 10, 0)) / 10, qids))
+
+        mean_average_precision, mean_reciprocal_rank = get_map_mrr(qids, predictions, true_labels, self.data_loader.device)
+        test_cross_entropy_loss /= len(batch.dataset.examples)
+
+        return [test_cross_entropy_loss, mean_average_precision, mean_reciprocal_rank], ['cross entropy loss', 'map', 'mrr']

mp_cnn/main.py

@@ -1,32 +1,23 @@
 import argparse
+import logging
 import os
+import pprint
 import random
 
 import numpy as np
 import torch
 import torch.optim as optim
 
-from dataset import MPCNNDatasetFactory
-from evaluation import MPCNNEvaluatorFactory
-from model import MPCNN
-from train import MPCNNTrainerFactory
-
-# logging setup
-import logging
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-ch = logging.StreamHandler()
-ch.setLevel(logging.DEBUG)
-formatter = logging.Formatter('%(levelname)s - %(message)s')
-ch.setFormatter(formatter)
-logger.addHandler(ch)
+from mp_cnn.dataset import MPCNNDatasetFactory
+from mp_cnn.evaluation import MPCNNEvaluatorFactory
+from mp_cnn.model import MPCNN
+from mp_cnn.train import MPCNNTrainerFactory
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN')
     parser.add_argument('model_outfile', help='file to save final model')
-    parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid]', default='sick')
+    parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid, trecqa]', default='sick')
     parser.add_argument('--word-vectors-dir', help='word vectors directory', default=os.path.join(os.pardir, os.pardir, 'data', 'GloVe'))
     parser.add_argument('--word-vectors-file', help='word vectors filename', default='glove.840B.300d.txt')
     parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true')
@@ -58,6 +49,18 @@ if __name__ == '__main__':
     if args.device != -1:
         torch.cuda.manual_seed(args.seed)
 
+    # logging setup
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.INFO)
+
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    logger.info(pprint.pformat(vars(args)))
+
     dataset_cls, embedding, train_loader, test_loader, dev_loader \
         = MPCNNDatasetFactory.get_dataset(args.dataset, args.word_vectors_dir, args.word_vectors_file, args.batch_size, args.device)
 
@@ -89,7 +92,8 @@ if __name__ == '__main__':
         'lr_reduce_factor': args.lr_reduce_factor,
         'patience': args.patience,
         'tensorboard': args.tensorboard,
-        'run_label': args.run_label
+        'run_label': args.run_label,
+        'logger': logger
     }
     trainer = MPCNNTrainerFactory.get_trainer(args.dataset, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
 

mp_cnn/train.py

@@ -1,242 +1,23 @@
-import math
-import time
-
-import torch
-import torch.nn.functional as F
-from torch.autograd import Variable
-from torch.optim.lr_scheduler import ReduceLROnPlateau
-from scipy.stats import pearsonr, spearmanr
-
-# logging setup
-import logging
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-ch = logging.StreamHandler()
-ch.setLevel(logging.DEBUG)
-formatter = logging.Formatter('%(levelname)s - %(message)s')
-ch.setFormatter(formatter)
-logger.addHandler(ch)
+from mp_cnn.trainers.sick_trainer import SICKTrainer
+from mp_cnn.trainers.msrvid_trainer import MSRVIDTrainer
+from mp_cnn.trainers.trecqa_trainer import TRECQATrainer
 
 
 class MPCNNTrainerFactory(object):
     """
     Get the corresponding Trainer class for a particular dataset.
     """
+    trainer_map = {
+        'sick': SICKTrainer,
+        'msrvid': MSRVIDTrainer,
+        'trecqa': TRECQATrainer
+    }
+
     @staticmethod
     def get_trainer(dataset_name, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
-        if dataset_name == 'sick':
-            return SICKTrainer(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
-        elif dataset_name == 'msrvid':
-            return MSRVIDTrainer(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
-        else:
-            raise ValueError('{} is not a valid dataset.'.format(dataset_name))
+        if dataset_name not in MPCNNTrainerFactory.trainer_map:
+            raise ValueError('{} is not implemented.'.format(dataset_name))
 
-
-class Trainer(object):
-
-    """
-    Abstraction for training a model on a Dataset.
-    """
-
-    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
-        self.model = model
-        self.optimizer = trainer_config['optimizer']
-        self.train_loader = train_loader
-        self.batch_size = trainer_config['batch_size']
-        self.log_interval = trainer_config['log_interval']
-        self.model_outfile = trainer_config['model_outfile']
-        self.lr_reduce_factor = trainer_config['lr_reduce_factor']
-        self.patience = trainer_config['patience']
-        self.use_tensorboard = trainer_config['tensorboard']
-        if self.use_tensorboard:
-            from tensorboardX import SummaryWriter
-            self.writer = SummaryWriter(log_dir=None, comment='' if trainer_config['run_label'] is None else trainer_config['run_label'])
-
-        self.train_evaluator = train_evaluator
-        self.test_evaluator = test_evaluator
-        self.dev_evaluator = dev_evaluator
-
-    def evaluate(self, evaluator, dataset_name):
-        scores, metric_names = evaluator.get_scores()
-        logger.info('Evaluation metrics for {}:'.format(dataset_name))
-        logger.info('\t'.join([' '] + metric_names))
-        logger.info('\t'.join([dataset_name] + list(map(str, scores))))
-        return scores
-
-    def train_epoch(self, epoch):
-        raise NotImplementedError()
-
-    def train(self, epochs):
-        raise NotImplementedError()
-
-
-class SICKTrainer(Trainer):
-
-    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
-        super(SICKTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
-
-    def train_epoch(self, epoch):
-        self.model.train()
-        total_loss = 0
-        for batch_idx, batch in enumerate(self.train_loader):
-            self.optimizer.zero_grad()
-            output = self.model(batch.a, batch.b, batch.ext_feats)
-            loss = F.kl_div(output, batch.label)
-            total_loss += loss.data[0]
-            loss.backward()
-            self.optimizer.step()
-            if batch_idx % self.log_interval == 0:
-                logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
-                    epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)),
-                    len(batch.dataset.examples),
-                    100. * batch_idx / (len(self.train_loader)), loss.data[0])
-                )
-
-        if self.use_tensorboard:
-            self.writer.add_scalar('sick/train/kl_div_loss', total_loss, epoch)
-
-        return total_loss
-
-    def train(self, epochs):
-        scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience)
-        epoch_times = []
-        prev_loss = -1
-        best_dev_score = -1
-        for epoch in range(1, epochs + 1):
-            start = time.time()
-            logger.info('Epoch {} started...'.format(epoch))
-            self.train_epoch(epoch)
-
-            dev_scores = self.evaluate(self.dev_evaluator, 'dev')
-            new_loss = dev_scores[2]
-
-            if self.use_tensorboard:
-                self.writer.add_scalar('sick/lr', self.optimizer.param_groups[0]['lr'], epoch)
-                self.writer.add_scalar('sick/dev/pearson_r', dev_scores[0], epoch)
-                self.writer.add_scalar('sick/dev/kl_div_loss', new_loss, epoch)
-
-            end = time.time()
-            duration = end - start
-            logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
-            epoch_times.append(duration)
-
-            if dev_scores[0] > best_dev_score:
-                best_dev_score = dev_scores[0]
-                torch.save(self.model, self.model_outfile)
-
-            if abs(prev_loss - new_loss) <= 0.0002:
-                logger.info('Early stopping. Loss changed by less than 0.0002.')
-                break
-
-            prev_loss = new_loss
-            scheduler.step(dev_scores[0])
-
-        logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))
-
-
-class MSRVIDTrainer(Trainer):
-
-    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
-        super(MSRVIDTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
-
-    def train_epoch(self, epoch):
-        self.model.train()
-        total_loss = 0
-
-        # since MSRVID doesn't have validation set, we manually leave-out some training data for validation
-        batches = math.ceil(len(self.train_loader.dataset.examples) / self.batch_size)
-        start_val_batch = math.floor(0.8 * batches)
-        left_out_val_a, left_out_val_b = [], []
-        left_out_val_ext_feats = []
-        left_out_val_labels = []
-
-        for batch_idx, batch in enumerate(self.train_loader):
-            if batch_idx >= start_val_batch:
-                left_out_val_a.append(batch.a)
-                left_out_val_b.append(batch.b)
-                left_out_val_ext_feats.append(batch.ext_feats)
-                left_out_val_labels.append(batch.label)
-                continue
-            self.optimizer.zero_grad()
-            output = self.model(batch.a, batch.b, batch.ext_feats)
-            loss = F.kl_div(output, batch.label)
-            total_loss += loss.data[0]
-            loss.backward()
-            self.optimizer.step()
-            if batch_idx % self.log_interval == 0:
-                logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
-                    epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)),
-                    len(batch.dataset.examples),
-                    100. * batch_idx / (len(self.train_loader)), loss.data[0])
-                )
-
-        self.evaluate(self.train_evaluator, 'train')
-
-        if self.use_tensorboard:
-            self.writer.add_scalar('msrvid/train/kl_div_loss', total_loss, epoch)
-
-        return left_out_val_a, left_out_val_b, left_out_val_ext_feats, left_out_val_labels
-
-    def train(self, epochs):
-        scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience)
-        epoch_times = []
-        prev_loss = -1
-        best_dev_score = -1
-        for epoch in range(1, epochs + 1):
-            start = time.time()
-            logger.info('Epoch {} started...'.format(epoch))
-            left_out_a, left_out_b, left_out_ext_feats, left_out_label = self.train_epoch(epoch)
-
-            # manually evaluating the validating set
-            all_predictions, all_true_labels = [], []
-            val_kl_div_loss = 0
-            for i in range(len(left_out_a)):
-                output = self.model(left_out_a[i], left_out_b[i], left_out_ext_feats[i])
-                val_kl_div_loss += F.kl_div(output, left_out_label[i], size_average=False).data[0]
-                predict_classes = torch.arange(0, self.train_loader.dataset.NUM_CLASSES).expand(len(left_out_a[i]), self.train_loader.dataset.NUM_CLASSES)
-                if self.train_loader.device != -1:
-                    with torch.cuda.device(self.train_loader.device):
-                        predict_classes = predict_classes.cuda()
-
-                predictions = (predict_classes * output.data.exp()).sum(dim=1)
-                true_labels = (predict_classes * left_out_label[i].data).sum(dim=1)
-                all_predictions.append(predictions)
-                all_true_labels.append(true_labels)
-
-            predictions = torch.cat(all_predictions).cpu().numpy()
-            true_labels = torch.cat(all_true_labels).cpu().numpy()
-            pearson_r = pearsonr(predictions, true_labels)[0]
-            val_kl_div_loss /= len(predictions)
-
-            if self.use_tensorboard:
-                self.writer.add_scalar('msrvid/dev/pearson_r', pearson_r, epoch)
-
-            for param_group in self.optimizer.param_groups:
-                logger.info('Validation size: %s Pearson\'s r: %s', output.size()[0], pearson_r)
-                logger.info('Learning rate: %s', param_group['lr'])
-
-                if self.use_tensorboard:
-                    self.writer.add_scalar('msrvid/lr', param_group['lr'], epoch)
-                    self.writer.add_scalar('msrvid/dev/kl_div_loss', val_kl_div_loss, epoch)
-                break
-
-            scheduler.step(pearson_r)
-
-            end = time.time()
-            duration = end - start
-            logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
-            epoch_times.append(duration)
-
-            if pearson_r > best_dev_score:
-                best_dev_score = pearson_r
-                torch.save(self.model, self.model_outfile)
-
-            if abs(prev_loss - val_kl_div_loss) <= 0.0005:
-                logger.info('Early stopping. Loss changed by less than 0.0005.')
-                break
-
-            prev_loss = val_kl_div_loss
-            self.evaluate(self.test_evaluator, 'test')
-
-        logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))
+        return MPCNNTrainerFactory.trainer_map[dataset_name](
+            model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator
+        )

mp_cnn/trainers/msrvid_trainer.py

@@ -0,0 +1,117 @@
+import math
+import time
+
+import torch
+import torch.nn.functional as F
+from torch.optim.lr_scheduler import ReduceLROnPlateau
+from scipy.stats import pearsonr
+
+from mp_cnn.trainers.trainer import Trainer
+
+
+class MSRVIDTrainer(Trainer):
+
+    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
+        super(MSRVIDTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
+
+    def train_epoch(self, epoch):
+        self.model.train()
+        total_loss = 0
+
+        # since MSRVID doesn't have validation set, we manually leave-out some training data for validation
+        batches = math.ceil(len(self.train_loader.dataset.examples) / self.batch_size)
+        start_val_batch = math.floor(0.8 * batches)
+        left_out_val_a, left_out_val_b = [], []
+        left_out_val_ext_feats = []
+        left_out_val_labels = []
+
+        for batch_idx, batch in enumerate(self.train_loader):
+            # msrvid does not contain a validation set, we leave out some training data for validation to do model selection
+            if batch_idx >= start_val_batch:
+                left_out_val_a.append(batch.sentence_1)
+                left_out_val_b.append(batch.sentence_2)
+                left_out_val_ext_feats.append(batch.ext_feats)
+                left_out_val_labels.append(batch.label)
+                continue
+            self.optimizer.zero_grad()
+            output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            loss = F.kl_div(output, batch.label)
+            total_loss += loss.data[0]
+            loss.backward()
+            self.optimizer.step()
+            if batch_idx % self.log_interval == 0:
+                self.logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
+                    epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)),
+                    len(batch.dataset.examples),
+                    100. * batch_idx / (len(self.train_loader)), loss.data[0])
+                )
+
+        self.evaluate(self.train_evaluator, 'train')
+
+        if self.use_tensorboard:
+            self.writer.add_scalar('msrvid/train/kl_div_loss', total_loss, epoch)
+
+        return left_out_val_a, left_out_val_b, left_out_val_ext_feats, left_out_val_labels
+
+    def train(self, epochs):
+        scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience)
+        epoch_times = []
+        prev_loss = -1
+        best_dev_score = -1
+        for epoch in range(1, epochs + 1):
+            start = time.time()
+            self.logger.info('Epoch {} started...'.format(epoch))
+            left_out_a, left_out_b, left_out_ext_feats, left_out_label = self.train_epoch(epoch)
+
+            # manually evaluating the validating set
+            all_predictions, all_true_labels = [], []
+            val_kl_div_loss = 0
+            for i in range(len(left_out_a)):
+                output = self.model(left_out_a[i], left_out_b[i], left_out_ext_feats[i])
+                val_kl_div_loss += F.kl_div(output, left_out_label[i], size_average=False).data[0]
+                predict_classes = torch.arange(0, self.train_loader.dataset.NUM_CLASSES).expand(len(left_out_a[i]), self.train_loader.dataset.NUM_CLASSES)
+                if self.train_loader.device != -1:
+                    with torch.cuda.device(self.train_loader.device):
+                        predict_classes = predict_classes.cuda()
+
+                predictions = (predict_classes * output.data.exp()).sum(dim=1)
+                true_labels = (predict_classes * left_out_label[i].data).sum(dim=1)
+                all_predictions.append(predictions)
+                all_true_labels.append(true_labels)
+
+            predictions = torch.cat(all_predictions).cpu().numpy()
+            true_labels = torch.cat(all_true_labels).cpu().numpy()
+            pearson_r = pearsonr(predictions, true_labels)[0]
+            val_kl_div_loss /= len(predictions)
+
+            if self.use_tensorboard:
+                self.writer.add_scalar('msrvid/dev/pearson_r', pearson_r, epoch)
+
+            for param_group in self.optimizer.param_groups:
+                self.logger.info('Validation size: %s Pearson\'s r: %s', output.size()[0], pearson_r)
+                self.logger.info('Learning rate: %s', param_group['lr'])
+
+                if self.use_tensorboard:
+                    self.writer.add_scalar('msrvid/lr', param_group['lr'], epoch)
+                    self.writer.add_scalar('msrvid/dev/kl_div_loss', val_kl_div_loss, epoch)
+                break
+
+            scheduler.step(pearson_r)
+
+            end = time.time()
+            duration = end - start
+            self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
+            epoch_times.append(duration)
+
+            if pearson_r > best_dev_score:
+                best_dev_score = pearson_r
+                torch.save(self.model, self.model_outfile)
+
+            if abs(prev_loss - val_kl_div_loss) <= 0.0005:
+                self.logger.info('Early stopping. Loss changed by less than 0.0005.')
+                break
+
+            prev_loss = val_kl_div_loss
+            self.evaluate(self.test_evaluator, 'test')
+
+        self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))

mp_cnn/trainers/sick_trainer.py

@@ -0,0 +1,71 @@
+import time
+
+import torch
+import torch.nn.functional as F
+from torch.optim.lr_scheduler import ReduceLROnPlateau
+
+from mp_cnn.trainers.trainer import Trainer
+
+
+class SICKTrainer(Trainer):
+
+    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
+        super(SICKTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
+
+    def train_epoch(self, epoch):
+        self.model.train()
+        total_loss = 0
+        for batch_idx, batch in enumerate(self.train_loader):
+            self.optimizer.zero_grad()
+            output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            loss = F.kl_div(output, batch.label)
+            total_loss += loss.data[0]
+            loss.backward()
+            self.optimizer.step()
+            if batch_idx % self.log_interval == 0:
+                self.logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
+                    epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)),
+                    len(batch.dataset.examples),
+                    100. * batch_idx / (len(self.train_loader)), loss.data[0])
+                )
+
+        if self.use_tensorboard:
+            self.writer.add_scalar('sick/train/kl_div_loss', total_loss, epoch)
+
+        return total_loss
+
+    def train(self, epochs):
+        scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience)
+        epoch_times = []
+        prev_loss = -1
+        best_dev_score = -1
+        for epoch in range(1, epochs + 1):
+            start = time.time()
+            self.logger.info('Epoch {} started...'.format(epoch))
+            self.train_epoch(epoch)
+
+            dev_scores = self.evaluate(self.dev_evaluator, 'dev')
+            new_loss = dev_scores[2]
+
+            if self.use_tensorboard:
+                self.writer.add_scalar('sick/lr', self.optimizer.param_groups[0]['lr'], epoch)
+                self.writer.add_scalar('sick/dev/pearson_r', dev_scores[0], epoch)
+                self.writer.add_scalar('sick/dev/kl_div_loss', new_loss, epoch)
+
+            end = time.time()
+            duration = end - start
+            self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
+            epoch_times.append(duration)
+
+            if dev_scores[0] > best_dev_score:
+                best_dev_score = dev_scores[0]
+                torch.save(self.model, self.model_outfile)
+
+            if abs(prev_loss - new_loss) <= 0.0002:
+                self.logger.info('Early stopping. Loss changed by less than 0.0002.')
+                break
+
+            prev_loss = new_loss
+            scheduler.step(dev_scores[0])
+
+        self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))

mp_cnn/trainers/trainer.py

@@ -0,0 +1,37 @@
+class Trainer(object):
+
+    """
+    Abstraction for training a model on a Dataset.
+    """
+
+    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
+        self.model = model
+        self.optimizer = trainer_config['optimizer']
+        self.train_loader = train_loader
+        self.batch_size = trainer_config['batch_size']
+        self.log_interval = trainer_config['log_interval']
+        self.model_outfile = trainer_config['model_outfile']
+        self.lr_reduce_factor = trainer_config['lr_reduce_factor']
+        self.patience = trainer_config['patience']
+        self.use_tensorboard = trainer_config['tensorboard']
+        if self.use_tensorboard:
+            from tensorboardX import SummaryWriter
+            self.writer = SummaryWriter(log_dir=None, comment='' if trainer_config['run_label'] is None else trainer_config['run_label'])
+        self.logger = trainer_config['logger']
+
+        self.train_evaluator = train_evaluator
+        self.test_evaluator = test_evaluator
+        self.dev_evaluator = dev_evaluator
+
+    def evaluate(self, evaluator, dataset_name):
+        scores, metric_names = evaluator.get_scores()
+        self.logger.info('Evaluation metrics for {}:'.format(dataset_name))
+        self.logger.info('\t'.join([' '] + metric_names))
+        self.logger.info('\t'.join([dataset_name] + list(map(str, scores))))
+        return scores
+
+    def train_epoch(self, epoch):
+        raise NotImplementedError()
+
+    def train(self, epochs):
+        raise NotImplementedError()

mp_cnn/trainers/trecqa_trainer.py

@@ -0,0 +1,76 @@
+import time
+
+import torch
+import torch.nn.functional as F
+from torch.optim.lr_scheduler import ReduceLROnPlateau
+
+from mp_cnn.trainers.trainer import Trainer
+
+
+class TRECQATrainer(Trainer):
+
+    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
+        super(TRECQATrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
+
+    def train_epoch(self, epoch):
+        self.model.train()
+        total_loss = 0
+        for batch_idx, batch in enumerate(self.train_loader):
+            self.optimizer.zero_grad()
+            output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            loss = F.cross_entropy(output, batch.label, size_average=False)
+            total_loss += loss.data[0]
+            loss.backward()
+            self.optimizer.step()
+            if batch_idx % self.log_interval == 0:
+                self.logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
+                    epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)),
+                    len(batch.dataset.examples),
+                    100. * batch_idx / (len(self.train_loader)), loss.data[0])
+                )
+
+        average_loss, mean_average_precision, mean_reciprocal_rank = self.evaluate(self.train_evaluator, 'train')
+
+        if self.use_tensorboard:
+            self.writer.add_scalar('trecqa/train/cross_entropy_loss', average_loss, epoch)
+            self.writer.add_scalar('trecqa/train/map', mean_average_precision, epoch)
+            self.writer.add_scalar('trecqa/train/mrr', mean_reciprocal_rank, epoch)
+
+        return total_loss
+
+    def train(self, epochs):
+        scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience)
+        epoch_times = []
+        prev_loss = -1
+        best_dev_score = -1
+        for epoch in range(1, epochs + 1):
+            start = time.time()
+            self.logger.info('Epoch {} started...'.format(epoch))
+            self.train_epoch(epoch)
+
+            dev_scores = self.evaluate(self.dev_evaluator, 'dev')
+            new_loss, mean_average_precision, mean_reciprocal_rank = dev_scores
+
+            if self.use_tensorboard:
+                self.writer.add_scalar('trecqa/lr', self.optimizer.param_groups[0]['lr'], epoch)
+                self.writer.add_scalar('trecqa/dev/cross_entropy_loss', new_loss, epoch)
+                self.writer.add_scalar('trecqa/dev/map', mean_average_precision, epoch)
+                self.writer.add_scalar('trecqa/dev/mrr', mean_reciprocal_rank, epoch)
+
+            end = time.time()
+            duration = end - start
+            self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
+            epoch_times.append(duration)
+
+            if dev_scores[0] > best_dev_score:
+                best_dev_score = dev_scores[0]
+                torch.save(self.model, self.model_outfile)
+
+            if abs(prev_loss - new_loss) <= 0.0002:
+                self.logger.info('Early stopping. Loss changed by less than 0.0002.')
+                break
+
+            prev_loss = new_loss
+            scheduler.step(dev_scores[0])
+
+        self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))

utils/get_trec_eval.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+VERSION=9.0.5
+
+wget https://github.com/usnistgov/trec_eval/archive/v${VERSION}.tar.gz
+tar -xvzf v${VERSION}.tar.gz
+cd trec_eval-${VERSION}
+make
+cd ..
+
+rm -rf v${VERSION}.tar.gz

utils/relevancy_metrics.py

@@ -0,0 +1,36 @@
+import os
+import subprocess
+import time
+
+
+def get_map_mrr(qids, predictions, labels, device=0):
+    """
+    Get the map and mrr using the trec_eval utility.
+    qids, predictions, labels should have the same length.
+    device is not a required parameter, it is only used to prevent potential naming conflicts when you
+    are calling this concurrently from different threads of execution.
+    :param qids: query ids of predictions and labels
+    :param predictions: iterable of predictions made by the models
+    :param labels: iterable of labels of the dataset
+    :param device: device (GPU index or -1 for CPU) for identification purposes only
+    """
+    qrel_fname = 'trecqa_{}_{}.qrel'.format(time.time(), device)
+    results_fname = 'trecqa_{}_{}.results'.format(time.time(), device)
+    qrel_template = '{qid} 0 {docno} {rel}\n'
+    results_template = '{qid} 0 {docno} 0 {sim} mpcnn\n'
+    with open(qrel_fname, 'w') as f1, open(results_fname, 'w') as f2:
+        docnos = range(len(qids))
+        for qid, docno, predicted, actual in zip(qids, docnos, predictions, labels):
+            f1.write(qrel_template.format(qid=qid, docno=docno, rel=actual))
+            f2.write(results_template.format(qid=qid, docno=docno, sim=predicted))
+
+    trec_eval_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'trec_eval-9.0.5/trec_eval')
+    trec_out = subprocess.check_output([trec_eval_path, '-m', 'map', '-m', 'recip_rank', qrel_fname, results_fname])
+    trec_out_lines = str(trec_out, 'utf-8').split('\n')
+    mean_average_precision = float(trec_out_lines[0].split('\t')[-1])
+    mean_reciprocal_rank = float(trec_out_lines[1].split('\t')[-1])
+
+    os.remove(qrel_fname)
+    os.remove(results_fname)
+
+    return mean_average_precision, mean_reciprocal_rank