add NCE to MP-CNN (#84)

* update nce-sm

* refactor code, update torchtext

* use shared evaluation

* refactor code, use shared data loader

* refactor code

* refactor code

* refactor code according to Michael's great suggestions

* update readme and requirement

* update datasets and readme

* update data loader

* add space between +

* update refactor code

* add nce-mp

* remove duplicate files

* update readme, refactor code according to mp_cnn and delete duplicate code, follow PEP8 standard

* refactor code, add/delete comments

* import exit from sys

datasets/castor_dataset.py

@@ -1,10 +1,12 @@
 from abc import ABCMeta, abstractmethod
 import os
 import numpy as np
+from sys import exit
 
 from torchtext.data.dataset import Dataset
 from torchtext.data.example import Example
 from torchtext.data.field import Field
+import torch
 
 from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features
 
@@ -49,3 +51,21 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta):
                 examples.append(example)
 
         super(CastorPairDataset, self).__init__(examples, fields)
+
+    @classmethod
+    def set_vectors(cls, field, vector_path):
+        if os.path.isfile(vector_path):
+            stoi, vectors, dim = torch.load(vector_path)
+            field.vocab.vectors = torch.Tensor(len(field.vocab), dim)
+
+            for i, token in enumerate(field.vocab.itos):
+                wv_index = stoi.get(token, None)
+                if wv_index is not None:
+                    field.vocab.vectors[i] = vectors[wv_index]
+                else:
+                    # initialize <unk> with uniform_(-0.05, 0.05) vectors
+                    field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05)
+        else:
+            print("Error: Need word embedding pt file")
+            exit(1)
+        return field

datasets/trecqa.py

@@ -36,24 +36,6 @@ class TRECQA(CastorPairDataset):
     def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs):
         return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs)
 
-    @classmethod
-    def set_vectors(cls, field, vector_path):
-        if os.path.isfile(vector_path):
-            stoi, vectors, dim = torch.load(vector_path)
-            field.vocab.vectors = torch.Tensor(len(field.vocab), dim)
-
-            for i, token in enumerate(field.vocab.itos):
-                wv_index = stoi.get(token, None)
-                if wv_index is not None:
-                    field.vocab.vectors[i] = vectors[wv_index]
-                else:
-                    # initialize <unk> with uniform_(-0.05, 0.05) vectors
-                    field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05)
-        else:
-            print("Error: Need word embedding pt file")
-            exit(1)
-        return field
-
     @classmethod
     def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file = False, vectors=None, unk_init=torch.Tensor.zero_):
         """

datasets/wikiqa.py

@@ -14,6 +14,7 @@ class WikiQA(CastorPairDataset):
     NAME = 'wikiqa'
     NUM_CLASSES = 2
     ID_FIELD = Field(sequential=False, tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True)
+    AID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
     TEXT_FIELD = Field(batch_first=True, tokenize=lambda x: x)  # tokenizer is identity since we already tokenized it to compute external features
     EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x)
     LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
@@ -33,22 +34,32 @@ class WikiQA(CastorPairDataset):
         return super(WikiQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs)
 
     @classmethod
-    def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, unk_init=torch.Tensor.zero_):
+    def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file=False, vectors=None,
+              unk_init=torch.Tensor.zero_):
         """
         :param path: directory containing train, test, dev files
         :param vectors_name: name of word vectors file
-        :param vectors_cache: directory containing word vectors file
+        :param vectors_dir: directory containing word vectors file
         :param batch_size: batch size
         :param device: GPU device
         :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes
+        :param pt_file: load cached embedding file from disk if it is true
         :param unk_init: function used to generate vector for OOV words
         :return:
         """
-        if vectors is None:
-            vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init)
 
         train, validation, test = cls.splits(path)
+        if not pt_file:
+            if vectors is None:
+                vectors = Vectors(name=vectors_name, cache=vectors_dir, unk_init=unk_init)
+            cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors)
+        else:
+            cls.TEXT_FIELD.build_vocab(train, validation, test)
+            cls.TEXT_FIELD = cls.set_vectors(cls.TEXT_FIELD, os.path.join(vectors_dir, vectors_name))
 
-        cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors)
+        cls.LABEL_FIELD.build_vocab(train, validation, test)
 
-        return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device)
+        cls.VOCAB_SIZE = len(cls.TEXT_FIELD.vocab)
+
+        return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle,
+                                     device=device)

mp_cnn/dataset.py

@@ -29,16 +29,16 @@ class MPCNNDatasetFactory(object):
     Get the corresponding Dataset class for a particular dataset.
     """
     @staticmethod
-    def get_dataset(dataset_name, word_vectors_dir, word_vectors_file, batch_size, device):
+    def get_dataset(dataset_name, word_vectors_dir, word_vectors_file, batch_size, device, castor_dir="../", utils_trecqa="utils/trec_eval-9.0.5/trec_eval"):
         if dataset_name == 'sick':
-            dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'sick/')
+            dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'sick/')
             train_loader, dev_loader, test_loader = SICK.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
             embedding_dim = SICK.TEXT_FIELD.vocab.vectors.size()
             embedding = nn.Embedding(embedding_dim[0], embedding_dim[1])
             embedding.weight = nn.Parameter(SICK.TEXT_FIELD.vocab.vectors)
             return SICK, embedding, train_loader, test_loader, dev_loader
         elif dataset_name == 'msrvid':
-            dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'msrvid/')
+            dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'msrvid/')
             dev_loader = None
             train_loader, test_loader = MSRVID.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
             embedding_dim = MSRVID.TEXT_FIELD.vocab.vectors.size()
@@ -46,18 +46,18 @@ class MPCNNDatasetFactory(object):
             embedding.weight = nn.Parameter(MSRVID.TEXT_FIELD.vocab.vectors)
             return MSRVID, embedding, train_loader, test_loader, dev_loader
         elif dataset_name == 'trecqa':
-            if not os.path.exists('../utils/trec_eval-9.0.5/trec_eval'):
+            if not os.path.exists(os.path.join(castor_dir, utils_trecqa)):
                 raise FileNotFoundError('TrecQA requires the trec_eval tool to run. Please run get_trec_eval.sh inside Castor/utils (as working directory) before continuing.')
-            dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'TrecQA/')
+            dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'TrecQA/')
             train_loader, dev_loader, test_loader = TRECQA.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
             embedding_dim = TRECQA.TEXT_FIELD.vocab.vectors.size()
             embedding = nn.Embedding(embedding_dim[0], embedding_dim[1])
             embedding.weight = nn.Parameter(TRECQA.TEXT_FIELD.vocab.vectors)
             return TRECQA, embedding, train_loader, test_loader, dev_loader
         elif dataset_name == 'wikiqa':
-            if not os.path.exists('../utils/trec_eval-9.0.5/trec_eval'):
+            if not os.path.exists(os.path.join(castor_dir, utils_trecqa)):
                 raise FileNotFoundError('TrecQA requires the trec_eval tool to run. Please run get_trec_eval.sh inside Castor/utils (as working directory) before continuing.')
-            dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'WikiQA/')
+            dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'WikiQA/')
             train_loader, dev_loader, test_loader = WikiQA.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
             embedding_dim = WikiQA.TEXT_FIELD.vocab.vectors.size()
             embedding = nn.Embedding(embedding_dim[0], embedding_dim[1])

mp_cnn/evaluation.py

@@ -2,7 +2,8 @@ from mp_cnn.evaluators.sick_evaluator import SICKEvaluator
 from mp_cnn.evaluators.msrvid_evaluator import MSRVIDEvaluator
 from mp_cnn.evaluators.trecqa_evaluator import TRECQAEvaluator
 from mp_cnn.evaluators.wikiqa_evaluator import WikiQAEvaluator
-
+from nce.nce_pairwise_mp.evaluators.trecqa_evaluator import TRECQAEvaluatorNCE
+from nce.nce_pairwise_mp.evaluators.wikiqa_evaluator import WikiQAEvaluatorNCE
 
 class MPCNNEvaluatorFactory(object):
     """
@@ -15,17 +16,27 @@ class MPCNNEvaluatorFactory(object):
         'wikiqa': WikiQAEvaluator
     }
 
+    evaluator_map_nce = {
+        'trecqa': TRECQAEvaluatorNCE,
+        'wikiqa': WikiQAEvaluatorNCE
+    }
+
     @staticmethod
-    def get_evaluator(dataset_cls, model, data_loader, batch_size, device):
+    def get_evaluator(dataset_cls, model, data_loader, batch_size, device, nce=False):
         if data_loader is None:
             return None
 
+        if nce:
+            evaluator_map = MPCNNEvaluatorFactory.evaluator_map_nce
+        else:
+            evaluator_map = MPCNNEvaluatorFactory.evaluator_map
+
         if not hasattr(dataset_cls, 'NAME'):
             raise ValueError('Invalid dataset. Dataset should have NAME attribute.')
 
-        if dataset_cls.NAME not in MPCNNEvaluatorFactory.evaluator_map:
+        if dataset_cls.NAME not in evaluator_map:
             raise ValueError('{} is not implemented.'.format(dataset_cls))
 
-        return MPCNNEvaluatorFactory.evaluator_map[dataset_cls.NAME](
+        return evaluator_map[dataset_cls.NAME](
             dataset_cls, model, data_loader, batch_size, device
         )

mp_cnn/model.py

@@ -46,10 +46,10 @@ class MPCNN(nn.Module):
             # comparison units from per-dim conv
             2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS_PER_DIM
         )
-        n_feat = n_feat_h + n_feat_v + EXT_FEATS
+        self.n_feat = n_feat_h + n_feat_v + EXT_FEATS
 
         self.final_layers = nn.Sequential(
-            nn.Linear(n_feat, hidden_layer_units),
+            nn.Linear(self.n_feat, hidden_layer_units),
             nn.Tanh(),
             nn.Dropout(dropout),
             nn.Linear(hidden_layer_units, num_classes),

mp_cnn/train.py

@@ -2,6 +2,8 @@ from mp_cnn.trainers.sick_trainer import SICKTrainer
 from mp_cnn.trainers.msrvid_trainer import MSRVIDTrainer
 from mp_cnn.trainers.trecqa_trainer import TRECQATrainer
 from mp_cnn.trainers.wikiqa_trainer import WikiQATrainer
+from nce.nce_pairwise_mp.trainers.trecqa_trainer import TRECQATrainerNCE
+from nce.nce_pairwise_mp.trainers.wikiqa_trainer import WikiQATrainerNCE
 
 
 class MPCNNTrainerFactory(object):
@@ -15,11 +17,21 @@ class MPCNNTrainerFactory(object):
         'wikiqa': WikiQATrainer
     }
 
+    trainer_map_nce = {
+        'trecqa': TRECQATrainerNCE,
+        'wikiqa': WikiQATrainerNCE
+    }
+
     @staticmethod
-    def get_trainer(dataset_name, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
-        if dataset_name not in MPCNNTrainerFactory.trainer_map:
+    def get_trainer(dataset_name, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None, nce=False):
+        if nce:
+            trainer_map = MPCNNTrainerFactory.trainer_map_nce
+        else:
+            trainer_map = MPCNNTrainerFactory.trainer_map
+
+        if dataset_name not in trainer_map:
             raise ValueError('{} is not implemented.'.format(dataset_name))
 
-        return MPCNNTrainerFactory.trainer_map[dataset_name](
+        return trainer_map[dataset_name](
             model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator
         )

mp_cnn/trainers/trainer.py

@@ -14,6 +14,7 @@ class Trainer(object):
         self.lr_reduce_factor = trainer_config['lr_reduce_factor']
         self.patience = trainer_config['patience']
         self.use_tensorboard = trainer_config['tensorboard']
+
         if self.use_tensorboard:
             from tensorboardX import SummaryWriter
             self.writer = SummaryWriter(log_dir=None, comment='' if trainer_config['run_label'] is None else trainer_config['run_label'])
@@ -25,7 +26,6 @@ class Trainer(object):
 
     def evaluate(self, evaluator, dataset_name):
         scores, metric_names = evaluator.get_scores()
-        self.logger.info('Evaluation metrics for {}:'.format(dataset_name))
         self.logger.info('\t'.join([' '] + metric_names))
         self.logger.info('\t'.join([dataset_name] + list(map(str, scores))))
         return scores

nce/NCE-Pairwise-SM/train.py

@@ -15,6 +15,7 @@ from datasets.trecqa import TRECQA
 from args import get_args
 from model import SmPlusPlus, PairwiseConv
 from utils.relevancy_metrics import get_map_mrr
+from utils.nce_neighbors import get_nearest_neg_id, get_random_neg_id, get_batch
 
 
 class UnknownWordVecCache(object):
@@ -125,46 +126,6 @@ def train_sm():
     os.makedirs(os.path.join(args.save_path, args.dataset), exist_ok=True)
     print(header)
 
-    # get the nearest negative samples to the positive sample by computing the feature difference
-    def get_nearest_neg_id(pos_feature, neg_dict, distance="cosine", k=1):
-        dis_list = []
-        pos_feature = pos_feature.data.cpu().numpy()
-        pos_feature_norm = pos_feature / np.sqrt(sum(pos_feature ** 2))
-        neg_list = []
-        for key in neg_dict:
-            if distance == "l2":
-                dis = np.sqrt(np.sum((np.array(pos_feature) - neg_dict[key]["feature"]) ** 2))
-            elif distance == "cosine":
-                neg_feature = np.array(neg_dict[key]["feature"])
-                feat_norm = neg_feature / np.sqrt(sum(neg_feature ** 2))
-                dis = 1 - feat_norm.dot(pos_feature_norm)
-            dis_list.append(dis)
-            neg_list.append(key)
-
-        k = min(k, len(neg_dict))
-        min_list = heapq.nsmallest(k, enumerate(dis_list), key=operator.itemgetter(1))
-        min_id_list = [neg_list[x[0]] for x in min_list]
-        return min_id_list
-
-    # get the negative samples randomly
-    def get_random_neg_id(q2neg, qid_i, k=5):
-        # question 1734 has no neg answer
-        if qid_i not in q2neg:
-            return []
-        k = min(k, len(q2neg[qid_i]))
-        ran = random.sample(q2neg[qid_i], k)
-        return ran
-
-    # pack the lists of question/answer/ext_feat into a torchtext batch
-    def get_batch(question, answer, ext_feat, size):
-        new_batch = data.Batch()
-        new_batch.batch_size = size
-        new_batch.dataset = batch.dataset
-        setattr(new_batch, "sentence_2", torch.stack(answer))
-        setattr(new_batch, "sentence_1", torch.stack(question))
-        setattr(new_batch, "ext_feats", torch.stack(ext_feat))
-        return new_batch
-
     while True:
         if early_stop:
             logger.log("Early Stopping. Epoch: {}, Best Dev Loss: {}".format(epoch, best_dev_loss))

nce/nce_pairwise_mp/README.md

@@ -0,0 +1,85 @@
+# MP-CNN PyTorch Implementation
+
+This is a PyTorch implementation of the following paper
+
+* Hua He, Kevin Gimpel, and Jimmy Lin. [Multi-Perspective Sentence Similarity Modeling with Convolutional Neural Networks](http://aclweb.org/anthology/D/D15/D15-1181.pdf). *Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP 2015)*, pages 1576-1586.
+* Jinfeng Rao, Hua He, and Jimmy Lin. [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks.](http://dl.acm.org/citation.cfm?id=2983872) *Proceedings of the 25th ACM International on Conference on Information and Knowledge Management (CIKM 2016)*, pages 1913-1916.
+
+
+The SICK and MSRVID datasets are available in https://github.com/castorini/data, as well as the GloVe word embeddings.
+
+Directory layout should be like this:
+```
+├── Castor
+│   ├── README.md
+│   ├── ...
+│   └── mp_cnn/
+├── data
+│   ├── README.md
+│   ├── ...
+│   ├── msrvid/
+│   ├── sick/
+│   └── GloVe/
+```
+
+## TrecQA Dataset
+
+To run MP-CNN on (Raw) TrecQA, you first need to run `./get_trec_eval.sh` in `utils` under the repo root while inside the `utils` directory. This will download and compile the official `trec_eval` tool used for evaluation.
+
+Then, you can run:
+```
+python train_script.py --dataset wikiqa --device -1
+```
+
+Metric|Without NCE (original paper) | only random sampling | only max sampling | Pair-wise+nagative sampling (original paper) | Pair-wise+random sampling| Pair-wise+nagative sampling | Pair-wise+nagative sampling+pair weighting
+-------|------|----------|------------|------------|------------|------|------
+MAP    |0.762 | 0.7579| 0.7678|0.780  | 0.7745   |0.7873|0.7683
+MRR    |0.830 |0.8239| 0.8387|0.834  | 0.8435  |0.8414|0.8253
+
+The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872).
+
+## WikiQA Dataset
+
+You also need `trec_eval` for this dataset, similar to TrecQA.
+
+Then, you can run:
+```
+python train_script.py --dataset trecqa --device -1
+```
+
+Metric|Without NCE (original paper) | only random sampling  | only max sampling| Pair-wise+nagative sampling (original paper)| Pair-wise+random sampling | Pair-wise+nagative sampling | Pair-wise+nagative sampling+pair weighting
+-------|-------|------|----------|------------|------------|------------|------------
+MAP    |0.693 | 0.6744| 0.6795 | 0.701| 0.7047    |0.7049| 0.7047
+MRR    |0.709 | 0.6898| 0.6951 |0.718 | 0.7172   |0.7192| 0.7211
+
+
+The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872).
+
+
+To see all options available and train with your parameters, use
+```
+python main.py --help
+```
+
+## Troubleshooting
+
+### ModuleNotFoundError: datasets
+```
+Traceback (most recent call last):
+  File "main.py", line 9, in <module>
+    from dataset import MPCNNDatasetFactory
+  File "/u/z3tu/castorini/Castor/mp_cnn/dataset.py", line 12, in <module>
+    from datasets.sick import SICK
+ModuleNotFoundError: No module named 'datasets'
+```
+
+You need to make sure the repository root is in your `PYTHONPATH` environment variable. One way to do this is while you are in the repo root (Castor) as your current working directory, run `export PYTHONPATH=$(pwd)`.
+
+## Optional Dependencies
+
+To optionally visualize the learning curve during training, we make use of https://github.com/lanpa/tensorboard-pytorch to connect to [TensorBoard](https://github.com/tensorflow/tensorboard). These projects require TensorFlow as a dependency, so you need to install TensorFlow before running the commands below. After these are installed, just add `--tensorboard` when running `main.py` and open TensorBoard in the browser.
+
+```sh
+pip install tensorboardX
+pip install tensorflow-tensorboard
+```

nce/nce_pairwise_mp/evaluators/qa_evaluator.py

@@ -0,0 +1,33 @@
+import numpy as np
+
+from mp_cnn.evaluators.evaluator import Evaluator
+from utils.relevancy_metrics import get_map_mrr
+
+
+class QAEvaluator(Evaluator):
+
+    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
+        super(QAEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device)
+
+    def get_scores(self):
+        self.model.eval()
+        qids = []
+        predictions = []
+        labels = []
+
+        for batch in self.data_loader:
+            scores = self.model.convModel(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            scores = self.model.linearLayer(scores)
+            qid_array = np.transpose(batch.id.cpu().data.numpy())
+            score_array = scores.cpu().data.numpy().reshape(-1)
+            true_label_array = np.transpose(batch.label.cpu().data.numpy())
+
+            qids.extend(qid_array.tolist())
+            predictions.extend(score_array.tolist())
+            labels.extend(true_label_array.tolist())
+
+            del scores
+
+        mean_average_precision, mean_reciprocal_rank = get_map_mrr(qids, predictions, labels, self.data_loader.device)
+
+        return [mean_average_precision, mean_reciprocal_rank], ['map', 'mrr']

nce/nce_pairwise_mp/evaluators/trecqa_evaluator.py

@@ -0,0 +1,7 @@
+from nce.nce_pairwise_mp.evaluators.qa_evaluator import QAEvaluator
+
+
+class TRECQAEvaluatorNCE(QAEvaluator):
+
+    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
+        super(TRECQAEvaluatorNCE, self).__init__(dataset_cls, model, data_loader, batch_size, device)

nce/nce_pairwise_mp/evaluators/wikiqa_evaluator.py

@@ -0,0 +1,7 @@
+from nce.nce_pairwise_mp.evaluators.qa_evaluator import QAEvaluator
+
+
+class WikiQAEvaluatorNCE(QAEvaluator):
+
+    def __init__(self, dataset_cls, model, data_loader, batch_size, device):
+        super(WikiQAEvaluatorNCE, self).__init__(dataset_cls, model, data_loader, batch_size, device)

nce/nce_pairwise_mp/main.py

@@ -0,0 +1,137 @@
+import argparse
+import logging
+import os
+import pprint
+import random
+
+import numpy as np
+import torch
+import torch.optim as optim
+
+from mp_cnn.dataset import MPCNNDatasetFactory
+from mp_cnn.evaluation import MPCNNEvaluatorFactory
+from nce.nce_pairwise_mp.model import MPCNN4NCE, PairwiseConv
+from mp_cnn.train import MPCNNTrainerFactory
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN')
+    parser.add_argument('model_outfile', help='file to save final model')
+    parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid, trecqa, wikiqa]', default='sick')
+    parser.add_argument('--word-vectors-dir', help='word vectors directory', default=os.path.join(os.pardir, os.pardir, os.pardir, 'data', 'GloVe'))
+    parser.add_argument('--word-vectors-file', help='word vectors filename', default='glove.840B.300d.txt')
+    parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true')
+    parser.add_argument('--device', type=int, default=0, help='GPU device, -1 for CPU (default: 0)')
+    parser.add_argument('--sparse-features', action='store_true', default=False, help='use sparse features (default: false)')
+    parser.add_argument('--batch-size', type=int, default=64, help='input batch size for training (default: 64)')
+    parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 10)')
+    parser.add_argument('--optimizer', type=str, default='adam', help='optimizer to use: adam or sgd (default: adam)')
+    parser.add_argument('--lr', type=float, default=0.001, help='learning rate (default: 0.001)')
+    parser.add_argument('--lr-reduce-factor', type=float, default=0.3, help='learning rate reduce factor after plateau (default: 0.3)')
+    parser.add_argument('--patience', type=float, default=2, help='learning rate patience after seeing plateau (default: 2)')
+    parser.add_argument('--momentum', type=float, default=0, help='momentum (default: 0)')
+    parser.add_argument('--epsilon', type=float, default=1e-8, help='Adam epsilon (default: 1e-8)')
+    parser.add_argument('--log-interval', type=int, default=10, help='how many batches to wait before logging training status (default: 10)')
+    parser.add_argument('--regularization', type=float, default=0.0001, help='Regularization for the optimizer (default: 0.0001)')
+    parser.add_argument('--max-window-size', type=int, default=3, help='windows sizes will be [1,max_window_size] and infinity (default: 300)')
+    parser.add_argument('--holistic-filters', type=int, default=300, help='number of holistic filters (default: 300)')
+    parser.add_argument('--per-dim-filters', type=int, default=20, help='number of per-dimension filters (default: 20)')
+    parser.add_argument('--hidden-units', type=int, default=150, help='number of hidden units in each of the two hidden layers (default: 150)')
+    parser.add_argument('--dropout', type=float, default=0.5, help='dropout probability (default: 0.5)')
+    parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)')
+    parser.add_argument('--tensorboard', action='store_true', default=False, help='use TensorBoard to visualize training (default: false)')
+    parser.add_argument('--run-label', type=str, help='label to describe run')
+    parser.add_argument('--dev_log_interval', type=int, default=100, help='how many batches to wait before logging validation status (default: 100)')
+    parser.add_argument('--neg_num', type=int, default=8, help='number of negative samples for each question')
+    parser.add_argument('--neg_sample', type=str, default="random", help='strategy of negative sampling, random or max')
+    parser.add_argument('--castor_dir', help='castor directory', default=os.path.join(os.pardir, os.pardir))
+    parser.add_argument('--utils_trecqa', help='trecqa util file', default="utils/trec_eval-9.0.5/trec_eval")
+
+
+    args = parser.parse_args()
+
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.device != -1:
+        torch.cuda.manual_seed(args.seed)
+
+    # logging setup
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.INFO)
+
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    logger.info(pprint.pformat(vars(args)))
+
+    dataset_cls, embedding, train_loader, test_loader, dev_loader = MPCNNDatasetFactory.get_dataset(args.dataset,
+                                                                                                    args.word_vectors_dir,
+                                                                                                    args.word_vectors_file,
+                                                                                                    args.batch_size,
+                                                                                                    args.device,
+                                                                                                    castor_dir=args.castor_dir,
+                                                                                                    utils_trecqa=args.utils_trecqa)
+
+    filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf]
+    model = MPCNN4NCE(embedding, args.holistic_filters, args.per_dim_filters, filter_widths,
+                    args.hidden_units, dataset_cls.NUM_CLASSES, args.dropout, args.sparse_features)
+
+    pw_model = PairwiseConv(model)
+
+    if args.device != -1:
+        with torch.cuda.device(args.device):
+            pw_model.cuda()
+
+    optimizer = None
+    if args.optimizer == 'adam':
+        optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.regularization, eps=args.epsilon)
+    elif args.optimizer == 'sgd':
+        optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.regularization)
+    else:
+        raise ValueError('optimizer not recognized: it should be either adam or sgd')
+
+    test_batch_size = 32
+    train_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, train_loader, args.batch_size, args.device, nce=True)
+    dev_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, dev_loader, args.batch_size, args.device, nce=True)
+    test_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, test_loader, test_batch_size, args.device, nce=True)
+
+    if args.device != -1:
+        margin_label = torch.autograd.Variable(torch.ones(1).cuda(device=args.device))
+    else:
+        margin_label = torch.autograd.Variable(torch.ones(1))
+
+    trainer_config = {
+        'optimizer': optimizer,
+        'batch_size': args.batch_size,
+        'log_interval': args.log_interval,
+        'dev_log_interval': args.dev_log_interval,
+        'model_outfile': args.model_outfile,
+        'lr_reduce_factor': args.lr_reduce_factor,
+        'patience': args.patience,
+        'tensorboard': args.tensorboard,
+        'run_label': args.run_label,
+        'logger': logger,
+        'neg_num': args.neg_num,
+        'neg_sample': args.neg_sample,
+        'margin_label': margin_label
+    }
+    trainer = MPCNNTrainerFactory.get_trainer(args.dataset, pw_model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator, nce=True)
+
+    if not args.skip_training:
+        total_params = 0
+        for param in pw_model.parameters():
+            size = [s for s in param.size()]
+            total_params += np.prod(size)
+        logger.info('Total number of parameters: %s', total_params)
+        trainer.train(args.epochs)
+
+    pw_model = torch.load(args.model_outfile)
+    saved_model_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, test_loader, args.batch_size, args.device)
+    scores, metric_names = saved_model_evaluator.get_scores()
+    logger.info('Evaluation metrics for test')
+    logger.info('\t'.join([' '] + metric_names))
+    logger.info('\t'.join(['test'] + list(map(str, scores))))

nce/nce_pairwise_mp/train_script.py

@@ -0,0 +1,44 @@
+import argparse
+from random import uniform
+from subprocess import call
+
+
+# set parameters from random uniform sampling
+def get_param():
+    for eps in [1e-8, 1e-6]:
+        for learning_rate in [0.0001, 0.0003, 0.0006, 0.001]:
+            for reg in [3e-5, 9e-5, 3e-4, 9e-4]:
+                yield learning_rate, eps, reg
+
+def run(device, epochs, neg_sample, neg_num, dataset, dev_log_interval, batch_size):
+    # for _ in range(count):
+    param_gen = get_param()
+    for learning_rate, eps, reg in param_gen:
+
+        filename = "grid_{dataset}_lr_{learning_rate}_eps_{eps}_reg_{reg}_device_{dev}.txt".format(
+            learning_rate=learning_rate, eps=eps, reg=reg, dev=device, dataset=dataset)
+        model_name = filename[:-4] + ".castor"
+
+        command = "python -u main.py saved_models/{model} --epochs {epo} --device {dev} --dataset {dataset} " \
+                  "--batch-size {batch_size} --lr {learning_rate} --epsilon {eps} --regularization {reg} --tensorboard " \
+                  "--run-label {label} --dev_log_interval {dev_log_interval} --neg_sample {neg_sample} --neg_num {neg_num}" \
+            .format(epo=epochs, model=model_name, dev=device, dataset=dataset, batch_size=batch_size,
+                    learning_rate=learning_rate, eps=eps,
+                    reg=reg, label=filename, dev_log_interval=dev_log_interval, neg_sample=neg_sample, neg_num=neg_num)
+
+        print("Running: " + command)
+        with open(filename, 'w') as outfile:
+            call(command, shell=True, stderr=outfile)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Hyper parameters sweeper')
+    parser.add_argument('--device', type=int, default=2, help='GPU device, -1 for CPU (default: 0)')
+    parser.add_argument('--epochs', type=int, default=8, help='number of epochs to run')
+    parser.add_argument('--neg_num', type=int, default=8, help='number of negative samples')
+    parser.add_argument('--neg_sample', type=str, default="max", help='strategy of negative samples')
+    parser.add_argument('--dataset', type=str, default="trecqa", help='dataset')
+    parser.add_argument('--dev_log_interval', type=int, default=150, help='number of negative samples')
+    parser.add_argument('--batch_size', type=int, default=32, help='batch size')
+    args = parser.parse_args()
+    run(args.device, args.epochs, neg_sample=args.neg_sample, neg_num=args.neg_num, dataset=args.dataset,
+        dev_log_interval=args.dev_log_interval, batch_size=args.batch_size)

nce/nce_pairwise_mp/trainers/qa_trainer.py

@@ -0,0 +1,219 @@
+import time
+import numpy as np
+
+import torch
+import torch.nn.functional as F
+from torch.optim.lr_scheduler import ReduceLROnPlateau
+
+from mp_cnn.trainers.trainer import Trainer
+from utils.nce_neighbors import get_nearest_neg_id, get_random_neg_id, get_batch
+
+class QATrainer(Trainer):
+
+    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None, weighting=False):
+        super(QATrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
+        self.loss = torch.nn.MarginRankingLoss(margin=1, size_average=True)
+        self.question2answer = {}
+        self.best_dev_map = 0
+        self.best_dev_mrr = 0
+        self.false_samples = {}
+        self.question2answer = {}
+        self.start = time.time()
+        self.q2neg = {}
+        self.iteration = 0
+        self.name = self.train_loader.dataset.NAME
+        self.dev_log_interval = trainer_config['dev_log_interval']
+        self.neg_num = trainer_config['neg_num'] if 'neg_num' in trainer_config else 0
+        self.neg_sample = trainer_config['neg_sample'] if 'neg_sample' in trainer_config else ''
+        self.log_template = 'Train Epoch:{} [{}/{}]\tLoss:{}  Acc:{}'
+        self.margin_label = trainer_config['margin_label']
+        self.dev_index = 1
+        self.weighting = weighting
+
+    def train_epoch(self, epoch):
+        self.model.train()
+        total_loss = 0
+        acc = 0
+        tot = 0
+        for batch_idx, batch in enumerate(self.train_loader):
+            self.iteration += 1
+
+            features = self.model.convModel(batch.sentence_1, batch.sentence_2, batch.ext_feats)
+            new_train_pos = {"answer": [], "question": [], "ext_feat": []}
+            new_train_neg = {"answer": [], "question": [], "ext_feat": []}
+            max_len_q = 0
+            max_len_a = 0
+
+            batch_near_list = []
+            batch_qid = []
+            batch_aid = []
+            new_near_score = []
+
+            for i in range(batch.batch_size):
+                label_i = batch.label[i].cpu().data.numpy()[0]
+                question_i = batch.sentence_1[i]
+                # question_i = question_i[question_i!=1] # remove padding 1 <pad>
+                answer_i = batch.sentence_2[i]
+                # answer_i = answer_i[answer_i!=1] # remove padding 1 <pad>
+                ext_feat_i = batch.ext_feats[i]
+                qid_i = batch.id[i].data.cpu().numpy()[0]
+                aid_i = batch.aid[i].data.cpu().numpy()[0]
+
+                if qid_i not in self.question2answer:
+                    self.question2answer[qid_i] = {"question": question_i, "pos": {}, "neg": {}}
+                if label_i == 1:
+
+                    if aid_i not in self.question2answer[qid_i]["pos"]:
+                        self.question2answer[qid_i]["pos"][aid_i] = {}
+
+                    self.question2answer[qid_i]["pos"][aid_i]["answer"] = answer_i
+                    self.question2answer[qid_i]["pos"][aid_i]["ext_feat"] = ext_feat_i
+
+                    # get neg samples in the first epoch but do not train
+                    if epoch == 1:
+                        continue
+                    # random generate sample in the first training epoch
+                    elif epoch == 2 or self.neg_sample == "random":
+                        near_list = get_random_neg_id(self.q2neg, qid_i, k=self.neg_num)
+                    else:
+                        near_list, near_score = get_nearest_neg_id(features[i], self.question2answer[qid_i]["neg"],
+                                                                   distance="cosine", k=self.neg_num, weight=True)
+                        new_near_score.extend(near_score)
+
+                    batch_near_list.extend(near_list)
+
+                    neg_size = len(near_list)
+                    if neg_size != 0:
+                        answer_i = answer_i[answer_i != 1]  # remove padding 1 <pad>
+                        question_i = question_i[question_i != 1]  # remove padding 1 <pad>
+                        for near_id in near_list:
+                            batch_qid.append(qid_i)
+                            batch_aid.append(aid_i)
+
+                            new_train_pos["answer"].append(answer_i)
+                            new_train_pos["question"].append(question_i)
+                            new_train_pos["ext_feat"].append(ext_feat_i)
+
+                            near_answer = self.question2answer[qid_i]["neg"][near_id]["answer"]
+                            if question_i.size(0) > max_len_q:
+                                max_len_q = question_i.size(0)
+                            if near_answer.size(0) > max_len_a:
+                                max_len_a = near_answer.size(0)
+                            if answer_i.size(0) > max_len_a:
+                                max_len_a = answer_i.size(0)
+
+                            ext_feat_neg = self.question2answer[qid_i]["neg"][near_id]["ext_feat"]
+                            new_train_neg["answer"].append(near_answer)
+                            new_train_neg["question"].append(question_i)
+                            new_train_neg["ext_feat"].append(ext_feat_neg)
+
+                elif label_i == 0:
+
+                    if aid_i not in self.question2answer[qid_i]["neg"]:
+                        answer_i = answer_i[answer_i != 1]
+                        self.question2answer[qid_i]["neg"][aid_i] = {"answer": answer_i}
+
+                    if "ext_feat" in self.question2answer[qid_i]["neg"][aid_i]:
+                        del self.question2answer[qid_i]["neg"][aid_i]["ext_feat"]
+                    self.question2answer[qid_i]["neg"][aid_i]["feature"] = features[i].data.cpu().numpy()
+                    self.question2answer[qid_i]["neg"][aid_i]["ext_feat"] = ext_feat_i
+
+
+                    if epoch == 1:
+                        if qid_i not in self.q2neg:
+                            self.q2neg[qid_i] = []
+
+                        self.q2neg[qid_i].append(aid_i)
+
+            del features
+            # pack the selected pos and neg samples into the torchtext batch and train
+            if epoch != 1:
+                true_batch_size = len(new_train_neg["answer"])
+                if true_batch_size != 0:
+                    for j in range(true_batch_size):
+                        new_train_neg["answer"][j] = F.pad(new_train_neg["answer"][j],
+                                                           (0, max_len_a - new_train_neg["answer"][j].size(0)),
+                                                           value=1)
+                        new_train_pos["answer"][j] = F.pad(new_train_pos["answer"][j],
+                                                           (0, max_len_a - new_train_pos["answer"][j].size(0)),
+                                                           value=1)
+                        new_train_pos["question"][j] = F.pad(new_train_pos["question"][j],
+                                                             (0, max_len_q - new_train_pos["question"][j].size(0)),
+                                                             value=1)
+                        new_train_neg["question"][j] = F.pad(new_train_neg["question"][j],
+                                                             (0, max_len_q - new_train_neg["question"][j].size(0)),
+                                                             value=1)
+
+
+                    pos_batch = get_batch(new_train_pos["question"], new_train_pos["answer"], new_train_pos["ext_feat"],
+                                          true_batch_size)
+                    neg_batch = get_batch(new_train_neg["question"], new_train_neg["answer"], new_train_neg["ext_feat"],
+                                          true_batch_size)
+
+                    self.model.train()
+                    self.optimizer.zero_grad()
+                    output = self.model([pos_batch, neg_batch])
+
+                    cmp = output[:, 0] > output[:, 1]
+                    acc += sum(cmp.data.cpu().numpy())
+                    tot += true_batch_size
+
+                    loss = self.loss(output[:, 0], output[:, 1], self.margin_label)
+                    if len(new_near_score) != 0 and self.weighting:
+                        # Element wise weighting hasn't been implemented for MarginRankingLoss by pytorch
+                        # Apply the mean score and set branch size to 1 as weighting implementation temporarily here
+                        # Reference: https://github.com/pytorch/pytorch/issues/264
+                        # loss *= torch.autograd.variable.Variable(torch.from_numpy(np.array(new_near_score)).cuda())
+                        loss *= np.mean(new_near_score)
+
+                    loss_num = loss.data.cpu().numpy()[0]
+                    total_loss += loss_num
+                    loss.backward()
+                    self.optimizer.step()
+
+                    del new_train_neg
+                    del new_train_pos
+
+                    if self.iteration % self.dev_log_interval == 1 and epoch != 1:
+                        dev_map, dev_mrr = self.evaluate(self.dev_evaluator, 'dev')
+                        test_map, test_mrr = self.evaluate(self.test_evaluator, 'test')
+
+                        if self.use_tensorboard:
+                            self.writer.add_scalar('{}/dev/map'.format(self.name), dev_map, self.dev_index)
+                            self.writer.add_scalar('{}/dev/mrr'.format(self.name), dev_mrr, self.dev_index)
+                            self.writer.add_scalar('{}/test/map'.format(self.name), test_map, self.dev_index)
+                            self.writer.add_scalar('{}/test/mrr'.format(self.name), test_mrr, self.dev_index)
+                            self.writer.add_scalar('{}/train/loss'.format(self.name), loss_num, self.dev_index)
+                            self.writer.add_scalar('{}/lr'.format(self.train_loader.dataset.NAME),
+                                                   self.optimizer.param_groups[0]['lr'], self.dev_index)
+
+                        self.dev_index += 1
+                        if self.best_dev_mrr < dev_mrr:
+                            torch.save(self.model, self.model_outfile)
+                            self.best_dev_mrr = dev_mrr
+                            self.best_dev_map = dev_map
+
+                    if self.iteration % self.log_interval == 1 and epoch != 1:
+                        # logger.info progress message
+                        self.logger.info(self.log_template.format(epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)),
+                                                                  len(batch.dataset.examples), loss_num, acc / tot))
+
+        return total_loss
+
+    def train(self, epochs):
+
+        scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience)
+        epoch_times = []
+        self.start = time.time()
+        for epoch in range(1, epochs + 1):
+            start = time.time()
+            train_loss = self.train_epoch(epoch)
+
+            end = time.time()
+            duration = end - start
+            self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
+            epoch_times.append(duration)
+
+            scheduler.step(train_loss)
+
+        self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))

nce/nce_pairwise_mp/trainers/trecqa_trainer.py

@@ -0,0 +1,7 @@
+from nce.nce_pairwise_mp.trainers.qa_trainer import QATrainer
+
+
+class TRECQATrainerNCE(QATrainer):
+
+    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
+        super(TRECQATrainerNCE, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)

nce/nce_pairwise_mp/trainers/wikiqa_trainer.py

@@ -0,0 +1,7 @@
+from nce.nce_pairwise_mp.trainers.qa_trainer import QATrainer
+
+
+class WikiQATrainerNCE(QATrainer):
+
+    def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None):
+        super(WikiQATrainerNCE, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)

utils/nce_neighbors.py

@@ -0,0 +1,51 @@
+import random
+import numpy as np
+import heapq
+import operator
+
+import torch
+from torchtext import data
+
+# get the nearest negative samples to the positive sample by computing the feature difference
+def get_nearest_neg_id(pos_feature, neg_dict, distance="cosine", k=1, weight=False):
+    dis_list = []
+    pos_feature = pos_feature.data.cpu().numpy()
+    pos_feature_norm = pos_feature / np.sqrt(sum(pos_feature ** 2))
+    neg_list = []
+    for key in neg_dict:
+        if distance == "l2":
+            dis = np.sqrt(np.sum((np.array(pos_feature) - neg_dict[key]["feature"]) ** 2))
+        elif distance == "cosine":
+            neg_feature = np.array(neg_dict[key]["feature"])
+            feat_norm = neg_feature / np.sqrt(sum(neg_feature ** 2))
+            dis = 1 - feat_norm.dot(pos_feature_norm)
+        dis_list.append(dis)
+        neg_list.append(key)
+
+    k = min(k, len(neg_dict))
+    min_list = heapq.nsmallest(k, enumerate(dis_list), key=operator.itemgetter(1))
+    # find the corresponding neg id
+    min_id_list = [neg_list[x[0]] for x in min_list]
+    if weight:
+        min_id_score = [1 - x[1] for x in min_list]
+        return min_id_list, min_id_score
+    else:
+        return min_id_list
+
+# get the negative samples randomly
+def get_random_neg_id(q2neg, qid_i, k=8):
+    # question 1734 in TrecQA has only one positive answer and no negative answer
+    if qid_i not in q2neg:
+        return []
+    k = min(k, len(q2neg[qid_i]))
+    ran = random.sample(q2neg[qid_i], k)
+    return ran
+
+# pack the lists of question/answer/ext_feat into a torchtext batch
+def get_batch(question, answer, ext_feat, size):
+    new_batch = data.Batch()
+    new_batch.batch_size = size
+    setattr(new_batch, "sentence_2", torch.stack(answer))
+    setattr(new_batch, "sentence_1", torch.stack(question))
+    setattr(new_batch, "ext_feats", torch.stack(ext_feat))
+    return new_batch