add NCE to MP-CNN (#84)

* update nce-sm

* refactor code, update torchtext

* use shared evaluation

* refactor code, use shared data loader

* refactor code

* refactor code

* refactor code according to Michael's great suggestions

* update readme and requirement

* update datasets and readme

* update data loader

* add space between +

* update refactor code

* add nce-mp

* remove duplicate files

* update readme, refactor code according to mp_cnn and delete duplicate code, follow PEP8 standard

* refactor code, add/delete comments

* import exit from sys

datasets/castor_dataset.py

@@ -1,10 +1,12 @@
 from abc import ABCMeta, abstractmethod
 import os
 import numpy as np
+from sys import exit
 
 from torchtext.data.dataset import Dataset
 from torchtext.data.example import Example
 from torchtext.data.field import Field
+import torch
 
 from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features
 
@@ -49,3 +51,21 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta):
                 examples.append(example)
 
         super(CastorPairDataset, self).__init__(examples, fields)
+
+    @classmethod
+    def set_vectors(cls, field, vector_path):
+        if os.path.isfile(vector_path):
+            stoi, vectors, dim = torch.load(vector_path)
+            field.vocab.vectors = torch.Tensor(len(field.vocab), dim)
+
+            for i, token in enumerate(field.vocab.itos):
+                wv_index = stoi.get(token, None)
+                if wv_index is not None:
+                    field.vocab.vectors[i] = vectors[wv_index]
+                else:
+                    # initialize <unk> with uniform_(-0.05, 0.05) vectors
+                    field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05)
+        else:
+            print("Error: Need word embedding pt file")
+            exit(1)
+        return field

datasets/trecqa.py

@@ -36,24 +36,6 @@ class TRECQA(CastorPairDataset):
     def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs):
         return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs)
 
-    @classmethod
-    def set_vectors(cls, field, vector_path):
-        if os.path.isfile(vector_path):
-            stoi, vectors, dim = torch.load(vector_path)
-            field.vocab.vectors = torch.Tensor(len(field.vocab), dim)
-
-            for i, token in enumerate(field.vocab.itos):
-                wv_index = stoi.get(token, None)
-                if wv_index is not None:
-                    field.vocab.vectors[i] = vectors[wv_index]
-                else:
-                    # initialize <unk> with uniform_(-0.05, 0.05) vectors
-                    field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05)
-        else:
-            print("Error: Need word embedding pt file")
-            exit(1)
-        return field
-
     @classmethod
     def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file = False, vectors=None, unk_init=torch.Tensor.zero_):
         """

datasets/wikiqa.py

@@ -14,6 +14,7 @@ class WikiQA(CastorPairDataset):
     NAME = 'wikiqa'
     NUM_CLASSES = 2
     ID_FIELD = Field(sequential=False, tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True)
+    AID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
     TEXT_FIELD = Field(batch_first=True, tokenize=lambda x: x)  # tokenizer is identity since we already tokenized it to compute external features
     EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x)
     LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
@@ -33,22 +34,32 @@ class WikiQA(CastorPairDataset):
         return super(WikiQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs)
 
     @classmethod
-    def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, unk_init=torch.Tensor.zero_):
+    def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file=False, vectors=None,
+              unk_init=torch.Tensor.zero_):
         """
         :param path: directory containing train, test, dev files
         :param vectors_name: name of word vectors file
-        :param vectors_cache: directory containing word vectors file
+        :param vectors_dir: directory containing word vectors file
         :param batch_size: batch size
         :param device: GPU device
         :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes
+        :param pt_file: load cached embedding file from disk if it is true
         :param unk_init: function used to generate vector for OOV words
         :return:
         """
-        if vectors is None:
-            vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init)
 
         train, validation, test = cls.splits(path)
+        if not pt_file:
+            if vectors is None:
+                vectors = Vectors(name=vectors_name, cache=vectors_dir, unk_init=unk_init)
+            cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors)
+        else:
+            cls.TEXT_FIELD.build_vocab(train, validation, test)
+            cls.TEXT_FIELD = cls.set_vectors(cls.TEXT_FIELD, os.path.join(vectors_dir, vectors_name))
 
-        cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors)
+        cls.LABEL_FIELD.build_vocab(train, validation, test)
 
-        return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device)
+        cls.VOCAB_SIZE = len(cls.TEXT_FIELD.vocab)
+
+        return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle,
+                                     device=device)