update nce-sm (#78)

* update nce-sm

* refactor code, update torchtext

* use shared evaluation

* refactor code, use shared data loader

* refactor code

* refactor code

* refactor code according to Michael's great suggestions

* update readme and requirement

* update datasets and readme

* update data loader

* add space between +

datasets/castor_dataset.py

@@ -1,5 +1,6 @@
 from abc import ABCMeta, abstractmethod
 import os
+import numpy as np
 
 from torchtext.data.dataset import Dataset
 from torchtext.data.example import Example
@@ -17,13 +18,15 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta):
     TEXT_FIELD = None
     EXT_FEATS_FIELD = None
     LABEL_FIELD = None
+    AID_FIELD = None
 
     @abstractmethod
-    def __init__(self, path):
+    def __init__(self, path, load_ext_feats=False):
         """
         Create a Castor dataset involving pairs of texts
         """
-        fields = [('id', self.ID_FIELD), ('sentence_1', self.TEXT_FIELD), ('sentence_2', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)]
+        fields = [('id', self.ID_FIELD), ('sentence_1', self.TEXT_FIELD), ('sentence_2', self.TEXT_FIELD), ('ext_feats',
+                self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD), ('aid', self.AID_FIELD)]
 
         examples = []
         with open(os.path.join(path, 'a.toks'), 'r') as f1, open(os.path.join(path, 'b.toks'), 'r') as f2:
@@ -31,13 +34,18 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta):
             sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2]
 
         word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2)
-        overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt)
+
+        if not load_ext_feats:
+            overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt)
+        else:
+            overlap_feats = np.loadtxt(os.path.join(path, 'overlap_feats.txt'))
 
         with open(os.path.join(path, 'id.txt'), 'r') as id_file, open(os.path.join(path, 'sim.txt'), 'r') as label_file:
-            for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file):
+            for i, (pair_id, l1, l2, ext_feats, label) in enumerate(zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file)):
                 pair_id = pair_id.rstrip('.\n')
                 label = label.rstrip('.\n')
-                example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields)
+                example_list = [pair_id, l1, l2, ext_feats, label, i + 1]
+                example = Example.fromlist(example_list, fields)
                 examples.append(example)
 
         super(CastorPairDataset, self).__init__(examples, fields)

datasets/trecqa.py

@@ -1,10 +1,11 @@
 import os
 
 import torch
-from torchtext.data.example import Example
 from torchtext.data.field import Field
 from torchtext.data.iterator import BucketIterator
+from torchtext.data.iterator import Iterator
 from torchtext.vocab import Vectors
+from torchtext.data import Pipeline
 
 from datasets.castor_dataset import CastorPairDataset
 from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features
@@ -14,9 +15,12 @@ class TRECQA(CastorPairDataset):
     NAME = 'trecqa'
     NUM_CLASSES = 2
     ID_FIELD = Field(sequential=False, tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True)
+    AID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
     TEXT_FIELD = Field(batch_first=True, tokenize=lambda x: x)  # tokenizer is identity since we already tokenized it to compute external features
-    EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x)
+    EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x,
+                            postprocessing=Pipeline(lambda arr, _, train: [float(y) for y in arr]))
     LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True)
+    VOCAB_SIZE = 0
 
     @staticmethod
     def sort_key(ex):
@@ -26,29 +30,54 @@ class TRECQA(CastorPairDataset):
         """
         Create a TRECQA dataset instance
         """
-        super(TRECQA, self).__init__(path)
+        super(TRECQA, self).__init__(path, load_ext_feats=True)
 
     @classmethod
     def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs):
         return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs)
 
     @classmethod
-    def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, unk_init=torch.Tensor.zero_):
+    def set_vectors(cls, field, vector_path):
+        if os.path.isfile(vector_path):
+            stoi, vectors, dim = torch.load(vector_path)
+            field.vocab.vectors = torch.Tensor(len(field.vocab), dim)
+
+            for i, token in enumerate(field.vocab.itos):
+                wv_index = stoi.get(token, None)
+                if wv_index is not None:
+                    field.vocab.vectors[i] = vectors[wv_index]
+                else:
+                    # initialize <unk> with uniform_(-0.05, 0.05) vectors
+                    field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05)
+        else:
+            print("Error: Need word embedding pt file")
+            exit(1)
+        return field
+
+    @classmethod
+    def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file = False, vectors=None, unk_init=torch.Tensor.zero_):
         """
         :param path: directory containing train, test, dev files
         :param vectors_name: name of word vectors file
-        :param vectors_cache: directory containing word vectors file
+        :param vectors_dir: directory containing word vectors file
         :param batch_size: batch size
         :param device: GPU device
         :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes
         :param unk_init: function used to generate vector for OOV words
         :return:
         """
-        if vectors is None:
-            vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init)
 
         train, validation, test = cls.splits(path)
+        if not pt_file:
+            if vectors is None:
+                vectors = Vectors(name=vectors_name, cache=vectors_dir, unk_init=unk_init)
+            cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors)
+        else:
+            cls.TEXT_FIELD.build_vocab(train, validation, test)
+            cls.TEXT_FIELD = cls.set_vectors(cls.TEXT_FIELD, os.path.join(vectors_dir, vectors_name))
 
-        cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors)
+        cls.LABEL_FIELD.build_vocab(train, validation, test)
+
+        cls.VOCAB_SIZE = len(cls.TEXT_FIELD.vocab)
 
         return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device)

nce/NCE-Pairwise-SM/README.md

@@ -0,0 +1,73 @@
+## NCE-SM model
+
+#### References:
++ Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural
+Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information 
+Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738
+
++ Jinfeng Rao, Hua He, and Jimmy Lin. [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks.](http://dl.acm.org/citation.cfm?id=2983872) *Proceedings of the 25th ACM International on Conference on Information and Knowledge Management (CIKM 2016)*, pages 1913-1916.
+
+
+The code uses torchtext for text processing. Set torchtext:
+```bash
+git clone https://github.com/pytorch/text.git
+cd text
+python setup.py install
+```
+
+Download the word2vec model from [here] (https://drive.google.com/file/d/0B2u_nClt6NbzUmhOZU55eEo4QWM/view?usp=sharing) 
+and copy it to the `Castor/data/word2vec` folder.
+
+### Training the model
+
+You can train the SM model for the 4 following configurations:
+1. __random__ - the word embedddings are initialized randomly and are tuned during training
+2. __static__ - the word embeddings are static (Severyn and Moschitti, SIGIR'15)
+3. __non-static__ - the word embeddings are tuned during training
+4. __multichannel__ - contains static and non-static channels for question and answer conv layers
+
+
+```bash
+python train.py --no_cuda --mode rand --batch_size 64 --neg_num 8 --dev_every 50 --patience 1000
+```
+
+NB: pass `--no_cuda` to use CPU
+
+The trained model will be save to:
+```
+saves/static_best_model.pt
+```
+
+### Testing the model
+
+```
+python main.py --trained_model saves/TREC/multichannel_best_model.pt --batch_size 64 --no_cuda
+```
+
+### Evaluation
+
+#### The performance on TrecQA dataset:
+
+##### Without NCE
+
+Metric |rand   |static|non-static|multichannel
+-------|-------|------|----------|------------
+MAP    |0.7441 |0.7524|0.7688    |0.7641
+MRR    |0.8172 |0.8012|0.8144    |0.8174
+
+##### Max Neg Sample
+
+To be added
+
+##### Pairwise + Max Neg Sample with neg_num = 8
+
+Metric |rand   |static|non-static|multichannel
+-------|-------|------|----------|------------
+MAP    |0.7427 |0.7546|0.7716    |0.7794
+MRR    |0.8151 |0.8061|0.8347    |0.8467
+
+
+#### The performance on WikiQA dataset:
+
+To be added
+

nce/NCE-Pairwise-SM/args.py

@@ -0,0 +1,34 @@
+from argparse import ArgumentParser
+
+def get_args():
+    parser = ArgumentParser(description="SM CNN")
+    parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda')
+    parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU
+    parser.add_argument('--epochs', type=int, default=30)
+    parser.add_argument('--batch_size', type=int, default=64)
+    parser.add_argument('--mode', type=str, default='static')
+    parser.add_argument('--lr', type=float, default=0.95)
+    parser.add_argument('--seed', type=int, default=3435)
+    parser.add_argument('--dataset', type=str, default='TREC')
+    parser.add_argument('--resume_snapshot', type=str, default=None)
+    parser.add_argument('--dev_every', type=int, default=100)
+    parser.add_argument('--log_every', type=int, default=10)
+    parser.add_argument('--patience', type=int, default=50)
+    parser.add_argument('--save_path', type=str,    default='saves')
+    parser.add_argument('--output_channel', type=int, default=150)
+    parser.add_argument('--filter_width', type=int, default=5)
+    parser.add_argument('--words_dim', type=int, default=50)
+    parser.add_argument('--dropout', type=float, default=0.5)
+    parser.add_argument('--epoch_decay', type=int, default=15)
+    parser.add_argument('--wordvec_dir', type=str, default='../../data/word2vec/')
+    parser.add_argument('--vector_cache', type=str, default='word2vec.trecqa.pt')
+    parser.add_argument('--trained_model', type=str, default="")
+    parser.add_argument('--weight_decay',type=float, default=1e-5)
+    parser.add_argument('--ext_feats_size', type=int, default=4)
+    parser.add_argument('--neg_num', type=int, default=5)
+    parser.add_argument('--neg_sample', type=str, default="random")
+    parser.add_argument('--eps', type=float, default=1e-6)
+    parser.add_argument('--optimizer', type=str, default="adadelta")
+
+    args = parser.parse_args()
+    return args

nce/NCE-Pairwise-SM/main.py

@@ -0,0 +1,81 @@
+import numpy as np
+import random
+import logging
+import os
+
+import torch
+from torchtext import data
+
+from args import get_args
+from trec_dataset import TrecDataset
+from utils.relevancy_metrics import get_map_mrr
+from datasets.trecqa import TRECQA
+from train import UnknownWordVecCache
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(levelname)s - %(message)s')
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+args = get_args()
+config = args
+
+torch.manual_seed(args.seed)
+np.random.seed(args.seed)
+random.seed(args.seed)
+
+if not args.cuda:
+    args.gpu = -1
+if torch.cuda.is_available() and args.cuda:
+    logger.info("Note: You are using GPU for training")
+    torch.cuda.set_device(args.gpu)
+    torch.cuda.manual_seed(args.seed)
+if torch.cuda.is_available() and not args.cuda:
+    logger.info("Warning: You have Cuda but do not use it. You are using CPU for training")
+
+
+if config.dataset == 'TREC':
+    dataset_root = os.path.join(os.pardir, 'data', 'TrecQA/')
+    train_iter, dev_iter, test_iter = TRECQA.iters(dataset_root, args.vector_cache, args.wordvec_dir, batch_size=args.batch_size, pt_file=True, device=args.gpu, unk_init=UnknownWordVecCache.unk)
+else:
+    logger.info("Unsupported dataset")
+    exit()
+
+config.target_class = 2
+config.questions_num = len(TRECQA.TEXT_FIELD.vocab)
+config.answers_num = len(TRECQA.TEXT_FIELD.vocab)
+
+if args.cuda:
+    model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu))
+else:
+    model = torch.load(args.trained_model, map_location=lambda storage,location: storage)
+
+
+def predict(test_mode, dataset_iter):
+    model.eval()
+    dataset_iter.init_epoch()
+    qids = []
+    predictions = []
+    labels = []
+    for dev_batch_idx, dev_batch in enumerate(dataset_iter):
+        qid_array = np.transpose(dev_batch.id.cpu().data.numpy())
+        true_label_array = np.transpose(dev_batch.label.cpu().data.numpy())
+        output = model.convModel(dev_batch)
+        scores = model.linearLayer(output)
+        score_array = scores.cpu().data.numpy().reshape(-1)
+        qids.extend(qid_array.tolist())
+        predictions.extend(score_array.tolist())
+        labels.extend(true_label_array.tolist())
+
+    dev_map, dev_mrr = get_map_mrr(qids, predictions, labels)
+
+    logger.info("{} {}".format(dev_map, dev_mrr))
+
+# Run the model on the dev set
+predict('dev', dataset_iter=dev_iter)
+
+# Run the model on the test set
+predict('test', dataset_iter=test_iter)

nce/NCE-Pairwise-SM/model.py

@@ -0,0 +1,103 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class PairwiseConv(nn.Module):
+    """docstring for PairwiseConv"""
+    def __init__(self, model):
+        super(PairwiseConv, self).__init__()
+        self.convModel = model
+        self.dropout = nn.Dropout(self.convModel.dropout)
+        self.linearLayer = nn.Linear(model.n_hidden, 1)
+        self.posModel = self.convModel
+        # share or copy ??
+        # https://discuss.pytorch.org/t/copying-nn-modules-without-shared-memory/113
+        # self.negModel = copy.deepcopy(self.posModel)
+        self.negModel = self.convModel
+
+    def forward(self, input):
+        pos = self.posModel(input[0])
+        neg = self.negModel(input[1])
+        pos = self.dropout(pos)
+        neg = self.dropout(neg)
+        pos = self.linearLayer(pos)
+        neg = self.linearLayer(neg)
+        combine = torch.cat([pos, neg], 1)
+        return combine
+
+class SmPlusPlus(nn.Module):
+    def __init__(self, config):
+        super(SmPlusPlus, self).__init__()
+        output_channel = config.output_channel
+        questions_num = config.questions_num
+        answers_num = config.answers_num
+        words_dim = config.words_dim
+        filter_width = config.filter_width
+        self.mode = config.mode
+        self.dropout = config.dropout
+
+        n_classes = config.target_class
+        ext_feats_size = config.ext_feats_size
+
+        if self.mode == 'multichannel':
+            input_channel = 2
+        else:
+            input_channel = 1
+
+        self.question_embed = nn.Embedding(questions_num, words_dim)
+        self.answer_embed = nn.Embedding(answers_num, words_dim)
+        self.static_question_embed = nn.Embedding(questions_num, words_dim)
+        self.nonstatic_question_embed = nn.Embedding(questions_num, words_dim)
+        self.static_answer_embed = nn.Embedding(answers_num, words_dim)
+        self.nonstatic_answer_embed = nn.Embedding(answers_num, words_dim)
+        self.static_question_embed.weight.requires_grad = False
+        self.static_answer_embed.weight.requires_grad = False
+
+        self.conv_q = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0))
+        self.conv_a = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0))
+
+        self.n_hidden = 2 * output_channel + ext_feats_size
+
+        self.combined_feature_vector = nn.Linear(self.n_hidden, self.n_hidden)
+        self.hidden = nn.Linear(self.n_hidden, n_classes)
+
+    def forward(self, x):
+        x_question = x.sentence_1
+        x_answer = x.sentence_2
+        x_ext = x.ext_feats
+
+        if self.mode == 'rand':
+            question = self.question_embed(x_question).unsqueeze(1)
+            answer = self.answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim)
+            x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
+            x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]  # max-over-time pooling
+        # actual SM model mode (Severyn & Moschitti, 2015)
+        elif self.mode == 'static':
+            question = self.static_question_embed(x_question).unsqueeze(1)
+            answer = self.static_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim)
+            x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
+            x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]  # max-over-time pooling
+        elif self.mode == 'non-static':
+            question = self.nonstatic_question_embed(x_question).unsqueeze(1)
+            answer = self.nonstatic_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim)
+            x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
+            x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]  # max-over-time pooling
+        elif self.mode == 'multichannel':
+            question_static = self.static_question_embed(x_question)
+            answer_static = self.static_answer_embed(x_answer)
+            question_nonstatic = self.nonstatic_question_embed(x_question)
+            answer_nonstatic = self.nonstatic_answer_embed(x_answer)
+            question = torch.stack([question_static, question_nonstatic], dim=1)
+            answer = torch.stack([answer_static, answer_nonstatic], dim=1)
+            x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)]
+            x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x]  # max-over-time pooling
+        else:
+            print("Unsupported Mode")
+            exit()
+
+        x.append(x_ext)
+        x = torch.cat(x, 1)
+        x = F.tanh(self.combined_feature_vector(x))
+
+        return x

nce/NCE-Pairwise-SM/overlap_features.py

@@ -0,0 +1,154 @@
+import numpy as np
+import string
+import pickle
+from collections import defaultdict
+from argparse import ArgumentParser
+
+from nltk.stem.porter import PorterStemmer
+
+def load_data(dname):
+    stemmer = PorterStemmer()
+    qids, questions, answers, labels = [], [], [], []
+    print('Load folder ' + dname)
+    with open(dname+'a.toks', encoding='utf-8') as f:
+        for line in f:
+            question = line.strip().split()
+            question = [stemmer.stem(word) for word in question]
+            questions.append(question)
+    with open(dname+'b.toks', encoding='utf-8') as f:
+        for line in f:
+            answer = line.strip().split()
+            answer_list = []
+            for word in answer:
+                try:
+                    answer_list.append(stemmer.stem(word))
+                except Exception as e:
+                    print("couldn't stem the word:" + word)
+            answers.append(answer_list)
+    with open(dname + 'id.txt', encoding='utf-8') as f:
+        for line in f:
+            qids.append(line.strip())
+    with open(dname + 'sim.txt', encoding='utf-8') as f:
+        for line in f:
+            labels.append(int(line.strip()))
+    return qids, questions, answers, labels
+
+def compute_overlap_features(questions, answers, word2df=None, stoplist=None):
+    word2df = word2df if word2df else {}
+    stoplist = stoplist if stoplist else set()
+    feats_overlap = []
+    for question, answer in zip(questions, answers):
+        q_set = set([q for q in question if q not in stoplist])
+        a_set = set([a for a in answer if a not in stoplist])
+        word_overlap = q_set.intersection(a_set)
+        if len(q_set) == 0 and len(a_set) == 0:
+            overlap = 0
+        else:
+            overlap = float(len(word_overlap)) / (len(q_set) + len(a_set))
+
+        word_overlap = q_set.intersection(a_set)
+        df_overlap = 0.0
+        for w in word_overlap:
+            df_overlap += word2df[w]
+
+        if len(q_set) == 0 and len(a_set) == 0:
+            df_overlap = 0
+        else:
+            df_overlap /= (len(q_set) + len(a_set))
+
+        feats_overlap.append(np.array([overlap, df_overlap]))
+    return np.array(feats_overlap)
+
+def compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length):
+    stoplist = stoplist if stoplist else []
+    q_indices, a_indices = [], []
+    for question, answer in zip(questions, answers):
+        q_set = set([q for q in question if q not in stoplist])
+        a_set = set([a for a in answer if a not in stoplist])
+        word_overlap = q_set.intersection(a_set)
+
+        q_idx = np.ones(q_max_sent_length) * 2
+        for i, q in enumerate(question):
+            value = 0
+            if q in word_overlap:
+                value = 1
+            q_idx[i] = value
+        q_indices.append(q_idx)
+
+        a_idx = np.ones(a_max_sent_length) * 2
+        for i, a in enumerate(answer):
+            value = 0
+            if a in word_overlap:
+                value = 1
+            a_idx[i] = value
+        a_indices.append(a_idx)
+
+    q_indices = np.vstack(q_indices).astype('int32')
+    a_indices = np.vstack(a_indices).astype('int32')
+
+    return q_indices, a_indices
+
+def compute_dfs(docs):
+    word2df = defaultdict(float)
+    for doc in docs:
+        for w in set(doc):
+            word2df[w] += 1.0
+    num_docs = len(docs)
+
+    for w, value in word2df.items():
+        word2df[w] = np.math.log(num_docs / value)   # bug feats fixed
+
+    return word2df
+
+if __name__ == '__main__':
+    parser = ArgumentParser(description='create TrecQA/WikiQA dataset')
+    parser.add_argument('--dir', help='path to the TrecQA|WikiQA data directory', default="../../data/TrecQA")
+    args = parser.parse_args()
+
+    stoplist = set([line.strip() for line in open('../../data/TrecQA/stopwords.txt', encoding='utf-8')])
+    punct = set(string.punctuation)
+    stoplist.update(punct)
+
+    all_questions, all_answers, all_qids = [], [], []
+    base_dir = args.dir
+
+    if 'TrecQA' in base_dir:
+        sub_dirs = ['train/', 'train-all/', 'raw-dev/', 'raw-test/']
+    elif 'WikiQA' in base_dir:
+        sub_dirs = ['train/', 'dev/', 'test/']
+    else:
+        print('Unsupported dataset')
+        exit()
+
+    for sub in sub_dirs:
+        qids, questions, answers, labels = load_data(base_dir + sub)
+        all_questions.extend(questions)
+        all_answers.extend(answers)
+        all_qids.extend(qids)
+
+    seen = set()
+    unique_questions = []
+    for q, qid in zip(all_questions, all_qids):
+        if qid not in seen:
+            seen.add(qid)
+            unique_questions.append(q)
+
+    docs = all_answers + unique_questions
+    word2dfs = compute_dfs(docs)
+    pickle.dump(word2dfs, open("word2dfs.p", "wb"))
+
+    q_max_sent_length = max(map(lambda x: len(x), all_questions))
+    a_max_sent_length = max(map(lambda x: len(x), all_answers))
+
+    for sub in sub_dirs:
+        qids, questions, answers, labels = load_data(base_dir + sub)
+
+        overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs)
+        overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs)
+        overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist])
+
+        with open(base_dir + sub + 'overlap_feats.txt', 'w') as f:
+            for i in range(overlap_feats.shape[0]):
+                for j in range(4):
+                    f.write(str(overlap_feats[i][j]) + ' ')
+                f.write('\n')

nce/NCE-Pairwise-SM/requirements.txt

@@ -0,0 +1,4 @@
+nltk==3.2.4
+numpy==1.13.1
+pytorch==0.2.0+5f864ca
+torchtext==0.2.0

nce/NCE-Pairwise-SM/train.py

@@ -0,0 +1,350 @@
+import time
+import os
+import numpy as np
+import random
+import heapq
+import operator
+import logging
+
+import torch
+import torch.nn as nn
+from torchtext import data
+from torch.nn import functional as F
+
+from datasets.trecqa import TRECQA
+from args import get_args
+from model import SmPlusPlus, PairwiseConv
+from utils.relevancy_metrics import get_map_mrr
+
+
+class UnknownWordVecCache(object):
+    """
+    Caches the first randomly generated word vector for a certain size to make it is reused.
+    """
+    cache = {}
+
+    @classmethod
+    def unk(cls, tensor):
+        size_tup = tuple(tensor.size())
+        if size_tup not in cls.cache:
+            cls.cache[size_tup] = torch.Tensor(tensor.size())
+            cls.cache[size_tup].uniform_(-0.05, 0.05)
+        return cls.cache[size_tup]
+
+
+def train_sm():
+
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.INFO)
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    args = get_args()
+    config = args
+    torch.backends.cudnn.deterministic = True
+
+    # Set random seed for reproducibility
+    torch.manual_seed(args.seed)
+    if not args.cuda:
+        args.gpu = -1
+    if torch.cuda.is_available() and args.cuda:
+        logger.info("Note: You are using GPU for training")
+        torch.cuda.set_device(args.gpu)
+        torch.cuda.manual_seed(args.seed)
+    if torch.cuda.is_available() and not args.cuda:
+        logger.info("You have Cuda but you're using CPU for training.")
+    np.random.seed(args.seed)
+    random.seed(args.seed)
+
+    dataset_root = os.path.join(os.pardir, 'data', 'TrecQA/')
+    train_iter, dev_iter, test_iter = TRECQA.iters(dataset_root, args.vector_cache, args.wordvec_dir, batch_size=args.batch_size,
+                                                   pt_file=True, device=args.gpu, unk_init=UnknownWordVecCache.unk) #
+
+    index2text = np.array(TRECQA.TEXT_FIELD.vocab.itos)
+
+    config.target_class = 2
+    config.questions_num = TRECQA.VOCAB_SIZE
+    config.answers_num = TRECQA.VOCAB_SIZE
+
+    logger.info("index2text: {}".format(index2text))
+    logger.info("Dataset: {}, Mode: {}".format(args.dataset, args.mode))
+    logger.info("VOCAB num: {}".format(TRECQA.VOCAB_SIZE))
+
+    if args.resume_snapshot:
+        if args.cuda:
+            pw_model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage.cuda(args.gpu))
+        else:
+            pw_model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage)
+    else:
+        model = SmPlusPlus(config)
+        model.static_question_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors)
+        model.nonstatic_question_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors)
+        model.static_answer_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors)
+        model.nonstatic_answer_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors)
+
+        if args.cuda:
+            model.cuda()
+            logger.info("Shift model to GPU")
+
+        pw_model = PairwiseConv(model)
+
+    parameter = filter(lambda p: p.requires_grad, pw_model.parameters())
+
+    if args.optimizer == "adadelta":
+        # the SM model originally follows SGD but Adadelta is used here
+        optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay, eps=args.eps)
+    # A good lr is required to use in the following optimizer
+    elif args.optimizer == "adam":
+        optimizer = torch.optim.Adam(parameter, lr=args.lr, weight_decay=args.weight_decay, eps=1e-8)
+    elif args.optimizer == "sgd":
+        optimizer = torch.optim.SGD(parameter, lr=0.001, momentum=0.9, weight_decay=args.weight_decay)
+    elif args.optimizer == "rmsprop":
+        optimizer = torch.optim.RMSprop(parameter, lr=0.0001, weight_decay=args.weight_decay)
+
+    marginRankingLoss = nn.MarginRankingLoss(margin=1, size_average=True)
+
+    early_stop = False
+    iterations = 0
+    iters_not_improved = 0
+    epoch = 0
+    q2neg = {} # a dict from qid to a list of aid
+    question2answer = {} # a dict from qid to the information of both pos and neg answers
+    best_dev_map = 0
+    best_dev_mrr = 0
+    false_samples = {}
+
+    start = time.time()
+    header = '  Time Epoch Iteration Progress    (%Epoch)  Average_Loss Train_Accuracy Dev/MAP  Dev/MRR'
+    dev_log_template = ' '.join(
+        '{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>11.6f},{:>11.6f},{:12.6f},{:8.4f}'.split(','))
+    log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>11.6f},{:>11.6f},'.split(','))
+    os.makedirs(args.save_path, exist_ok=True)
+    os.makedirs(os.path.join(args.save_path, args.dataset), exist_ok=True)
+    print(header)
+
+    # get the nearest negative samples to the positive sample by computing the feature difference
+    def get_nearest_neg_id(pos_feature, neg_dict, distance="cosine", k=1):
+        dis_list = []
+        pos_feature = pos_feature.data.cpu().numpy()
+        pos_feature_norm = pos_feature / np.sqrt(sum(pos_feature ** 2))
+        neg_list = []
+        for key in neg_dict:
+            if distance == "l2":
+                dis = np.sqrt(np.sum((np.array(pos_feature) - neg_dict[key]["feature"]) ** 2))
+            elif distance == "cosine":
+                neg_feature = np.array(neg_dict[key]["feature"])
+                feat_norm = neg_feature / np.sqrt(sum(neg_feature ** 2))
+                dis = 1 - feat_norm.dot(pos_feature_norm)
+            dis_list.append(dis)
+            neg_list.append(key)
+
+        k = min(k, len(neg_dict))
+        min_list = heapq.nsmallest(k, enumerate(dis_list), key=operator.itemgetter(1))
+        min_id_list = [neg_list[x[0]] for x in min_list]
+        return min_id_list
+
+    # get the negative samples randomly
+    def get_random_neg_id(q2neg, qid_i, k=5):
+        # question 1734 has no neg answer
+        if qid_i not in q2neg:
+            return []
+        k = min(k, len(q2neg[qid_i]))
+        ran = random.sample(q2neg[qid_i], k)
+        return ran
+
+    # pack the lists of question/answer/ext_feat into a torchtext batch
+    def get_batch(question, answer, ext_feat, size):
+        new_batch = data.Batch()
+        new_batch.batch_size = size
+        new_batch.dataset = batch.dataset
+        setattr(new_batch, "sentence_2", torch.stack(answer))
+        setattr(new_batch, "sentence_1", torch.stack(question))
+        setattr(new_batch, "ext_feats", torch.stack(ext_feat))
+        return new_batch
+
+    while True:
+        if early_stop:
+            logger.log("Early Stopping. Epoch: {}, Best Dev Loss: {}".format(epoch, best_dev_loss))
+            break
+        epoch += 1
+        train_iter.init_epoch()
+        '''
+        batch size issue: padding is a choice (add or delete them in both train and test)
+                        associated with the batch size. Currently, it seems to affect the result a lot.
+        '''
+        acc = 0
+        tot = 0
+        for batch_idx, batch in enumerate(iter(train_iter)):
+            if epoch != 1:
+                iterations += 1
+            loss_num = 0
+            pw_model.train()
+
+            new_train = {"ext_feat": [], "question": [], "answer": [], "label": []}
+            features = pw_model.convModel(batch)
+            new_train_pos = {"answer": [], "question": [], "ext_feat": []}
+            new_train_neg = {"answer": [], "question": [], "ext_feat": []}
+            max_len_q = 0
+            max_len_a = 0
+
+            batch_near_list = []
+            batch_qid = []
+            batch_aid = []
+
+            for i in range(batch.batch_size):
+                label_i = batch.label[i].cpu().data.numpy()[0]
+                question_i = batch.sentence_1[i]
+                # question_i = question_i[question_i!=1] # remove padding 1 <pad>
+                answer_i = batch.sentence_2[i]
+                # answer_i = answer_i[answer_i!=1] # remove padding 1 <pad>
+                ext_feat_i = batch.ext_feats[i]
+                qid_i = batch.id[i].data.cpu().numpy()[0]
+                aid_i = batch.aid[i].data.cpu().numpy()[0]
+
+                if qid_i not in question2answer:
+                    question2answer[qid_i] = {"question": question_i, "pos": {}, "neg": {}}
+                if label_i == 1:
+
+                    if aid_i not in question2answer[qid_i]["pos"]:
+                        question2answer[qid_i]["pos"][aid_i] = {}
+
+                    question2answer[qid_i]["pos"][aid_i]["answer"] = answer_i
+                    question2answer[qid_i]["pos"][aid_i]["ext_feat"] = ext_feat_i
+
+                    # get neg samples in the first epoch but do not train
+                    if epoch == 1:
+                        continue
+                    # random generate sample in the first training epoch
+                    elif epoch == 2 or args.neg_sample == "random":
+                        near_list = get_random_neg_id(q2neg, qid_i, k=args.neg_num)
+                    else:
+                        debug_qid = qid_i
+                        near_list = get_nearest_neg_id(features[i], question2answer[qid_i]["neg"], distance="cosine", k=args.neg_num)
+
+                    batch_near_list.extend(near_list)
+
+                    neg_size = len(near_list)
+                    if neg_size != 0:
+                        answer_i = answer_i[answer_i != 1] # remove padding 1 <pad>
+                        question_i = question_i[question_i != 1] # remove padding 1 <pad>
+                        for near_id in near_list:
+                            batch_qid.append(qid_i)
+                            batch_aid.append(aid_i)
+
+                            new_train_pos["answer"].append(answer_i)
+                            new_train_pos["question"].append(question_i)
+                            new_train_pos["ext_feat"].append(ext_feat_i)
+
+                            near_answer = question2answer[qid_i]["neg"][near_id]["answer"]
+                            if question_i.size()[0] > max_len_q:
+                                max_len_q = question_i.size()[0]
+                            if near_answer.size()[0] > max_len_a:
+                                max_len_a = near_answer.size()[0]
+                            if answer_i.size()[0] > max_len_a:
+                                max_len_a = answer_i.size()[0]
+
+                            ext_feat_neg = question2answer[qid_i]["neg"][near_id]["ext_feat"]
+                            new_train_neg["answer"].append(near_answer)
+                            new_train_neg["question"].append(question_i)
+                            new_train_neg["ext_feat"].append(ext_feat_neg)
+
+                elif label_i == 0:
+
+                    if aid_i not in question2answer[qid_i]["neg"]:
+                        answer_i = answer_i[answer_i != 1]
+                        question2answer[qid_i]["neg"][aid_i] = {"answer": answer_i}
+
+                    question2answer[qid_i]["neg"][aid_i]["feature"] = features[i].data.cpu().numpy()
+                    question2answer[qid_i]["neg"][aid_i]["ext_feat"] = ext_feat_i
+
+                    if epoch == 1:
+                        if qid_i not in q2neg:
+                            q2neg[qid_i] = []
+
+                        q2neg[qid_i].append(aid_i)
+
+            # pack the selected pos and neg samples into the torchtext batch and train
+            if epoch != 1:
+                true_batch_size = len(new_train_neg["answer"])
+                if true_batch_size != 0:
+                    for j in range(true_batch_size):
+                        new_train_neg["answer"][j] = F.pad(new_train_neg["answer"][j],
+                                                           (0, max_len_a - new_train_neg["answer"][j].size()[0]), value=1)
+                        new_train_pos["answer"][j] = F.pad(new_train_pos["answer"][j],
+                                                           (0, max_len_a - new_train_pos["answer"][j].size()[0]), value=1)
+                        new_train_pos["question"][j] = F.pad(new_train_pos["question"][j],
+                                                           (0, max_len_q - new_train_pos["question"][j].size()[0]), value=1)
+                        new_train_neg["question"][j] = F.pad(new_train_neg["question"][j],
+                                                           (0, max_len_q - new_train_neg["question"][j].size()[0]), value=1)
+
+                    pos_batch = get_batch(new_train_pos["question"], new_train_pos["answer"], new_train_pos["ext_feat"],
+                                          true_batch_size)
+                    neg_batch = get_batch(new_train_neg["question"], new_train_neg["answer"], new_train_neg["ext_feat"],
+                                          true_batch_size)
+
+                    optimizer.zero_grad()
+                    output = pw_model([pos_batch, neg_batch])
+
+                    cmp = output[:, 0] > output[:, 1]
+                    acc += sum(cmp.data.cpu().numpy())
+                    tot += true_batch_size
+
+                    loss = marginRankingLoss(output[:, 0], output[:, 1], torch.autograd.Variable(torch.ones(1)))
+                    loss_num = loss.data.numpy()[0]
+                    loss.backward()
+                    optimizer.step()
+
+            # Evaluate performance on validation set
+            if iterations % args.dev_every == 1 and epoch != 1:
+                # switch model into evaluation mode
+                pw_model.eval()
+                dev_iter.init_epoch()
+                qids = []
+                predictions = []
+                labels = []
+
+                for dev_batch_idx, dev_batch in enumerate(dev_iter):
+                    '''
+                    # dev singlely or in a batch? -> in a batch
+                    but dev singlely is equal to dev_size = 1
+                    '''
+                    scores = pw_model.convModel(dev_batch)
+                    scores = pw_model.linearLayer(scores)
+                    qid_array = np.transpose(dev_batch.id.cpu().data.numpy())
+                    score_array = scores.cpu().data.numpy().reshape(-1)
+                    true_label_array = np.transpose(dev_batch.label.cpu().data.numpy())
+
+                    qids.extend(qid_array.tolist())
+                    predictions.extend(score_array.tolist())
+                    labels.extend(true_label_array.tolist())
+
+                dev_map, dev_mrr = get_map_mrr(qids, predictions, labels)
+                print(dev_log_template.format(time.time() - start,
+                                              epoch, iterations, 1 + batch_idx, len(train_iter),
+                                              100. * (1 + batch_idx) / len(train_iter),
+                                              loss_num, acc / tot, dev_map, dev_mrr))
+                if best_dev_mrr < dev_mrr:
+                    snapshot_path = os.path.join(args.save_path, args.dataset, args.mode + '_best_model.pt')
+                    torch.save(pw_model, snapshot_path)
+                    iters_not_improved = 0
+                    best_dev_mrr = dev_mrr
+                else:
+                    iters_not_improved += 1
+                    if iters_not_improved >= args.patience:
+                        early_stop = True
+                        break
+
+            if iterations % args.log_every == 1 and epoch != 1:
+                # logger.info progress message
+                print(log_template.format(time.time() - start,
+                                          epoch, iterations, 1 + batch_idx, len(train_iter),
+                                          100. * (1 + batch_idx) / len(train_iter),
+                                         loss_num,  acc / tot))
+                acc = 0
+                tot = 0
+
+if __name__ == '__main__':
+    train_sm()