MP-CNN: Add Additional Features (#51)

* MP-CNN: support external features

* MP-CNN: calculate overlap in same way as SM-model

* MP-CNN: properly calculate idf overlap

mp_cnn/dataset.py

@@ -1,14 +1,19 @@
+from collections import defaultdict
 from enum import Enum
 import math
 import os
 
+import nltk
+import numpy as np
 import torch
 from torch.autograd import Variable
-import torch.nn.functional as F
 import torch.utils.data as data
 
 import preprocessing
 
+nltk.download('stopwords', quiet=True)
+from nltk.corpus import stopwords
+
 # logging setup
 import logging
 logger = logging.getLogger(__name__)
@@ -94,6 +99,7 @@ class MPCNNDataset(data.Dataset):
         """
         sent_a = self._load(self.dataset_dir, 'a.txt')
         sent_b = self._load(self.dataset_dir, 'b.txt')
+        word_to_doc_cnt = defaultdict(int)
 
         # obtain max sentence length to use as dimension for padding to support batching
         sent_a_tokens, sent_b_tokens = [], []
@@ -104,12 +110,36 @@ class MPCNNDataset(data.Dataset):
             sent_a_tokens.append(sa_tokens)
             sent_b_tokens.append(sb_tokens)
 
+            unique_tokens = set(sa_tokens) | set(sb_tokens)
+            for t in unique_tokens:
+                word_to_doc_cnt[t] += 1
+
         self.sentences = []
+        stoplist = set(stopwords.words('english'))
+        num_docs = len(word_to_doc_cnt)
         for i in range(len(sent_a)):
             sent_pair = {}
             sent_pair['a'] = self._get_sentence_embeddings(sent_a_tokens[i], word_index, embedding)
             sent_pair['b'] = self._get_sentence_embeddings(sent_b_tokens[i], word_index, embedding)
+
+            tokens_a_set, tokens_b_set = set(sent_a_tokens[i]), set(sent_b_tokens[i])
+            intersect = tokens_a_set & tokens_b_set
+            overlap = len(intersect) / (len(tokens_a_set) + len(tokens_b_set))
+            idf_intersect = sum(np.math.log(num_docs / word_to_doc_cnt[w]) for w in intersect)
+            idf_weighted_overlap = idf_intersect / (len(tokens_a_set) + len(tokens_b_set))
+
+            tokens_a_set_no_stop = set(w for w in sent_a_tokens[i] if w not in stoplist)
+            tokens_b_set_no_stop = set(w for w in sent_b_tokens[i] if w not in stoplist)
+            intersect_no_stop = tokens_a_set_no_stop & tokens_b_set_no_stop
+            overlap_no_stop = len(intersect_no_stop) / (len(tokens_a_set_no_stop) + len(tokens_b_set_no_stop))
+            idf_intersect_no_stop = sum(np.math.log(num_docs / word_to_doc_cnt[w]) for w in intersect_no_stop)
+            idf_weighted_overlap_no_stop = idf_intersect_no_stop / (len(tokens_a_set_no_stop) + len(tokens_b_set_no_stop))
+            ext_feats = torch.Tensor([overlap, idf_weighted_overlap, overlap_no_stop, idf_weighted_overlap_no_stop])
+            ext_feats = ext_feats.cuda() if self.cuda else ext_feats
+            sent_pair['ext_feats'] = ext_feats
+
             self.sentences.append(sent_pair)
+
         self.labels = self._load(self.dataset_dir, 'sim.txt', float)
 
     def _load(self, dataset_dir, fname, type_converter=str):

mp_cnn/evaluation.py

@@ -57,8 +57,9 @@ class SICKEvaluator(Evaluator):
         true_labels = []
         for sentences, labels in self.data_loader:
             sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True)
+            ext_feats = Variable(sentences['ext_feats'], volatile=True)
             labels = Variable(labels, volatile=True)
-            output = self.model(sent_a, sent_b)
+            output = self.model(sent_a, sent_b, ext_feats)
             test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0]
             # handle last batch which might have smaller size
             if len(predict_classes) != len(sent_a):
@@ -94,8 +95,9 @@ class MSRVIDEvaluator(Evaluator):
         true_labels = []
         for sentences, labels in self.data_loader:
             sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True)
+            ext_feats = Variable(sentences['ext_feats'], volatile=True)
             labels = Variable(labels, volatile=True)
-            output = self.model(sent_a, sent_b)
+            output = self.model(sent_a, sent_b, ext_feats)
             test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0]
             # handle last batch which might have smaller size
             if len(predict_classes) != len(sent_a):

mp_cnn/main.py

@@ -29,6 +29,7 @@ if __name__ == '__main__':
     parser.add_argument('--word-vectors-file', help='word vectors file', default=os.path.join(os.pardir, os.pardir, 'data', 'GloVe', 'glove.840B.300d.txt'))
     parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true')
     parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training (default: false)')
+    parser.add_argument('--sparse-features', action='store_true', default=False, help='use sparse features (default: false)')
     parser.add_argument('--batch-size', type=int, default=64, help='input batch size for training (default: 64)')
     parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 10)')
     parser.add_argument('--optimizer', type=str, default='adam', help='optimizer to use: adam or sgd (default: adam)')
@@ -58,7 +59,7 @@ if __name__ == '__main__':
 
     filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf]
     input_channels = 300
-    model = MPCNN(input_channels, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes, args.dropout)
+    model = MPCNN(input_channels, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes, args.dropout, args.sparse_features)
     if args.cuda:
         model.cuda()
     optimizer = None

mp_cnn/model.py

@@ -6,13 +6,14 @@ import torch.nn.functional as F
 
 class MPCNN(nn.Module):
 
-    def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes, dropout):
+    def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes, dropout, ext_feats):
         super(MPCNN, self).__init__()
 
         self.n_word_dim = n_word_dim
         self.n_holistic_filters = n_holistic_filters
         self.n_per_dim_filters = n_per_dim_filters
         self.filter_widths = filter_widths
+        self.ext_feats = ext_feats
         holistic_conv_layers = []
         per_dim_conv_layers = []
 
@@ -35,6 +36,7 @@ class MPCNN(nn.Module):
 
         # compute number of inputs to first hidden layer
         COMP_1_COMPONENTS_HOLISTIC, COMP_1_COMPONENTS_PER_DIM, COMP_2_COMPONENTS = 2 + n_holistic_filters, 2 + n_word_dim, 2
+        EXT_FEATS = 4 if ext_feats else 0
         n_feat_h = 3 * len(self.filter_widths) * COMP_2_COMPONENTS
         n_feat_v = (
             # comparison units from holistic conv for min, max, mean pooling for non-infinite widths
@@ -44,7 +46,7 @@ class MPCNN(nn.Module):
             # comparison units from per-dim conv
             2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS_PER_DIM
         )
-        n_feat = n_feat_h + n_feat_v
+        n_feat = n_feat_h + n_feat_v + EXT_FEATS
 
         self.final_layers = nn.Sequential(
             nn.Linear(n_feat, hidden_layer_units),
@@ -119,7 +121,7 @@ class MPCNN(nn.Module):
 
         return torch.cat(comparison_feats, dim=1)
 
-    def forward(self, sent1, sent2):
+    def forward(self, sent1, sent2, ext_feats):
         # Sentence modeling module
         sent1_block_a, sent1_block_b = self._get_blocks_for_sentence(sent1)
         sent2_block_a, sent2_block_b = self._get_blocks_for_sentence(sent2)
@@ -127,7 +129,8 @@ class MPCNN(nn.Module):
         # Similarity measurement layer
         feat_h = self._algo_1_horiz_comp(sent1_block_a, sent2_block_a)
         feat_v = self._algo_2_vert_comp(sent1_block_a, sent2_block_a, sent1_block_b, sent2_block_b)
-        feat_all = torch.cat([feat_h, feat_v], dim=1)
+        combined_feats = [feat_h, feat_v, ext_feats] if self.ext_feats else [feat_h, feat_v]
+        feat_all = torch.cat(combined_feats, dim=1)
 
         preds = self.final_layers(feat_all)
         return preds

mp_cnn/train.py

@@ -77,9 +77,10 @@ class SICKTrainer(Trainer):
         total_loss = 0
         for batch_idx, (sentences, labels) in enumerate(self.train_loader):
             sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b'])
+            ext_feats = Variable(sentences['ext_feats'])
             labels = Variable(labels)
             self.optimizer.zero_grad()
-            output = self.model(sent_a, sent_b)
+            output = self.model(sent_a, sent_b, ext_feats)
             loss = F.kl_div(output, labels)
             total_loss += loss.data[0]
             loss.backward()
@@ -130,10 +131,12 @@ class MSRVIDTrainer(Trainer):
         batches = math.ceil(len(self.train_loader.dataset) / self.batch_size)
         start_val_batch = math.floor(0.8 * batches)
         left_out_val_a, left_out_val_b = [], []
+        left_out_ext_feats = []
         left_out_val_labels = []
 
         for batch_idx, (sentences, labels) in enumerate(self.train_loader):
             sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b'])
+            ext_feats = Variable(sentences['ext_feats'])
             labels = Variable(labels)
             if batch_idx >= start_val_batch:
                 left_out_val_a.append(sent_a)
@@ -141,7 +144,7 @@ class MSRVIDTrainer(Trainer):
                 left_out_val_labels.append(labels)
                 continue
             self.optimizer.zero_grad()
-            output = self.model(sent_a, sent_b)
+            output = self.model(sent_a, sent_b, ext_feats)
             loss = F.kl_div(output, labels)
             loss.backward()
             self.optimizer.step()
@@ -155,7 +158,7 @@ class MSRVIDTrainer(Trainer):
             del loss, output
 
         self.evaluate(self.train_evaluator, 'train')
-        return left_out_val_a, left_out_val_b, left_out_val_labels
+        return left_out_val_a, left_out_val_b, left_out_ext_feats, left_out_val_labels
 
     def train(self, epochs):
         scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience)
@@ -164,13 +167,14 @@ class MSRVIDTrainer(Trainer):
         for epoch in range(1, epochs + 1):
             start = time.time()
             logger.info('Epoch {} started...'.format(epoch))
-            left_out_a, left_out_b, left_out_label = self.train_epoch(epoch)
+            left_out_a, left_out_b, left_out_ext_feats, left_out_label = self.train_epoch(epoch)
 
             # manually evaluating the validating set
             left_out_a = torch.cat(left_out_a)
             left_out_b = torch.cat(left_out_b)
+            left_out_ext_feats = torch.cat(left_out_ext_feats)
             left_out_label = torch.cat(left_out_label)
-            output = self.model(left_out_a, left_out_b)
+            output = self.model(left_out_a, left_out_b, left_out_ext_feats)
             predict_classes = torch.arange(0, 6).expand(len(left_out_a), 6).cuda()
             true_labels = (predict_classes * left_out_label.data).sum(dim=1)
             predictions = (predict_classes * output.data.exp()).sum(dim=1)