diff --git a/mp_cnn/dataset.py b/mp_cnn/dataset.py index 0f3e20b..cba0877 100644 --- a/mp_cnn/dataset.py +++ b/mp_cnn/dataset.py @@ -1,14 +1,19 @@ +from collections import defaultdict from enum import Enum import math import os +import nltk +import numpy as np import torch from torch.autograd import Variable -import torch.nn.functional as F import torch.utils.data as data import preprocessing +nltk.download('stopwords', quiet=True) +from nltk.corpus import stopwords + # logging setup import logging logger = logging.getLogger(__name__) @@ -94,6 +99,7 @@ class MPCNNDataset(data.Dataset): """ sent_a = self._load(self.dataset_dir, 'a.txt') sent_b = self._load(self.dataset_dir, 'b.txt') + word_to_doc_cnt = defaultdict(int) # obtain max sentence length to use as dimension for padding to support batching sent_a_tokens, sent_b_tokens = [], [] @@ -104,12 +110,36 @@ class MPCNNDataset(data.Dataset): sent_a_tokens.append(sa_tokens) sent_b_tokens.append(sb_tokens) + unique_tokens = set(sa_tokens) | set(sb_tokens) + for t in unique_tokens: + word_to_doc_cnt[t] += 1 + self.sentences = [] + stoplist = set(stopwords.words('english')) + num_docs = len(word_to_doc_cnt) for i in range(len(sent_a)): sent_pair = {} sent_pair['a'] = self._get_sentence_embeddings(sent_a_tokens[i], word_index, embedding) sent_pair['b'] = self._get_sentence_embeddings(sent_b_tokens[i], word_index, embedding) + + tokens_a_set, tokens_b_set = set(sent_a_tokens[i]), set(sent_b_tokens[i]) + intersect = tokens_a_set & tokens_b_set + overlap = len(intersect) / (len(tokens_a_set) + len(tokens_b_set)) + idf_intersect = sum(np.math.log(num_docs / word_to_doc_cnt[w]) for w in intersect) + idf_weighted_overlap = idf_intersect / (len(tokens_a_set) + len(tokens_b_set)) + + tokens_a_set_no_stop = set(w for w in sent_a_tokens[i] if w not in stoplist) + tokens_b_set_no_stop = set(w for w in sent_b_tokens[i] if w not in stoplist) + intersect_no_stop = tokens_a_set_no_stop & tokens_b_set_no_stop + overlap_no_stop = len(intersect_no_stop) / (len(tokens_a_set_no_stop) + len(tokens_b_set_no_stop)) + idf_intersect_no_stop = sum(np.math.log(num_docs / word_to_doc_cnt[w]) for w in intersect_no_stop) + idf_weighted_overlap_no_stop = idf_intersect_no_stop / (len(tokens_a_set_no_stop) + len(tokens_b_set_no_stop)) + ext_feats = torch.Tensor([overlap, idf_weighted_overlap, overlap_no_stop, idf_weighted_overlap_no_stop]) + ext_feats = ext_feats.cuda() if self.cuda else ext_feats + sent_pair['ext_feats'] = ext_feats + self.sentences.append(sent_pair) + self.labels = self._load(self.dataset_dir, 'sim.txt', float) def _load(self, dataset_dir, fname, type_converter=str): diff --git a/mp_cnn/evaluation.py b/mp_cnn/evaluation.py index 7bdd653..0e8bb0e 100644 --- a/mp_cnn/evaluation.py +++ b/mp_cnn/evaluation.py @@ -57,8 +57,9 @@ class SICKEvaluator(Evaluator): true_labels = [] for sentences, labels in self.data_loader: sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True) + ext_feats = Variable(sentences['ext_feats'], volatile=True) labels = Variable(labels, volatile=True) - output = self.model(sent_a, sent_b) + output = self.model(sent_a, sent_b, ext_feats) test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0] # handle last batch which might have smaller size if len(predict_classes) != len(sent_a): @@ -94,8 +95,9 @@ class MSRVIDEvaluator(Evaluator): true_labels = [] for sentences, labels in self.data_loader: sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True) + ext_feats = Variable(sentences['ext_feats'], volatile=True) labels = Variable(labels, volatile=True) - output = self.model(sent_a, sent_b) + output = self.model(sent_a, sent_b, ext_feats) test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0] # handle last batch which might have smaller size if len(predict_classes) != len(sent_a): diff --git a/mp_cnn/main.py b/mp_cnn/main.py index adf153b..9312dfc 100644 --- a/mp_cnn/main.py +++ b/mp_cnn/main.py @@ -29,6 +29,7 @@ if __name__ == '__main__': parser.add_argument('--word-vectors-file', help='word vectors file', default=os.path.join(os.pardir, os.pardir, 'data', 'GloVe', 'glove.840B.300d.txt')) parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training (default: false)') + parser.add_argument('--sparse-features', action='store_true', default=False, help='use sparse features (default: false)') parser.add_argument('--batch-size', type=int, default=64, help='input batch size for training (default: 64)') parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 10)') parser.add_argument('--optimizer', type=str, default='adam', help='optimizer to use: adam or sgd (default: adam)') @@ -58,7 +59,7 @@ if __name__ == '__main__': filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf] input_channels = 300 - model = MPCNN(input_channels, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes, args.dropout) + model = MPCNN(input_channels, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes, args.dropout, args.sparse_features) if args.cuda: model.cuda() optimizer = None diff --git a/mp_cnn/model.py b/mp_cnn/model.py index 300671f..a9e569e 100644 --- a/mp_cnn/model.py +++ b/mp_cnn/model.py @@ -6,13 +6,14 @@ import torch.nn.functional as F class MPCNN(nn.Module): - def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes, dropout): + def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes, dropout, ext_feats): super(MPCNN, self).__init__() self.n_word_dim = n_word_dim self.n_holistic_filters = n_holistic_filters self.n_per_dim_filters = n_per_dim_filters self.filter_widths = filter_widths + self.ext_feats = ext_feats holistic_conv_layers = [] per_dim_conv_layers = [] @@ -35,6 +36,7 @@ class MPCNN(nn.Module): # compute number of inputs to first hidden layer COMP_1_COMPONENTS_HOLISTIC, COMP_1_COMPONENTS_PER_DIM, COMP_2_COMPONENTS = 2 + n_holistic_filters, 2 + n_word_dim, 2 + EXT_FEATS = 4 if ext_feats else 0 n_feat_h = 3 * len(self.filter_widths) * COMP_2_COMPONENTS n_feat_v = ( # comparison units from holistic conv for min, max, mean pooling for non-infinite widths @@ -44,7 +46,7 @@ class MPCNN(nn.Module): # comparison units from per-dim conv 2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS_PER_DIM ) - n_feat = n_feat_h + n_feat_v + n_feat = n_feat_h + n_feat_v + EXT_FEATS self.final_layers = nn.Sequential( nn.Linear(n_feat, hidden_layer_units), @@ -119,7 +121,7 @@ class MPCNN(nn.Module): return torch.cat(comparison_feats, dim=1) - def forward(self, sent1, sent2): + def forward(self, sent1, sent2, ext_feats): # Sentence modeling module sent1_block_a, sent1_block_b = self._get_blocks_for_sentence(sent1) sent2_block_a, sent2_block_b = self._get_blocks_for_sentence(sent2) @@ -127,7 +129,8 @@ class MPCNN(nn.Module): # Similarity measurement layer feat_h = self._algo_1_horiz_comp(sent1_block_a, sent2_block_a) feat_v = self._algo_2_vert_comp(sent1_block_a, sent2_block_a, sent1_block_b, sent2_block_b) - feat_all = torch.cat([feat_h, feat_v], dim=1) + combined_feats = [feat_h, feat_v, ext_feats] if self.ext_feats else [feat_h, feat_v] + feat_all = torch.cat(combined_feats, dim=1) preds = self.final_layers(feat_all) return preds diff --git a/mp_cnn/train.py b/mp_cnn/train.py index 4768ee7..b20d24f 100644 --- a/mp_cnn/train.py +++ b/mp_cnn/train.py @@ -77,9 +77,10 @@ class SICKTrainer(Trainer): total_loss = 0 for batch_idx, (sentences, labels) in enumerate(self.train_loader): sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b']) + ext_feats = Variable(sentences['ext_feats']) labels = Variable(labels) self.optimizer.zero_grad() - output = self.model(sent_a, sent_b) + output = self.model(sent_a, sent_b, ext_feats) loss = F.kl_div(output, labels) total_loss += loss.data[0] loss.backward() @@ -130,10 +131,12 @@ class MSRVIDTrainer(Trainer): batches = math.ceil(len(self.train_loader.dataset) / self.batch_size) start_val_batch = math.floor(0.8 * batches) left_out_val_a, left_out_val_b = [], [] + left_out_ext_feats = [] left_out_val_labels = [] for batch_idx, (sentences, labels) in enumerate(self.train_loader): sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b']) + ext_feats = Variable(sentences['ext_feats']) labels = Variable(labels) if batch_idx >= start_val_batch: left_out_val_a.append(sent_a) @@ -141,7 +144,7 @@ class MSRVIDTrainer(Trainer): left_out_val_labels.append(labels) continue self.optimizer.zero_grad() - output = self.model(sent_a, sent_b) + output = self.model(sent_a, sent_b, ext_feats) loss = F.kl_div(output, labels) loss.backward() self.optimizer.step() @@ -155,7 +158,7 @@ class MSRVIDTrainer(Trainer): del loss, output self.evaluate(self.train_evaluator, 'train') - return left_out_val_a, left_out_val_b, left_out_val_labels + return left_out_val_a, left_out_val_b, left_out_ext_feats, left_out_val_labels def train(self, epochs): scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) @@ -164,13 +167,14 @@ class MSRVIDTrainer(Trainer): for epoch in range(1, epochs + 1): start = time.time() logger.info('Epoch {} started...'.format(epoch)) - left_out_a, left_out_b, left_out_label = self.train_epoch(epoch) + left_out_a, left_out_b, left_out_ext_feats, left_out_label = self.train_epoch(epoch) # manually evaluating the validating set left_out_a = torch.cat(left_out_a) left_out_b = torch.cat(left_out_b) + left_out_ext_feats = torch.cat(left_out_ext_feats) left_out_label = torch.cat(left_out_label) - output = self.model(left_out_a, left_out_b) + output = self.model(left_out_a, left_out_b, left_out_ext_feats) predict_classes = torch.arange(0, 6).expand(len(left_out_a), 6).cuda() true_labels = (predict_classes * left_out_label.data).sum(dim=1) predictions = (predict_classes * output.data.exp()).sum(dim=1)