diff --git a/mp_cnn/README.md b/mp_cnn/README.md index bb563f6..1e0b78b 100644 --- a/mp_cnn/README.md +++ b/mp_cnn/README.md @@ -20,17 +20,27 @@ Directory layout should be like this: │ └── GloVe/ ``` -To run MP-CNN on the SICK dataset, use the following command: +To run MP-CNN on the SICK dataset, use the following command. `--dropout 0` is for mimicking the original paper, although adding dropout can improve performance. ``` -python main.py mpcnn.sick.model.castor --dataset sick --batch-size 32 --epochs 15 +python main.py mpcnn.sick.model.castor --dataset sick --epochs 19 --epsilon 1e-7 --dropout 0 ``` +| Implementation and config | Pearson's r | Spearman's p | +| -------------------------------- |:-------------:|:-------------:| +| Paper | 0.8686 | 0.8047 | +| PyTorch using above config | 0.8763 | 0.8215 | + To run MP-CNN on the MSRVID dataset, use the following command: ``` -python main.py mpcnn.msrvid.model.castor --dataset msrvid --batch-size 8 --epochs 30 --epsilon 0.01 +python main.py mpcnn.msrvid.model.castor --dataset msrvid --batch-size 16 --epsilon 1e-7 --epochs 32 --dropout 0 --regularization 0.0025 ``` +| Implementation and config | Pearson's r | +| -------------------------------- |:-------------:| +| Paper | 0.9090 | +| PyTorch using above config | 0.9050 | + These are not the optimal hyperparameters but they are decent. This README will be updated with more optimal hyperparameters and results in the future. To see all options available, use diff --git a/mp_cnn/dataset.py b/mp_cnn/dataset.py index 51becdc..0f3e20b 100644 --- a/mp_cnn/dataset.py +++ b/mp_cnn/dataset.py @@ -85,6 +85,8 @@ class MPCNNDataset(data.Dataset): self.cuda = cuda self.max_length = -10000 + self.unk = torch.Tensor(300) + self.unk.normal_(0, 0.01) def initialize(self, word_index, embedding): """ @@ -127,6 +129,8 @@ class MPCNNDataset(data.Dataset): if token in word_index: found_pos.append(i) found_emb_idx.append(word_index[token]) + else: + sentence_embedding[:, i] = self.unk found_word_vecs = embedding(Variable(torch.LongTensor(found_emb_idx))) for i, v in enumerate(found_pos): diff --git a/mp_cnn/evaluation.py b/mp_cnn/evaluation.py index ec2e026..7bdd653 100644 --- a/mp_cnn/evaluation.py +++ b/mp_cnn/evaluation.py @@ -68,6 +68,8 @@ class SICKEvaluator(Evaluator): true_labels.append((predict_classes * labels.data).sum(dim=1)) predictions.append((predict_classes * output.data.exp()).sum(dim=1)) + del output + predictions = torch.cat(predictions).cpu().numpy() true_labels = torch.cat(true_labels).cpu().numpy() test_kl_div_loss /= len(self.data_loader.dataset) @@ -103,6 +105,8 @@ class MSRVIDEvaluator(Evaluator): true_labels.append((predict_classes * labels.data).sum(dim=1)) predictions.append((predict_classes * output.data.exp()).sum(dim=1)) + del output + predictions = torch.cat(predictions).cpu().numpy() true_labels = torch.cat(true_labels).cpu().numpy() test_kl_div_loss /= len(self.data_loader.dataset) diff --git a/mp_cnn/main.py b/mp_cnn/main.py index 92f5cfe..adf153b 100644 --- a/mp_cnn/main.py +++ b/mp_cnn/main.py @@ -1,16 +1,11 @@ import argparse -import math import os -import time import numpy as np -from scipy.stats import pearsonr, spearmanr import torch -import torch.nn.functional as F -from torch.autograd import Variable import torch.optim as optim -from dataset import DatasetType, MPCNNDatasetFactory +from dataset import MPCNNDatasetFactory from evaluation import MPCNNEvaluatorFactory from model import MPCNN from train import MPCNNTrainerFactory @@ -27,26 +22,30 @@ ch.setFormatter(formatter) logger.addHandler(ch) - if __name__ == '__main__': parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN') parser.add_argument('model_outfile', help='file to save final model') parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid]', default='sick') parser.add_argument('--word-vectors-file', help='word vectors file', default=os.path.join(os.pardir, os.pardir, 'data', 'GloVe', 'glove.840B.300d.txt')) parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true') - parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') - parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') - parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') - parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.001)') - parser.add_argument('--epsilon', type=float, default=1e-8, metavar='M', help='Adam epsilon (default: 1e-8)') - parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') - parser.add_argument('--sample', type=int, default=0, metavar='N', help='how many examples to take from each dataset, meant for quickly testing entire end-to-end pipeline (default: all)') - parser.add_argument('--regularization', type=float, default=0.0001, metavar='REG', help='Regularization for the optimizer (default: 0.0001)') - parser.add_argument('--max-window-size', type=int, default=3, metavar='N', help='windows sizes will be [1,max_window_size] and infinity') - parser.add_argument('--holistic-filters', type=int, default=300, metavar='N', help='number of holistic filters') - parser.add_argument('--per-dim-filters', type=int, default=20, metavar='N', help='number of per-dimension filters') - parser.add_argument('--hidden-units', type=int, default=150, metavar='N', help='number of hidden units in each of the two hidden layers') - parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') + parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training (default: false)') + parser.add_argument('--batch-size', type=int, default=64, help='input batch size for training (default: 64)') + parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 10)') + parser.add_argument('--optimizer', type=str, default='adam', help='optimizer to use: adam or sgd (default: adam)') + parser.add_argument('--lr', type=float, default=0.001, help='learning rate (default: 0.001)') + parser.add_argument('--lr-reduce-factor', type=float, default=0.3, help='learning rate reduce factor after plateau (default: 0.3)') + parser.add_argument('--patience', type=float, default=2, help='learning rate patience after seeing plateau (default: 2)') + parser.add_argument('--momentum', type=float, default=0, help='momentum (default: 0)') + parser.add_argument('--epsilon', type=float, default=1e-8, help='Adam epsilon (default: 1e-8)') + parser.add_argument('--log-interval', type=int, default=10, help='how many batches to wait before logging training status (default: 10)') + parser.add_argument('--sample', type=int, default=0, help='how many examples to take from each dataset, meant for quickly testing entire end-to-end pipeline (default: all)') + parser.add_argument('--regularization', type=float, default=0.0001, help='Regularization for the optimizer (default: 0.0001)') + parser.add_argument('--max-window-size', type=int, default=3, help='windows sizes will be [1,max_window_size] and infinity (default: 300)') + parser.add_argument('--holistic-filters', type=int, default=300, help='number of holistic filters (default: 300)') + parser.add_argument('--per-dim-filters', type=int, default=20, help='number of per-dimension filters (default: 20)') + parser.add_argument('--hidden-units', type=int, default=150, help='number of hidden units in each of the two hidden layers (default: 150)') + parser.add_argument('--dropout', type=float, default=0.5, help='dropout probability (default: 0.5)') + parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() @@ -58,15 +57,22 @@ if __name__ == '__main__': train_loader, test_loader, dev_loader = MPCNNDatasetFactory.get_dataset(args.dataset, args.word_vectors_file, args.batch_size, args.cuda, args.sample) filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf] - model = MPCNN(300, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes) + input_channels = 300 + model = MPCNN(input_channels, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes, args.dropout) if args.cuda: model.cuda() - optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.regularization, eps=args.epsilon) + optimizer = None + if args.optimizer == 'adam': + optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.regularization, eps=args.epsilon) + elif args.optimizer == 'sgd': + optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.regularization) + else: + raise ValueError('optimizer not recognized: it should be either adam or sgd') train_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, train_loader, args.batch_size, args.cuda) test_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, test_loader, args.batch_size, args.cuda) dev_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, dev_loader, args.batch_size, args.cuda) - trainer = MPCNNTrainerFactory.get_trainer(args.dataset, model, optimizer, train_loader, args.batch_size, args.sample, args.log_interval, args.model_outfile, train_evaluator, test_evaluator, dev_evaluator) + trainer = MPCNNTrainerFactory.get_trainer(args.dataset, model, optimizer, train_loader, args.batch_size, args.sample, args.log_interval, args.model_outfile, args.lr_reduce_factor, args.patience, train_evaluator, test_evaluator, dev_evaluator) if not args.skip_training: total_params = 0 diff --git a/mp_cnn/model.py b/mp_cnn/model.py index 6e88aeb..300671f 100644 --- a/mp_cnn/model.py +++ b/mp_cnn/model.py @@ -2,15 +2,15 @@ import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from torch.autograd import Variable class MPCNN(nn.Module): - def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes): + def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes, dropout): super(MPCNN, self).__init__() self.n_word_dim = n_word_dim + self.n_holistic_filters = n_holistic_filters self.n_per_dim_filters = n_per_dim_filters self.filter_widths = filter_widths holistic_conv_layers = [] @@ -34,14 +34,22 @@ class MPCNN(nn.Module): self.per_dim_conv_layers = nn.ModuleList(per_dim_conv_layers) # compute number of inputs to first hidden layer - COMP_1_COMPONENTS, COMP_2_COMPONENTS = 2 + n_word_dim, 2 + COMP_1_COMPONENTS_HOLISTIC, COMP_1_COMPONENTS_PER_DIM, COMP_2_COMPONENTS = 2 + n_holistic_filters, 2 + n_word_dim, 2 n_feat_h = 3 * len(self.filter_widths) * COMP_2_COMPONENTS - n_feat_v = 3 * (len(self.filter_widths) ** 2) * COMP_1_COMPONENTS + 2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS + n_feat_v = ( + # comparison units from holistic conv for min, max, mean pooling for non-infinite widths + 3 * ((len(self.filter_widths) - 1) ** 2) * COMP_1_COMPONENTS_HOLISTIC + + # comparison units from holistic conv for min, max, mean pooling for infinite widths + 3 * 3 + + # comparison units from per-dim conv + 2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS_PER_DIM + ) n_feat = n_feat_h + n_feat_v self.final_layers = nn.Sequential( nn.Linear(n_feat, hidden_layer_units), nn.Tanh(), + nn.Dropout(dropout), nn.Linear(hidden_layer_units, num_classes), nn.LogSoftmax() ) @@ -50,21 +58,25 @@ class MPCNN(nn.Module): block_a = {} block_b = {} for ws in self.filter_widths: - holistic_conv_out = self.holistic_conv_layers[ws - 1](sent) if not np.isinf(ws) else sent - block_a[ws] = { - 'max': F.max_pool1d(holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim), - 'min': F.max_pool1d(-1 * holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim), - 'mean': F.avg_pool1d(holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim) - } - - # only compute per-dimension convolution for non-infinity widths if np.isinf(ws): + block_a[ws] = { + 'max': F.max_pool1d(sent.view(sent.size(0), 1, -1), sent.size(1) * sent.size(2)).view(sent.size(0), -1), + 'min': F.max_pool1d(-1 * sent.view(sent.size(0), 1, -1), sent.size(1) * sent.size(2)).view(sent.size(0), -1), + 'mean': F.avg_pool1d(sent.view(sent.size(0), 1, -1), sent.size(1) * sent.size(2)).view(sent.size(0), -1) + } continue + holistic_conv_out = self.holistic_conv_layers[ws - 1](sent) + block_a[ws] = { + 'max': F.max_pool1d(holistic_conv_out, holistic_conv_out.size(2)).view(-1, self.n_holistic_filters), + 'min': F.max_pool1d(-1 * holistic_conv_out, holistic_conv_out.size(2)).view(-1, self.n_holistic_filters), + 'mean': F.avg_pool1d(holistic_conv_out, holistic_conv_out.size(2)).view(-1, self.n_holistic_filters) + } + per_dim_conv_out = self.per_dim_conv_layers[ws - 1](sent) block_b[ws] = { - 'max': F.max_pool1d(per_dim_conv_out, per_dim_conv_out.size()[2]).view(-1, self.n_word_dim, self.n_per_dim_filters), - 'min': F.max_pool1d(-1 * per_dim_conv_out, per_dim_conv_out.size()[2]).view(-1, self.n_word_dim, self.n_per_dim_filters) + 'max': F.max_pool1d(per_dim_conv_out, per_dim_conv_out.size(2)).view(-1, self.n_word_dim, self.n_per_dim_filters), + 'min': F.max_pool1d(-1 * per_dim_conv_out, per_dim_conv_out.size(2)).view(-1, self.n_word_dim, self.n_per_dim_filters) } return block_a, block_b @@ -81,24 +93,26 @@ class MPCNN(nn.Module): def _algo_2_vert_comp(self, sent1_block_a, sent2_block_a, sent1_block_b, sent2_block_b): comparison_feats = [] + ws_no_inf = [w for w in self.filter_widths if not np.isinf(w)] for pool in ('max', 'min', 'mean'): for ws1 in self.filter_widths: x1 = sent1_block_a[ws1][pool] batch_size = x1.size()[0] for ws2 in self.filter_widths: x2 = sent2_block_a[ws2][pool] - comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1)) - comparison_feats.append(F.pairwise_distance(x1, x2)) - comparison_feats.append(torch.abs(x1 - x2)) + if (not np.isinf(ws1) and not np.isinf(ws2)) or (np.isinf(ws1) and np.isinf(ws2)): + comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1)) + comparison_feats.append(F.pairwise_distance(x1, x2)) + comparison_feats.append(torch.abs(x1 - x2)) for pool in ('max', 'min'): - ws_no_inf = [w for w in self.filter_widths if not np.isinf(w)] for ws in ws_no_inf: oG_1B = sent1_block_b[ws][pool] oG_2B = sent2_block_b[ws][pool] for i in range(0, self.n_per_dim_filters): x1 = oG_1B[:, :, i] x2 = oG_2B[:, :, i] + batch_size = x1.size()[0] comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1)) comparison_feats.append(F.pairwise_distance(x1, x2)) comparison_feats.append(torch.abs(x1 - x2)) diff --git a/mp_cnn/train.py b/mp_cnn/train.py index fe1d71b..4768ee7 100644 --- a/mp_cnn/train.py +++ b/mp_cnn/train.py @@ -4,7 +4,6 @@ import time import torch import torch.nn.functional as F from torch.autograd import Variable -import torch.optim as optim from torch.optim.lr_scheduler import ReduceLROnPlateau from scipy.stats import pearsonr, spearmanr @@ -25,11 +24,11 @@ class MPCNNTrainerFactory(object): Get the corresponding Trainer class for a particular dataset. """ @staticmethod - def get_trainer(dataset_name, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): + def get_trainer(dataset_name, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator=None): if dataset_name == 'sick': - return SICKTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + return SICKTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator) elif dataset_name == 'msrvid': - return MSRVIDTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + return MSRVIDTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile,lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator) else: raise ValueError('{} is not a valid dataset.'.format(dataset_name)) @@ -40,7 +39,7 @@ class Trainer(object): Abstraction for training a model on a Dataset. """ - def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): + def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator=None): self.model = model self.optimizer = optimizer self.train_loader = train_loader @@ -48,6 +47,8 @@ class Trainer(object): self.sample = sample self.log_interval = log_interval self.model_outfile = model_outfile + self.lr_reduce_factor = lr_reduce_factor + self.patience = patience self.train_evaluator = train_evaluator self.test_evaluator = test_evaluator self.dev_evaluator = dev_evaluator @@ -68,8 +69,8 @@ class Trainer(object): class SICKTrainer(Trainer): - def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): - super(SICKTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator=None): + super(SICKTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator) def train_epoch(self, epoch): self.model.train() @@ -89,10 +90,13 @@ class SICKTrainer(Trainer): len(self.train_loader.dataset) if not self.sample else self.sample, 100. * batch_idx / (len(self.train_loader) if not self.sample else math.ceil(self.sample / self.batch_size)), loss.data[0]) ) + + del loss, output + return total_loss def train(self, epochs): - scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=0.3, patience=2) + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) epoch_times = [] best_dev_score = -1 for epoch in range(1, epochs + 1): @@ -116,8 +120,8 @@ class SICKTrainer(Trainer): class MSRVIDTrainer(Trainer): - def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): - super(MSRVIDTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator=None): + super(MSRVIDTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, lr_reduce_factor, patience, train_evaluator, test_evaluator, dev_evaluator) def train_epoch(self, epoch): self.model.train() @@ -148,11 +152,13 @@ class MSRVIDTrainer(Trainer): 100. * batch_idx / (len(self.train_loader) if not self.sample else math.ceil(self.sample / self.batch_size)), loss.data[0]) ) + del loss, output + self.evaluate(self.train_evaluator, 'train') return left_out_val_a, left_out_val_b, left_out_val_labels def train(self, epochs): - scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=0.3, patience=2) + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) epoch_times = [] best_dev_score = -1 for epoch in range(1, epochs + 1):