From 449d715ab0255145e3f50413af13ce49ef2a0528 Mon Sep 17 00:00:00 2001 From: Michael Tu Date: Fri, 4 Aug 2017 19:37:34 -0400 Subject: [PATCH] MP-CNN PyTorch Re-implementation (#37) Re-implementation of MP-CNN in PyTorch. --- mp_cnn/README.md | 39 ++++++++ mp_cnn/__init__.py | 0 mp_cnn/dataset.py | 184 ++++++++++++++++++++++++++++++++++++++ mp_cnn/evaluation.py | 110 +++++++++++++++++++++++ mp_cnn/main.py | 84 ++++++++++++++++++ mp_cnn/model.py | 119 +++++++++++++++++++++++++ mp_cnn/preprocessing.py | 55 ++++++++++++ mp_cnn/train.py | 191 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 782 insertions(+) create mode 100644 mp_cnn/README.md create mode 100644 mp_cnn/__init__.py create mode 100644 mp_cnn/dataset.py create mode 100644 mp_cnn/evaluation.py create mode 100644 mp_cnn/main.py create mode 100644 mp_cnn/model.py create mode 100644 mp_cnn/preprocessing.py create mode 100644 mp_cnn/train.py diff --git a/mp_cnn/README.md b/mp_cnn/README.md new file mode 100644 index 0000000..bb563f6 --- /dev/null +++ b/mp_cnn/README.md @@ -0,0 +1,39 @@ +# MP-CNN PyTorch Implementation + +This is a PyTorch implementation of the following paper + +* Hua He, Kevin Gimpel, and Jimmy Lin. [Multi-Perspective Sentence Similarity Modeling with Convolutional Neural Networks](http://aclweb.org/anthology/D/D15/D15-1181.pdf). *Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP 2015)*, pages 1576-1586. + +The SICK and MSRVID datasets are available in https://github.com/castorini/data, as well as the GloVe word embeddings. + +Directory layout should be like this: +``` +├── Castor +│ ├── README.md +│ ├── ... +│ └── mp_cnn/ +├── data +│ ├── README.md +│ ├── ... +│ ├── msrvid/ +│ ├── sick/ +│ └── GloVe/ +``` + +To run MP-CNN on the SICK dataset, use the following command: + +``` +python main.py mpcnn.sick.model.castor --dataset sick --batch-size 32 --epochs 15 +``` + +To run MP-CNN on the MSRVID dataset, use the following command: +``` +python main.py mpcnn.msrvid.model.castor --dataset msrvid --batch-size 8 --epochs 30 --epsilon 0.01 +``` + +These are not the optimal hyperparameters but they are decent. This README will be updated with more optimal hyperparameters and results in the future. + +To see all options available, use +``` +python main.py --help +``` diff --git a/mp_cnn/__init__.py b/mp_cnn/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mp_cnn/dataset.py b/mp_cnn/dataset.py new file mode 100644 index 0000000..51becdc --- /dev/null +++ b/mp_cnn/dataset.py @@ -0,0 +1,184 @@ +from enum import Enum +import math +import os + +import torch +from torch.autograd import Variable +import torch.nn.functional as F +import torch.utils.data as data + +import preprocessing + +# logging setup +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + + +class DatasetType(Enum): + TRAIN = 1 + TEST = 2 + DEV = 3 + + +class MPCNNDatasetFactory(object): + """ + Get the corresponding Dataset class for a particular dataset. + """ + @staticmethod + def get_dataset(dataset_name, word_vectors_file, batch_size, cuda, sample): + extra_args = {'shuffle': True} + dev_loader = None + if sample: + sample_indices = list(range(sample)) + subset_random_sampler = data.sampler.SubsetRandomSampler(sample_indices) + extra_args['sampler'] = subset_random_sampler + extra_args['shuffle'] = False + if dataset_name == 'sick': + train_loader = torch.utils.data.DataLoader(SICKDataset(DatasetType.TRAIN, cuda), batch_size=batch_size, **extra_args) + test_loader = torch.utils.data.DataLoader(SICKDataset(DatasetType.TEST, cuda), batch_size=batch_size, **extra_args) + dev_loader = torch.utils.data.DataLoader(SICKDataset(DatasetType.DEV, cuda), batch_size=batch_size, **extra_args) + elif dataset_name == 'msrvid': + train_loader = torch.utils.data.DataLoader(MSRVIDDataset(DatasetType.TRAIN, cuda), batch_size=batch_size, **extra_args) + test_loader = torch.utils.data.DataLoader(MSRVIDDataset(DatasetType.TEST, cuda), batch_size=batch_size, **extra_args) + else: + raise ValueError('{} is not a valid dataset.'.format(dataset_name)) + + word_index, embedding = preprocessing.get_glove_embedding(word_vectors_file, train_loader.dataset.dataset_root) + logger.info('Finished loading GloVe embedding for vocab in data...') + + train_loader.dataset.initialize(word_index, embedding) + test_loader.dataset.initialize(word_index, embedding) + if dev_loader is not None: + dev_loader.dataset.initialize(word_index, embedding) + return train_loader, test_loader, dev_loader + + +class MPCNNDataset(data.Dataset): + train_folder = 'train' + test_folder = 'test' + dev_folder = 'dev' + # subclass will override fields below + dataset_root = None + num_classes = None + + def __init__(self, dataset_type, cuda): + if not isinstance(dataset_type, DatasetType): + raise ValueError('dataset_type ({}) must be of type DatasetType enum'.format(dataset_type)) + + if dataset_type == DatasetType.TRAIN: + subfolder = MPCNNDataset.train_folder + elif dataset_type == DatasetType.TEST: + subfolder = MPCNNDataset.test_folder + else: + subfolder = MPCNNDataset.dev_folder + + self.dataset_dir = os.path.join(self.dataset_root, subfolder) + if not os.path.exists(self.dataset_dir): + raise RuntimeError('{} does not exist'.format(self.dataset_dir)) + + self.cuda = cuda + self.max_length = -10000 + + def initialize(self, word_index, embedding): + """ + Convert sentences into sentence embeddings. + """ + sent_a = self._load(self.dataset_dir, 'a.txt') + sent_b = self._load(self.dataset_dir, 'b.txt') + + # obtain max sentence length to use as dimension for padding to support batching + sent_a_tokens, sent_b_tokens = [], [] + for i in range(len(sent_a)): + sa_tokens = sent_a[i].split(' ') + sb_tokens = sent_b[i].split(' ') + self.max_length = max(self.max_length, len(sa_tokens), len(sb_tokens)) + sent_a_tokens.append(sa_tokens) + sent_b_tokens.append(sb_tokens) + + self.sentences = [] + for i in range(len(sent_a)): + sent_pair = {} + sent_pair['a'] = self._get_sentence_embeddings(sent_a_tokens[i], word_index, embedding) + sent_pair['b'] = self._get_sentence_embeddings(sent_b_tokens[i], word_index, embedding) + self.sentences.append(sent_pair) + self.labels = self._load(self.dataset_dir, 'sim.txt', float) + + def _load(self, dataset_dir, fname, type_converter=str): + data = [] + with open(os.path.join(dataset_dir, fname), 'r') as f: + for line in f: + stripped_line = line.rstrip('.\n') + item = type_converter(stripped_line) + data.append(item) + return data + + def _get_sentence_embeddings(self, tokens, word_index, embedding): + sentence_embedding = torch.zeros(300, self.max_length) + sentence_embedding[:, :len(tokens)].normal_(0, 1) + found_pos, found_emb_idx = [], [] + for i, token in enumerate(tokens): + if token in word_index: + found_pos.append(i) + found_emb_idx.append(word_index[token]) + + found_word_vecs = embedding(Variable(torch.LongTensor(found_emb_idx))) + for i, v in enumerate(found_pos): + sentence_embedding[:, v] = found_word_vecs[i].data + return sentence_embedding.cuda() if self.cuda else sentence_embedding + + def __getitem__(self, idx): + return self.sentences[idx], self.labels[idx] + + def __len__(self): + return len(self.labels) + + +class SICKDataset(MPCNNDataset): + + dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'sick') + num_classes = 5 + + def __init__(self, dataset_type, cuda): + super(SICKDataset, self).__init__(dataset_type, cuda) + + def initialize(self, word_index, embedding): + super(SICKDataset, self).initialize(word_index, embedding) + new_labels = torch.zeros(self.__len__(), self.num_classes) + for i, sim in enumerate(self.labels): + ceil, floor = math.ceil(sim), math.floor(sim) + if ceil == floor: + new_labels[i][floor - 1] = 1 + else: + new_labels[i][floor - 1] = ceil - sim + new_labels[i][ceil - 1] = sim - floor + + self.labels = new_labels.cuda() if self.cuda else new_labels + + +class MSRVIDDataset(MPCNNDataset): + + dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'msrvid') + num_classes = 6 + + def __init__(self, dataset_type, cuda): + super(MSRVIDDataset, self).__init__(dataset_type, cuda) + + def initialize(self, word_index, embedding): + super(MSRVIDDataset, self).initialize(word_index, embedding) + new_labels = torch.zeros(self.__len__(), self.num_classes) + for i, sim in enumerate(self.labels): + ceil, floor = math.ceil(sim), math.floor(sim) + if ceil == floor: + new_labels[i][floor] = 1 + else: + new_labels[i][floor] = ceil - sim + new_labels[i][ceil] = sim - floor + + self.labels = new_labels.cuda() if self.cuda else new_labels diff --git a/mp_cnn/evaluation.py b/mp_cnn/evaluation.py new file mode 100644 index 0000000..ec2e026 --- /dev/null +++ b/mp_cnn/evaluation.py @@ -0,0 +1,110 @@ +from scipy.stats import pearsonr, spearmanr +import torch +import torch.nn.functional as F +from torch.autograd import Variable + + +class MPCNNEvaluatorFactory(object): + """ + Get the corresponding Evaluator class for a particular dataset. + """ + @staticmethod + def get_evaluator(dataset_name, model, data_loader, batch_size, cuda): + if data_loader is None: + return None + + if dataset_name == 'sick': + return SICKEvaluator(model, data_loader, batch_size, cuda) + elif dataset_name == 'msrvid': + return MSRVIDEvaluator(model, data_loader, batch_size, cuda) + else: + raise ValueError('{} is not a valid dataset.'.format(dataset_name)) + + +class Evaluator(object): + """ + Evaluates performance of model on a Dataset, using metrics specific to the Dataset. + """ + + def __init__(self, model, data_loader, batch_size, cuda): + self.model = model + self.data_loader = data_loader + self.batch_size = batch_size + self.cuda = cuda + + def get_scores(self): + """ + Get the scores used to evaluate the model. + Should return ([score1, score2, ..], [score1_name, score2_name, ...]). + The first score is the primary score used to determine if the model has improved. + """ + raise NotImplementedError('Evaluator subclass needs to implement get_score') + + +class SICKEvaluator(Evaluator): + + def __init__(self, model, data_loader, batch_size, cuda): + super(SICKEvaluator, self).__init__(model, data_loader, batch_size, cuda) + + def get_scores(self): + self.model.eval() + num_classes = self.data_loader.dataset.num_classes + predict_classes = torch.arange(1, num_classes + 1).expand(self.batch_size, num_classes) + if self.cuda: + predict_classes = predict_classes.cuda() + test_kl_div_loss = 0 + predictions = [] + true_labels = [] + for sentences, labels in self.data_loader: + sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True) + labels = Variable(labels, volatile=True) + output = self.model(sent_a, sent_b) + test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0] + # handle last batch which might have smaller size + if len(predict_classes) != len(sent_a): + predict_classes = torch.arange(1, num_classes + 1).expand(len(sent_a), num_classes) + if self.cuda: + predict_classes = predict_classes.cuda() + true_labels.append((predict_classes * labels.data).sum(dim=1)) + predictions.append((predict_classes * output.data.exp()).sum(dim=1)) + + predictions = torch.cat(predictions).cpu().numpy() + true_labels = torch.cat(true_labels).cpu().numpy() + test_kl_div_loss /= len(self.data_loader.dataset) + pearson_r = pearsonr(predictions, true_labels)[0] + spearman_r = spearmanr(predictions, true_labels)[0] + return [pearson_r, spearman_r, test_kl_div_loss], ['pearson_r', 'spearman_r', 'KL-divergence loss'] + + +class MSRVIDEvaluator(Evaluator): + + def __init__(self, model, data_loader, batch_size, cuda): + super(MSRVIDEvaluator, self).__init__(model, data_loader, batch_size, cuda) + + def get_scores(self): + self.model.eval() + num_classes = self.data_loader.dataset.num_classes + predict_classes = torch.arange(0, num_classes).expand(self.batch_size, num_classes) + if self.cuda: + predict_classes = predict_classes.cuda() + test_kl_div_loss = 0 + predictions = [] + true_labels = [] + for sentences, labels in self.data_loader: + sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True) + labels = Variable(labels, volatile=True) + output = self.model(sent_a, sent_b) + test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0] + # handle last batch which might have smaller size + if len(predict_classes) != len(sent_a): + predict_classes = torch.arange(0, num_classes).expand(len(sent_a), num_classes) + if self.cuda: + predict_classes = predict_classes.cuda() + true_labels.append((predict_classes * labels.data).sum(dim=1)) + predictions.append((predict_classes * output.data.exp()).sum(dim=1)) + + predictions = torch.cat(predictions).cpu().numpy() + true_labels = torch.cat(true_labels).cpu().numpy() + test_kl_div_loss /= len(self.data_loader.dataset) + pearson_r = pearsonr(predictions, true_labels)[0] + return [pearson_r, test_kl_div_loss], ['pearson_r', 'KL-divergence loss'] diff --git a/mp_cnn/main.py b/mp_cnn/main.py new file mode 100644 index 0000000..92f5cfe --- /dev/null +++ b/mp_cnn/main.py @@ -0,0 +1,84 @@ +import argparse +import math +import os +import time + +import numpy as np +from scipy.stats import pearsonr, spearmanr +import torch +import torch.nn.functional as F +from torch.autograd import Variable +import torch.optim as optim + +from dataset import DatasetType, MPCNNDatasetFactory +from evaluation import MPCNNEvaluatorFactory +from model import MPCNN +from train import MPCNNTrainerFactory + +# logging setup +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN') + parser.add_argument('model_outfile', help='file to save final model') + parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid]', default='sick') + parser.add_argument('--word-vectors-file', help='word vectors file', default=os.path.join(os.pardir, os.pardir, 'data', 'GloVe', 'glove.840B.300d.txt')) + parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true') + parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') + parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') + parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') + parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.001)') + parser.add_argument('--epsilon', type=float, default=1e-8, metavar='M', help='Adam epsilon (default: 1e-8)') + parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') + parser.add_argument('--sample', type=int, default=0, metavar='N', help='how many examples to take from each dataset, meant for quickly testing entire end-to-end pipeline (default: all)') + parser.add_argument('--regularization', type=float, default=0.0001, metavar='REG', help='Regularization for the optimizer (default: 0.0001)') + parser.add_argument('--max-window-size', type=int, default=3, metavar='N', help='windows sizes will be [1,max_window_size] and infinity') + parser.add_argument('--holistic-filters', type=int, default=300, metavar='N', help='number of holistic filters') + parser.add_argument('--per-dim-filters', type=int, default=20, metavar='N', help='number of per-dimension filters') + parser.add_argument('--hidden-units', type=int, default=150, metavar='N', help='number of hidden units in each of the two hidden layers') + parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') + args = parser.parse_args() + args.cuda = not args.no_cuda and torch.cuda.is_available() + + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.cuda: + torch.cuda.manual_seed(args.seed) + + train_loader, test_loader, dev_loader = MPCNNDatasetFactory.get_dataset(args.dataset, args.word_vectors_file, args.batch_size, args.cuda, args.sample) + + filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf] + model = MPCNN(300, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes) + if args.cuda: + model.cuda() + optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.regularization, eps=args.epsilon) + train_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, train_loader, args.batch_size, args.cuda) + test_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, test_loader, args.batch_size, args.cuda) + dev_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, dev_loader, args.batch_size, args.cuda) + + trainer = MPCNNTrainerFactory.get_trainer(args.dataset, model, optimizer, train_loader, args.batch_size, args.sample, args.log_interval, args.model_outfile, train_evaluator, test_evaluator, dev_evaluator) + + if not args.skip_training: + total_params = 0 + for param in model.parameters(): + size = [s for s in param.size()] + total_params += np.prod(size) + logger.info('Total number of parameters: %s', total_params) + trainer.train(args.epochs) + + model = torch.load(args.model_outfile) + test_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, test_loader, args.batch_size, args.cuda) + scores, metric_names = test_evaluator.get_scores() + logger.info('Evaluation metrics for test') + logger.info('\t'.join([' '] + metric_names)) + logger.info('\t'.join(['test'] + list(map(str, scores)))) diff --git a/mp_cnn/model.py b/mp_cnn/model.py new file mode 100644 index 0000000..6e88aeb --- /dev/null +++ b/mp_cnn/model.py @@ -0,0 +1,119 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Variable + + +class MPCNN(nn.Module): + + def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes): + super(MPCNN, self).__init__() + + self.n_word_dim = n_word_dim + self.n_per_dim_filters = n_per_dim_filters + self.filter_widths = filter_widths + holistic_conv_layers = [] + per_dim_conv_layers = [] + + for ws in filter_widths: + if np.isinf(ws): + continue + + holistic_conv_layers.append(nn.Sequential( + nn.Conv1d(n_word_dim, n_holistic_filters, ws), + nn.Tanh() + )) + + per_dim_conv_layers.append(nn.Sequential( + nn.Conv1d(n_word_dim, n_word_dim * n_per_dim_filters, ws, groups=n_word_dim), + nn.Tanh() + )) + + self.holistic_conv_layers = nn.ModuleList(holistic_conv_layers) + self.per_dim_conv_layers = nn.ModuleList(per_dim_conv_layers) + + # compute number of inputs to first hidden layer + COMP_1_COMPONENTS, COMP_2_COMPONENTS = 2 + n_word_dim, 2 + n_feat_h = 3 * len(self.filter_widths) * COMP_2_COMPONENTS + n_feat_v = 3 * (len(self.filter_widths) ** 2) * COMP_1_COMPONENTS + 2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS + n_feat = n_feat_h + n_feat_v + + self.final_layers = nn.Sequential( + nn.Linear(n_feat, hidden_layer_units), + nn.Tanh(), + nn.Linear(hidden_layer_units, num_classes), + nn.LogSoftmax() + ) + + def _get_blocks_for_sentence(self, sent): + block_a = {} + block_b = {} + for ws in self.filter_widths: + holistic_conv_out = self.holistic_conv_layers[ws - 1](sent) if not np.isinf(ws) else sent + block_a[ws] = { + 'max': F.max_pool1d(holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim), + 'min': F.max_pool1d(-1 * holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim), + 'mean': F.avg_pool1d(holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim) + } + + # only compute per-dimension convolution for non-infinity widths + if np.isinf(ws): + continue + + per_dim_conv_out = self.per_dim_conv_layers[ws - 1](sent) + block_b[ws] = { + 'max': F.max_pool1d(per_dim_conv_out, per_dim_conv_out.size()[2]).view(-1, self.n_word_dim, self.n_per_dim_filters), + 'min': F.max_pool1d(-1 * per_dim_conv_out, per_dim_conv_out.size()[2]).view(-1, self.n_word_dim, self.n_per_dim_filters) + } + return block_a, block_b + + def _algo_1_horiz_comp(self, sent1_block_a, sent2_block_a): + comparison_feats = [] + for pool in ('max', 'min', 'mean'): + for ws in self.filter_widths: + x1 = sent1_block_a[ws][pool] + x2 = sent2_block_a[ws][pool] + batch_size = x1.size()[0] + comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1)) + comparison_feats.append(F.pairwise_distance(x1, x2)) + return torch.cat(comparison_feats, dim=1) + + def _algo_2_vert_comp(self, sent1_block_a, sent2_block_a, sent1_block_b, sent2_block_b): + comparison_feats = [] + for pool in ('max', 'min', 'mean'): + for ws1 in self.filter_widths: + x1 = sent1_block_a[ws1][pool] + batch_size = x1.size()[0] + for ws2 in self.filter_widths: + x2 = sent2_block_a[ws2][pool] + comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1)) + comparison_feats.append(F.pairwise_distance(x1, x2)) + comparison_feats.append(torch.abs(x1 - x2)) + + for pool in ('max', 'min'): + ws_no_inf = [w for w in self.filter_widths if not np.isinf(w)] + for ws in ws_no_inf: + oG_1B = sent1_block_b[ws][pool] + oG_2B = sent2_block_b[ws][pool] + for i in range(0, self.n_per_dim_filters): + x1 = oG_1B[:, :, i] + x2 = oG_2B[:, :, i] + comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1)) + comparison_feats.append(F.pairwise_distance(x1, x2)) + comparison_feats.append(torch.abs(x1 - x2)) + + return torch.cat(comparison_feats, dim=1) + + def forward(self, sent1, sent2): + # Sentence modeling module + sent1_block_a, sent1_block_b = self._get_blocks_for_sentence(sent1) + sent2_block_a, sent2_block_b = self._get_blocks_for_sentence(sent2) + + # Similarity measurement layer + feat_h = self._algo_1_horiz_comp(sent1_block_a, sent2_block_a) + feat_v = self._algo_2_vert_comp(sent1_block_a, sent2_block_a, sent1_block_b, sent2_block_b) + feat_all = torch.cat([feat_h, feat_v], dim=1) + + preds = self.final_layers(feat_all) + return preds diff --git a/mp_cnn/preprocessing.py b/mp_cnn/preprocessing.py new file mode 100644 index 0000000..245e57f --- /dev/null +++ b/mp_cnn/preprocessing.py @@ -0,0 +1,55 @@ +""" +Preprocessing utilities such as preparing embeddings. +""" +import os + +import numpy as np +import torch +import torch.nn as nn + + +def get_vocab(data_dir): + """ + Get vocabulary as set of words. + """ + vocab = set() + with open(os.path.join(data_dir, 'vocab-cased.txt'), 'r') as f: + for line in f: + word = line.rstrip() + vocab.add(word) + return vocab + + +def get_embedding_index(vocab, glove_file): + """ + Get dictionary mapping word to its word vector. + """ + embedding_index = {} + with open(glove_file, 'r') as f: + for line in f: + parts = line.split() + word = parts[0] + if word not in vocab or len(parts) != 301: + continue + vec = np.asarray(parts[1:], dtype='float32') + embedding_index[word] = vec + return embedding_index + + +def get_glove_embedding(glove_file, data_dir): + """ + Get embedding for the words in the data set. + """ + vocab = get_vocab(data_dir) + embedding_index = get_embedding_index(vocab, glove_file) + word_index = {w: i for i, w in enumerate(embedding_index.keys())} + + embedding_matrix = np.zeros((len(word_index), 300)) + for word, i in word_index.items(): + embedding_matrix[i] = embedding_index.get(word) + + embedding_tensor = torch.from_numpy(embedding_matrix) + embedding = nn.Embedding(len(embedding_index), 300) + embedding.weight = nn.Parameter(embedding_tensor) + return word_index, embedding + diff --git a/mp_cnn/train.py b/mp_cnn/train.py new file mode 100644 index 0000000..fe1d71b --- /dev/null +++ b/mp_cnn/train.py @@ -0,0 +1,191 @@ +import math +import time + +import torch +import torch.nn.functional as F +from torch.autograd import Variable +import torch.optim as optim +from torch.optim.lr_scheduler import ReduceLROnPlateau +from scipy.stats import pearsonr, spearmanr + +# logging setup +import logging +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + + +class MPCNNTrainerFactory(object): + """ + Get the corresponding Trainer class for a particular dataset. + """ + @staticmethod + def get_trainer(dataset_name, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): + if dataset_name == 'sick': + return SICKTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + elif dataset_name == 'msrvid': + return MSRVIDTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + else: + raise ValueError('{} is not a valid dataset.'.format(dataset_name)) + + +class Trainer(object): + + """ + Abstraction for training a model on a Dataset. + """ + + def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): + self.model = model + self.optimizer = optimizer + self.train_loader = train_loader + self.batch_size = batch_size + self.sample = sample + self.log_interval = log_interval + self.model_outfile = model_outfile + self.train_evaluator = train_evaluator + self.test_evaluator = test_evaluator + self.dev_evaluator = dev_evaluator + + def evaluate(self, evaluator, dataset_name): + scores, metric_names = evaluator.get_scores() + logger.info('Evaluation metrics for {}:'.format(dataset_name)) + logger.info('\t'.join([' '] + metric_names)) + logger.info('\t'.join([dataset_name] + list(map(str, scores)))) + return scores + + def train_epoch(self, epoch): + raise NotImplementedError() + + def train(self, epochs): + raise NotImplementedError() + + +class SICKTrainer(Trainer): + + def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): + super(SICKTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + + def train_epoch(self, epoch): + self.model.train() + total_loss = 0 + for batch_idx, (sentences, labels) in enumerate(self.train_loader): + sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b']) + labels = Variable(labels) + self.optimizer.zero_grad() + output = self.model(sent_a, sent_b) + loss = F.kl_div(output, labels) + total_loss += loss.data[0] + loss.backward() + self.optimizer.step() + if batch_idx % self.log_interval == 0: + logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, min(batch_idx * self.batch_size, len(self.train_loader.dataset)), + len(self.train_loader.dataset) if not self.sample else self.sample, + 100. * batch_idx / (len(self.train_loader) if not self.sample else math.ceil(self.sample / self.batch_size)), loss.data[0]) + ) + return total_loss + + def train(self, epochs): + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=0.3, patience=2) + epoch_times = [] + best_dev_score = -1 + for epoch in range(1, epochs + 1): + start = time.time() + logger.info('Epoch {} started...'.format(epoch)) + self.train_epoch(epoch) + + dev_scores = self.evaluate(self.dev_evaluator, 'dev') + end = time.time() + duration = end - start + logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) + epoch_times.append(duration) + + if dev_scores[0] > best_dev_score: + best_dev_score = dev_scores[0] + torch.save(self.model, self.model_outfile) + scheduler.step(dev_scores[0]) + + logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) + + +class MSRVIDTrainer(Trainer): + + def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None): + super(MSRVIDTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator) + + def train_epoch(self, epoch): + self.model.train() + + # since MSRVID doesn't have validation set, we manually leave-out some training data for validation + batches = math.ceil(len(self.train_loader.dataset) / self.batch_size) + start_val_batch = math.floor(0.8 * batches) + left_out_val_a, left_out_val_b = [], [] + left_out_val_labels = [] + + for batch_idx, (sentences, labels) in enumerate(self.train_loader): + sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b']) + labels = Variable(labels) + if batch_idx >= start_val_batch: + left_out_val_a.append(sent_a) + left_out_val_b.append(sent_b) + left_out_val_labels.append(labels) + continue + self.optimizer.zero_grad() + output = self.model(sent_a, sent_b) + loss = F.kl_div(output, labels) + loss.backward() + self.optimizer.step() + if batch_idx % self.log_interval == 0: + logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, min(batch_idx * self.batch_size, len(self.train_loader.dataset)), + len(self.train_loader.dataset) if not self.sample else self.sample, + 100. * batch_idx / (len(self.train_loader) if not self.sample else math.ceil(self.sample / self.batch_size)), loss.data[0]) + ) + + self.evaluate(self.train_evaluator, 'train') + return left_out_val_a, left_out_val_b, left_out_val_labels + + def train(self, epochs): + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=0.3, patience=2) + epoch_times = [] + best_dev_score = -1 + for epoch in range(1, epochs + 1): + start = time.time() + logger.info('Epoch {} started...'.format(epoch)) + left_out_a, left_out_b, left_out_label = self.train_epoch(epoch) + + # manually evaluating the validating set + left_out_a = torch.cat(left_out_a) + left_out_b = torch.cat(left_out_b) + left_out_label = torch.cat(left_out_label) + output = self.model(left_out_a, left_out_b) + predict_classes = torch.arange(0, 6).expand(len(left_out_a), 6).cuda() + true_labels = (predict_classes * left_out_label.data).sum(dim=1) + predictions = (predict_classes * output.data.exp()).sum(dim=1) + predictions = predictions.cpu().numpy() + true_labels = true_labels.cpu().numpy() + pearson_r = pearsonr(predictions, true_labels)[0] + for param_group in self.optimizer.param_groups: + logger.info('Validation size: %s Pearson\'s r: %s', output.size()[0], pearson_r) + logger.info('Learning rate: %s', param_group['lr']) + break + scheduler.step(pearson_r) + + end = time.time() + duration = end - start + logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) + epoch_times.append(duration) + + if pearson_r > best_dev_score: + best_dev_score = pearson_r + torch.save(self.model, self.model_outfile) + + self.evaluate(self.test_evaluator, 'test') + + logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))