mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
MP-CNN PyTorch Re-implementation (#37)
Re-implementation of MP-CNN in PyTorch.
This commit is contained in:
Michael Tu
@@ -0,0 +1,39 @@
|
||||
# MP-CNN PyTorch Implementation
|
||||
|
||||
This is a PyTorch implementation of the following paper
|
||||
|
||||
* Hua He, Kevin Gimpel, and Jimmy Lin. [Multi-Perspective Sentence Similarity Modeling with Convolutional Neural Networks](http://aclweb.org/anthology/D/D15/D15-1181.pdf). *Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP 2015)*, pages 1576-1586.
|
||||
|
||||
The SICK and MSRVID datasets are available in https://github.com/castorini/data, as well as the GloVe word embeddings.
|
||||
|
||||
Directory layout should be like this:
|
||||
```
|
||||
├── Castor
|
||||
│ ├── README.md
|
||||
│ ├── ...
|
||||
│ └── mp_cnn/
|
||||
├── data
|
||||
│ ├── README.md
|
||||
│ ├── ...
|
||||
│ ├── msrvid/
|
||||
│ ├── sick/
|
||||
│ └── GloVe/
|
||||
```
|
||||
|
||||
To run MP-CNN on the SICK dataset, use the following command:
|
||||
|
||||
```
|
||||
python main.py mpcnn.sick.model.castor --dataset sick --batch-size 32 --epochs 15
|
||||
```
|
||||
|
||||
To run MP-CNN on the MSRVID dataset, use the following command:
|
||||
```
|
||||
python main.py mpcnn.msrvid.model.castor --dataset msrvid --batch-size 8 --epochs 30 --epsilon 0.01
|
||||
```
|
||||
|
||||
These are not the optimal hyperparameters but they are decent. This README will be updated with more optimal hyperparameters and results in the future.
|
||||
|
||||
To see all options available, use
|
||||
```
|
||||
python main.py --help
|
||||
```
|
||||
@@ -0,0 +1,184 @@
|
||||
from enum import Enum
|
||||
import math
|
||||
import os
|
||||
|
||||
import torch
|
||||
from torch.autograd import Variable
|
||||
import torch.nn.functional as F
|
||||
import torch.utils.data as data
|
||||
|
||||
import preprocessing
|
||||
|
||||
# logging setup
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
|
||||
class DatasetType(Enum):
|
||||
TRAIN = 1
|
||||
TEST = 2
|
||||
DEV = 3
|
||||
|
||||
|
||||
class MPCNNDatasetFactory(object):
|
||||
"""
|
||||
Get the corresponding Dataset class for a particular dataset.
|
||||
"""
|
||||
@staticmethod
|
||||
def get_dataset(dataset_name, word_vectors_file, batch_size, cuda, sample):
|
||||
extra_args = {'shuffle': True}
|
||||
dev_loader = None
|
||||
if sample:
|
||||
sample_indices = list(range(sample))
|
||||
subset_random_sampler = data.sampler.SubsetRandomSampler(sample_indices)
|
||||
extra_args['sampler'] = subset_random_sampler
|
||||
extra_args['shuffle'] = False
|
||||
if dataset_name == 'sick':
|
||||
train_loader = torch.utils.data.DataLoader(SICKDataset(DatasetType.TRAIN, cuda), batch_size=batch_size, **extra_args)
|
||||
test_loader = torch.utils.data.DataLoader(SICKDataset(DatasetType.TEST, cuda), batch_size=batch_size, **extra_args)
|
||||
dev_loader = torch.utils.data.DataLoader(SICKDataset(DatasetType.DEV, cuda), batch_size=batch_size, **extra_args)
|
||||
elif dataset_name == 'msrvid':
|
||||
train_loader = torch.utils.data.DataLoader(MSRVIDDataset(DatasetType.TRAIN, cuda), batch_size=batch_size, **extra_args)
|
||||
test_loader = torch.utils.data.DataLoader(MSRVIDDataset(DatasetType.TEST, cuda), batch_size=batch_size, **extra_args)
|
||||
else:
|
||||
raise ValueError('{} is not a valid dataset.'.format(dataset_name))
|
||||
|
||||
word_index, embedding = preprocessing.get_glove_embedding(word_vectors_file, train_loader.dataset.dataset_root)
|
||||
logger.info('Finished loading GloVe embedding for vocab in data...')
|
||||
|
||||
train_loader.dataset.initialize(word_index, embedding)
|
||||
test_loader.dataset.initialize(word_index, embedding)
|
||||
if dev_loader is not None:
|
||||
dev_loader.dataset.initialize(word_index, embedding)
|
||||
return train_loader, test_loader, dev_loader
|
||||
|
||||
|
||||
class MPCNNDataset(data.Dataset):
|
||||
train_folder = 'train'
|
||||
test_folder = 'test'
|
||||
dev_folder = 'dev'
|
||||
# subclass will override fields below
|
||||
dataset_root = None
|
||||
num_classes = None
|
||||
|
||||
def __init__(self, dataset_type, cuda):
|
||||
if not isinstance(dataset_type, DatasetType):
|
||||
raise ValueError('dataset_type ({}) must be of type DatasetType enum'.format(dataset_type))
|
||||
|
||||
if dataset_type == DatasetType.TRAIN:
|
||||
subfolder = MPCNNDataset.train_folder
|
||||
elif dataset_type == DatasetType.TEST:
|
||||
subfolder = MPCNNDataset.test_folder
|
||||
else:
|
||||
subfolder = MPCNNDataset.dev_folder
|
||||
|
||||
self.dataset_dir = os.path.join(self.dataset_root, subfolder)
|
||||
if not os.path.exists(self.dataset_dir):
|
||||
raise RuntimeError('{} does not exist'.format(self.dataset_dir))
|
||||
|
||||
self.cuda = cuda
|
||||
self.max_length = -10000
|
||||
|
||||
def initialize(self, word_index, embedding):
|
||||
"""
|
||||
Convert sentences into sentence embeddings.
|
||||
"""
|
||||
sent_a = self._load(self.dataset_dir, 'a.txt')
|
||||
sent_b = self._load(self.dataset_dir, 'b.txt')
|
||||
|
||||
# obtain max sentence length to use as dimension for padding to support batching
|
||||
sent_a_tokens, sent_b_tokens = [], []
|
||||
for i in range(len(sent_a)):
|
||||
sa_tokens = sent_a[i].split(' ')
|
||||
sb_tokens = sent_b[i].split(' ')
|
||||
self.max_length = max(self.max_length, len(sa_tokens), len(sb_tokens))
|
||||
sent_a_tokens.append(sa_tokens)
|
||||
sent_b_tokens.append(sb_tokens)
|
||||
|
||||
self.sentences = []
|
||||
for i in range(len(sent_a)):
|
||||
sent_pair = {}
|
||||
sent_pair['a'] = self._get_sentence_embeddings(sent_a_tokens[i], word_index, embedding)
|
||||
sent_pair['b'] = self._get_sentence_embeddings(sent_b_tokens[i], word_index, embedding)
|
||||
self.sentences.append(sent_pair)
|
||||
self.labels = self._load(self.dataset_dir, 'sim.txt', float)
|
||||
|
||||
def _load(self, dataset_dir, fname, type_converter=str):
|
||||
data = []
|
||||
with open(os.path.join(dataset_dir, fname), 'r') as f:
|
||||
for line in f:
|
||||
stripped_line = line.rstrip('.\n')
|
||||
item = type_converter(stripped_line)
|
||||
data.append(item)
|
||||
return data
|
||||
|
||||
def _get_sentence_embeddings(self, tokens, word_index, embedding):
|
||||
sentence_embedding = torch.zeros(300, self.max_length)
|
||||
sentence_embedding[:, :len(tokens)].normal_(0, 1)
|
||||
found_pos, found_emb_idx = [], []
|
||||
for i, token in enumerate(tokens):
|
||||
if token in word_index:
|
||||
found_pos.append(i)
|
||||
found_emb_idx.append(word_index[token])
|
||||
|
||||
found_word_vecs = embedding(Variable(torch.LongTensor(found_emb_idx)))
|
||||
for i, v in enumerate(found_pos):
|
||||
sentence_embedding[:, v] = found_word_vecs[i].data
|
||||
return sentence_embedding.cuda() if self.cuda else sentence_embedding
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return self.sentences[idx], self.labels[idx]
|
||||
|
||||
def __len__(self):
|
||||
return len(self.labels)
|
||||
|
||||
|
||||
class SICKDataset(MPCNNDataset):
|
||||
|
||||
dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'sick')
|
||||
num_classes = 5
|
||||
|
||||
def __init__(self, dataset_type, cuda):
|
||||
super(SICKDataset, self).__init__(dataset_type, cuda)
|
||||
|
||||
def initialize(self, word_index, embedding):
|
||||
super(SICKDataset, self).initialize(word_index, embedding)
|
||||
new_labels = torch.zeros(self.__len__(), self.num_classes)
|
||||
for i, sim in enumerate(self.labels):
|
||||
ceil, floor = math.ceil(sim), math.floor(sim)
|
||||
if ceil == floor:
|
||||
new_labels[i][floor - 1] = 1
|
||||
else:
|
||||
new_labels[i][floor - 1] = ceil - sim
|
||||
new_labels[i][ceil - 1] = sim - floor
|
||||
|
||||
self.labels = new_labels.cuda() if self.cuda else new_labels
|
||||
|
||||
|
||||
class MSRVIDDataset(MPCNNDataset):
|
||||
|
||||
dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'msrvid')
|
||||
num_classes = 6
|
||||
|
||||
def __init__(self, dataset_type, cuda):
|
||||
super(MSRVIDDataset, self).__init__(dataset_type, cuda)
|
||||
|
||||
def initialize(self, word_index, embedding):
|
||||
super(MSRVIDDataset, self).initialize(word_index, embedding)
|
||||
new_labels = torch.zeros(self.__len__(), self.num_classes)
|
||||
for i, sim in enumerate(self.labels):
|
||||
ceil, floor = math.ceil(sim), math.floor(sim)
|
||||
if ceil == floor:
|
||||
new_labels[i][floor] = 1
|
||||
else:
|
||||
new_labels[i][floor] = ceil - sim
|
||||
new_labels[i][ceil] = sim - floor
|
||||
|
||||
self.labels = new_labels.cuda() if self.cuda else new_labels
|
||||
@@ -0,0 +1,110 @@
|
||||
from scipy.stats import pearsonr, spearmanr
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
|
||||
|
||||
class MPCNNEvaluatorFactory(object):
|
||||
"""
|
||||
Get the corresponding Evaluator class for a particular dataset.
|
||||
"""
|
||||
@staticmethod
|
||||
def get_evaluator(dataset_name, model, data_loader, batch_size, cuda):
|
||||
if data_loader is None:
|
||||
return None
|
||||
|
||||
if dataset_name == 'sick':
|
||||
return SICKEvaluator(model, data_loader, batch_size, cuda)
|
||||
elif dataset_name == 'msrvid':
|
||||
return MSRVIDEvaluator(model, data_loader, batch_size, cuda)
|
||||
else:
|
||||
raise ValueError('{} is not a valid dataset.'.format(dataset_name))
|
||||
|
||||
|
||||
class Evaluator(object):
|
||||
"""
|
||||
Evaluates performance of model on a Dataset, using metrics specific to the Dataset.
|
||||
"""
|
||||
|
||||
def __init__(self, model, data_loader, batch_size, cuda):
|
||||
self.model = model
|
||||
self.data_loader = data_loader
|
||||
self.batch_size = batch_size
|
||||
self.cuda = cuda
|
||||
|
||||
def get_scores(self):
|
||||
"""
|
||||
Get the scores used to evaluate the model.
|
||||
Should return ([score1, score2, ..], [score1_name, score2_name, ...]).
|
||||
The first score is the primary score used to determine if the model has improved.
|
||||
"""
|
||||
raise NotImplementedError('Evaluator subclass needs to implement get_score')
|
||||
|
||||
|
||||
class SICKEvaluator(Evaluator):
|
||||
|
||||
def __init__(self, model, data_loader, batch_size, cuda):
|
||||
super(SICKEvaluator, self).__init__(model, data_loader, batch_size, cuda)
|
||||
|
||||
def get_scores(self):
|
||||
self.model.eval()
|
||||
num_classes = self.data_loader.dataset.num_classes
|
||||
predict_classes = torch.arange(1, num_classes + 1).expand(self.batch_size, num_classes)
|
||||
if self.cuda:
|
||||
predict_classes = predict_classes.cuda()
|
||||
test_kl_div_loss = 0
|
||||
predictions = []
|
||||
true_labels = []
|
||||
for sentences, labels in self.data_loader:
|
||||
sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True)
|
||||
labels = Variable(labels, volatile=True)
|
||||
output = self.model(sent_a, sent_b)
|
||||
test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0]
|
||||
# handle last batch which might have smaller size
|
||||
if len(predict_classes) != len(sent_a):
|
||||
predict_classes = torch.arange(1, num_classes + 1).expand(len(sent_a), num_classes)
|
||||
if self.cuda:
|
||||
predict_classes = predict_classes.cuda()
|
||||
true_labels.append((predict_classes * labels.data).sum(dim=1))
|
||||
predictions.append((predict_classes * output.data.exp()).sum(dim=1))
|
||||
|
||||
predictions = torch.cat(predictions).cpu().numpy()
|
||||
true_labels = torch.cat(true_labels).cpu().numpy()
|
||||
test_kl_div_loss /= len(self.data_loader.dataset)
|
||||
pearson_r = pearsonr(predictions, true_labels)[0]
|
||||
spearman_r = spearmanr(predictions, true_labels)[0]
|
||||
return [pearson_r, spearman_r, test_kl_div_loss], ['pearson_r', 'spearman_r', 'KL-divergence loss']
|
||||
|
||||
|
||||
class MSRVIDEvaluator(Evaluator):
|
||||
|
||||
def __init__(self, model, data_loader, batch_size, cuda):
|
||||
super(MSRVIDEvaluator, self).__init__(model, data_loader, batch_size, cuda)
|
||||
|
||||
def get_scores(self):
|
||||
self.model.eval()
|
||||
num_classes = self.data_loader.dataset.num_classes
|
||||
predict_classes = torch.arange(0, num_classes).expand(self.batch_size, num_classes)
|
||||
if self.cuda:
|
||||
predict_classes = predict_classes.cuda()
|
||||
test_kl_div_loss = 0
|
||||
predictions = []
|
||||
true_labels = []
|
||||
for sentences, labels in self.data_loader:
|
||||
sent_a, sent_b = Variable(sentences['a'], volatile=True), Variable(sentences['b'], volatile=True)
|
||||
labels = Variable(labels, volatile=True)
|
||||
output = self.model(sent_a, sent_b)
|
||||
test_kl_div_loss += F.kl_div(output, labels, size_average=False).data[0]
|
||||
# handle last batch which might have smaller size
|
||||
if len(predict_classes) != len(sent_a):
|
||||
predict_classes = torch.arange(0, num_classes).expand(len(sent_a), num_classes)
|
||||
if self.cuda:
|
||||
predict_classes = predict_classes.cuda()
|
||||
true_labels.append((predict_classes * labels.data).sum(dim=1))
|
||||
predictions.append((predict_classes * output.data.exp()).sum(dim=1))
|
||||
|
||||
predictions = torch.cat(predictions).cpu().numpy()
|
||||
true_labels = torch.cat(true_labels).cpu().numpy()
|
||||
test_kl_div_loss /= len(self.data_loader.dataset)
|
||||
pearson_r = pearsonr(predictions, true_labels)[0]
|
||||
return [pearson_r, test_kl_div_loss], ['pearson_r', 'KL-divergence loss']
|
||||
@@ -0,0 +1,84 @@
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
from scipy.stats import pearsonr, spearmanr
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
import torch.optim as optim
|
||||
|
||||
from dataset import DatasetType, MPCNNDatasetFactory
|
||||
from evaluation import MPCNNEvaluatorFactory
|
||||
from model import MPCNN
|
||||
from train import MPCNNTrainerFactory
|
||||
|
||||
# logging setup
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN')
|
||||
parser.add_argument('model_outfile', help='file to save final model')
|
||||
parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid]', default='sick')
|
||||
parser.add_argument('--word-vectors-file', help='word vectors file', default=os.path.join(os.pardir, os.pardir, 'data', 'GloVe', 'glove.840B.300d.txt'))
|
||||
parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true')
|
||||
parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training')
|
||||
parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)')
|
||||
parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)')
|
||||
parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.001)')
|
||||
parser.add_argument('--epsilon', type=float, default=1e-8, metavar='M', help='Adam epsilon (default: 1e-8)')
|
||||
parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status')
|
||||
parser.add_argument('--sample', type=int, default=0, metavar='N', help='how many examples to take from each dataset, meant for quickly testing entire end-to-end pipeline (default: all)')
|
||||
parser.add_argument('--regularization', type=float, default=0.0001, metavar='REG', help='Regularization for the optimizer (default: 0.0001)')
|
||||
parser.add_argument('--max-window-size', type=int, default=3, metavar='N', help='windows sizes will be [1,max_window_size] and infinity')
|
||||
parser.add_argument('--holistic-filters', type=int, default=300, metavar='N', help='number of holistic filters')
|
||||
parser.add_argument('--per-dim-filters', type=int, default=20, metavar='N', help='number of per-dimension filters')
|
||||
parser.add_argument('--hidden-units', type=int, default=150, metavar='N', help='number of hidden units in each of the two hidden layers')
|
||||
parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)')
|
||||
args = parser.parse_args()
|
||||
args.cuda = not args.no_cuda and torch.cuda.is_available()
|
||||
|
||||
np.random.seed(args.seed)
|
||||
torch.manual_seed(args.seed)
|
||||
if args.cuda:
|
||||
torch.cuda.manual_seed(args.seed)
|
||||
|
||||
train_loader, test_loader, dev_loader = MPCNNDatasetFactory.get_dataset(args.dataset, args.word_vectors_file, args.batch_size, args.cuda, args.sample)
|
||||
|
||||
filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf]
|
||||
model = MPCNN(300, args.holistic_filters, args.per_dim_filters, filter_widths, args.hidden_units, train_loader.dataset.num_classes)
|
||||
if args.cuda:
|
||||
model.cuda()
|
||||
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.regularization, eps=args.epsilon)
|
||||
train_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, train_loader, args.batch_size, args.cuda)
|
||||
test_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, test_loader, args.batch_size, args.cuda)
|
||||
dev_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, dev_loader, args.batch_size, args.cuda)
|
||||
|
||||
trainer = MPCNNTrainerFactory.get_trainer(args.dataset, model, optimizer, train_loader, args.batch_size, args.sample, args.log_interval, args.model_outfile, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
if not args.skip_training:
|
||||
total_params = 0
|
||||
for param in model.parameters():
|
||||
size = [s for s in param.size()]
|
||||
total_params += np.prod(size)
|
||||
logger.info('Total number of parameters: %s', total_params)
|
||||
trainer.train(args.epochs)
|
||||
|
||||
model = torch.load(args.model_outfile)
|
||||
test_evaluator = MPCNNEvaluatorFactory.get_evaluator(args.dataset, model, test_loader, args.batch_size, args.cuda)
|
||||
scores, metric_names = test_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for test')
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join(['test'] + list(map(str, scores))))
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
|
||||
|
||||
class MPCNN(nn.Module):
|
||||
|
||||
def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes):
|
||||
super(MPCNN, self).__init__()
|
||||
|
||||
self.n_word_dim = n_word_dim
|
||||
self.n_per_dim_filters = n_per_dim_filters
|
||||
self.filter_widths = filter_widths
|
||||
holistic_conv_layers = []
|
||||
per_dim_conv_layers = []
|
||||
|
||||
for ws in filter_widths:
|
||||
if np.isinf(ws):
|
||||
continue
|
||||
|
||||
holistic_conv_layers.append(nn.Sequential(
|
||||
nn.Conv1d(n_word_dim, n_holistic_filters, ws),
|
||||
nn.Tanh()
|
||||
))
|
||||
|
||||
per_dim_conv_layers.append(nn.Sequential(
|
||||
nn.Conv1d(n_word_dim, n_word_dim * n_per_dim_filters, ws, groups=n_word_dim),
|
||||
nn.Tanh()
|
||||
))
|
||||
|
||||
self.holistic_conv_layers = nn.ModuleList(holistic_conv_layers)
|
||||
self.per_dim_conv_layers = nn.ModuleList(per_dim_conv_layers)
|
||||
|
||||
# compute number of inputs to first hidden layer
|
||||
COMP_1_COMPONENTS, COMP_2_COMPONENTS = 2 + n_word_dim, 2
|
||||
n_feat_h = 3 * len(self.filter_widths) * COMP_2_COMPONENTS
|
||||
n_feat_v = 3 * (len(self.filter_widths) ** 2) * COMP_1_COMPONENTS + 2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS
|
||||
n_feat = n_feat_h + n_feat_v
|
||||
|
||||
self.final_layers = nn.Sequential(
|
||||
nn.Linear(n_feat, hidden_layer_units),
|
||||
nn.Tanh(),
|
||||
nn.Linear(hidden_layer_units, num_classes),
|
||||
nn.LogSoftmax()
|
||||
)
|
||||
|
||||
def _get_blocks_for_sentence(self, sent):
|
||||
block_a = {}
|
||||
block_b = {}
|
||||
for ws in self.filter_widths:
|
||||
holistic_conv_out = self.holistic_conv_layers[ws - 1](sent) if not np.isinf(ws) else sent
|
||||
block_a[ws] = {
|
||||
'max': F.max_pool1d(holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim),
|
||||
'min': F.max_pool1d(-1 * holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim),
|
||||
'mean': F.avg_pool1d(holistic_conv_out, holistic_conv_out.size()[2]).view(-1, self.n_word_dim)
|
||||
}
|
||||
|
||||
# only compute per-dimension convolution for non-infinity widths
|
||||
if np.isinf(ws):
|
||||
continue
|
||||
|
||||
per_dim_conv_out = self.per_dim_conv_layers[ws - 1](sent)
|
||||
block_b[ws] = {
|
||||
'max': F.max_pool1d(per_dim_conv_out, per_dim_conv_out.size()[2]).view(-1, self.n_word_dim, self.n_per_dim_filters),
|
||||
'min': F.max_pool1d(-1 * per_dim_conv_out, per_dim_conv_out.size()[2]).view(-1, self.n_word_dim, self.n_per_dim_filters)
|
||||
}
|
||||
return block_a, block_b
|
||||
|
||||
def _algo_1_horiz_comp(self, sent1_block_a, sent2_block_a):
|
||||
comparison_feats = []
|
||||
for pool in ('max', 'min', 'mean'):
|
||||
for ws in self.filter_widths:
|
||||
x1 = sent1_block_a[ws][pool]
|
||||
x2 = sent2_block_a[ws][pool]
|
||||
batch_size = x1.size()[0]
|
||||
comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1))
|
||||
comparison_feats.append(F.pairwise_distance(x1, x2))
|
||||
return torch.cat(comparison_feats, dim=1)
|
||||
|
||||
def _algo_2_vert_comp(self, sent1_block_a, sent2_block_a, sent1_block_b, sent2_block_b):
|
||||
comparison_feats = []
|
||||
for pool in ('max', 'min', 'mean'):
|
||||
for ws1 in self.filter_widths:
|
||||
x1 = sent1_block_a[ws1][pool]
|
||||
batch_size = x1.size()[0]
|
||||
for ws2 in self.filter_widths:
|
||||
x2 = sent2_block_a[ws2][pool]
|
||||
comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1))
|
||||
comparison_feats.append(F.pairwise_distance(x1, x2))
|
||||
comparison_feats.append(torch.abs(x1 - x2))
|
||||
|
||||
for pool in ('max', 'min'):
|
||||
ws_no_inf = [w for w in self.filter_widths if not np.isinf(w)]
|
||||
for ws in ws_no_inf:
|
||||
oG_1B = sent1_block_b[ws][pool]
|
||||
oG_2B = sent2_block_b[ws][pool]
|
||||
for i in range(0, self.n_per_dim_filters):
|
||||
x1 = oG_1B[:, :, i]
|
||||
x2 = oG_2B[:, :, i]
|
||||
comparison_feats.append(F.cosine_similarity(x1, x2).view(batch_size, 1))
|
||||
comparison_feats.append(F.pairwise_distance(x1, x2))
|
||||
comparison_feats.append(torch.abs(x1 - x2))
|
||||
|
||||
return torch.cat(comparison_feats, dim=1)
|
||||
|
||||
def forward(self, sent1, sent2):
|
||||
# Sentence modeling module
|
||||
sent1_block_a, sent1_block_b = self._get_blocks_for_sentence(sent1)
|
||||
sent2_block_a, sent2_block_b = self._get_blocks_for_sentence(sent2)
|
||||
|
||||
# Similarity measurement layer
|
||||
feat_h = self._algo_1_horiz_comp(sent1_block_a, sent2_block_a)
|
||||
feat_v = self._algo_2_vert_comp(sent1_block_a, sent2_block_a, sent1_block_b, sent2_block_b)
|
||||
feat_all = torch.cat([feat_h, feat_v], dim=1)
|
||||
|
||||
preds = self.final_layers(feat_all)
|
||||
return preds
|
||||
@@ -0,0 +1,55 @@
|
||||
"""
|
||||
Preprocessing utilities such as preparing embeddings.
|
||||
"""
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
def get_vocab(data_dir):
|
||||
"""
|
||||
Get vocabulary as set of words.
|
||||
"""
|
||||
vocab = set()
|
||||
with open(os.path.join(data_dir, 'vocab-cased.txt'), 'r') as f:
|
||||
for line in f:
|
||||
word = line.rstrip()
|
||||
vocab.add(word)
|
||||
return vocab
|
||||
|
||||
|
||||
def get_embedding_index(vocab, glove_file):
|
||||
"""
|
||||
Get dictionary mapping word to its word vector.
|
||||
"""
|
||||
embedding_index = {}
|
||||
with open(glove_file, 'r') as f:
|
||||
for line in f:
|
||||
parts = line.split()
|
||||
word = parts[0]
|
||||
if word not in vocab or len(parts) != 301:
|
||||
continue
|
||||
vec = np.asarray(parts[1:], dtype='float32')
|
||||
embedding_index[word] = vec
|
||||
return embedding_index
|
||||
|
||||
|
||||
def get_glove_embedding(glove_file, data_dir):
|
||||
"""
|
||||
Get embedding for the words in the data set.
|
||||
"""
|
||||
vocab = get_vocab(data_dir)
|
||||
embedding_index = get_embedding_index(vocab, glove_file)
|
||||
word_index = {w: i for i, w in enumerate(embedding_index.keys())}
|
||||
|
||||
embedding_matrix = np.zeros((len(word_index), 300))
|
||||
for word, i in word_index.items():
|
||||
embedding_matrix[i] = embedding_index.get(word)
|
||||
|
||||
embedding_tensor = torch.from_numpy(embedding_matrix)
|
||||
embedding = nn.Embedding(len(embedding_index), 300)
|
||||
embedding.weight = nn.Parameter(embedding_tensor)
|
||||
return word_index, embedding
|
||||
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
import math
|
||||
import time
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
import torch.optim as optim
|
||||
from torch.optim.lr_scheduler import ReduceLROnPlateau
|
||||
from scipy.stats import pearsonr, spearmanr
|
||||
|
||||
# logging setup
|
||||
import logging
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
|
||||
class MPCNNTrainerFactory(object):
|
||||
"""
|
||||
Get the corresponding Trainer class for a particular dataset.
|
||||
"""
|
||||
@staticmethod
|
||||
def get_trainer(dataset_name, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None):
|
||||
if dataset_name == 'sick':
|
||||
return SICKTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator)
|
||||
elif dataset_name == 'msrvid':
|
||||
return MSRVIDTrainer(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator)
|
||||
else:
|
||||
raise ValueError('{} is not a valid dataset.'.format(dataset_name))
|
||||
|
||||
|
||||
class Trainer(object):
|
||||
|
||||
"""
|
||||
Abstraction for training a model on a Dataset.
|
||||
"""
|
||||
|
||||
def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None):
|
||||
self.model = model
|
||||
self.optimizer = optimizer
|
||||
self.train_loader = train_loader
|
||||
self.batch_size = batch_size
|
||||
self.sample = sample
|
||||
self.log_interval = log_interval
|
||||
self.model_outfile = model_outfile
|
||||
self.train_evaluator = train_evaluator
|
||||
self.test_evaluator = test_evaluator
|
||||
self.dev_evaluator = dev_evaluator
|
||||
|
||||
def evaluate(self, evaluator, dataset_name):
|
||||
scores, metric_names = evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}:'.format(dataset_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([dataset_name] + list(map(str, scores))))
|
||||
return scores
|
||||
|
||||
def train_epoch(self, epoch):
|
||||
raise NotImplementedError()
|
||||
|
||||
def train(self, epochs):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class SICKTrainer(Trainer):
|
||||
|
||||
def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None):
|
||||
super(SICKTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
def train_epoch(self, epoch):
|
||||
self.model.train()
|
||||
total_loss = 0
|
||||
for batch_idx, (sentences, labels) in enumerate(self.train_loader):
|
||||
sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b'])
|
||||
labels = Variable(labels)
|
||||
self.optimizer.zero_grad()
|
||||
output = self.model(sent_a, sent_b)
|
||||
loss = F.kl_div(output, labels)
|
||||
total_loss += loss.data[0]
|
||||
loss.backward()
|
||||
self.optimizer.step()
|
||||
if batch_idx % self.log_interval == 0:
|
||||
logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
|
||||
epoch, min(batch_idx * self.batch_size, len(self.train_loader.dataset)),
|
||||
len(self.train_loader.dataset) if not self.sample else self.sample,
|
||||
100. * batch_idx / (len(self.train_loader) if not self.sample else math.ceil(self.sample / self.batch_size)), loss.data[0])
|
||||
)
|
||||
return total_loss
|
||||
|
||||
def train(self, epochs):
|
||||
scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=0.3, patience=2)
|
||||
epoch_times = []
|
||||
best_dev_score = -1
|
||||
for epoch in range(1, epochs + 1):
|
||||
start = time.time()
|
||||
logger.info('Epoch {} started...'.format(epoch))
|
||||
self.train_epoch(epoch)
|
||||
|
||||
dev_scores = self.evaluate(self.dev_evaluator, 'dev')
|
||||
end = time.time()
|
||||
duration = end - start
|
||||
logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
|
||||
epoch_times.append(duration)
|
||||
|
||||
if dev_scores[0] > best_dev_score:
|
||||
best_dev_score = dev_scores[0]
|
||||
torch.save(self.model, self.model_outfile)
|
||||
scheduler.step(dev_scores[0])
|
||||
|
||||
logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))
|
||||
|
||||
|
||||
class MSRVIDTrainer(Trainer):
|
||||
|
||||
def __init__(self, model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator=None):
|
||||
super(MSRVIDTrainer, self).__init__(model, optimizer, train_loader, batch_size, sample, log_interval, model_outfile, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
def train_epoch(self, epoch):
|
||||
self.model.train()
|
||||
|
||||
# since MSRVID doesn't have validation set, we manually leave-out some training data for validation
|
||||
batches = math.ceil(len(self.train_loader.dataset) / self.batch_size)
|
||||
start_val_batch = math.floor(0.8 * batches)
|
||||
left_out_val_a, left_out_val_b = [], []
|
||||
left_out_val_labels = []
|
||||
|
||||
for batch_idx, (sentences, labels) in enumerate(self.train_loader):
|
||||
sent_a, sent_b = Variable(sentences['a']), Variable(sentences['b'])
|
||||
labels = Variable(labels)
|
||||
if batch_idx >= start_val_batch:
|
||||
left_out_val_a.append(sent_a)
|
||||
left_out_val_b.append(sent_b)
|
||||
left_out_val_labels.append(labels)
|
||||
continue
|
||||
self.optimizer.zero_grad()
|
||||
output = self.model(sent_a, sent_b)
|
||||
loss = F.kl_div(output, labels)
|
||||
loss.backward()
|
||||
self.optimizer.step()
|
||||
if batch_idx % self.log_interval == 0:
|
||||
logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
|
||||
epoch, min(batch_idx * self.batch_size, len(self.train_loader.dataset)),
|
||||
len(self.train_loader.dataset) if not self.sample else self.sample,
|
||||
100. * batch_idx / (len(self.train_loader) if not self.sample else math.ceil(self.sample / self.batch_size)), loss.data[0])
|
||||
)
|
||||
|
||||
self.evaluate(self.train_evaluator, 'train')
|
||||
return left_out_val_a, left_out_val_b, left_out_val_labels
|
||||
|
||||
def train(self, epochs):
|
||||
scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=0.3, patience=2)
|
||||
epoch_times = []
|
||||
best_dev_score = -1
|
||||
for epoch in range(1, epochs + 1):
|
||||
start = time.time()
|
||||
logger.info('Epoch {} started...'.format(epoch))
|
||||
left_out_a, left_out_b, left_out_label = self.train_epoch(epoch)
|
||||
|
||||
# manually evaluating the validating set
|
||||
left_out_a = torch.cat(left_out_a)
|
||||
left_out_b = torch.cat(left_out_b)
|
||||
left_out_label = torch.cat(left_out_label)
|
||||
output = self.model(left_out_a, left_out_b)
|
||||
predict_classes = torch.arange(0, 6).expand(len(left_out_a), 6).cuda()
|
||||
true_labels = (predict_classes * left_out_label.data).sum(dim=1)
|
||||
predictions = (predict_classes * output.data.exp()).sum(dim=1)
|
||||
predictions = predictions.cpu().numpy()
|
||||
true_labels = true_labels.cpu().numpy()
|
||||
pearson_r = pearsonr(predictions, true_labels)[0]
|
||||
for param_group in self.optimizer.param_groups:
|
||||
logger.info('Validation size: %s Pearson\'s r: %s', output.size()[0], pearson_r)
|
||||
logger.info('Learning rate: %s', param_group['lr'])
|
||||
break
|
||||
scheduler.step(pearson_r)
|
||||
|
||||
end = time.time()
|
||||
duration = end - start
|
||||
logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60))
|
||||
epoch_times.append(duration)
|
||||
|
||||
if pearson_r > best_dev_score:
|
||||
best_dev_score = pearson_r
|
||||
torch.save(self.model, self.model_outfile)
|
||||
|
||||
self.evaluate(self.test_evaluator, 'test')
|
||||
|
||||
logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60))
|
||||
Reference in New Issue
Block a user