mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
add DecAtt model (#170)
* add DecAtt model * update readme, add dropout * fix more comments * add trecqa, wikiqa results * remove extraneous comment
This commit is contained in:
Victor Yang
@@ -0,0 +1,68 @@
|
||||
# DecAtt
|
||||
|
||||
This is a PyTorch reimplementation of the following paper:
|
||||
|
||||
```
|
||||
@inproceedings{parikh-EtAl:2016:EMNLP2016,
|
||||
author = {Parikh, Ankur and T\"{a}ckstr\"{o}m, Oscar and Das, Dipanjan and Uszkoreit, Jakob},
|
||||
title = {A Decomposable Attention Model for Natural Language Inference},
|
||||
booktitle = {Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing (EMNLP)},
|
||||
year = {2016}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
Please ensure you have followed instructions in the main [README](../README.md) doc before running any further commands in this doc.
|
||||
The commands in this doc assume you are under the root directory of the Castor repo.
|
||||
|
||||
## SICK Dataset
|
||||
|
||||
To run DecAtt on the SICK dataset, use the following command. `--dropout 0` is for mimicking the original paper, although adding dropout can improve results. If you have any problems running it check the Troubleshooting section below.
|
||||
|
||||
```
|
||||
python -m decatt decatt.sick.model --dataset sick --epochs 500 --regularization 5e-4 --lr 0.001 --lr-reduce-factor 0.5 --dropout 0.1
|
||||
```
|
||||
|
||||
| Implementation and config | Pearson's r | Spearman's p | MSE |
|
||||
| -------------------------------- |:-------------:|:-------------:|:----------:|
|
||||
| PyTorch using above config | 0.80094564 | 0.7184082390455326 | 0.3711671233177185 |
|
||||
|
||||
## TrecQA Dataset
|
||||
|
||||
To run DecAtt on the TrecQA dataset, use the following command:
|
||||
```
|
||||
python -m decatt decatt.trecqa.model --dataset trecqa --epochs 500 --regularization 5e-4 --lr 0.001 --lr-reduce-factor 0.5 --dropout 0.1
|
||||
```
|
||||
|
||||
| Implementation and config | map | mrr |
|
||||
| -------------------------------- |:------:|:------:|
|
||||
| PyTorch using above config | 0.6536 | 0.6848 |
|
||||
|
||||
This are the TrecQA raw dataset results. The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872).
|
||||
|
||||
## WikiQA Dataset
|
||||
|
||||
You also need `trec_eval` for this dataset, similar to TrecQA.
|
||||
|
||||
Then, you can run:
|
||||
```
|
||||
python -m decatt decatt.wikiqa.model --dataset wikiqa --epochs 500 --regularization 5e-4 --lr 0.001 --lr-reduce-factor 0.5 --dropout 0.1
|
||||
```
|
||||
| Implementation and config | map | mrr |
|
||||
| -------------------------------- |:------:|:------:|
|
||||
| PyTorch using above config | 0.6462 | 0.6603 |
|
||||
|
||||
|
||||
To see all options available, use
|
||||
```
|
||||
python -m decatt --help
|
||||
```
|
||||
|
||||
## Optional Dependencies
|
||||
|
||||
To optionally visualize the learning curve during training, we make use of https://github.com/lanpa/tensorboard-pytorch to connect to [TensorBoard](https://github.com/tensorflow/tensorboard). These projects require TensorFlow as a dependency, so you need to install TensorFlow before running the commands below. After these are installed, just add `--tensorboard` when running the training commands and open TensorBoard in the browser.
|
||||
|
||||
```sh
|
||||
pip install tensorboardX
|
||||
pip install tensorflow-tensorboard
|
||||
```
|
||||
@@ -0,0 +1,149 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import pprint
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.optim as optim
|
||||
|
||||
from common.dataset import DatasetFactory
|
||||
from common.evaluation import EvaluatorFactory
|
||||
from common.train import TrainerFactory
|
||||
from utils.serialization import load_checkpoint
|
||||
from .model import DecAtt
|
||||
|
||||
|
||||
def get_logger():
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, keep_results=False):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device,
|
||||
keep_results=keep_results)
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([split_name] + list(map(str, scores))))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN')
|
||||
parser.add_argument('model_outfile', help='file to save final model')
|
||||
parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid, trecqa, wikiqa]', default='sick')
|
||||
parser.add_argument('--word-vectors-dir', help='word vectors directory',
|
||||
default=os.path.join(os.pardir, 'Castor-data', 'embeddings', 'GloVe'))
|
||||
parser.add_argument('--word-vectors-file', help='word vectors filename', default='glove.840B.300d.txt')
|
||||
parser.add_argument('--word-vectors-dim', type=int, default=300,
|
||||
help='number of dimensions of word vectors (default: 300)')
|
||||
parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true')
|
||||
parser.add_argument('--device', type=int, default=0, help='GPU device, -1 for CPU (default: 0)')
|
||||
parser.add_argument('--wide-conv', action='store_true', default=False,
|
||||
help='use wide convolution instead of narrow convolution (default: false)')
|
||||
parser.add_argument('--sparse-features', action='store_true',
|
||||
default=False, help='use sparse features (default: false)')
|
||||
parser.add_argument('--batch-size', type=int, default=64, help='input batch size for training (default: 64)')
|
||||
parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 10)')
|
||||
parser.add_argument('--optimizer', type=str, default='adam', help='optimizer to use: adam or sgd (default: adam)')
|
||||
parser.add_argument('--lr', type=float, default=0.001, help='learning rate (default: 0.001)')
|
||||
parser.add_argument('--lr-reduce-factor', type=float, default=0.3,
|
||||
help='learning rate reduce factor after plateau (default: 0.3)')
|
||||
parser.add_argument('--patience', type=float, default=2,
|
||||
help='learning rate patience after seeing plateau (default: 2)')
|
||||
parser.add_argument('--momentum', type=float, default=0, help='momentum (default: 0)')
|
||||
parser.add_argument('--epsilon', type=float, default=1e-8, help='Optimizer epsilon (default: 1e-8)')
|
||||
parser.add_argument('--log-interval', type=int, default=10,
|
||||
help='how many batches to wait before logging training status (default: 10)')
|
||||
parser.add_argument('--regularization', type=float, default=0.0001,
|
||||
help='Regularization for the optimizer (default: 0.0001)')
|
||||
parser.add_argument('--max-window-size', type=int, default=3,
|
||||
help='windows sizes will be [1,max_window_size] and infinity (default: 3)')
|
||||
parser.add_argument('--dropout', type=float, default=0.5, help='dropout probability (default: 0.1)')
|
||||
parser.add_argument('--maxlen', type=int, default=60, help='maximum length of text (default: 60)')
|
||||
parser.add_argument('--seed', type=int, default=1234, help='random seed (default: 1234)')
|
||||
parser.add_argument('--tensorboard', action='store_true', default=False,
|
||||
help='use TensorBoard to visualize training (default: false)')
|
||||
parser.add_argument('--run-label', type=str, help='label to describe run')
|
||||
parser.add_argument('--keep-results', action='store_true',
|
||||
help='store the output score and qrel files into disk for the test set')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
device = torch.device(f'cuda:{args.device}' if torch.cuda.is_available() and args.device >= 0 else 'cpu')
|
||||
|
||||
random.seed(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
torch.manual_seed(args.seed)
|
||||
if args.device != -1:
|
||||
torch.cuda.manual_seed(args.seed)
|
||||
|
||||
logger = get_logger()
|
||||
logger.info(pprint.pformat(vars(args)))
|
||||
|
||||
dataset_cls, embedding, train_loader, test_loader, dev_loader \
|
||||
= DatasetFactory.get_dataset(args.dataset, args.word_vectors_dir, args.word_vectors_file, args.batch_size, args.device)
|
||||
|
||||
filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf]
|
||||
ext_feats = dataset_cls.EXT_FEATS if args.sparse_features else 0
|
||||
|
||||
model = DecAtt(embedding_size=args.word_vectors_dim, device=args.device, num_units=args.word_vectors_dim,
|
||||
num_classes=dataset_cls.NUM_CLASSES, dropout=args.dropout, max_sentence_length=args.maxlen)
|
||||
|
||||
model = model.to(device)
|
||||
embedding = embedding.to(device)
|
||||
|
||||
optimizer = None
|
||||
if args.optimizer == 'adam':
|
||||
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.regularization, eps=args.epsilon)
|
||||
elif args.optimizer == 'sgd':
|
||||
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.regularization)
|
||||
else:
|
||||
raise ValueError('optimizer not recognized: it should be either adam or sgd')
|
||||
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, train_loader, args.batch_size,
|
||||
args.device)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, test_loader, args.batch_size,
|
||||
args.device)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, dev_loader, args.batch_size,
|
||||
args.device)
|
||||
|
||||
trainer_config = {
|
||||
'optimizer': optimizer,
|
||||
'batch_size': args.batch_size,
|
||||
'log_interval': args.log_interval,
|
||||
'model_outfile': args.model_outfile,
|
||||
'lr_reduce_factor': args.lr_reduce_factor,
|
||||
'patience': args.patience,
|
||||
'tensorboard': args.tensorboard,
|
||||
'run_label': args.run_label,
|
||||
'logger': logger
|
||||
}
|
||||
trainer = TrainerFactory.get_trainer(args.dataset, model, embedding, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
if not args.skip_training:
|
||||
total_params = 0
|
||||
for param in model.parameters():
|
||||
size = [s for s in param.size()]
|
||||
total_params += np.prod(size)
|
||||
logger.info('Total number of parameters: %s', total_params)
|
||||
trainer.train(args.epochs)
|
||||
|
||||
_, _, state_dict, _, _ = load_checkpoint(args.model_outfile)
|
||||
|
||||
for k, tensor in state_dict.items():
|
||||
state_dict[k] = tensor.to(device)
|
||||
|
||||
model.load_state_dict(state_dict)
|
||||
if dev_loader:
|
||||
evaluate_dataset('dev', dataset_cls, model, embedding, dev_loader, args.batch_size, args.device)
|
||||
evaluate_dataset('test', dataset_cls, model, embedding, test_loader, args.batch_size, args.device, args.keep_results)
|
||||
+169
@@ -0,0 +1,169 @@
|
||||
import sys
|
||||
import math
|
||||
import numpy as np
|
||||
from datetime import datetime
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch.autograd import Variable
|
||||
|
||||
|
||||
class DecAtt(nn.Module):
|
||||
def __init__(self, num_units, num_classes, embedding_size, dropout, device=0,
|
||||
training=True, project_input=True,
|
||||
use_intra_attention=False, distance_biases=10, max_sentence_length=30):
|
||||
"""
|
||||
Create the model based on MLP networks.
|
||||
|
||||
:param num_units: size of the networks
|
||||
:param num_classes: number of classes in the problem
|
||||
:param embedding_size: size of each word embedding
|
||||
:param use_intra_attention: whether to use intra-attention model
|
||||
:param training: whether to create training tensors (optimizer)
|
||||
:p/word_embeddingaram project_input: whether to project input embeddings to a
|
||||
different dimensionality
|
||||
:param distance_biases: number of different distances with biases used
|
||||
in the intra-attention model
|
||||
"""
|
||||
super().__init__()
|
||||
self.arch = "DecAtt"
|
||||
self.num_units = num_units
|
||||
self.num_classes = num_classes
|
||||
self.project_input = project_input
|
||||
self.embedding_size = embedding_size
|
||||
self.distance_biases = distance_biases
|
||||
self.intra_attention = False
|
||||
self.max_sentence_length = max_sentence_length
|
||||
self.device = device
|
||||
|
||||
self.bias_embedding = nn.Embedding(max_sentence_length,1)
|
||||
self.linear_layer_project = nn.Linear(embedding_size, num_units, bias=False)
|
||||
#self.linear_layer_intra = nn.Sequential(nn.Linear(num_units, num_units), nn.ReLU(), nn.Linear(num_units, num_units), nn.ReLU())
|
||||
|
||||
self.linear_layer_attend = nn.Sequential(nn.Dropout(p=dropout), nn.Linear(num_units, num_units), nn.ReLU(),
|
||||
nn.Dropout(p=dropout), nn.Linear(num_units, num_units), nn.ReLU())
|
||||
|
||||
self.linear_layer_compare = nn.Sequential(nn.Dropout(p=dropout), nn.Linear(num_units*2, num_units), nn.ReLU(),
|
||||
nn.Dropout(p=dropout), nn.Linear(num_units, num_units), nn.ReLU())
|
||||
|
||||
self.linear_layer_aggregate = nn.Sequential(nn.Dropout(p=dropout), nn.Linear(num_units*2, num_units), nn.ReLU(),
|
||||
nn.Dropout(p=dropout), nn.Linear(num_units, num_units), nn.ReLU(),
|
||||
nn.Linear(num_units, num_classes), nn.LogSoftmax())
|
||||
self.init_weight()
|
||||
|
||||
def init_weight(self):
|
||||
self.linear_layer_project.weight.data.normal_(0, 0.01)
|
||||
self.linear_layer_attend[1].weight.data.normal_(0, 0.01)
|
||||
self.linear_layer_attend[1].bias.data.fill_(0)
|
||||
self.linear_layer_attend[4].weight.data.normal_(0, 0.01)
|
||||
self.linear_layer_attend[4].bias.data.fill_(0)
|
||||
self.linear_layer_compare[1].weight.data.normal_(0, 0.01)
|
||||
self.linear_layer_compare[1].bias.data.fill_(0)
|
||||
self.linear_layer_compare[4].weight.data.normal_(0, 0.01)
|
||||
self.linear_layer_compare[4].bias.data.fill_(0)
|
||||
self.linear_layer_aggregate[1].weight.data.normal_(0, 0.01)
|
||||
self.linear_layer_aggregate[1].bias.data.fill_(0)
|
||||
self.linear_layer_aggregate[4].weight.data.normal_(0, 0.01)
|
||||
self.linear_layer_aggregate[4].bias.data.fill_(0)
|
||||
#self.word_embedding.weight.data.copy_(torch.from_numpy(self.pretrained_emb))
|
||||
|
||||
def attention_softmax3d(self, raw_attentions):
|
||||
reshaped_attentions = raw_attentions.view(-1, raw_attentions.size(2))
|
||||
out = nn.functional.softmax(reshaped_attentions, dim=1)
|
||||
return out.view(raw_attentions.size(0),raw_attentions.size(1),raw_attentions.size(2))
|
||||
|
||||
def _transformation_input(self, embed_sent):
|
||||
embed_sent = self.linear_layer_project(embed_sent)
|
||||
result = embed_sent
|
||||
if self.intra_attention:
|
||||
f_intra = self.linear_layer_intra(embed_sent)
|
||||
f_intra_t = torch.transpose(f_intra, 1, 2)
|
||||
raw_attentions = torch.matmul(f_intra, f_intra_t)
|
||||
time_steps = embed_sent.size(1)
|
||||
r = torch.arange(0, time_steps)
|
||||
r_matrix = r.view(1,-1).expand(time_steps,time_steps)
|
||||
raw_index = r_matrix-r.view(-1,1)
|
||||
clipped_index = torch.clamp(raw_index,0,self.distance_biases-1)
|
||||
clipped_index = Variable(clipped_index.long())
|
||||
if torch.cuda.is_available():
|
||||
clipped_index = clipped_index.to(self.device)
|
||||
bias = self.bias_embedding(clipped_index)
|
||||
bias = torch.squeeze(bias)
|
||||
raw_attentions += bias
|
||||
attentions = self.attention_softmax3d(raw_attentions)
|
||||
attended = torch.matmul(attentions, embed_sent)
|
||||
result = torch.cat([embed_sent,attended],2)
|
||||
return result
|
||||
|
||||
def attend(self, sent1, sent2, lsize_list, rsize_list):
|
||||
"""
|
||||
Compute inter-sentence attention. This is step 1 (attend) in the paper
|
||||
|
||||
:param sent1: tensor in shape (batch, time_steps, num_units),
|
||||
the projected sentence 1
|
||||
:param sent2: tensor in shape (batch, time_steps, num_units)
|
||||
:return: a tuple of 3-d tensors, alfa and beta.
|
||||
"""
|
||||
repr1 = self.linear_layer_attend(sent1)
|
||||
repr2 = self.linear_layer_attend(sent2)
|
||||
repr2 = torch.transpose(repr2,1,2)
|
||||
raw_attentions = torch.matmul(repr1, repr2)
|
||||
|
||||
#self.mask = generate_mask(lsize_list, rsize_list)
|
||||
# masked = mask(self.raw_attentions, rsize_list)
|
||||
#masked = raw_attentions * self.mask
|
||||
att_sent1 = self.attention_softmax3d(raw_attentions)
|
||||
beta = torch.matmul(att_sent1, sent2) #input2_soft
|
||||
|
||||
raw_attentions_t = torch.transpose(raw_attentions,1,2).contiguous()
|
||||
#self.mask_t = torch.transpose(self.mask, 1, 2).contiguous()
|
||||
# masked = mask(raw_attentions_t, lsize_list)
|
||||
#masked = raw_attentions_t * self.mask_t
|
||||
att_sent2 = self.attention_softmax3d(raw_attentions_t)
|
||||
alpha = torch.matmul(att_sent2,sent1) #input1_soft
|
||||
|
||||
return alpha, beta
|
||||
|
||||
def compare(self, sentence, soft_alignment):
|
||||
"""
|
||||
Apply a feed forward network to compare o ne sentence to its
|
||||
soft alignment with the other.
|
||||
|
||||
:param sentence: embedded and projected sentence,
|
||||
shape (batch, time_steps, num_units)
|
||||
:param soft_alignment: tensor with shape (batch, time_steps, num_units)
|
||||
:return: a tensor (batch, time_steps, num_units)
|
||||
"""
|
||||
sent_alignment = torch.cat([sentence, soft_alignment],2)
|
||||
out = self.linear_layer_compare(sent_alignment)
|
||||
#out, (state, _) = self.lstm_compare(out)
|
||||
return out
|
||||
|
||||
def aggregate(self, v1, v2):
|
||||
"""
|
||||
Aggregate the representations induced from both sentences and their
|
||||
representations
|
||||
|
||||
:param v1: tensor with shape (batch, time_steps, num_units)
|
||||
:param v2: tensor with shape (batch, time_steps, num_units)
|
||||
:return: logits over classes, shape (batch, num_classes)
|
||||
"""
|
||||
v1_sum = torch.sum(v1,1)
|
||||
v2_sum = torch.sum(v2,1)
|
||||
out = self.linear_layer_aggregate(torch.cat([v1_sum,v2_sum],1))
|
||||
return out
|
||||
|
||||
def forward(self, sent1, sent2, ext_feats=None, word_to_doc_count=None, raw_sent1=None, raw_sent2=None):
|
||||
lsize_list = [len(s.split(" ")) for s in raw_sent1]
|
||||
rsize_list = [len(s.split(" ")) for s in raw_sent2]
|
||||
sent1 = sent1.permute(0, 2, 1)
|
||||
sent2 = sent2.permute(0, 2, 1)
|
||||
sent1 = self._transformation_input(sent1)
|
||||
sent2 = self._transformation_input(sent2)
|
||||
alpha, beta = self.attend(sent1, sent2, lsize_list, rsize_list)
|
||||
v1 = self.compare(sent1, beta)
|
||||
v2 = self.compare(sent2, alpha)
|
||||
logits = self.aggregate(v1, v2)
|
||||
return logits
|
||||
|
||||
Reference in New Issue
Block a user