From 62f8abec7f9b12b1bac2175f84b576942f77f4e8 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Fri, 25 May 2018 11:52:25 -0400 Subject: [PATCH] Add README (#110) * Make *QA/MSRVID work with VDPWI * Add README --- common/trainers/msrvid_trainer.py | 6 ++-- common/trainers/qa_trainer.py | 7 +++-- vdpwi/README.md | 48 +++++++++++++++++++++++++++++++ vdpwi/__main__.py | 10 +++---- vdpwi/model.py | 2 +- 5 files changed, 63 insertions(+), 10 deletions(-) create mode 100644 vdpwi/README.md diff --git a/common/trainers/msrvid_trainer.py b/common/trainers/msrvid_trainer.py index 68a6212..7946b62 100644 --- a/common/trainers/msrvid_trainer.py +++ b/common/trainers/msrvid_trainer.py @@ -56,7 +56,8 @@ class MSRVIDTrainer(Trainer): return left_out_val_a, left_out_val_b, left_out_val_ext_feats, left_out_val_labels def train(self, epochs): - scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) + if self.lr_reduce_factor != 1 and self.lr_reduce_factor != None: + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) epoch_times = [] prev_loss = -1 best_dev_score = -1 @@ -100,7 +101,8 @@ class MSRVIDTrainer(Trainer): self.writer.add_scalar('msrvid/dev/kl_div_loss', val_kl_div_loss, epoch) break - scheduler.step(pearson_r) + if scheduler is not None: + scheduler.step(pearson_r) end = time.time() duration = end - start diff --git a/common/trainers/qa_trainer.py b/common/trainers/qa_trainer.py index bc36872..7fc130c 100644 --- a/common/trainers/qa_trainer.py +++ b/common/trainers/qa_trainer.py @@ -40,7 +40,9 @@ class QATrainer(Trainer): return total_loss def train(self, epochs): - scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) + scheduler = None + if self.lr_reduce_factor != 1 and self.lr_reduce_factor != None: + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) epoch_times = [] prev_loss = -1 best_dev_score = -1 @@ -72,6 +74,7 @@ class QATrainer(Trainer): break prev_loss = new_loss - scheduler.step(mean_average_precision) + if scheduler is not None: + scheduler.step(mean_average_precision) self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) diff --git a/vdpwi/README.md b/vdpwi/README.md new file mode 100644 index 0000000..0ca71fe --- /dev/null +++ b/vdpwi/README.md @@ -0,0 +1,48 @@ +# VDPWI PyTorch Implementation + +This is a PyTorch implementation of the following paper + +* Hua He and Jimmy Lin. [Pairwise Word Interaction Modeling with Deep Neural Networks for Semantic Similarity Measurement.](http://www.aclweb.org/anthology/N16-1108) *Proceedings of the 15th Annual Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (NAACL/HLT 2016)*, pages 937-948. + + +Please ensure you have followed instructions in the main [README](../README.md) doc before running any further commands in this doc. + +## SICK Dataset + +To run VDPWI on the SICK dataset, use the following command. If you have any problems running it check the Troubleshooting section below. + +``` +python -m vdpwi vdpwi.sick.model.castor --dataset sick --epochs 19 --epsilon 1e-7 +``` + +## MSRVID Dataset + +To run VDPWI on the MSRVID dataset, use the following command: +``` +python -m vdpwi vdpwi.msrvid.model.castor --dataset msrvid --batch-size 16 --epochs 32 --regularization 0.0025 +``` + +## TrecQA Dataset + +To run VDPWI on (Raw) TrecQA, you first need to run `./get_trec_eval.sh` in `utils` under the repo root while inside the `utils` directory. This will download and compile the official `trec_eval` tool used for evaluation. + +Then, you can run: +``` +python -m vdpwi vdpwi.trecqa.model --dataset trecqa --epochs 5 --regularization 0.0005 --eps 0.1 +``` + +The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872). + +## WikiQA Dataset + +You also need `trec_eval` for this dataset, similar to TrecQA. + +Then, you can run: +``` +python -m vdpwi vdpwi.wikiqa.model --epochs 10 --dataset wikiqa --batch-size 64 --lr 0.0004 --regularization 0.02 +``` + +To see all options available, use +``` +python -m vdpwi --help +``` diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py index 7b3494e..c8522ef 100644 --- a/vdpwi/__main__.py +++ b/vdpwi/__main__.py @@ -33,17 +33,17 @@ if __name__ == '__main__': parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true') parser.add_argument('--device', type=int, default=0, help='GPU device, -1 for CPU (default: 0)') parser.add_argument('--sparse-features', action='store_true', default=False, help='use sparse features (default: false)') - parser.add_argument('--batch-size', type=int, default=64, help='input batch size for training (default: 64)') + parser.add_argument('--batch-size', type=int, default=16, help='input batch size for training (default: 64)') parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 10)') - parser.add_argument('--optimizer', type=str, default='adam', help='optimizer to use: adam or sgd (default: adam)') + parser.add_argument('--optimizer', type=str, default='rmsprop', help='optimizer to use: adam, sgd, or rmsprop (default: adam)') parser.add_argument('--lr', type=float, default=5E-4, help='learning rate (default: 0.001)') - parser.add_argument('--lr-reduce-factor', type=float, default=1, help='learning rate reduce factor after plateau (default: 0.3)') + parser.add_argument('--lr-reduce-factor', type=float, default=0.3, help='learning rate reduce factor after plateau (default: 0.3)') parser.add_argument('--patience', type=float, default=2, help='learning rate patience after seeing plateau (default: 2)') parser.add_argument('--momentum', type=float, default=0.1, help='momentum (default: 0.1)') parser.add_argument('--epsilon', type=float, default=1e-8, help='Adam epsilon (default: 1e-8)') parser.add_argument('--log-interval', type=int, default=10, help='how many batches to wait before logging training status (default: 10)') parser.add_argument('--regularization', type=float, default=1E-5, help='Regularization for the optimizer (default: 0.00001)') - parser.add_argument('--hidden-units', type=int, default=150, help='number of hidden units in the RNN') + parser.add_argument('--hidden-units', type=int, default=250, help='number of hidden units in the RNN') parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') parser.add_argument('--tensorboard', action='store_true', default=False, help='use TensorBoard to visualize training (default: false)') parser.add_argument('--run-label', type=str, help='label to describe run') @@ -98,7 +98,7 @@ if __name__ == '__main__': elif args.optimizer == 'sgd': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.regularization) elif args.optimizer == "rmsprop": - optimizer = optim.RMSprop(model.parameters(), lr=args.lr, momentum=args.momentum, alpha=config.decay, + optimizer = optim.RMSprop(model.parameters(), lr=args.lr, momentum=args.momentum, alpha=args.decay, weight_decay=args.regularization) else: raise ValueError('optimizer not recognized: it should be one of adam, sgd, or rmsprop') diff --git a/vdpwi/model.py b/vdpwi/model.py index 32b4d1f..4c0dd7f 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -32,7 +32,7 @@ class ResNet(nn.Module): x += old_x old_x = x x = torch.mean(x.view(x.size(0), x.size(1), -1), 2) - return self.output(x) + return F.log_softmax(self.output(x), 1) class VDPWIConvNet(nn.Module): def __init__(self, config):