From 4470983504b560b105d4ddb8a4449bd62e39dc21 Mon Sep 17 00:00:00 2001 From: Michael Tu Date: Sat, 4 Nov 2017 18:22:40 -0400 Subject: [PATCH] TrecQA for MP-CNN (#77) * Add TrecQA dataset and modularize MP-CNN infra * Stylistic improvements * Fix and warn about trec_eval path issue * Update README for MP-CNN * Update incorrect map/mrr * MP-CNN: address code review comments * Create common Castor pair Dataset class * Move map and mrr computation to Castor utils * Make map mrr utility trec_eval path more general --- .gitignore | 1 + __init__.py | 0 datasets/castor_dataset.py | 43 +++++ datasets/msrvid.py | 30 +--- datasets/sick.py | 30 +--- datasets/trecqa.py | 54 ++++++ mp_cnn/README.md | 24 ++- mp_cnn/dataset.py | 33 ++-- mp_cnn/evaluation.py | 120 ++----------- mp_cnn/evaluators/__init__.py | 0 mp_cnn/evaluators/evaluator.py | 19 ++ mp_cnn/evaluators/msrvid_evaluator.py | 42 +++++ mp_cnn/evaluators/sick_evaluator.py | 43 +++++ mp_cnn/evaluators/trecqa_evaluator.py | 34 ++++ mp_cnn/main.py | 38 ++-- mp_cnn/train.py | 247 ++------------------------ mp_cnn/trainers/__init__.py | 0 mp_cnn/trainers/msrvid_trainer.py | 117 ++++++++++++ mp_cnn/trainers/sick_trainer.py | 71 ++++++++ mp_cnn/trainers/trainer.py | 37 ++++ mp_cnn/trainers/trecqa_trainer.py | 76 ++++++++ utils/__init__.py | 0 utils/get_trec_eval.sh | 11 ++ utils/relevancy_metrics.py | 36 ++++ 24 files changed, 678 insertions(+), 428 deletions(-) create mode 100644 __init__.py create mode 100644 datasets/castor_dataset.py create mode 100644 datasets/trecqa.py create mode 100644 mp_cnn/evaluators/__init__.py create mode 100644 mp_cnn/evaluators/evaluator.py create mode 100644 mp_cnn/evaluators/msrvid_evaluator.py create mode 100644 mp_cnn/evaluators/sick_evaluator.py create mode 100644 mp_cnn/evaluators/trecqa_evaluator.py create mode 100644 mp_cnn/trainers/__init__.py create mode 100644 mp_cnn/trainers/msrvid_trainer.py create mode 100644 mp_cnn/trainers/sick_trainer.py create mode 100644 mp_cnn/trainers/trainer.py create mode 100644 mp_cnn/trainers/trecqa_trainer.py create mode 100644 utils/__init__.py create mode 100755 utils/get_trec_eval.sh create mode 100644 utils/relevancy_metrics.py diff --git a/.gitignore b/.gitignore index f3f2148..aa58290 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,6 @@ __pycache__ *idfsim *.swp trec_eval.9.0/ +trec_eval-9.0.5 *.pt text/ diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datasets/castor_dataset.py b/datasets/castor_dataset.py new file mode 100644 index 0000000..1d83b32 --- /dev/null +++ b/datasets/castor_dataset.py @@ -0,0 +1,43 @@ +from abc import ABCMeta, abstractmethod +import os + +from torchtext.data.dataset import Dataset +from torchtext.data.example import Example +from torchtext.data.field import Field + +from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features + + +class CastorPairDataset(Dataset, metaclass=ABCMeta): + + # Child classes must define + NAME = None + NUM_CLASSES = None + ID_FIELD = None + TEXT_FIELD = None + EXT_FEATS_FIELD = None + LABEL_FIELD = None + + @abstractmethod + def __init__(self, path): + """ + Create a Castor dataset involving pairs of texts + """ + fields = [('id', self.ID_FIELD), ('sentence_1', self.TEXT_FIELD), ('sentence_2', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)] + + examples = [] + with open(os.path.join(path, 'a.toks'), 'r') as f1, open(os.path.join(path, 'b.toks'), 'r') as f2: + sent_list_1 = [l.rstrip('.\n').split(' ') for l in f1] + sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2] + + word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2) + overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt) + + with open(os.path.join(path, 'id.txt'), 'r') as id_file, open(os.path.join(path, 'sim.txt'), 'r') as label_file: + for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file): + pair_id = pair_id.rstrip('.\n') + label = label.rstrip('.\n') + example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields) + examples.append(example) + + super(CastorPairDataset, self).__init__(examples, fields) diff --git a/datasets/msrvid.py b/datasets/msrvid.py index 7b5cb9f..c7f389d 100644 --- a/datasets/msrvid.py +++ b/datasets/msrvid.py @@ -3,13 +3,13 @@ import os import numpy as np import torch -from torchtext.data.dataset import Dataset from torchtext.data.example import Example from torchtext.data.field import Field from torchtext.data.iterator import BucketIterator from torchtext.data.pipeline import Pipeline from torchtext.vocab import Vectors +from datasets.castor_dataset import CastorPairDataset from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features @@ -28,7 +28,7 @@ def get_class_probs(sim, *args): return class_probs -class MSRVID(Dataset): +class MSRVID(CastorPairDataset): NAME = 'msrvid' NUM_CLASSES = 6 ID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True) @@ -38,35 +38,13 @@ class MSRVID(Dataset): @staticmethod def sort_key(ex): - return len(ex.a) + return len(ex.sentence_1) def __init__(self, path): """ Create a MSRVID dataset instance """ - fields = [('id', self.ID_FIELD), ('a', self.TEXT_FIELD), ('b', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)] - - examples = [] - f1 = open(os.path.join(path, 'a.txt'), 'r') - f2 = open(os.path.join(path, 'b.txt'), 'r') - id_file = open(os.path.join(path, 'id.txt'), 'r') - label_file = open(os.path.join(path, 'sim.txt'), 'r') - - sent_list_1 = [l.rstrip('.\n').split(' ') for l in f1] - sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2] - - word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2) - overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt) - - for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file): - pair_id = pair_id.rstrip('.\n') - label = label.rstrip('.\n') - example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields) - examples.append(example) - - map(lambda f: f.close(), [f1, f2, label_file]) - - super(MSRVID, self).__init__(examples, fields) + super(MSRVID, self).__init__(path) @classmethod def splits(cls, path, train='train', test='test', **kwargs): diff --git a/datasets/sick.py b/datasets/sick.py index e860ae5..d9a0132 100644 --- a/datasets/sick.py +++ b/datasets/sick.py @@ -3,13 +3,13 @@ import os import numpy as np import torch -from torchtext.data.dataset import Dataset from torchtext.data.example import Example from torchtext.data.field import Field from torchtext.data.iterator import BucketIterator from torchtext.data.pipeline import Pipeline from torchtext.vocab import Vectors +from datasets.castor_dataset import CastorPairDataset from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features @@ -28,7 +28,7 @@ def get_class_probs(sim, *args): return class_probs -class SICK(Dataset): +class SICK(CastorPairDataset): NAME = 'sick' NUM_CLASSES = 5 ID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True) @@ -38,35 +38,13 @@ class SICK(Dataset): @staticmethod def sort_key(ex): - return len(ex.a) + return len(ex.sentence_1) def __init__(self, path): """ Create a SICK dataset instance """ - fields = [('id', self.ID_FIELD), ('a', self.TEXT_FIELD), ('b', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)] - - examples = [] - f1 = open(os.path.join(path, 'a.txt'), 'r') - f2 = open(os.path.join(path, 'b.txt'), 'r') - id_file = open(os.path.join(path, 'id.txt'), 'r') - label_file = open(os.path.join(path, 'sim.txt'), 'r') - - sent_list_1 = [l.rstrip('.\n').split(' ') for l in f1] - sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2] - - word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2) - overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt) - - for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file): - pair_id = pair_id.rstrip('.\n') - label = label.rstrip('.\n') - example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields) - examples.append(example) - - map(lambda f: f.close(), [f1, f2, label_file]) - - super(SICK, self).__init__(examples, fields) + super(SICK, self).__init__(path) @classmethod def splits(cls, path, train='train', validation='dev', test='test', **kwargs): diff --git a/datasets/trecqa.py b/datasets/trecqa.py new file mode 100644 index 0000000..e7d0627 --- /dev/null +++ b/datasets/trecqa.py @@ -0,0 +1,54 @@ +import os + +import torch +from torchtext.data.example import Example +from torchtext.data.field import Field +from torchtext.data.iterator import BucketIterator +from torchtext.vocab import Vectors + +from datasets.castor_dataset import CastorPairDataset +from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features + + +class TRECQA(CastorPairDataset): + NAME = 'trecqa' + NUM_CLASSES = 2 + ID_FIELD = Field(sequential=False, tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True) + TEXT_FIELD = Field(batch_first=True, tokenize=lambda x: x) # tokenizer is identity since we already tokenized it to compute external features + EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x) + LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True) + + @staticmethod + def sort_key(ex): + return len(ex.sentence_1) + + def __init__(self, path): + """ + Create a TRECQA dataset instance + """ + super(TRECQA, self).__init__(path) + + @classmethod + def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs): + return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs) + + @classmethod + def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, unk_init=torch.Tensor.zero_): + """ + :param path: directory containing train, test, dev files + :param vectors_name: name of word vectors file + :param vectors_cache: directory containing word vectors file + :param batch_size: batch size + :param device: GPU device + :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes + :param unk_init: function used to generate vector for OOV words + :return: + """ + if vectors is None: + vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init) + + train, validation, test = cls.splits(path) + + cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors) + + return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device) diff --git a/mp_cnn/README.md b/mp_cnn/README.md index 8e69180..4c4132c 100644 --- a/mp_cnn/README.md +++ b/mp_cnn/README.md @@ -20,6 +20,8 @@ Directory layout should be like this: │ └── GloVe/ ``` +## SICK Dataset + To run MP-CNN on the SICK dataset, use the following command. `--dropout 0` is for mimicking the original paper, although adding dropout can improve performance. If you have any problems running it check the Troubleshooting section below. ``` @@ -29,7 +31,9 @@ python main.py mpcnn.sick.model.castor --dataset sick --epochs 19 --epsilon 1e-7 | Implementation and config | Pearson's r | Spearman's p | | -------------------------------- |:-------------:|:-------------:| | Paper | 0.8686 | 0.8047 | -| PyTorch using above config | 0.8763 | 0.8215 | +| PyTorch using above config | 0.8684 | 0.8083 | + +## MSRVID Dataset To run MP-CNN on the MSRVID dataset, use the following command: ``` @@ -39,7 +43,23 @@ python main.py mpcnn.msrvid.model.castor --dataset msrvid --batch-size 16 --epsi | Implementation and config | Pearson's r | | -------------------------------- |:-------------:| | Paper | 0.9090 | -| PyTorch using above config | 0.9050 | +| PyTorch using above config | 0.8911 | + +## TrecQA Dataset + +To run MP-CNN on (Raw) TrecQA, you first need to run `./get_trec_eval.sh` in `utils` under the repo root while inside the `utils` directory. This will download and compile the official `trec_eval` tool used for evaluation. + +Then, you can run: +``` +python main.py mpcnn.trecqa.model --dataset trecqa --epochs 5 --regularization 0.0005 --dropout 0.5 --eps 0.1 +``` + +| Implementation and config | map | mrr | +| -------------------------------- |:------:|:------:| +| Paper | 0.762 | 0.830 | +| PyTorch using above config | 0.7904 | 0.8223 | + +The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872). These are not the optimal hyperparameters but they are decent. This README will be updated with more optimal hyperparameters and results in the future. diff --git a/mp_cnn/dataset.py b/mp_cnn/dataset.py index 15d0ebd..a1239df 100644 --- a/mp_cnn/dataset.py +++ b/mp_cnn/dataset.py @@ -1,30 +1,14 @@ -from collections import defaultdict -from enum import Enum -import math import os -import numpy as np import torch -from torch.autograd import Variable import torch.nn as nn -import torch.utils.data as data from datasets.sick import SICK from datasets.msrvid import MSRVID - -# logging setup -import logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) +from datasets.trecqa import TRECQA -class UnknownWorcVecCache(object): +class UnknownWordVecCache(object): """ Caches the first randomly generated word vector for a certain size to make it is reused. """ @@ -47,7 +31,7 @@ class MPCNNDatasetFactory(object): def get_dataset(dataset_name, word_vectors_dir, word_vectors_file, batch_size, device): if dataset_name == 'sick': dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'sick/') - train_loader, dev_loader, test_loader = SICK.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWorcVecCache.unk) + train_loader, dev_loader, test_loader = SICK.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding_dim = SICK.TEXT_FIELD.vocab.vectors.size() embedding = nn.Embedding(embedding_dim[0], embedding_dim[1]) embedding.weight = nn.Parameter(SICK.TEXT_FIELD.vocab.vectors) @@ -55,11 +39,20 @@ class MPCNNDatasetFactory(object): elif dataset_name == 'msrvid': dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'msrvid/') dev_loader = None - train_loader, test_loader = MSRVID.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWorcVecCache.unk) + train_loader, test_loader = MSRVID.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding_dim = MSRVID.TEXT_FIELD.vocab.vectors.size() embedding = nn.Embedding(embedding_dim[0], embedding_dim[1]) embedding.weight = nn.Parameter(MSRVID.TEXT_FIELD.vocab.vectors) return MSRVID, embedding, train_loader, test_loader, dev_loader + elif dataset_name == 'trecqa': + if not os.path.exists('../utils/trec_eval-9.0.5/trec_eval'): + raise FileNotFoundError('TrecQA requires the trec_eval tool to run. Please run get_trec_eval.sh inside Castor/utils (as working directory) before continuing.') + dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'TrecQA/') + train_loader, dev_loader, test_loader = TRECQA.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) + embedding_dim = TRECQA.TEXT_FIELD.vocab.vectors.size() + embedding = nn.Embedding(embedding_dim[0], embedding_dim[1]) + embedding.weight = nn.Parameter(TRECQA.TEXT_FIELD.vocab.vectors) + return TRECQA, embedding, train_loader, test_loader, dev_loader else: raise ValueError('{} is not a valid dataset.'.format(dataset_name)) diff --git a/mp_cnn/evaluation.py b/mp_cnn/evaluation.py index 2e9a81c..47fc05d 100644 --- a/mp_cnn/evaluation.py +++ b/mp_cnn/evaluation.py @@ -1,117 +1,29 @@ -from scipy.stats import pearsonr, spearmanr -import torch -import torch.nn.functional as F -from torch.autograd import Variable +from mp_cnn.evaluators.sick_evaluator import SICKEvaluator +from mp_cnn.evaluators.msrvid_evaluator import MSRVIDEvaluator +from mp_cnn.evaluators.trecqa_evaluator import TRECQAEvaluator class MPCNNEvaluatorFactory(object): """ Get the corresponding Evaluator class for a particular dataset. """ + evaluator_map = { + 'sick': SICKEvaluator, + 'msrvid': MSRVIDEvaluator, + 'trecqa': TRECQAEvaluator + } + @staticmethod def get_evaluator(dataset_cls, model, data_loader, batch_size, device): if data_loader is None: return None - if hasattr(dataset_cls, 'NAME') and dataset_cls.NAME == 'sick': - return SICKEvaluator(dataset_cls, model, data_loader, batch_size, device) - elif hasattr(dataset_cls, 'NAME') and dataset_cls.NAME == 'msrvid': - return MSRVIDEvaluator(dataset_cls, model, data_loader, batch_size, device) - else: - raise ValueError('{} is not a valid dataset.'.format(dataset_cls)) + if not hasattr(dataset_cls, 'NAME'): + raise ValueError('Invalid dataset. Dataset should have NAME attribute.') + if dataset_cls.NAME not in MPCNNEvaluatorFactory.evaluator_map: + raise ValueError('{} is not implemented.'.format(dataset_cls)) -class Evaluator(object): - """ - Evaluates performance of model on a Dataset, using metrics specific to the Dataset. - """ - - def __init__(self, dataset_cls, model, data_loader, batch_size, device): - self.dataset_cls = dataset_cls - self.model = model - self.data_loader = data_loader - self.batch_size = batch_size - self.device = device - - def get_scores(self): - """ - Get the scores used to evaluate the model. - Should return ([score1, score2, ..], [score1_name, score2_name, ...]). - The first score is the primary score used to determine if the model has improved. - """ - raise NotImplementedError('Evaluator subclass needs to implement get_score') - - -class SICKEvaluator(Evaluator): - - def __init__(self, dataset_cls, model, data_loader, batch_size, device): - super(SICKEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device) - - def get_scores(self): - self.model.eval() - num_classes = self.dataset_cls.NUM_CLASSES - predict_classes = torch.arange(1, num_classes + 1).expand(self.batch_size, num_classes) - test_kl_div_loss = 0 - predictions = [] - true_labels = [] - - for batch in self.data_loader: - output = self.model(batch.a, batch.b, batch.ext_feats) - test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0] - # handle last batch which might have smaller size - if len(predict_classes) != len(batch.a): - predict_classes = torch.arange(1, num_classes + 1).expand(len(batch.a), num_classes) - - if self.data_loader.device != -1: - with torch.cuda.device(self.device): - predict_classes = predict_classes.cuda() - - true_labels.append((predict_classes * batch.label.data).sum(dim=1)) - predictions.append((predict_classes * output.data.exp()).sum(dim=1)) - - del output - - predictions = torch.cat(predictions).cpu().numpy() - true_labels = torch.cat(true_labels).cpu().numpy() - test_kl_div_loss /= len(batch.dataset.examples) - pearson_r = pearsonr(predictions, true_labels)[0] - spearman_r = spearmanr(predictions, true_labels)[0] - - return [pearson_r, spearman_r, test_kl_div_loss], ['pearson_r', 'spearman_r', 'KL-divergence loss'] - - -class MSRVIDEvaluator(Evaluator): - - def __init__(self, dataset_cls, model, data_loader, batch_size, device): - super(MSRVIDEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device) - - def get_scores(self): - self.model.eval() - num_classes = self.dataset_cls.NUM_CLASSES - predict_classes = torch.arange(0, num_classes).expand(self.batch_size, num_classes) - test_kl_div_loss = 0 - predictions = [] - true_labels = [] - - for batch in self.data_loader: - output = self.model(batch.a, batch.b, batch.ext_feats) - test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0] - # handle last batch which might have smaller size - if len(predict_classes) != len(batch.a): - predict_classes = torch.arange(0, num_classes).expand(len(batch.a), num_classes) - - if self.data_loader.device != -1: - with torch.cuda.device(self.device): - predict_classes = predict_classes.cuda() - - true_labels.append((predict_classes * batch.label.data).sum(dim=1)) - predictions.append((predict_classes * output.data.exp()).sum(dim=1)) - - del output - - predictions = torch.cat(predictions).cpu().numpy() - true_labels = torch.cat(true_labels).cpu().numpy() - test_kl_div_loss /= len(batch.dataset.examples) - pearson_r = pearsonr(predictions, true_labels)[0] - - return [pearson_r, test_kl_div_loss], ['pearson_r', 'KL-divergence loss'] + return MPCNNEvaluatorFactory.evaluator_map[dataset_cls.NAME]( + dataset_cls, model, data_loader, batch_size, device + ) diff --git a/mp_cnn/evaluators/__init__.py b/mp_cnn/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mp_cnn/evaluators/evaluator.py b/mp_cnn/evaluators/evaluator.py new file mode 100644 index 0000000..ad3fbba --- /dev/null +++ b/mp_cnn/evaluators/evaluator.py @@ -0,0 +1,19 @@ +class Evaluator(object): + """ + Evaluates performance of model on a Dataset, using metrics specific to the Dataset. + """ + + def __init__(self, dataset_cls, model, data_loader, batch_size, device): + self.dataset_cls = dataset_cls + self.model = model + self.data_loader = data_loader + self.batch_size = batch_size + self.device = device + + def get_scores(self): + """ + Get the scores used to evaluate the model. + Should return ([score1, score2, ..], [score1_name, score2_name, ...]). + The first score is the primary score used to determine if the model has improved. + """ + raise NotImplementedError('Evaluator subclass needs to implement get_score') diff --git a/mp_cnn/evaluators/msrvid_evaluator.py b/mp_cnn/evaluators/msrvid_evaluator.py new file mode 100644 index 0000000..3af565f --- /dev/null +++ b/mp_cnn/evaluators/msrvid_evaluator.py @@ -0,0 +1,42 @@ +from scipy.stats import pearsonr +import torch +import torch.nn.functional as F + +from mp_cnn.evaluators.evaluator import Evaluator + + +class MSRVIDEvaluator(Evaluator): + + def __init__(self, dataset_cls, model, data_loader, batch_size, device): + super(MSRVIDEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device) + + def get_scores(self): + self.model.eval() + num_classes = self.dataset_cls.NUM_CLASSES + predict_classes = torch.arange(0, num_classes).expand(self.batch_size, num_classes) + test_kl_div_loss = 0 + predictions = [] + true_labels = [] + + for batch in self.data_loader: + output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats) + test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0] + # handle last batch which might have smaller size + if len(predict_classes) != len(batch.sentence_1): + predict_classes = torch.arange(0, num_classes).expand(len(batch.sentence_1), num_classes) + + if self.data_loader.device != -1: + with torch.cuda.device(self.device): + predict_classes = predict_classes.cuda() + + true_labels.append((predict_classes * batch.label.data).sum(dim=1)) + predictions.append((predict_classes * output.data.exp()).sum(dim=1)) + + del output + + predictions = torch.cat(predictions).cpu().numpy() + true_labels = torch.cat(true_labels).cpu().numpy() + test_kl_div_loss /= len(batch.dataset.examples) + pearson_r = pearsonr(predictions, true_labels)[0] + + return [pearson_r, test_kl_div_loss], ['pearson_r', 'KL-divergence loss'] diff --git a/mp_cnn/evaluators/sick_evaluator.py b/mp_cnn/evaluators/sick_evaluator.py new file mode 100644 index 0000000..1101b07 --- /dev/null +++ b/mp_cnn/evaluators/sick_evaluator.py @@ -0,0 +1,43 @@ +from scipy.stats import pearsonr, spearmanr +import torch +import torch.nn.functional as F + +from mp_cnn.evaluators.evaluator import Evaluator + + +class SICKEvaluator(Evaluator): + + def __init__(self, dataset_cls, model, data_loader, batch_size, device): + super(SICKEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device) + + def get_scores(self): + self.model.eval() + num_classes = self.dataset_cls.NUM_CLASSES + predict_classes = torch.arange(1, num_classes + 1).expand(self.batch_size, num_classes) + test_kl_div_loss = 0 + predictions = [] + true_labels = [] + + for batch in self.data_loader: + output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats) + test_kl_div_loss += F.kl_div(output, batch.label, size_average=False).data[0] + # handle last batch which might have smaller size + if len(predict_classes) != len(batch.sentence_1): + predict_classes = torch.arange(1, num_classes + 1).expand(len(batch.sentence_1), num_classes) + + if self.data_loader.device != -1: + with torch.cuda.device(self.device): + predict_classes = predict_classes.cuda() + + true_labels.append((predict_classes * batch.label.data).sum(dim=1)) + predictions.append((predict_classes * output.data.exp()).sum(dim=1)) + + del output + + predictions = torch.cat(predictions).cpu().numpy() + true_labels = torch.cat(true_labels).cpu().numpy() + test_kl_div_loss /= len(batch.dataset.examples) + pearson_r = pearsonr(predictions, true_labels)[0] + spearman_r = spearmanr(predictions, true_labels)[0] + + return [pearson_r, spearman_r, test_kl_div_loss], ['pearson_r', 'spearman_r', 'KL-divergence loss'] diff --git a/mp_cnn/evaluators/trecqa_evaluator.py b/mp_cnn/evaluators/trecqa_evaluator.py new file mode 100644 index 0000000..9391745 --- /dev/null +++ b/mp_cnn/evaluators/trecqa_evaluator.py @@ -0,0 +1,34 @@ +import torch.nn.functional as F + +from mp_cnn.evaluators.evaluator import Evaluator +from utils.relevancy_metrics import get_map_mrr + + +class TRECQAEvaluator(Evaluator): + + def __init__(self, dataset_cls, model, data_loader, batch_size, device): + super(TRECQAEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device) + + def get_scores(self): + self.model.eval() + test_cross_entropy_loss = 0 + qids = [] + true_labels = [] + predictions = [] + + for batch in self.data_loader: + qids.extend(batch.id.data.cpu().numpy()) + output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats) + test_cross_entropy_loss += F.cross_entropy(output, batch.label, size_average=False).data[0] + + true_labels.extend(batch.label.data.cpu().numpy()) + predictions.extend(output.data.exp()[:, 1].cpu().numpy()) + + del output + + qids = list(map(lambda n: int(round(n * 10, 0)) / 10, qids)) + + mean_average_precision, mean_reciprocal_rank = get_map_mrr(qids, predictions, true_labels, self.data_loader.device) + test_cross_entropy_loss /= len(batch.dataset.examples) + + return [test_cross_entropy_loss, mean_average_precision, mean_reciprocal_rank], ['cross entropy loss', 'map', 'mrr'] diff --git a/mp_cnn/main.py b/mp_cnn/main.py index 32c387b..53e33d9 100644 --- a/mp_cnn/main.py +++ b/mp_cnn/main.py @@ -1,32 +1,23 @@ import argparse +import logging import os +import pprint import random import numpy as np import torch import torch.optim as optim -from dataset import MPCNNDatasetFactory -from evaluation import MPCNNEvaluatorFactory -from model import MPCNN -from train import MPCNNTrainerFactory - -# logging setup -import logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) +from mp_cnn.dataset import MPCNNDatasetFactory +from mp_cnn.evaluation import MPCNNEvaluatorFactory +from mp_cnn.model import MPCNN +from mp_cnn.train import MPCNNTrainerFactory if __name__ == '__main__': parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN') parser.add_argument('model_outfile', help='file to save final model') - parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid]', default='sick') + parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid, trecqa]', default='sick') parser.add_argument('--word-vectors-dir', help='word vectors directory', default=os.path.join(os.pardir, os.pardir, 'data', 'GloVe')) parser.add_argument('--word-vectors-file', help='word vectors filename', default='glove.840B.300d.txt') parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true') @@ -58,6 +49,18 @@ if __name__ == '__main__': if args.device != -1: torch.cuda.manual_seed(args.seed) + # logging setup + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + logger.info(pprint.pformat(vars(args))) + dataset_cls, embedding, train_loader, test_loader, dev_loader \ = MPCNNDatasetFactory.get_dataset(args.dataset, args.word_vectors_dir, args.word_vectors_file, args.batch_size, args.device) @@ -89,7 +92,8 @@ if __name__ == '__main__': 'lr_reduce_factor': args.lr_reduce_factor, 'patience': args.patience, 'tensorboard': args.tensorboard, - 'run_label': args.run_label + 'run_label': args.run_label, + 'logger': logger } trainer = MPCNNTrainerFactory.get_trainer(args.dataset, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) diff --git a/mp_cnn/train.py b/mp_cnn/train.py index acbc6c8..039297b 100644 --- a/mp_cnn/train.py +++ b/mp_cnn/train.py @@ -1,242 +1,23 @@ -import math -import time - -import torch -import torch.nn.functional as F -from torch.autograd import Variable -from torch.optim.lr_scheduler import ReduceLROnPlateau -from scipy.stats import pearsonr, spearmanr - -# logging setup -import logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(levelname)s - %(message)s') -ch.setFormatter(formatter) -logger.addHandler(ch) +from mp_cnn.trainers.sick_trainer import SICKTrainer +from mp_cnn.trainers.msrvid_trainer import MSRVIDTrainer +from mp_cnn.trainers.trecqa_trainer import TRECQATrainer class MPCNNTrainerFactory(object): """ Get the corresponding Trainer class for a particular dataset. """ + trainer_map = { + 'sick': SICKTrainer, + 'msrvid': MSRVIDTrainer, + 'trecqa': TRECQATrainer + } + @staticmethod def get_trainer(dataset_name, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): - if dataset_name == 'sick': - return SICKTrainer(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) - elif dataset_name == 'msrvid': - return MSRVIDTrainer(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) - else: - raise ValueError('{} is not a valid dataset.'.format(dataset_name)) + if dataset_name not in MPCNNTrainerFactory.trainer_map: + raise ValueError('{} is not implemented.'.format(dataset_name)) - -class Trainer(object): - - """ - Abstraction for training a model on a Dataset. - """ - - def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): - self.model = model - self.optimizer = trainer_config['optimizer'] - self.train_loader = train_loader - self.batch_size = trainer_config['batch_size'] - self.log_interval = trainer_config['log_interval'] - self.model_outfile = trainer_config['model_outfile'] - self.lr_reduce_factor = trainer_config['lr_reduce_factor'] - self.patience = trainer_config['patience'] - self.use_tensorboard = trainer_config['tensorboard'] - if self.use_tensorboard: - from tensorboardX import SummaryWriter - self.writer = SummaryWriter(log_dir=None, comment='' if trainer_config['run_label'] is None else trainer_config['run_label']) - - self.train_evaluator = train_evaluator - self.test_evaluator = test_evaluator - self.dev_evaluator = dev_evaluator - - def evaluate(self, evaluator, dataset_name): - scores, metric_names = evaluator.get_scores() - logger.info('Evaluation metrics for {}:'.format(dataset_name)) - logger.info('\t'.join([' '] + metric_names)) - logger.info('\t'.join([dataset_name] + list(map(str, scores)))) - return scores - - def train_epoch(self, epoch): - raise NotImplementedError() - - def train(self, epochs): - raise NotImplementedError() - - -class SICKTrainer(Trainer): - - def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): - super(SICKTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) - - def train_epoch(self, epoch): - self.model.train() - total_loss = 0 - for batch_idx, batch in enumerate(self.train_loader): - self.optimizer.zero_grad() - output = self.model(batch.a, batch.b, batch.ext_feats) - loss = F.kl_div(output, batch.label) - total_loss += loss.data[0] - loss.backward() - self.optimizer.step() - if batch_idx % self.log_interval == 0: - logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( - epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)), - len(batch.dataset.examples), - 100. * batch_idx / (len(self.train_loader)), loss.data[0]) - ) - - if self.use_tensorboard: - self.writer.add_scalar('sick/train/kl_div_loss', total_loss, epoch) - - return total_loss - - def train(self, epochs): - scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) - epoch_times = [] - prev_loss = -1 - best_dev_score = -1 - for epoch in range(1, epochs + 1): - start = time.time() - logger.info('Epoch {} started...'.format(epoch)) - self.train_epoch(epoch) - - dev_scores = self.evaluate(self.dev_evaluator, 'dev') - new_loss = dev_scores[2] - - if self.use_tensorboard: - self.writer.add_scalar('sick/lr', self.optimizer.param_groups[0]['lr'], epoch) - self.writer.add_scalar('sick/dev/pearson_r', dev_scores[0], epoch) - self.writer.add_scalar('sick/dev/kl_div_loss', new_loss, epoch) - - end = time.time() - duration = end - start - logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) - epoch_times.append(duration) - - if dev_scores[0] > best_dev_score: - best_dev_score = dev_scores[0] - torch.save(self.model, self.model_outfile) - - if abs(prev_loss - new_loss) <= 0.0002: - logger.info('Early stopping. Loss changed by less than 0.0002.') - break - - prev_loss = new_loss - scheduler.step(dev_scores[0]) - - logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) - - -class MSRVIDTrainer(Trainer): - - def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): - super(MSRVIDTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) - - def train_epoch(self, epoch): - self.model.train() - total_loss = 0 - - # since MSRVID doesn't have validation set, we manually leave-out some training data for validation - batches = math.ceil(len(self.train_loader.dataset.examples) / self.batch_size) - start_val_batch = math.floor(0.8 * batches) - left_out_val_a, left_out_val_b = [], [] - left_out_val_ext_feats = [] - left_out_val_labels = [] - - for batch_idx, batch in enumerate(self.train_loader): - if batch_idx >= start_val_batch: - left_out_val_a.append(batch.a) - left_out_val_b.append(batch.b) - left_out_val_ext_feats.append(batch.ext_feats) - left_out_val_labels.append(batch.label) - continue - self.optimizer.zero_grad() - output = self.model(batch.a, batch.b, batch.ext_feats) - loss = F.kl_div(output, batch.label) - total_loss += loss.data[0] - loss.backward() - self.optimizer.step() - if batch_idx % self.log_interval == 0: - logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( - epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)), - len(batch.dataset.examples), - 100. * batch_idx / (len(self.train_loader)), loss.data[0]) - ) - - self.evaluate(self.train_evaluator, 'train') - - if self.use_tensorboard: - self.writer.add_scalar('msrvid/train/kl_div_loss', total_loss, epoch) - - return left_out_val_a, left_out_val_b, left_out_val_ext_feats, left_out_val_labels - - def train(self, epochs): - scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) - epoch_times = [] - prev_loss = -1 - best_dev_score = -1 - for epoch in range(1, epochs + 1): - start = time.time() - logger.info('Epoch {} started...'.format(epoch)) - left_out_a, left_out_b, left_out_ext_feats, left_out_label = self.train_epoch(epoch) - - # manually evaluating the validating set - all_predictions, all_true_labels = [], [] - val_kl_div_loss = 0 - for i in range(len(left_out_a)): - output = self.model(left_out_a[i], left_out_b[i], left_out_ext_feats[i]) - val_kl_div_loss += F.kl_div(output, left_out_label[i], size_average=False).data[0] - predict_classes = torch.arange(0, self.train_loader.dataset.NUM_CLASSES).expand(len(left_out_a[i]), self.train_loader.dataset.NUM_CLASSES) - if self.train_loader.device != -1: - with torch.cuda.device(self.train_loader.device): - predict_classes = predict_classes.cuda() - - predictions = (predict_classes * output.data.exp()).sum(dim=1) - true_labels = (predict_classes * left_out_label[i].data).sum(dim=1) - all_predictions.append(predictions) - all_true_labels.append(true_labels) - - predictions = torch.cat(all_predictions).cpu().numpy() - true_labels = torch.cat(all_true_labels).cpu().numpy() - pearson_r = pearsonr(predictions, true_labels)[0] - val_kl_div_loss /= len(predictions) - - if self.use_tensorboard: - self.writer.add_scalar('msrvid/dev/pearson_r', pearson_r, epoch) - - for param_group in self.optimizer.param_groups: - logger.info('Validation size: %s Pearson\'s r: %s', output.size()[0], pearson_r) - logger.info('Learning rate: %s', param_group['lr']) - - if self.use_tensorboard: - self.writer.add_scalar('msrvid/lr', param_group['lr'], epoch) - self.writer.add_scalar('msrvid/dev/kl_div_loss', val_kl_div_loss, epoch) - break - - scheduler.step(pearson_r) - - end = time.time() - duration = end - start - logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) - epoch_times.append(duration) - - if pearson_r > best_dev_score: - best_dev_score = pearson_r - torch.save(self.model, self.model_outfile) - - if abs(prev_loss - val_kl_div_loss) <= 0.0005: - logger.info('Early stopping. Loss changed by less than 0.0005.') - break - - prev_loss = val_kl_div_loss - self.evaluate(self.test_evaluator, 'test') - - logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) + return MPCNNTrainerFactory.trainer_map[dataset_name]( + model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator + ) diff --git a/mp_cnn/trainers/__init__.py b/mp_cnn/trainers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mp_cnn/trainers/msrvid_trainer.py b/mp_cnn/trainers/msrvid_trainer.py new file mode 100644 index 0000000..de0807d --- /dev/null +++ b/mp_cnn/trainers/msrvid_trainer.py @@ -0,0 +1,117 @@ +import math +import time + +import torch +import torch.nn.functional as F +from torch.optim.lr_scheduler import ReduceLROnPlateau +from scipy.stats import pearsonr + +from mp_cnn.trainers.trainer import Trainer + + +class MSRVIDTrainer(Trainer): + + def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): + super(MSRVIDTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) + + def train_epoch(self, epoch): + self.model.train() + total_loss = 0 + + # since MSRVID doesn't have validation set, we manually leave-out some training data for validation + batches = math.ceil(len(self.train_loader.dataset.examples) / self.batch_size) + start_val_batch = math.floor(0.8 * batches) + left_out_val_a, left_out_val_b = [], [] + left_out_val_ext_feats = [] + left_out_val_labels = [] + + for batch_idx, batch in enumerate(self.train_loader): + # msrvid does not contain a validation set, we leave out some training data for validation to do model selection + if batch_idx >= start_val_batch: + left_out_val_a.append(batch.sentence_1) + left_out_val_b.append(batch.sentence_2) + left_out_val_ext_feats.append(batch.ext_feats) + left_out_val_labels.append(batch.label) + continue + self.optimizer.zero_grad() + output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats) + loss = F.kl_div(output, batch.label) + total_loss += loss.data[0] + loss.backward() + self.optimizer.step() + if batch_idx % self.log_interval == 0: + self.logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)), + len(batch.dataset.examples), + 100. * batch_idx / (len(self.train_loader)), loss.data[0]) + ) + + self.evaluate(self.train_evaluator, 'train') + + if self.use_tensorboard: + self.writer.add_scalar('msrvid/train/kl_div_loss', total_loss, epoch) + + return left_out_val_a, left_out_val_b, left_out_val_ext_feats, left_out_val_labels + + def train(self, epochs): + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) + epoch_times = [] + prev_loss = -1 + best_dev_score = -1 + for epoch in range(1, epochs + 1): + start = time.time() + self.logger.info('Epoch {} started...'.format(epoch)) + left_out_a, left_out_b, left_out_ext_feats, left_out_label = self.train_epoch(epoch) + + # manually evaluating the validating set + all_predictions, all_true_labels = [], [] + val_kl_div_loss = 0 + for i in range(len(left_out_a)): + output = self.model(left_out_a[i], left_out_b[i], left_out_ext_feats[i]) + val_kl_div_loss += F.kl_div(output, left_out_label[i], size_average=False).data[0] + predict_classes = torch.arange(0, self.train_loader.dataset.NUM_CLASSES).expand(len(left_out_a[i]), self.train_loader.dataset.NUM_CLASSES) + if self.train_loader.device != -1: + with torch.cuda.device(self.train_loader.device): + predict_classes = predict_classes.cuda() + + predictions = (predict_classes * output.data.exp()).sum(dim=1) + true_labels = (predict_classes * left_out_label[i].data).sum(dim=1) + all_predictions.append(predictions) + all_true_labels.append(true_labels) + + predictions = torch.cat(all_predictions).cpu().numpy() + true_labels = torch.cat(all_true_labels).cpu().numpy() + pearson_r = pearsonr(predictions, true_labels)[0] + val_kl_div_loss /= len(predictions) + + if self.use_tensorboard: + self.writer.add_scalar('msrvid/dev/pearson_r', pearson_r, epoch) + + for param_group in self.optimizer.param_groups: + self.logger.info('Validation size: %s Pearson\'s r: %s', output.size()[0], pearson_r) + self.logger.info('Learning rate: %s', param_group['lr']) + + if self.use_tensorboard: + self.writer.add_scalar('msrvid/lr', param_group['lr'], epoch) + self.writer.add_scalar('msrvid/dev/kl_div_loss', val_kl_div_loss, epoch) + break + + scheduler.step(pearson_r) + + end = time.time() + duration = end - start + self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) + epoch_times.append(duration) + + if pearson_r > best_dev_score: + best_dev_score = pearson_r + torch.save(self.model, self.model_outfile) + + if abs(prev_loss - val_kl_div_loss) <= 0.0005: + self.logger.info('Early stopping. Loss changed by less than 0.0005.') + break + + prev_loss = val_kl_div_loss + self.evaluate(self.test_evaluator, 'test') + + self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) diff --git a/mp_cnn/trainers/sick_trainer.py b/mp_cnn/trainers/sick_trainer.py new file mode 100644 index 0000000..5ff8870 --- /dev/null +++ b/mp_cnn/trainers/sick_trainer.py @@ -0,0 +1,71 @@ +import time + +import torch +import torch.nn.functional as F +from torch.optim.lr_scheduler import ReduceLROnPlateau + +from mp_cnn.trainers.trainer import Trainer + + +class SICKTrainer(Trainer): + + def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): + super(SICKTrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) + + def train_epoch(self, epoch): + self.model.train() + total_loss = 0 + for batch_idx, batch in enumerate(self.train_loader): + self.optimizer.zero_grad() + output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats) + loss = F.kl_div(output, batch.label) + total_loss += loss.data[0] + loss.backward() + self.optimizer.step() + if batch_idx % self.log_interval == 0: + self.logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)), + len(batch.dataset.examples), + 100. * batch_idx / (len(self.train_loader)), loss.data[0]) + ) + + if self.use_tensorboard: + self.writer.add_scalar('sick/train/kl_div_loss', total_loss, epoch) + + return total_loss + + def train(self, epochs): + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) + epoch_times = [] + prev_loss = -1 + best_dev_score = -1 + for epoch in range(1, epochs + 1): + start = time.time() + self.logger.info('Epoch {} started...'.format(epoch)) + self.train_epoch(epoch) + + dev_scores = self.evaluate(self.dev_evaluator, 'dev') + new_loss = dev_scores[2] + + if self.use_tensorboard: + self.writer.add_scalar('sick/lr', self.optimizer.param_groups[0]['lr'], epoch) + self.writer.add_scalar('sick/dev/pearson_r', dev_scores[0], epoch) + self.writer.add_scalar('sick/dev/kl_div_loss', new_loss, epoch) + + end = time.time() + duration = end - start + self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) + epoch_times.append(duration) + + if dev_scores[0] > best_dev_score: + best_dev_score = dev_scores[0] + torch.save(self.model, self.model_outfile) + + if abs(prev_loss - new_loss) <= 0.0002: + self.logger.info('Early stopping. Loss changed by less than 0.0002.') + break + + prev_loss = new_loss + scheduler.step(dev_scores[0]) + + self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) diff --git a/mp_cnn/trainers/trainer.py b/mp_cnn/trainers/trainer.py new file mode 100644 index 0000000..c3a7378 --- /dev/null +++ b/mp_cnn/trainers/trainer.py @@ -0,0 +1,37 @@ +class Trainer(object): + + """ + Abstraction for training a model on a Dataset. + """ + + def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): + self.model = model + self.optimizer = trainer_config['optimizer'] + self.train_loader = train_loader + self.batch_size = trainer_config['batch_size'] + self.log_interval = trainer_config['log_interval'] + self.model_outfile = trainer_config['model_outfile'] + self.lr_reduce_factor = trainer_config['lr_reduce_factor'] + self.patience = trainer_config['patience'] + self.use_tensorboard = trainer_config['tensorboard'] + if self.use_tensorboard: + from tensorboardX import SummaryWriter + self.writer = SummaryWriter(log_dir=None, comment='' if trainer_config['run_label'] is None else trainer_config['run_label']) + self.logger = trainer_config['logger'] + + self.train_evaluator = train_evaluator + self.test_evaluator = test_evaluator + self.dev_evaluator = dev_evaluator + + def evaluate(self, evaluator, dataset_name): + scores, metric_names = evaluator.get_scores() + self.logger.info('Evaluation metrics for {}:'.format(dataset_name)) + self.logger.info('\t'.join([' '] + metric_names)) + self.logger.info('\t'.join([dataset_name] + list(map(str, scores)))) + return scores + + def train_epoch(self, epoch): + raise NotImplementedError() + + def train(self, epochs): + raise NotImplementedError() diff --git a/mp_cnn/trainers/trecqa_trainer.py b/mp_cnn/trainers/trecqa_trainer.py new file mode 100644 index 0000000..69e6c5c --- /dev/null +++ b/mp_cnn/trainers/trecqa_trainer.py @@ -0,0 +1,76 @@ +import time + +import torch +import torch.nn.functional as F +from torch.optim.lr_scheduler import ReduceLROnPlateau + +from mp_cnn.trainers.trainer import Trainer + + +class TRECQATrainer(Trainer): + + def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): + super(TRECQATrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) + + def train_epoch(self, epoch): + self.model.train() + total_loss = 0 + for batch_idx, batch in enumerate(self.train_loader): + self.optimizer.zero_grad() + output = self.model(batch.sentence_1, batch.sentence_2, batch.ext_feats) + loss = F.cross_entropy(output, batch.label, size_average=False) + total_loss += loss.data[0] + loss.backward() + self.optimizer.step() + if batch_idx % self.log_interval == 0: + self.logger.info('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)), + len(batch.dataset.examples), + 100. * batch_idx / (len(self.train_loader)), loss.data[0]) + ) + + average_loss, mean_average_precision, mean_reciprocal_rank = self.evaluate(self.train_evaluator, 'train') + + if self.use_tensorboard: + self.writer.add_scalar('trecqa/train/cross_entropy_loss', average_loss, epoch) + self.writer.add_scalar('trecqa/train/map', mean_average_precision, epoch) + self.writer.add_scalar('trecqa/train/mrr', mean_reciprocal_rank, epoch) + + return total_loss + + def train(self, epochs): + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) + epoch_times = [] + prev_loss = -1 + best_dev_score = -1 + for epoch in range(1, epochs + 1): + start = time.time() + self.logger.info('Epoch {} started...'.format(epoch)) + self.train_epoch(epoch) + + dev_scores = self.evaluate(self.dev_evaluator, 'dev') + new_loss, mean_average_precision, mean_reciprocal_rank = dev_scores + + if self.use_tensorboard: + self.writer.add_scalar('trecqa/lr', self.optimizer.param_groups[0]['lr'], epoch) + self.writer.add_scalar('trecqa/dev/cross_entropy_loss', new_loss, epoch) + self.writer.add_scalar('trecqa/dev/map', mean_average_precision, epoch) + self.writer.add_scalar('trecqa/dev/mrr', mean_reciprocal_rank, epoch) + + end = time.time() + duration = end - start + self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) + epoch_times.append(duration) + + if dev_scores[0] > best_dev_score: + best_dev_score = dev_scores[0] + torch.save(self.model, self.model_outfile) + + if abs(prev_loss - new_loss) <= 0.0002: + self.logger.info('Early stopping. Loss changed by less than 0.0002.') + break + + prev_loss = new_loss + scheduler.step(dev_scores[0]) + + self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/get_trec_eval.sh b/utils/get_trec_eval.sh new file mode 100755 index 0000000..2d92fab --- /dev/null +++ b/utils/get_trec_eval.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +VERSION=9.0.5 + +wget https://github.com/usnistgov/trec_eval/archive/v${VERSION}.tar.gz +tar -xvzf v${VERSION}.tar.gz +cd trec_eval-${VERSION} +make +cd .. + +rm -rf v${VERSION}.tar.gz diff --git a/utils/relevancy_metrics.py b/utils/relevancy_metrics.py new file mode 100644 index 0000000..f617ffb --- /dev/null +++ b/utils/relevancy_metrics.py @@ -0,0 +1,36 @@ +import os +import subprocess +import time + + +def get_map_mrr(qids, predictions, labels, device=0): + """ + Get the map and mrr using the trec_eval utility. + qids, predictions, labels should have the same length. + device is not a required parameter, it is only used to prevent potential naming conflicts when you + are calling this concurrently from different threads of execution. + :param qids: query ids of predictions and labels + :param predictions: iterable of predictions made by the models + :param labels: iterable of labels of the dataset + :param device: device (GPU index or -1 for CPU) for identification purposes only + """ + qrel_fname = 'trecqa_{}_{}.qrel'.format(time.time(), device) + results_fname = 'trecqa_{}_{}.results'.format(time.time(), device) + qrel_template = '{qid} 0 {docno} {rel}\n' + results_template = '{qid} 0 {docno} 0 {sim} mpcnn\n' + with open(qrel_fname, 'w') as f1, open(results_fname, 'w') as f2: + docnos = range(len(qids)) + for qid, docno, predicted, actual in zip(qids, docnos, predictions, labels): + f1.write(qrel_template.format(qid=qid, docno=docno, rel=actual)) + f2.write(results_template.format(qid=qid, docno=docno, sim=predicted)) + + trec_eval_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'trec_eval-9.0.5/trec_eval') + trec_out = subprocess.check_output([trec_eval_path, '-m', 'map', '-m', 'recip_rank', qrel_fname, results_fname]) + trec_out_lines = str(trec_out, 'utf-8').split('\n') + mean_average_precision = float(trec_out_lines[0].split('\t')[-1]) + mean_reciprocal_rank = float(trec_out_lines[1].split('\t')[-1]) + + os.remove(qrel_fname) + os.remove(results_fname) + + return mean_average_precision, mean_reciprocal_rank