diff --git a/datasets/castor_dataset.py b/datasets/castor_dataset.py index a02cd0f..849d0fd 100644 --- a/datasets/castor_dataset.py +++ b/datasets/castor_dataset.py @@ -1,10 +1,12 @@ from abc import ABCMeta, abstractmethod import os import numpy as np +from sys import exit from torchtext.data.dataset import Dataset from torchtext.data.example import Example from torchtext.data.field import Field +import torch from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features @@ -49,3 +51,21 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta): examples.append(example) super(CastorPairDataset, self).__init__(examples, fields) + + @classmethod + def set_vectors(cls, field, vector_path): + if os.path.isfile(vector_path): + stoi, vectors, dim = torch.load(vector_path) + field.vocab.vectors = torch.Tensor(len(field.vocab), dim) + + for i, token in enumerate(field.vocab.itos): + wv_index = stoi.get(token, None) + if wv_index is not None: + field.vocab.vectors[i] = vectors[wv_index] + else: + # initialize with uniform_(-0.05, 0.05) vectors + field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05) + else: + print("Error: Need word embedding pt file") + exit(1) + return field diff --git a/datasets/trecqa.py b/datasets/trecqa.py index e7984a5..59183f8 100644 --- a/datasets/trecqa.py +++ b/datasets/trecqa.py @@ -36,24 +36,6 @@ class TRECQA(CastorPairDataset): def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs): return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs) - @classmethod - def set_vectors(cls, field, vector_path): - if os.path.isfile(vector_path): - stoi, vectors, dim = torch.load(vector_path) - field.vocab.vectors = torch.Tensor(len(field.vocab), dim) - - for i, token in enumerate(field.vocab.itos): - wv_index = stoi.get(token, None) - if wv_index is not None: - field.vocab.vectors[i] = vectors[wv_index] - else: - # initialize with uniform_(-0.05, 0.05) vectors - field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05) - else: - print("Error: Need word embedding pt file") - exit(1) - return field - @classmethod def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file = False, vectors=None, unk_init=torch.Tensor.zero_): """ diff --git a/datasets/wikiqa.py b/datasets/wikiqa.py index 2bf98e3..5420f55 100644 --- a/datasets/wikiqa.py +++ b/datasets/wikiqa.py @@ -14,6 +14,7 @@ class WikiQA(CastorPairDataset): NAME = 'wikiqa' NUM_CLASSES = 2 ID_FIELD = Field(sequential=False, tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True) + AID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True) TEXT_FIELD = Field(batch_first=True, tokenize=lambda x: x) # tokenizer is identity since we already tokenized it to compute external features EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x) LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True) @@ -33,22 +34,32 @@ class WikiQA(CastorPairDataset): return super(WikiQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs) @classmethod - def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, unk_init=torch.Tensor.zero_): + def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file=False, vectors=None, + unk_init=torch.Tensor.zero_): """ :param path: directory containing train, test, dev files :param vectors_name: name of word vectors file - :param vectors_cache: directory containing word vectors file + :param vectors_dir: directory containing word vectors file :param batch_size: batch size :param device: GPU device :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes + :param pt_file: load cached embedding file from disk if it is true :param unk_init: function used to generate vector for OOV words :return: """ - if vectors is None: - vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init) train, validation, test = cls.splits(path) + if not pt_file: + if vectors is None: + vectors = Vectors(name=vectors_name, cache=vectors_dir, unk_init=unk_init) + cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors) + else: + cls.TEXT_FIELD.build_vocab(train, validation, test) + cls.TEXT_FIELD = cls.set_vectors(cls.TEXT_FIELD, os.path.join(vectors_dir, vectors_name)) - cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors) + cls.LABEL_FIELD.build_vocab(train, validation, test) - return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device) + cls.VOCAB_SIZE = len(cls.TEXT_FIELD.vocab) + + return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, + device=device) \ No newline at end of file diff --git a/mp_cnn/dataset.py b/mp_cnn/dataset.py index ebe3d04..0ffb2a0 100644 --- a/mp_cnn/dataset.py +++ b/mp_cnn/dataset.py @@ -29,16 +29,16 @@ class MPCNNDatasetFactory(object): Get the corresponding Dataset class for a particular dataset. """ @staticmethod - def get_dataset(dataset_name, word_vectors_dir, word_vectors_file, batch_size, device): + def get_dataset(dataset_name, word_vectors_dir, word_vectors_file, batch_size, device, castor_dir="../", utils_trecqa="utils/trec_eval-9.0.5/trec_eval"): if dataset_name == 'sick': - dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'sick/') + dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'sick/') train_loader, dev_loader, test_loader = SICK.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding_dim = SICK.TEXT_FIELD.vocab.vectors.size() embedding = nn.Embedding(embedding_dim[0], embedding_dim[1]) embedding.weight = nn.Parameter(SICK.TEXT_FIELD.vocab.vectors) return SICK, embedding, train_loader, test_loader, dev_loader elif dataset_name == 'msrvid': - dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'msrvid/') + dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'msrvid/') dev_loader = None train_loader, test_loader = MSRVID.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding_dim = MSRVID.TEXT_FIELD.vocab.vectors.size() @@ -46,18 +46,18 @@ class MPCNNDatasetFactory(object): embedding.weight = nn.Parameter(MSRVID.TEXT_FIELD.vocab.vectors) return MSRVID, embedding, train_loader, test_loader, dev_loader elif dataset_name == 'trecqa': - if not os.path.exists('../utils/trec_eval-9.0.5/trec_eval'): + if not os.path.exists(os.path.join(castor_dir, utils_trecqa)): raise FileNotFoundError('TrecQA requires the trec_eval tool to run. Please run get_trec_eval.sh inside Castor/utils (as working directory) before continuing.') - dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'TrecQA/') + dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'TrecQA/') train_loader, dev_loader, test_loader = TRECQA.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding_dim = TRECQA.TEXT_FIELD.vocab.vectors.size() embedding = nn.Embedding(embedding_dim[0], embedding_dim[1]) embedding.weight = nn.Parameter(TRECQA.TEXT_FIELD.vocab.vectors) return TRECQA, embedding, train_loader, test_loader, dev_loader elif dataset_name == 'wikiqa': - if not os.path.exists('../utils/trec_eval-9.0.5/trec_eval'): + if not os.path.exists(os.path.join(castor_dir, utils_trecqa)): raise FileNotFoundError('TrecQA requires the trec_eval tool to run. Please run get_trec_eval.sh inside Castor/utils (as working directory) before continuing.') - dataset_root = os.path.join(os.pardir, os.pardir, 'data', 'WikiQA/') + dataset_root = os.path.join(os.pardir, castor_dir, 'data', 'WikiQA/') train_loader, dev_loader, test_loader = WikiQA.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding_dim = WikiQA.TEXT_FIELD.vocab.vectors.size() embedding = nn.Embedding(embedding_dim[0], embedding_dim[1]) diff --git a/mp_cnn/evaluation.py b/mp_cnn/evaluation.py index 1dfa9c1..9163ce6 100644 --- a/mp_cnn/evaluation.py +++ b/mp_cnn/evaluation.py @@ -2,7 +2,8 @@ from mp_cnn.evaluators.sick_evaluator import SICKEvaluator from mp_cnn.evaluators.msrvid_evaluator import MSRVIDEvaluator from mp_cnn.evaluators.trecqa_evaluator import TRECQAEvaluator from mp_cnn.evaluators.wikiqa_evaluator import WikiQAEvaluator - +from nce.nce_pairwise_mp.evaluators.trecqa_evaluator import TRECQAEvaluatorNCE +from nce.nce_pairwise_mp.evaluators.wikiqa_evaluator import WikiQAEvaluatorNCE class MPCNNEvaluatorFactory(object): """ @@ -15,17 +16,27 @@ class MPCNNEvaluatorFactory(object): 'wikiqa': WikiQAEvaluator } + evaluator_map_nce = { + 'trecqa': TRECQAEvaluatorNCE, + 'wikiqa': WikiQAEvaluatorNCE + } + @staticmethod - def get_evaluator(dataset_cls, model, data_loader, batch_size, device): + def get_evaluator(dataset_cls, model, data_loader, batch_size, device, nce=False): if data_loader is None: return None + if nce: + evaluator_map = MPCNNEvaluatorFactory.evaluator_map_nce + else: + evaluator_map = MPCNNEvaluatorFactory.evaluator_map + if not hasattr(dataset_cls, 'NAME'): raise ValueError('Invalid dataset. Dataset should have NAME attribute.') - if dataset_cls.NAME not in MPCNNEvaluatorFactory.evaluator_map: + if dataset_cls.NAME not in evaluator_map: raise ValueError('{} is not implemented.'.format(dataset_cls)) - return MPCNNEvaluatorFactory.evaluator_map[dataset_cls.NAME]( + return evaluator_map[dataset_cls.NAME]( dataset_cls, model, data_loader, batch_size, device ) diff --git a/mp_cnn/model.py b/mp_cnn/model.py index 834d364..651cdd2 100644 --- a/mp_cnn/model.py +++ b/mp_cnn/model.py @@ -46,10 +46,10 @@ class MPCNN(nn.Module): # comparison units from per-dim conv 2 * (len(self.filter_widths) - 1) * n_per_dim_filters * COMP_1_COMPONENTS_PER_DIM ) - n_feat = n_feat_h + n_feat_v + EXT_FEATS + self.n_feat = n_feat_h + n_feat_v + EXT_FEATS self.final_layers = nn.Sequential( - nn.Linear(n_feat, hidden_layer_units), + nn.Linear(self.n_feat, hidden_layer_units), nn.Tanh(), nn.Dropout(dropout), nn.Linear(hidden_layer_units, num_classes), diff --git a/mp_cnn/train.py b/mp_cnn/train.py index be22982..428574f 100644 --- a/mp_cnn/train.py +++ b/mp_cnn/train.py @@ -2,6 +2,8 @@ from mp_cnn.trainers.sick_trainer import SICKTrainer from mp_cnn.trainers.msrvid_trainer import MSRVIDTrainer from mp_cnn.trainers.trecqa_trainer import TRECQATrainer from mp_cnn.trainers.wikiqa_trainer import WikiQATrainer +from nce.nce_pairwise_mp.trainers.trecqa_trainer import TRECQATrainerNCE +from nce.nce_pairwise_mp.trainers.wikiqa_trainer import WikiQATrainerNCE class MPCNNTrainerFactory(object): @@ -15,11 +17,21 @@ class MPCNNTrainerFactory(object): 'wikiqa': WikiQATrainer } + trainer_map_nce = { + 'trecqa': TRECQATrainerNCE, + 'wikiqa': WikiQATrainerNCE + } + @staticmethod - def get_trainer(dataset_name, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): - if dataset_name not in MPCNNTrainerFactory.trainer_map: + def get_trainer(dataset_name, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None, nce=False): + if nce: + trainer_map = MPCNNTrainerFactory.trainer_map_nce + else: + trainer_map = MPCNNTrainerFactory.trainer_map + + if dataset_name not in trainer_map: raise ValueError('{} is not implemented.'.format(dataset_name)) - return MPCNNTrainerFactory.trainer_map[dataset_name]( + return trainer_map[dataset_name]( model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator ) diff --git a/mp_cnn/trainers/trainer.py b/mp_cnn/trainers/trainer.py index c3a7378..f16108a 100644 --- a/mp_cnn/trainers/trainer.py +++ b/mp_cnn/trainers/trainer.py @@ -14,6 +14,7 @@ class Trainer(object): self.lr_reduce_factor = trainer_config['lr_reduce_factor'] self.patience = trainer_config['patience'] self.use_tensorboard = trainer_config['tensorboard'] + if self.use_tensorboard: from tensorboardX import SummaryWriter self.writer = SummaryWriter(log_dir=None, comment='' if trainer_config['run_label'] is None else trainer_config['run_label']) @@ -25,7 +26,6 @@ class Trainer(object): def evaluate(self, evaluator, dataset_name): scores, metric_names = evaluator.get_scores() - self.logger.info('Evaluation metrics for {}:'.format(dataset_name)) self.logger.info('\t'.join([' '] + metric_names)) self.logger.info('\t'.join([dataset_name] + list(map(str, scores)))) return scores diff --git a/nce/NCE-Pairwise-SM/train.py b/nce/NCE-Pairwise-SM/train.py index a2d66a6..ff51e55 100644 --- a/nce/NCE-Pairwise-SM/train.py +++ b/nce/NCE-Pairwise-SM/train.py @@ -15,6 +15,7 @@ from datasets.trecqa import TRECQA from args import get_args from model import SmPlusPlus, PairwiseConv from utils.relevancy_metrics import get_map_mrr +from utils.nce_neighbors import get_nearest_neg_id, get_random_neg_id, get_batch class UnknownWordVecCache(object): @@ -125,46 +126,6 @@ def train_sm(): os.makedirs(os.path.join(args.save_path, args.dataset), exist_ok=True) print(header) - # get the nearest negative samples to the positive sample by computing the feature difference - def get_nearest_neg_id(pos_feature, neg_dict, distance="cosine", k=1): - dis_list = [] - pos_feature = pos_feature.data.cpu().numpy() - pos_feature_norm = pos_feature / np.sqrt(sum(pos_feature ** 2)) - neg_list = [] - for key in neg_dict: - if distance == "l2": - dis = np.sqrt(np.sum((np.array(pos_feature) - neg_dict[key]["feature"]) ** 2)) - elif distance == "cosine": - neg_feature = np.array(neg_dict[key]["feature"]) - feat_norm = neg_feature / np.sqrt(sum(neg_feature ** 2)) - dis = 1 - feat_norm.dot(pos_feature_norm) - dis_list.append(dis) - neg_list.append(key) - - k = min(k, len(neg_dict)) - min_list = heapq.nsmallest(k, enumerate(dis_list), key=operator.itemgetter(1)) - min_id_list = [neg_list[x[0]] for x in min_list] - return min_id_list - - # get the negative samples randomly - def get_random_neg_id(q2neg, qid_i, k=5): - # question 1734 has no neg answer - if qid_i not in q2neg: - return [] - k = min(k, len(q2neg[qid_i])) - ran = random.sample(q2neg[qid_i], k) - return ran - - # pack the lists of question/answer/ext_feat into a torchtext batch - def get_batch(question, answer, ext_feat, size): - new_batch = data.Batch() - new_batch.batch_size = size - new_batch.dataset = batch.dataset - setattr(new_batch, "sentence_2", torch.stack(answer)) - setattr(new_batch, "sentence_1", torch.stack(question)) - setattr(new_batch, "ext_feats", torch.stack(ext_feat)) - return new_batch - while True: if early_stop: logger.log("Early Stopping. Epoch: {}, Best Dev Loss: {}".format(epoch, best_dev_loss)) diff --git a/nce/nce_pairwise_mp/README.md b/nce/nce_pairwise_mp/README.md new file mode 100644 index 0000000..f9dbc0a --- /dev/null +++ b/nce/nce_pairwise_mp/README.md @@ -0,0 +1,85 @@ +# MP-CNN PyTorch Implementation + +This is a PyTorch implementation of the following paper + +* Hua He, Kevin Gimpel, and Jimmy Lin. [Multi-Perspective Sentence Similarity Modeling with Convolutional Neural Networks](http://aclweb.org/anthology/D/D15/D15-1181.pdf). *Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing (EMNLP 2015)*, pages 1576-1586. +* Jinfeng Rao, Hua He, and Jimmy Lin. [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks.](http://dl.acm.org/citation.cfm?id=2983872) *Proceedings of the 25th ACM International on Conference on Information and Knowledge Management (CIKM 2016)*, pages 1913-1916. + + +The SICK and MSRVID datasets are available in https://github.com/castorini/data, as well as the GloVe word embeddings. + +Directory layout should be like this: +``` +├── Castor +│ ├── README.md +│ ├── ... +│ └── mp_cnn/ +├── data +│ ├── README.md +│ ├── ... +│ ├── msrvid/ +│ ├── sick/ +│ └── GloVe/ +``` + +## TrecQA Dataset + +To run MP-CNN on (Raw) TrecQA, you first need to run `./get_trec_eval.sh` in `utils` under the repo root while inside the `utils` directory. This will download and compile the official `trec_eval` tool used for evaluation. + +Then, you can run: +``` +python train_script.py --dataset wikiqa --device -1 +``` + +Metric|Without NCE (original paper) | only random sampling | only max sampling | Pair-wise+nagative sampling (original paper) | Pair-wise+random sampling| Pair-wise+nagative sampling | Pair-wise+nagative sampling+pair weighting +-------|------|----------|------------|------------|------------|------|------ +MAP |0.762 | 0.7579| 0.7678|0.780 | 0.7745 |0.7873|0.7683 +MRR |0.830 |0.8239| 0.8387|0.834 | 0.8435 |0.8414|0.8253 + +The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872). + +## WikiQA Dataset + +You also need `trec_eval` for this dataset, similar to TrecQA. + +Then, you can run: +``` +python train_script.py --dataset trecqa --device -1 +``` + +Metric|Without NCE (original paper) | only random sampling | only max sampling| Pair-wise+nagative sampling (original paper)| Pair-wise+random sampling | Pair-wise+nagative sampling | Pair-wise+nagative sampling+pair weighting +-------|-------|------|----------|------------|------------|------------|------------ +MAP |0.693 | 0.6744| 0.6795 | 0.701| 0.7047 |0.7049| 0.7047 +MRR |0.709 | 0.6898| 0.6951 |0.718 | 0.7172 |0.7192| 0.7211 + + +The paper results are reported in [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks](https://dl.acm.org/citation.cfm?id=2983872). + + +To see all options available and train with your parameters, use +``` +python main.py --help +``` + +## Troubleshooting + +### ModuleNotFoundError: datasets +``` +Traceback (most recent call last): + File "main.py", line 9, in + from dataset import MPCNNDatasetFactory + File "/u/z3tu/castorini/Castor/mp_cnn/dataset.py", line 12, in + from datasets.sick import SICK +ModuleNotFoundError: No module named 'datasets' +``` + +You need to make sure the repository root is in your `PYTHONPATH` environment variable. One way to do this is while you are in the repo root (Castor) as your current working directory, run `export PYTHONPATH=$(pwd)`. + +## Optional Dependencies + +To optionally visualize the learning curve during training, we make use of https://github.com/lanpa/tensorboard-pytorch to connect to [TensorBoard](https://github.com/tensorflow/tensorboard). These projects require TensorFlow as a dependency, so you need to install TensorFlow before running the commands below. After these are installed, just add `--tensorboard` when running `main.py` and open TensorBoard in the browser. + +```sh +pip install tensorboardX +pip install tensorflow-tensorboard +``` diff --git a/nce/nce_pairwise_mp/__init__.py b/nce/nce_pairwise_mp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nce/nce_pairwise_mp/evaluators/__init__.py b/nce/nce_pairwise_mp/evaluators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nce/nce_pairwise_mp/evaluators/qa_evaluator.py b/nce/nce_pairwise_mp/evaluators/qa_evaluator.py new file mode 100644 index 0000000..d8db2bb --- /dev/null +++ b/nce/nce_pairwise_mp/evaluators/qa_evaluator.py @@ -0,0 +1,33 @@ +import numpy as np + +from mp_cnn.evaluators.evaluator import Evaluator +from utils.relevancy_metrics import get_map_mrr + + +class QAEvaluator(Evaluator): + + def __init__(self, dataset_cls, model, data_loader, batch_size, device): + super(QAEvaluator, self).__init__(dataset_cls, model, data_loader, batch_size, device) + + def get_scores(self): + self.model.eval() + qids = [] + predictions = [] + labels = [] + + for batch in self.data_loader: + scores = self.model.convModel(batch.sentence_1, batch.sentence_2, batch.ext_feats) + scores = self.model.linearLayer(scores) + qid_array = np.transpose(batch.id.cpu().data.numpy()) + score_array = scores.cpu().data.numpy().reshape(-1) + true_label_array = np.transpose(batch.label.cpu().data.numpy()) + + qids.extend(qid_array.tolist()) + predictions.extend(score_array.tolist()) + labels.extend(true_label_array.tolist()) + + del scores + + mean_average_precision, mean_reciprocal_rank = get_map_mrr(qids, predictions, labels, self.data_loader.device) + + return [mean_average_precision, mean_reciprocal_rank], ['map', 'mrr'] diff --git a/nce/nce_pairwise_mp/evaluators/trecqa_evaluator.py b/nce/nce_pairwise_mp/evaluators/trecqa_evaluator.py new file mode 100644 index 0000000..ba08b4f --- /dev/null +++ b/nce/nce_pairwise_mp/evaluators/trecqa_evaluator.py @@ -0,0 +1,7 @@ +from nce.nce_pairwise_mp.evaluators.qa_evaluator import QAEvaluator + + +class TRECQAEvaluatorNCE(QAEvaluator): + + def __init__(self, dataset_cls, model, data_loader, batch_size, device): + super(TRECQAEvaluatorNCE, self).__init__(dataset_cls, model, data_loader, batch_size, device) diff --git a/nce/nce_pairwise_mp/evaluators/wikiqa_evaluator.py b/nce/nce_pairwise_mp/evaluators/wikiqa_evaluator.py new file mode 100644 index 0000000..0263e28 --- /dev/null +++ b/nce/nce_pairwise_mp/evaluators/wikiqa_evaluator.py @@ -0,0 +1,7 @@ +from nce.nce_pairwise_mp.evaluators.qa_evaluator import QAEvaluator + + +class WikiQAEvaluatorNCE(QAEvaluator): + + def __init__(self, dataset_cls, model, data_loader, batch_size, device): + super(WikiQAEvaluatorNCE, self).__init__(dataset_cls, model, data_loader, batch_size, device) diff --git a/nce/nce_pairwise_mp/main.py b/nce/nce_pairwise_mp/main.py new file mode 100644 index 0000000..ee57d56 --- /dev/null +++ b/nce/nce_pairwise_mp/main.py @@ -0,0 +1,137 @@ +import argparse +import logging +import os +import pprint +import random + +import numpy as np +import torch +import torch.optim as optim + +from mp_cnn.dataset import MPCNNDatasetFactory +from mp_cnn.evaluation import MPCNNEvaluatorFactory +from nce.nce_pairwise_mp.model import MPCNN4NCE, PairwiseConv +from mp_cnn.train import MPCNNTrainerFactory + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='PyTorch implementation of Multi-Perspective CNN') + parser.add_argument('model_outfile', help='file to save final model') + parser.add_argument('--dataset', help='dataset to use, one of [sick, msrvid, trecqa, wikiqa]', default='sick') + parser.add_argument('--word-vectors-dir', help='word vectors directory', default=os.path.join(os.pardir, os.pardir, os.pardir, 'data', 'GloVe')) + parser.add_argument('--word-vectors-file', help='word vectors filename', default='glove.840B.300d.txt') + parser.add_argument('--skip-training', help='will load pre-trained model', action='store_true') + parser.add_argument('--device', type=int, default=0, help='GPU device, -1 for CPU (default: 0)') + parser.add_argument('--sparse-features', action='store_true', default=False, help='use sparse features (default: false)') + parser.add_argument('--batch-size', type=int, default=64, help='input batch size for training (default: 64)') + parser.add_argument('--epochs', type=int, default=10, help='number of epochs to train (default: 10)') + parser.add_argument('--optimizer', type=str, default='adam', help='optimizer to use: adam or sgd (default: adam)') + parser.add_argument('--lr', type=float, default=0.001, help='learning rate (default: 0.001)') + parser.add_argument('--lr-reduce-factor', type=float, default=0.3, help='learning rate reduce factor after plateau (default: 0.3)') + parser.add_argument('--patience', type=float, default=2, help='learning rate patience after seeing plateau (default: 2)') + parser.add_argument('--momentum', type=float, default=0, help='momentum (default: 0)') + parser.add_argument('--epsilon', type=float, default=1e-8, help='Adam epsilon (default: 1e-8)') + parser.add_argument('--log-interval', type=int, default=10, help='how many batches to wait before logging training status (default: 10)') + parser.add_argument('--regularization', type=float, default=0.0001, help='Regularization for the optimizer (default: 0.0001)') + parser.add_argument('--max-window-size', type=int, default=3, help='windows sizes will be [1,max_window_size] and infinity (default: 300)') + parser.add_argument('--holistic-filters', type=int, default=300, help='number of holistic filters (default: 300)') + parser.add_argument('--per-dim-filters', type=int, default=20, help='number of per-dimension filters (default: 20)') + parser.add_argument('--hidden-units', type=int, default=150, help='number of hidden units in each of the two hidden layers (default: 150)') + parser.add_argument('--dropout', type=float, default=0.5, help='dropout probability (default: 0.5)') + parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') + parser.add_argument('--tensorboard', action='store_true', default=False, help='use TensorBoard to visualize training (default: false)') + parser.add_argument('--run-label', type=str, help='label to describe run') + parser.add_argument('--dev_log_interval', type=int, default=100, help='how many batches to wait before logging validation status (default: 100)') + parser.add_argument('--neg_num', type=int, default=8, help='number of negative samples for each question') + parser.add_argument('--neg_sample', type=str, default="random", help='strategy of negative sampling, random or max') + parser.add_argument('--castor_dir', help='castor directory', default=os.path.join(os.pardir, os.pardir)) + parser.add_argument('--utils_trecqa', help='trecqa util file', default="utils/trec_eval-9.0.5/trec_eval") + + + args = parser.parse_args() + + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.device != -1: + torch.cuda.manual_seed(args.seed) + + # logging setup + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + logger.info(pprint.pformat(vars(args))) + + dataset_cls, embedding, train_loader, test_loader, dev_loader = MPCNNDatasetFactory.get_dataset(args.dataset, + args.word_vectors_dir, + args.word_vectors_file, + args.batch_size, + args.device, + castor_dir=args.castor_dir, + utils_trecqa=args.utils_trecqa) + + filter_widths = list(range(1, args.max_window_size + 1)) + [np.inf] + model = MPCNN4NCE(embedding, args.holistic_filters, args.per_dim_filters, filter_widths, + args.hidden_units, dataset_cls.NUM_CLASSES, args.dropout, args.sparse_features) + + pw_model = PairwiseConv(model) + + if args.device != -1: + with torch.cuda.device(args.device): + pw_model.cuda() + + optimizer = None + if args.optimizer == 'adam': + optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.regularization, eps=args.epsilon) + elif args.optimizer == 'sgd': + optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.regularization) + else: + raise ValueError('optimizer not recognized: it should be either adam or sgd') + + test_batch_size = 32 + train_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, train_loader, args.batch_size, args.device, nce=True) + dev_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, dev_loader, args.batch_size, args.device, nce=True) + test_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, test_loader, test_batch_size, args.device, nce=True) + + if args.device != -1: + margin_label = torch.autograd.Variable(torch.ones(1).cuda(device=args.device)) + else: + margin_label = torch.autograd.Variable(torch.ones(1)) + + trainer_config = { + 'optimizer': optimizer, + 'batch_size': args.batch_size, + 'log_interval': args.log_interval, + 'dev_log_interval': args.dev_log_interval, + 'model_outfile': args.model_outfile, + 'lr_reduce_factor': args.lr_reduce_factor, + 'patience': args.patience, + 'tensorboard': args.tensorboard, + 'run_label': args.run_label, + 'logger': logger, + 'neg_num': args.neg_num, + 'neg_sample': args.neg_sample, + 'margin_label': margin_label + } + trainer = MPCNNTrainerFactory.get_trainer(args.dataset, pw_model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator, nce=True) + + if not args.skip_training: + total_params = 0 + for param in pw_model.parameters(): + size = [s for s in param.size()] + total_params += np.prod(size) + logger.info('Total number of parameters: %s', total_params) + trainer.train(args.epochs) + + pw_model = torch.load(args.model_outfile) + saved_model_evaluator = MPCNNEvaluatorFactory.get_evaluator(dataset_cls, pw_model, test_loader, args.batch_size, args.device) + scores, metric_names = saved_model_evaluator.get_scores() + logger.info('Evaluation metrics for test') + logger.info('\t'.join([' '] + metric_names)) + logger.info('\t'.join(['test'] + list(map(str, scores)))) diff --git a/nce/nce_pairwise_mp/train_script.py b/nce/nce_pairwise_mp/train_script.py new file mode 100644 index 0000000..9d7153e --- /dev/null +++ b/nce/nce_pairwise_mp/train_script.py @@ -0,0 +1,44 @@ +import argparse +from random import uniform +from subprocess import call + + +# set parameters from random uniform sampling +def get_param(): + for eps in [1e-8, 1e-6]: + for learning_rate in [0.0001, 0.0003, 0.0006, 0.001]: + for reg in [3e-5, 9e-5, 3e-4, 9e-4]: + yield learning_rate, eps, reg + +def run(device, epochs, neg_sample, neg_num, dataset, dev_log_interval, batch_size): + # for _ in range(count): + param_gen = get_param() + for learning_rate, eps, reg in param_gen: + + filename = "grid_{dataset}_lr_{learning_rate}_eps_{eps}_reg_{reg}_device_{dev}.txt".format( + learning_rate=learning_rate, eps=eps, reg=reg, dev=device, dataset=dataset) + model_name = filename[:-4] + ".castor" + + command = "python -u main.py saved_models/{model} --epochs {epo} --device {dev} --dataset {dataset} " \ + "--batch-size {batch_size} --lr {learning_rate} --epsilon {eps} --regularization {reg} --tensorboard " \ + "--run-label {label} --dev_log_interval {dev_log_interval} --neg_sample {neg_sample} --neg_num {neg_num}" \ + .format(epo=epochs, model=model_name, dev=device, dataset=dataset, batch_size=batch_size, + learning_rate=learning_rate, eps=eps, + reg=reg, label=filename, dev_log_interval=dev_log_interval, neg_sample=neg_sample, neg_num=neg_num) + + print("Running: " + command) + with open(filename, 'w') as outfile: + call(command, shell=True, stderr=outfile) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Hyper parameters sweeper') + parser.add_argument('--device', type=int, default=2, help='GPU device, -1 for CPU (default: 0)') + parser.add_argument('--epochs', type=int, default=8, help='number of epochs to run') + parser.add_argument('--neg_num', type=int, default=8, help='number of negative samples') + parser.add_argument('--neg_sample', type=str, default="max", help='strategy of negative samples') + parser.add_argument('--dataset', type=str, default="trecqa", help='dataset') + parser.add_argument('--dev_log_interval', type=int, default=150, help='number of negative samples') + parser.add_argument('--batch_size', type=int, default=32, help='batch size') + args = parser.parse_args() + run(args.device, args.epochs, neg_sample=args.neg_sample, neg_num=args.neg_num, dataset=args.dataset, + dev_log_interval=args.dev_log_interval, batch_size=args.batch_size) \ No newline at end of file diff --git a/nce/nce_pairwise_mp/trainers/__init__.py b/nce/nce_pairwise_mp/trainers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nce/nce_pairwise_mp/trainers/qa_trainer.py b/nce/nce_pairwise_mp/trainers/qa_trainer.py new file mode 100644 index 0000000..df9a04b --- /dev/null +++ b/nce/nce_pairwise_mp/trainers/qa_trainer.py @@ -0,0 +1,219 @@ +import time +import numpy as np + +import torch +import torch.nn.functional as F +from torch.optim.lr_scheduler import ReduceLROnPlateau + +from mp_cnn.trainers.trainer import Trainer +from utils.nce_neighbors import get_nearest_neg_id, get_random_neg_id, get_batch + +class QATrainer(Trainer): + + def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None, weighting=False): + super(QATrainer, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) + self.loss = torch.nn.MarginRankingLoss(margin=1, size_average=True) + self.question2answer = {} + self.best_dev_map = 0 + self.best_dev_mrr = 0 + self.false_samples = {} + self.question2answer = {} + self.start = time.time() + self.q2neg = {} + self.iteration = 0 + self.name = self.train_loader.dataset.NAME + self.dev_log_interval = trainer_config['dev_log_interval'] + self.neg_num = trainer_config['neg_num'] if 'neg_num' in trainer_config else 0 + self.neg_sample = trainer_config['neg_sample'] if 'neg_sample' in trainer_config else '' + self.log_template = 'Train Epoch:{} [{}/{}]\tLoss:{} Acc:{}' + self.margin_label = trainer_config['margin_label'] + self.dev_index = 1 + self.weighting = weighting + + def train_epoch(self, epoch): + self.model.train() + total_loss = 0 + acc = 0 + tot = 0 + for batch_idx, batch in enumerate(self.train_loader): + self.iteration += 1 + + features = self.model.convModel(batch.sentence_1, batch.sentence_2, batch.ext_feats) + new_train_pos = {"answer": [], "question": [], "ext_feat": []} + new_train_neg = {"answer": [], "question": [], "ext_feat": []} + max_len_q = 0 + max_len_a = 0 + + batch_near_list = [] + batch_qid = [] + batch_aid = [] + new_near_score = [] + + for i in range(batch.batch_size): + label_i = batch.label[i].cpu().data.numpy()[0] + question_i = batch.sentence_1[i] + # question_i = question_i[question_i!=1] # remove padding 1 + answer_i = batch.sentence_2[i] + # answer_i = answer_i[answer_i!=1] # remove padding 1 + ext_feat_i = batch.ext_feats[i] + qid_i = batch.id[i].data.cpu().numpy()[0] + aid_i = batch.aid[i].data.cpu().numpy()[0] + + if qid_i not in self.question2answer: + self.question2answer[qid_i] = {"question": question_i, "pos": {}, "neg": {}} + if label_i == 1: + + if aid_i not in self.question2answer[qid_i]["pos"]: + self.question2answer[qid_i]["pos"][aid_i] = {} + + self.question2answer[qid_i]["pos"][aid_i]["answer"] = answer_i + self.question2answer[qid_i]["pos"][aid_i]["ext_feat"] = ext_feat_i + + # get neg samples in the first epoch but do not train + if epoch == 1: + continue + # random generate sample in the first training epoch + elif epoch == 2 or self.neg_sample == "random": + near_list = get_random_neg_id(self.q2neg, qid_i, k=self.neg_num) + else: + near_list, near_score = get_nearest_neg_id(features[i], self.question2answer[qid_i]["neg"], + distance="cosine", k=self.neg_num, weight=True) + new_near_score.extend(near_score) + + batch_near_list.extend(near_list) + + neg_size = len(near_list) + if neg_size != 0: + answer_i = answer_i[answer_i != 1] # remove padding 1 + question_i = question_i[question_i != 1] # remove padding 1 + for near_id in near_list: + batch_qid.append(qid_i) + batch_aid.append(aid_i) + + new_train_pos["answer"].append(answer_i) + new_train_pos["question"].append(question_i) + new_train_pos["ext_feat"].append(ext_feat_i) + + near_answer = self.question2answer[qid_i]["neg"][near_id]["answer"] + if question_i.size(0) > max_len_q: + max_len_q = question_i.size(0) + if near_answer.size(0) > max_len_a: + max_len_a = near_answer.size(0) + if answer_i.size(0) > max_len_a: + max_len_a = answer_i.size(0) + + ext_feat_neg = self.question2answer[qid_i]["neg"][near_id]["ext_feat"] + new_train_neg["answer"].append(near_answer) + new_train_neg["question"].append(question_i) + new_train_neg["ext_feat"].append(ext_feat_neg) + + elif label_i == 0: + + if aid_i not in self.question2answer[qid_i]["neg"]: + answer_i = answer_i[answer_i != 1] + self.question2answer[qid_i]["neg"][aid_i] = {"answer": answer_i} + + if "ext_feat" in self.question2answer[qid_i]["neg"][aid_i]: + del self.question2answer[qid_i]["neg"][aid_i]["ext_feat"] + self.question2answer[qid_i]["neg"][aid_i]["feature"] = features[i].data.cpu().numpy() + self.question2answer[qid_i]["neg"][aid_i]["ext_feat"] = ext_feat_i + + + if epoch == 1: + if qid_i not in self.q2neg: + self.q2neg[qid_i] = [] + + self.q2neg[qid_i].append(aid_i) + + del features + # pack the selected pos and neg samples into the torchtext batch and train + if epoch != 1: + true_batch_size = len(new_train_neg["answer"]) + if true_batch_size != 0: + for j in range(true_batch_size): + new_train_neg["answer"][j] = F.pad(new_train_neg["answer"][j], + (0, max_len_a - new_train_neg["answer"][j].size(0)), + value=1) + new_train_pos["answer"][j] = F.pad(new_train_pos["answer"][j], + (0, max_len_a - new_train_pos["answer"][j].size(0)), + value=1) + new_train_pos["question"][j] = F.pad(new_train_pos["question"][j], + (0, max_len_q - new_train_pos["question"][j].size(0)), + value=1) + new_train_neg["question"][j] = F.pad(new_train_neg["question"][j], + (0, max_len_q - new_train_neg["question"][j].size(0)), + value=1) + + + pos_batch = get_batch(new_train_pos["question"], new_train_pos["answer"], new_train_pos["ext_feat"], + true_batch_size) + neg_batch = get_batch(new_train_neg["question"], new_train_neg["answer"], new_train_neg["ext_feat"], + true_batch_size) + + self.model.train() + self.optimizer.zero_grad() + output = self.model([pos_batch, neg_batch]) + + cmp = output[:, 0] > output[:, 1] + acc += sum(cmp.data.cpu().numpy()) + tot += true_batch_size + + loss = self.loss(output[:, 0], output[:, 1], self.margin_label) + if len(new_near_score) != 0 and self.weighting: + # Element wise weighting hasn't been implemented for MarginRankingLoss by pytorch + # Apply the mean score and set branch size to 1 as weighting implementation temporarily here + # Reference: https://github.com/pytorch/pytorch/issues/264 + # loss *= torch.autograd.variable.Variable(torch.from_numpy(np.array(new_near_score)).cuda()) + loss *= np.mean(new_near_score) + + loss_num = loss.data.cpu().numpy()[0] + total_loss += loss_num + loss.backward() + self.optimizer.step() + + del new_train_neg + del new_train_pos + + if self.iteration % self.dev_log_interval == 1 and epoch != 1: + dev_map, dev_mrr = self.evaluate(self.dev_evaluator, 'dev') + test_map, test_mrr = self.evaluate(self.test_evaluator, 'test') + + if self.use_tensorboard: + self.writer.add_scalar('{}/dev/map'.format(self.name), dev_map, self.dev_index) + self.writer.add_scalar('{}/dev/mrr'.format(self.name), dev_mrr, self.dev_index) + self.writer.add_scalar('{}/test/map'.format(self.name), test_map, self.dev_index) + self.writer.add_scalar('{}/test/mrr'.format(self.name), test_mrr, self.dev_index) + self.writer.add_scalar('{}/train/loss'.format(self.name), loss_num, self.dev_index) + self.writer.add_scalar('{}/lr'.format(self.train_loader.dataset.NAME), + self.optimizer.param_groups[0]['lr'], self.dev_index) + + self.dev_index += 1 + if self.best_dev_mrr < dev_mrr: + torch.save(self.model, self.model_outfile) + self.best_dev_mrr = dev_mrr + self.best_dev_map = dev_map + + if self.iteration % self.log_interval == 1 and epoch != 1: + # logger.info progress message + self.logger.info(self.log_template.format(epoch, min(batch_idx * self.batch_size, len(batch.dataset.examples)), + len(batch.dataset.examples), loss_num, acc / tot)) + + return total_loss + + def train(self, epochs): + + scheduler = ReduceLROnPlateau(self.optimizer, mode='max', factor=self.lr_reduce_factor, patience=self.patience) + epoch_times = [] + self.start = time.time() + for epoch in range(1, epochs + 1): + start = time.time() + train_loss = self.train_epoch(epoch) + + end = time.time() + duration = end - start + self.logger.info('Epoch {} finished in {:.2f} minutes'.format(epoch, duration / 60)) + epoch_times.append(duration) + + scheduler.step(train_loss) + + self.logger.info('Training took {:.2f} minutes overall...'.format(sum(epoch_times) / 60)) diff --git a/nce/nce_pairwise_mp/trainers/trecqa_trainer.py b/nce/nce_pairwise_mp/trainers/trecqa_trainer.py new file mode 100644 index 0000000..0c8b808 --- /dev/null +++ b/nce/nce_pairwise_mp/trainers/trecqa_trainer.py @@ -0,0 +1,7 @@ +from nce.nce_pairwise_mp.trainers.qa_trainer import QATrainer + + +class TRECQATrainerNCE(QATrainer): + + def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): + super(TRECQATrainerNCE, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) diff --git a/nce/nce_pairwise_mp/trainers/wikiqa_trainer.py b/nce/nce_pairwise_mp/trainers/wikiqa_trainer.py new file mode 100644 index 0000000..75bb2a2 --- /dev/null +++ b/nce/nce_pairwise_mp/trainers/wikiqa_trainer.py @@ -0,0 +1,7 @@ +from nce.nce_pairwise_mp.trainers.qa_trainer import QATrainer + + +class WikiQATrainerNCE(QATrainer): + + def __init__(self, model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator=None): + super(WikiQATrainerNCE, self).__init__(model, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) diff --git a/utils/nce_neighbors.py b/utils/nce_neighbors.py new file mode 100644 index 0000000..a1bf083 --- /dev/null +++ b/utils/nce_neighbors.py @@ -0,0 +1,51 @@ +import random +import numpy as np +import heapq +import operator + +import torch +from torchtext import data + +# get the nearest negative samples to the positive sample by computing the feature difference +def get_nearest_neg_id(pos_feature, neg_dict, distance="cosine", k=1, weight=False): + dis_list = [] + pos_feature = pos_feature.data.cpu().numpy() + pos_feature_norm = pos_feature / np.sqrt(sum(pos_feature ** 2)) + neg_list = [] + for key in neg_dict: + if distance == "l2": + dis = np.sqrt(np.sum((np.array(pos_feature) - neg_dict[key]["feature"]) ** 2)) + elif distance == "cosine": + neg_feature = np.array(neg_dict[key]["feature"]) + feat_norm = neg_feature / np.sqrt(sum(neg_feature ** 2)) + dis = 1 - feat_norm.dot(pos_feature_norm) + dis_list.append(dis) + neg_list.append(key) + + k = min(k, len(neg_dict)) + min_list = heapq.nsmallest(k, enumerate(dis_list), key=operator.itemgetter(1)) + # find the corresponding neg id + min_id_list = [neg_list[x[0]] for x in min_list] + if weight: + min_id_score = [1 - x[1] for x in min_list] + return min_id_list, min_id_score + else: + return min_id_list + +# get the negative samples randomly +def get_random_neg_id(q2neg, qid_i, k=8): + # question 1734 in TrecQA has only one positive answer and no negative answer + if qid_i not in q2neg: + return [] + k = min(k, len(q2neg[qid_i])) + ran = random.sample(q2neg[qid_i], k) + return ran + +# pack the lists of question/answer/ext_feat into a torchtext batch +def get_batch(question, answer, ext_feat, size): + new_batch = data.Batch() + new_batch.batch_size = size + setattr(new_batch, "sentence_2", torch.stack(answer)) + setattr(new_batch, "sentence_1", torch.stack(question)) + setattr(new_batch, "ext_feats", torch.stack(ext_feat)) + return new_batch \ No newline at end of file