diff --git a/datasets/castor_dataset.py b/datasets/castor_dataset.py index 1d83b32..a02cd0f 100644 --- a/datasets/castor_dataset.py +++ b/datasets/castor_dataset.py @@ -1,5 +1,6 @@ from abc import ABCMeta, abstractmethod import os +import numpy as np from torchtext.data.dataset import Dataset from torchtext.data.example import Example @@ -17,13 +18,15 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta): TEXT_FIELD = None EXT_FEATS_FIELD = None LABEL_FIELD = None + AID_FIELD = None @abstractmethod - def __init__(self, path): + def __init__(self, path, load_ext_feats=False): """ Create a Castor dataset involving pairs of texts """ - fields = [('id', self.ID_FIELD), ('sentence_1', self.TEXT_FIELD), ('sentence_2', self.TEXT_FIELD), ('ext_feats', self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD)] + fields = [('id', self.ID_FIELD), ('sentence_1', self.TEXT_FIELD), ('sentence_2', self.TEXT_FIELD), ('ext_feats', + self.EXT_FEATS_FIELD), ('label', self.LABEL_FIELD), ('aid', self.AID_FIELD)] examples = [] with open(os.path.join(path, 'a.toks'), 'r') as f1, open(os.path.join(path, 'b.toks'), 'r') as f2: @@ -31,13 +34,18 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta): sent_list_2 = [l.rstrip('.\n').split(' ') for l in f2] word_to_doc_cnt = get_pairwise_word_to_doc_freq(sent_list_1, sent_list_2) - overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt) + + if not load_ext_feats: + overlap_feats = get_pairwise_overlap_features(sent_list_1, sent_list_2, word_to_doc_cnt) + else: + overlap_feats = np.loadtxt(os.path.join(path, 'overlap_feats.txt')) with open(os.path.join(path, 'id.txt'), 'r') as id_file, open(os.path.join(path, 'sim.txt'), 'r') as label_file: - for pair_id, l1, l2, ext_feats, label in zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file): + for i, (pair_id, l1, l2, ext_feats, label) in enumerate(zip(id_file, sent_list_1, sent_list_2, overlap_feats, label_file)): pair_id = pair_id.rstrip('.\n') label = label.rstrip('.\n') - example = Example.fromlist([pair_id, l1, l2, ext_feats, label], fields) + example_list = [pair_id, l1, l2, ext_feats, label, i + 1] + example = Example.fromlist(example_list, fields) examples.append(example) super(CastorPairDataset, self).__init__(examples, fields) diff --git a/datasets/trecqa.py b/datasets/trecqa.py index e7d0627..e7984a5 100644 --- a/datasets/trecqa.py +++ b/datasets/trecqa.py @@ -1,10 +1,11 @@ import os import torch -from torchtext.data.example import Example from torchtext.data.field import Field from torchtext.data.iterator import BucketIterator +from torchtext.data.iterator import Iterator from torchtext.vocab import Vectors +from torchtext.data import Pipeline from datasets.castor_dataset import CastorPairDataset from datasets.idf_utils import get_pairwise_word_to_doc_freq, get_pairwise_overlap_features @@ -14,9 +15,12 @@ class TRECQA(CastorPairDataset): NAME = 'trecqa' NUM_CLASSES = 2 ID_FIELD = Field(sequential=False, tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True) + AID_FIELD = Field(sequential=False, use_vocab=False, batch_first=True) TEXT_FIELD = Field(batch_first=True, tokenize=lambda x: x) # tokenizer is identity since we already tokenized it to compute external features - EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x) + EXT_FEATS_FIELD = Field(tensor_type=torch.FloatTensor, use_vocab=False, batch_first=True, tokenize=lambda x: x, + postprocessing=Pipeline(lambda arr, _, train: [float(y) for y in arr])) LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True) + VOCAB_SIZE = 0 @staticmethod def sort_key(ex): @@ -26,29 +30,54 @@ class TRECQA(CastorPairDataset): """ Create a TRECQA dataset instance """ - super(TRECQA, self).__init__(path) + super(TRECQA, self).__init__(path, load_ext_feats=True) @classmethod def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs): return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs) @classmethod - def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, unk_init=torch.Tensor.zero_): + def set_vectors(cls, field, vector_path): + if os.path.isfile(vector_path): + stoi, vectors, dim = torch.load(vector_path) + field.vocab.vectors = torch.Tensor(len(field.vocab), dim) + + for i, token in enumerate(field.vocab.itos): + wv_index = stoi.get(token, None) + if wv_index is not None: + field.vocab.vectors[i] = vectors[wv_index] + else: + # initialize with uniform_(-0.05, 0.05) vectors + field.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.05, 0.05) + else: + print("Error: Need word embedding pt file") + exit(1) + return field + + @classmethod + def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file = False, vectors=None, unk_init=torch.Tensor.zero_): """ :param path: directory containing train, test, dev files :param vectors_name: name of word vectors file - :param vectors_cache: directory containing word vectors file + :param vectors_dir: directory containing word vectors file :param batch_size: batch size :param device: GPU device :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes :param unk_init: function used to generate vector for OOV words :return: """ - if vectors is None: - vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init) train, validation, test = cls.splits(path) + if not pt_file: + if vectors is None: + vectors = Vectors(name=vectors_name, cache=vectors_dir, unk_init=unk_init) + cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors) + else: + cls.TEXT_FIELD.build_vocab(train, validation, test) + cls.TEXT_FIELD = cls.set_vectors(cls.TEXT_FIELD, os.path.join(vectors_dir, vectors_name)) - cls.TEXT_FIELD.build_vocab(train, validation, test, vectors=vectors) + cls.LABEL_FIELD.build_vocab(train, validation, test) + + cls.VOCAB_SIZE = len(cls.TEXT_FIELD.vocab) return BucketIterator.splits((train, validation, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device) diff --git a/nce/NCE-Pairwise-SM/README.md b/nce/NCE-Pairwise-SM/README.md new file mode 100644 index 0000000..5cd66fc --- /dev/null +++ b/nce/NCE-Pairwise-SM/README.md @@ -0,0 +1,73 @@ +## NCE-SM model + +#### References: ++ Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural +Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information +Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 + ++ Jinfeng Rao, Hua He, and Jimmy Lin. [Noise-Contrastive Estimation for Answer Selection with Deep Neural Networks.](http://dl.acm.org/citation.cfm?id=2983872) *Proceedings of the 25th ACM International on Conference on Information and Knowledge Management (CIKM 2016)*, pages 1913-1916. + + +The code uses torchtext for text processing. Set torchtext: +```bash +git clone https://github.com/pytorch/text.git +cd text +python setup.py install +``` + +Download the word2vec model from [here] (https://drive.google.com/file/d/0B2u_nClt6NbzUmhOZU55eEo4QWM/view?usp=sharing) +and copy it to the `Castor/data/word2vec` folder. + +### Training the model + +You can train the SM model for the 4 following configurations: +1. __random__ - the word embedddings are initialized randomly and are tuned during training +2. __static__ - the word embeddings are static (Severyn and Moschitti, SIGIR'15) +3. __non-static__ - the word embeddings are tuned during training +4. __multichannel__ - contains static and non-static channels for question and answer conv layers + + +```bash +python train.py --no_cuda --mode rand --batch_size 64 --neg_num 8 --dev_every 50 --patience 1000 +``` + +NB: pass `--no_cuda` to use CPU + +The trained model will be save to: +``` +saves/static_best_model.pt +``` + +### Testing the model + +``` +python main.py --trained_model saves/TREC/multichannel_best_model.pt --batch_size 64 --no_cuda +``` + +### Evaluation + +#### The performance on TrecQA dataset: + +##### Without NCE + +Metric |rand |static|non-static|multichannel +-------|-------|------|----------|------------ +MAP |0.7441 |0.7524|0.7688 |0.7641 +MRR |0.8172 |0.8012|0.8144 |0.8174 + +##### Max Neg Sample + +To be added + +##### Pairwise + Max Neg Sample with neg_num = 8 + +Metric |rand |static|non-static|multichannel +-------|-------|------|----------|------------ +MAP |0.7427 |0.7546|0.7716 |0.7794 +MRR |0.8151 |0.8061|0.8347 |0.8467 + + +#### The performance on WikiQA dataset: + +To be added + diff --git a/nce/NCE-Pairwise-SM/args.py b/nce/NCE-Pairwise-SM/args.py new file mode 100644 index 0000000..4460b73 --- /dev/null +++ b/nce/NCE-Pairwise-SM/args.py @@ -0,0 +1,34 @@ +from argparse import ArgumentParser + +def get_args(): + parser = ArgumentParser(description="SM CNN") + parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') + parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU + parser.add_argument('--epochs', type=int, default=30) + parser.add_argument('--batch_size', type=int, default=64) + parser.add_argument('--mode', type=str, default='static') + parser.add_argument('--lr', type=float, default=0.95) + parser.add_argument('--seed', type=int, default=3435) + parser.add_argument('--dataset', type=str, default='TREC') + parser.add_argument('--resume_snapshot', type=str, default=None) + parser.add_argument('--dev_every', type=int, default=100) + parser.add_argument('--log_every', type=int, default=10) + parser.add_argument('--patience', type=int, default=50) + parser.add_argument('--save_path', type=str, default='saves') + parser.add_argument('--output_channel', type=int, default=150) + parser.add_argument('--filter_width', type=int, default=5) + parser.add_argument('--words_dim', type=int, default=50) + parser.add_argument('--dropout', type=float, default=0.5) + parser.add_argument('--epoch_decay', type=int, default=15) + parser.add_argument('--wordvec_dir', type=str, default='../../data/word2vec/') + parser.add_argument('--vector_cache', type=str, default='word2vec.trecqa.pt') + parser.add_argument('--trained_model', type=str, default="") + parser.add_argument('--weight_decay',type=float, default=1e-5) + parser.add_argument('--ext_feats_size', type=int, default=4) + parser.add_argument('--neg_num', type=int, default=5) + parser.add_argument('--neg_sample', type=str, default="random") + parser.add_argument('--eps', type=float, default=1e-6) + parser.add_argument('--optimizer', type=str, default="adadelta") + + args = parser.parse_args() + return args diff --git a/nce/NCE-Pairwise-SM/main.py b/nce/NCE-Pairwise-SM/main.py new file mode 100644 index 0000000..a43435f --- /dev/null +++ b/nce/NCE-Pairwise-SM/main.py @@ -0,0 +1,81 @@ +import numpy as np +import random +import logging +import os + +import torch +from torchtext import data + +from args import get_args +from trec_dataset import TrecDataset +from utils.relevancy_metrics import get_map_mrr +from datasets.trecqa import TRECQA +from train import UnknownWordVecCache + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(levelname)s - %(message)s') +ch.setFormatter(formatter) +logger.addHandler(ch) + +args = get_args() +config = args + +torch.manual_seed(args.seed) +np.random.seed(args.seed) +random.seed(args.seed) + +if not args.cuda: + args.gpu = -1 +if torch.cuda.is_available() and args.cuda: + logger.info("Note: You are using GPU for training") + torch.cuda.set_device(args.gpu) + torch.cuda.manual_seed(args.seed) +if torch.cuda.is_available() and not args.cuda: + logger.info("Warning: You have Cuda but do not use it. You are using CPU for training") + + +if config.dataset == 'TREC': + dataset_root = os.path.join(os.pardir, 'data', 'TrecQA/') + train_iter, dev_iter, test_iter = TRECQA.iters(dataset_root, args.vector_cache, args.wordvec_dir, batch_size=args.batch_size, pt_file=True, device=args.gpu, unk_init=UnknownWordVecCache.unk) +else: + logger.info("Unsupported dataset") + exit() + +config.target_class = 2 +config.questions_num = len(TRECQA.TEXT_FIELD.vocab) +config.answers_num = len(TRECQA.TEXT_FIELD.vocab) + +if args.cuda: + model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu)) +else: + model = torch.load(args.trained_model, map_location=lambda storage,location: storage) + + +def predict(test_mode, dataset_iter): + model.eval() + dataset_iter.init_epoch() + qids = [] + predictions = [] + labels = [] + for dev_batch_idx, dev_batch in enumerate(dataset_iter): + qid_array = np.transpose(dev_batch.id.cpu().data.numpy()) + true_label_array = np.transpose(dev_batch.label.cpu().data.numpy()) + output = model.convModel(dev_batch) + scores = model.linearLayer(output) + score_array = scores.cpu().data.numpy().reshape(-1) + qids.extend(qid_array.tolist()) + predictions.extend(score_array.tolist()) + labels.extend(true_label_array.tolist()) + + dev_map, dev_mrr = get_map_mrr(qids, predictions, labels) + + logger.info("{} {}".format(dev_map, dev_mrr)) + +# Run the model on the dev set +predict('dev', dataset_iter=dev_iter) + +# Run the model on the test set +predict('test', dataset_iter=test_iter) diff --git a/nce/NCE-Pairwise-SM/model.py b/nce/NCE-Pairwise-SM/model.py new file mode 100644 index 0000000..2c9738c --- /dev/null +++ b/nce/NCE-Pairwise-SM/model.py @@ -0,0 +1,103 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class PairwiseConv(nn.Module): + """docstring for PairwiseConv""" + def __init__(self, model): + super(PairwiseConv, self).__init__() + self.convModel = model + self.dropout = nn.Dropout(self.convModel.dropout) + self.linearLayer = nn.Linear(model.n_hidden, 1) + self.posModel = self.convModel + # share or copy ?? + # https://discuss.pytorch.org/t/copying-nn-modules-without-shared-memory/113 + # self.negModel = copy.deepcopy(self.posModel) + self.negModel = self.convModel + + def forward(self, input): + pos = self.posModel(input[0]) + neg = self.negModel(input[1]) + pos = self.dropout(pos) + neg = self.dropout(neg) + pos = self.linearLayer(pos) + neg = self.linearLayer(neg) + combine = torch.cat([pos, neg], 1) + return combine + +class SmPlusPlus(nn.Module): + def __init__(self, config): + super(SmPlusPlus, self).__init__() + output_channel = config.output_channel + questions_num = config.questions_num + answers_num = config.answers_num + words_dim = config.words_dim + filter_width = config.filter_width + self.mode = config.mode + self.dropout = config.dropout + + n_classes = config.target_class + ext_feats_size = config.ext_feats_size + + if self.mode == 'multichannel': + input_channel = 2 + else: + input_channel = 1 + + self.question_embed = nn.Embedding(questions_num, words_dim) + self.answer_embed = nn.Embedding(answers_num, words_dim) + self.static_question_embed = nn.Embedding(questions_num, words_dim) + self.nonstatic_question_embed = nn.Embedding(questions_num, words_dim) + self.static_answer_embed = nn.Embedding(answers_num, words_dim) + self.nonstatic_answer_embed = nn.Embedding(answers_num, words_dim) + self.static_question_embed.weight.requires_grad = False + self.static_answer_embed.weight.requires_grad = False + + self.conv_q = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0)) + self.conv_a = nn.Conv2d(input_channel, output_channel, (filter_width, words_dim), padding=(filter_width - 1, 0)) + + self.n_hidden = 2 * output_channel + ext_feats_size + + self.combined_feature_vector = nn.Linear(self.n_hidden, self.n_hidden) + self.hidden = nn.Linear(self.n_hidden, n_classes) + + def forward(self, x): + x_question = x.sentence_1 + x_answer = x.sentence_2 + x_ext = x.ext_feats + + if self.mode == 'rand': + question = self.question_embed(x_question).unsqueeze(1) + answer = self.answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + # actual SM model mode (Severyn & Moschitti, 2015) + elif self.mode == 'static': + question = self.static_question_embed(x_question).unsqueeze(1) + answer = self.static_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + elif self.mode == 'non-static': + question = self.nonstatic_question_embed(x_question).unsqueeze(1) + answer = self.nonstatic_answer_embed(x_answer).unsqueeze(1) # (batch, sent_len, embed_dim) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + elif self.mode == 'multichannel': + question_static = self.static_question_embed(x_question) + answer_static = self.static_answer_embed(x_answer) + question_nonstatic = self.nonstatic_question_embed(x_question) + answer_nonstatic = self.nonstatic_answer_embed(x_answer) + question = torch.stack([question_static, question_nonstatic], dim=1) + answer = torch.stack([answer_static, answer_nonstatic], dim=1) + x = [F.tanh(self.conv_q(question)).squeeze(3), F.tanh(self.conv_a(answer)).squeeze(3)] + x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling + else: + print("Unsupported Mode") + exit() + + x.append(x_ext) + x = torch.cat(x, 1) + x = F.tanh(self.combined_feature_vector(x)) + + return x \ No newline at end of file diff --git a/nce/NCE-Pairwise-SM/overlap_features.py b/nce/NCE-Pairwise-SM/overlap_features.py new file mode 100644 index 0000000..9996bac --- /dev/null +++ b/nce/NCE-Pairwise-SM/overlap_features.py @@ -0,0 +1,154 @@ +import numpy as np +import string +import pickle +from collections import defaultdict +from argparse import ArgumentParser + +from nltk.stem.porter import PorterStemmer + +def load_data(dname): + stemmer = PorterStemmer() + qids, questions, answers, labels = [], [], [], [] + print('Load folder ' + dname) + with open(dname+'a.toks', encoding='utf-8') as f: + for line in f: + question = line.strip().split() + question = [stemmer.stem(word) for word in question] + questions.append(question) + with open(dname+'b.toks', encoding='utf-8') as f: + for line in f: + answer = line.strip().split() + answer_list = [] + for word in answer: + try: + answer_list.append(stemmer.stem(word)) + except Exception as e: + print("couldn't stem the word:" + word) + answers.append(answer_list) + with open(dname + 'id.txt', encoding='utf-8') as f: + for line in f: + qids.append(line.strip()) + with open(dname + 'sim.txt', encoding='utf-8') as f: + for line in f: + labels.append(int(line.strip())) + return qids, questions, answers, labels + +def compute_overlap_features(questions, answers, word2df=None, stoplist=None): + word2df = word2df if word2df else {} + stoplist = stoplist if stoplist else set() + feats_overlap = [] + for question, answer in zip(questions, answers): + q_set = set([q for q in question if q not in stoplist]) + a_set = set([a for a in answer if a not in stoplist]) + word_overlap = q_set.intersection(a_set) + if len(q_set) == 0 and len(a_set) == 0: + overlap = 0 + else: + overlap = float(len(word_overlap)) / (len(q_set) + len(a_set)) + + word_overlap = q_set.intersection(a_set) + df_overlap = 0.0 + for w in word_overlap: + df_overlap += word2df[w] + + if len(q_set) == 0 and len(a_set) == 0: + df_overlap = 0 + else: + df_overlap /= (len(q_set) + len(a_set)) + + feats_overlap.append(np.array([overlap, df_overlap])) + return np.array(feats_overlap) + +def compute_overlap_idx(questions, answers, stoplist, q_max_sent_length, a_max_sent_length): + stoplist = stoplist if stoplist else [] + q_indices, a_indices = [], [] + for question, answer in zip(questions, answers): + q_set = set([q for q in question if q not in stoplist]) + a_set = set([a for a in answer if a not in stoplist]) + word_overlap = q_set.intersection(a_set) + + q_idx = np.ones(q_max_sent_length) * 2 + for i, q in enumerate(question): + value = 0 + if q in word_overlap: + value = 1 + q_idx[i] = value + q_indices.append(q_idx) + + a_idx = np.ones(a_max_sent_length) * 2 + for i, a in enumerate(answer): + value = 0 + if a in word_overlap: + value = 1 + a_idx[i] = value + a_indices.append(a_idx) + + q_indices = np.vstack(q_indices).astype('int32') + a_indices = np.vstack(a_indices).astype('int32') + + return q_indices, a_indices + +def compute_dfs(docs): + word2df = defaultdict(float) + for doc in docs: + for w in set(doc): + word2df[w] += 1.0 + num_docs = len(docs) + + for w, value in word2df.items(): + word2df[w] = np.math.log(num_docs / value) # bug feats fixed + + return word2df + +if __name__ == '__main__': + parser = ArgumentParser(description='create TrecQA/WikiQA dataset') + parser.add_argument('--dir', help='path to the TrecQA|WikiQA data directory', default="../../data/TrecQA") + args = parser.parse_args() + + stoplist = set([line.strip() for line in open('../../data/TrecQA/stopwords.txt', encoding='utf-8')]) + punct = set(string.punctuation) + stoplist.update(punct) + + all_questions, all_answers, all_qids = [], [], [] + base_dir = args.dir + + if 'TrecQA' in base_dir: + sub_dirs = ['train/', 'train-all/', 'raw-dev/', 'raw-test/'] + elif 'WikiQA' in base_dir: + sub_dirs = ['train/', 'dev/', 'test/'] + else: + print('Unsupported dataset') + exit() + + for sub in sub_dirs: + qids, questions, answers, labels = load_data(base_dir + sub) + all_questions.extend(questions) + all_answers.extend(answers) + all_qids.extend(qids) + + seen = set() + unique_questions = [] + for q, qid in zip(all_questions, all_qids): + if qid not in seen: + seen.add(qid) + unique_questions.append(q) + + docs = all_answers + unique_questions + word2dfs = compute_dfs(docs) + pickle.dump(word2dfs, open("word2dfs.p", "wb")) + + q_max_sent_length = max(map(lambda x: len(x), all_questions)) + a_max_sent_length = max(map(lambda x: len(x), all_answers)) + + for sub in sub_dirs: + qids, questions, answers, labels = load_data(base_dir + sub) + + overlap_feats = compute_overlap_features(questions, answers, stoplist=None, word2df=word2dfs) + overlap_feats_stoplist = compute_overlap_features(questions, answers, stoplist=stoplist, word2df=word2dfs) + overlap_feats = np.hstack([overlap_feats, overlap_feats_stoplist]) + + with open(base_dir + sub + 'overlap_feats.txt', 'w') as f: + for i in range(overlap_feats.shape[0]): + for j in range(4): + f.write(str(overlap_feats[i][j]) + ' ') + f.write('\n') diff --git a/nce/NCE-Pairwise-SM/requirements.txt b/nce/NCE-Pairwise-SM/requirements.txt new file mode 100644 index 0000000..706303b --- /dev/null +++ b/nce/NCE-Pairwise-SM/requirements.txt @@ -0,0 +1,4 @@ +nltk==3.2.4 +numpy==1.13.1 +pytorch==0.2.0+5f864ca +torchtext==0.2.0 diff --git a/nce/NCE-Pairwise-SM/train.py b/nce/NCE-Pairwise-SM/train.py new file mode 100644 index 0000000..a2d66a6 --- /dev/null +++ b/nce/NCE-Pairwise-SM/train.py @@ -0,0 +1,350 @@ +import time +import os +import numpy as np +import random +import heapq +import operator +import logging + +import torch +import torch.nn as nn +from torchtext import data +from torch.nn import functional as F + +from datasets.trecqa import TRECQA +from args import get_args +from model import SmPlusPlus, PairwiseConv +from utils.relevancy_metrics import get_map_mrr + + +class UnknownWordVecCache(object): + """ + Caches the first randomly generated word vector for a certain size to make it is reused. + """ + cache = {} + + @classmethod + def unk(cls, tensor): + size_tup = tuple(tensor.size()) + if size_tup not in cls.cache: + cls.cache[size_tup] = torch.Tensor(tensor.size()) + cls.cache[size_tup].uniform_(-0.05, 0.05) + return cls.cache[size_tup] + + +def train_sm(): + + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + args = get_args() + config = args + torch.backends.cudnn.deterministic = True + + # Set random seed for reproducibility + torch.manual_seed(args.seed) + if not args.cuda: + args.gpu = -1 + if torch.cuda.is_available() and args.cuda: + logger.info("Note: You are using GPU for training") + torch.cuda.set_device(args.gpu) + torch.cuda.manual_seed(args.seed) + if torch.cuda.is_available() and not args.cuda: + logger.info("You have Cuda but you're using CPU for training.") + np.random.seed(args.seed) + random.seed(args.seed) + + dataset_root = os.path.join(os.pardir, 'data', 'TrecQA/') + train_iter, dev_iter, test_iter = TRECQA.iters(dataset_root, args.vector_cache, args.wordvec_dir, batch_size=args.batch_size, + pt_file=True, device=args.gpu, unk_init=UnknownWordVecCache.unk) # + + index2text = np.array(TRECQA.TEXT_FIELD.vocab.itos) + + config.target_class = 2 + config.questions_num = TRECQA.VOCAB_SIZE + config.answers_num = TRECQA.VOCAB_SIZE + + logger.info("index2text: {}".format(index2text)) + logger.info("Dataset: {}, Mode: {}".format(args.dataset, args.mode)) + logger.info("VOCAB num: {}".format(TRECQA.VOCAB_SIZE)) + + if args.resume_snapshot: + if args.cuda: + pw_model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage.cuda(args.gpu)) + else: + pw_model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage) + else: + model = SmPlusPlus(config) + model.static_question_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors) + model.nonstatic_question_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors) + model.static_answer_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors) + model.nonstatic_answer_embed.weight.data.copy_(TRECQA.TEXT_FIELD.vocab.vectors) + + if args.cuda: + model.cuda() + logger.info("Shift model to GPU") + + pw_model = PairwiseConv(model) + + parameter = filter(lambda p: p.requires_grad, pw_model.parameters()) + + if args.optimizer == "adadelta": + # the SM model originally follows SGD but Adadelta is used here + optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay, eps=args.eps) + # A good lr is required to use in the following optimizer + elif args.optimizer == "adam": + optimizer = torch.optim.Adam(parameter, lr=args.lr, weight_decay=args.weight_decay, eps=1e-8) + elif args.optimizer == "sgd": + optimizer = torch.optim.SGD(parameter, lr=0.001, momentum=0.9, weight_decay=args.weight_decay) + elif args.optimizer == "rmsprop": + optimizer = torch.optim.RMSprop(parameter, lr=0.0001, weight_decay=args.weight_decay) + + marginRankingLoss = nn.MarginRankingLoss(margin=1, size_average=True) + + early_stop = False + iterations = 0 + iters_not_improved = 0 + epoch = 0 + q2neg = {} # a dict from qid to a list of aid + question2answer = {} # a dict from qid to the information of both pos and neg answers + best_dev_map = 0 + best_dev_mrr = 0 + false_samples = {} + + start = time.time() + header = ' Time Epoch Iteration Progress (%Epoch) Average_Loss Train_Accuracy Dev/MAP Dev/MRR' + dev_log_template = ' '.join( + '{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>11.6f},{:>11.6f},{:12.6f},{:8.4f}'.split(',')) + log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>11.6f},{:>11.6f},'.split(',')) + os.makedirs(args.save_path, exist_ok=True) + os.makedirs(os.path.join(args.save_path, args.dataset), exist_ok=True) + print(header) + + # get the nearest negative samples to the positive sample by computing the feature difference + def get_nearest_neg_id(pos_feature, neg_dict, distance="cosine", k=1): + dis_list = [] + pos_feature = pos_feature.data.cpu().numpy() + pos_feature_norm = pos_feature / np.sqrt(sum(pos_feature ** 2)) + neg_list = [] + for key in neg_dict: + if distance == "l2": + dis = np.sqrt(np.sum((np.array(pos_feature) - neg_dict[key]["feature"]) ** 2)) + elif distance == "cosine": + neg_feature = np.array(neg_dict[key]["feature"]) + feat_norm = neg_feature / np.sqrt(sum(neg_feature ** 2)) + dis = 1 - feat_norm.dot(pos_feature_norm) + dis_list.append(dis) + neg_list.append(key) + + k = min(k, len(neg_dict)) + min_list = heapq.nsmallest(k, enumerate(dis_list), key=operator.itemgetter(1)) + min_id_list = [neg_list[x[0]] for x in min_list] + return min_id_list + + # get the negative samples randomly + def get_random_neg_id(q2neg, qid_i, k=5): + # question 1734 has no neg answer + if qid_i not in q2neg: + return [] + k = min(k, len(q2neg[qid_i])) + ran = random.sample(q2neg[qid_i], k) + return ran + + # pack the lists of question/answer/ext_feat into a torchtext batch + def get_batch(question, answer, ext_feat, size): + new_batch = data.Batch() + new_batch.batch_size = size + new_batch.dataset = batch.dataset + setattr(new_batch, "sentence_2", torch.stack(answer)) + setattr(new_batch, "sentence_1", torch.stack(question)) + setattr(new_batch, "ext_feats", torch.stack(ext_feat)) + return new_batch + + while True: + if early_stop: + logger.log("Early Stopping. Epoch: {}, Best Dev Loss: {}".format(epoch, best_dev_loss)) + break + epoch += 1 + train_iter.init_epoch() + ''' + batch size issue: padding is a choice (add or delete them in both train and test) + associated with the batch size. Currently, it seems to affect the result a lot. + ''' + acc = 0 + tot = 0 + for batch_idx, batch in enumerate(iter(train_iter)): + if epoch != 1: + iterations += 1 + loss_num = 0 + pw_model.train() + + new_train = {"ext_feat": [], "question": [], "answer": [], "label": []} + features = pw_model.convModel(batch) + new_train_pos = {"answer": [], "question": [], "ext_feat": []} + new_train_neg = {"answer": [], "question": [], "ext_feat": []} + max_len_q = 0 + max_len_a = 0 + + batch_near_list = [] + batch_qid = [] + batch_aid = [] + + for i in range(batch.batch_size): + label_i = batch.label[i].cpu().data.numpy()[0] + question_i = batch.sentence_1[i] + # question_i = question_i[question_i!=1] # remove padding 1 + answer_i = batch.sentence_2[i] + # answer_i = answer_i[answer_i!=1] # remove padding 1 + ext_feat_i = batch.ext_feats[i] + qid_i = batch.id[i].data.cpu().numpy()[0] + aid_i = batch.aid[i].data.cpu().numpy()[0] + + if qid_i not in question2answer: + question2answer[qid_i] = {"question": question_i, "pos": {}, "neg": {}} + if label_i == 1: + + if aid_i not in question2answer[qid_i]["pos"]: + question2answer[qid_i]["pos"][aid_i] = {} + + question2answer[qid_i]["pos"][aid_i]["answer"] = answer_i + question2answer[qid_i]["pos"][aid_i]["ext_feat"] = ext_feat_i + + # get neg samples in the first epoch but do not train + if epoch == 1: + continue + # random generate sample in the first training epoch + elif epoch == 2 or args.neg_sample == "random": + near_list = get_random_neg_id(q2neg, qid_i, k=args.neg_num) + else: + debug_qid = qid_i + near_list = get_nearest_neg_id(features[i], question2answer[qid_i]["neg"], distance="cosine", k=args.neg_num) + + batch_near_list.extend(near_list) + + neg_size = len(near_list) + if neg_size != 0: + answer_i = answer_i[answer_i != 1] # remove padding 1 + question_i = question_i[question_i != 1] # remove padding 1 + for near_id in near_list: + batch_qid.append(qid_i) + batch_aid.append(aid_i) + + new_train_pos["answer"].append(answer_i) + new_train_pos["question"].append(question_i) + new_train_pos["ext_feat"].append(ext_feat_i) + + near_answer = question2answer[qid_i]["neg"][near_id]["answer"] + if question_i.size()[0] > max_len_q: + max_len_q = question_i.size()[0] + if near_answer.size()[0] > max_len_a: + max_len_a = near_answer.size()[0] + if answer_i.size()[0] > max_len_a: + max_len_a = answer_i.size()[0] + + ext_feat_neg = question2answer[qid_i]["neg"][near_id]["ext_feat"] + new_train_neg["answer"].append(near_answer) + new_train_neg["question"].append(question_i) + new_train_neg["ext_feat"].append(ext_feat_neg) + + elif label_i == 0: + + if aid_i not in question2answer[qid_i]["neg"]: + answer_i = answer_i[answer_i != 1] + question2answer[qid_i]["neg"][aid_i] = {"answer": answer_i} + + question2answer[qid_i]["neg"][aid_i]["feature"] = features[i].data.cpu().numpy() + question2answer[qid_i]["neg"][aid_i]["ext_feat"] = ext_feat_i + + if epoch == 1: + if qid_i not in q2neg: + q2neg[qid_i] = [] + + q2neg[qid_i].append(aid_i) + + # pack the selected pos and neg samples into the torchtext batch and train + if epoch != 1: + true_batch_size = len(new_train_neg["answer"]) + if true_batch_size != 0: + for j in range(true_batch_size): + new_train_neg["answer"][j] = F.pad(new_train_neg["answer"][j], + (0, max_len_a - new_train_neg["answer"][j].size()[0]), value=1) + new_train_pos["answer"][j] = F.pad(new_train_pos["answer"][j], + (0, max_len_a - new_train_pos["answer"][j].size()[0]), value=1) + new_train_pos["question"][j] = F.pad(new_train_pos["question"][j], + (0, max_len_q - new_train_pos["question"][j].size()[0]), value=1) + new_train_neg["question"][j] = F.pad(new_train_neg["question"][j], + (0, max_len_q - new_train_neg["question"][j].size()[0]), value=1) + + pos_batch = get_batch(new_train_pos["question"], new_train_pos["answer"], new_train_pos["ext_feat"], + true_batch_size) + neg_batch = get_batch(new_train_neg["question"], new_train_neg["answer"], new_train_neg["ext_feat"], + true_batch_size) + + optimizer.zero_grad() + output = pw_model([pos_batch, neg_batch]) + + cmp = output[:, 0] > output[:, 1] + acc += sum(cmp.data.cpu().numpy()) + tot += true_batch_size + + loss = marginRankingLoss(output[:, 0], output[:, 1], torch.autograd.Variable(torch.ones(1))) + loss_num = loss.data.numpy()[0] + loss.backward() + optimizer.step() + + # Evaluate performance on validation set + if iterations % args.dev_every == 1 and epoch != 1: + # switch model into evaluation mode + pw_model.eval() + dev_iter.init_epoch() + qids = [] + predictions = [] + labels = [] + + for dev_batch_idx, dev_batch in enumerate(dev_iter): + ''' + # dev singlely or in a batch? -> in a batch + but dev singlely is equal to dev_size = 1 + ''' + scores = pw_model.convModel(dev_batch) + scores = pw_model.linearLayer(scores) + qid_array = np.transpose(dev_batch.id.cpu().data.numpy()) + score_array = scores.cpu().data.numpy().reshape(-1) + true_label_array = np.transpose(dev_batch.label.cpu().data.numpy()) + + qids.extend(qid_array.tolist()) + predictions.extend(score_array.tolist()) + labels.extend(true_label_array.tolist()) + + dev_map, dev_mrr = get_map_mrr(qids, predictions, labels) + print(dev_log_template.format(time.time() - start, + epoch, iterations, 1 + batch_idx, len(train_iter), + 100. * (1 + batch_idx) / len(train_iter), + loss_num, acc / tot, dev_map, dev_mrr)) + if best_dev_mrr < dev_mrr: + snapshot_path = os.path.join(args.save_path, args.dataset, args.mode + '_best_model.pt') + torch.save(pw_model, snapshot_path) + iters_not_improved = 0 + best_dev_mrr = dev_mrr + else: + iters_not_improved += 1 + if iters_not_improved >= args.patience: + early_stop = True + break + + if iterations % args.log_every == 1 and epoch != 1: + # logger.info progress message + print(log_template.format(time.time() - start, + epoch, iterations, 1 + batch_idx, len(train_iter), + 100. * (1 + batch_idx) / len(train_iter), + loss_num, acc / tot)) + acc = 0 + tot = 0 + +if __name__ == '__main__': + train_sm() \ No newline at end of file