diff --git a/.gitignore b/.gitignore index caed9fa..007c4f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_Store .idea/ *idfsim +*.swp diff --git a/sm_cnn/README.md b/sm_cnn/README.md index 99c7e38..49b69e0 100644 --- a/sm_cnn/README.md +++ b/sm_cnn/README.md @@ -1,4 +1,4 @@ -## SM model +## SM model #### References: 1. Aliaksei _S_everyn and Alessandro _M_oschitti. 2015. Learning to Rank Short Text Pairs with Convolutional Deep Neural Networks. In Proceedings of the 38th International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR '15). ACM, New York, NY, USA, 373-382. DOI: http://dx.doi.org/10.1145/2766462.2767738 @@ -20,9 +20,9 @@ Please install the requirements. See [Castor/README.md](../README.md) for pytorc ``` mkdir castorini cd castorini -git clone https://github.com/castorini/data.git -git clone https://github.com/castorini/models.git -git clone https://github.com/castorini/Castor.git +git clone https://github.com/castorini/data.git +git clone https://github.com/castorini/models.git +git clone https://github.com/castorini/Castor.git ``` This should generate: @@ -67,11 +67,13 @@ cd .. To train the S&M model on TrecQA ``` -python main.py ../../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor --paper-ext-features +python main.py ../../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor --paper-ext-feats ``` +**To use the GPU, add `--cuda`.** + The final model will be saved to ```../../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor``` -_NOTE:_ On first run, the program will create a memory-mapped cache for word e mbeddings (943MB) in ``data/word2vec``. +_NOTE:_ On first run, the program will create a memory-mapped cache for word e mbeddings (943MB) in ``data/word2vec``. The cache allows for faster loading of data in future runs. Run ```python main.py -h``` for more default options. diff --git a/sm_cnn/main.py b/sm_cnn/main.py index 39fa302..ec2c4c0 100644 --- a/sm_cnn/main.py +++ b/sm_cnn/main.py @@ -91,7 +91,7 @@ if __name__ == "__main__": ap.add_argument('--paper-ext-feats-stem', action="store_true", \ help="external features as per the paper") # system arguments - # TODO: add arguments for CUDA + ap.add_argument('--cuda', action='store_true', help='use CUDA if available') ap.add_argument('--num_threads', help="the number of simultaneous processes to run", \ type=int, default=4) # training arguments @@ -136,10 +136,10 @@ if __name__ == "__main__": vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file) # instantiate model - net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats) + net = QAModel(vec_dim, args.filter_width, args.num_conv_filters, args.no_ext_feats, cuda=args.cuda) # initialize the trainer - trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim) + trainer = Trainer(net, args.eta, args.mom, args.no_loss_reg, vec_dim, args.cuda) logger.info("Loading input data...") # load input data trainer.load_input_data(args.dataset_folder, cache_file, train_set, dev_set, test_set) @@ -156,7 +156,7 @@ if __name__ == "__main__": ext_feats_for_splits = \ set_external_features_as_per_paper_and_stem(trainer, args.index_for_corpusIDF) - + if not args.skip_training: best_map = 0.0 best_model = 0 @@ -192,7 +192,7 @@ if __name__ == "__main__": logger.info('Best dev MAP in training phase = {:.4f}'.format(best_map)) trained_model = QAModel.load(args.model_outfile) - evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim) + evaluator = Trainer(trained_model, args.eta, args.mom, args.no_loss_reg, vec_dim, args.cuda) for split in [test_set, dev_set]: evaluator.load_input_data(args.dataset_folder, cache_file, None, None, split) diff --git a/sm_cnn/model.py b/sm_cnn/model.py index 71ac9d9..3fe3755 100644 --- a/sm_cnn/model.py +++ b/sm_cnn/model.py @@ -27,7 +27,7 @@ class QAModel(nn.Module): def __init__(self, input_n_dim, filter_width, \ - conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2): + conv_filters=100, no_ext_feats=False, ext_feats_size=4, n_classes=2, cuda=False): super(QAModel, self).__init__() self.no_ext_feats = no_ext_feats @@ -53,6 +53,11 @@ class QAModel(nn.Module): self.hidden = nn.Linear(n_hidden, n_classes) self.logsoftmax = nn.LogSoftmax() + if cuda and torch.cuda.is_available(): + self.conv_q, self.conv_a = self.conv_q.cuda(), self.conv_a.cuda() + self.combined_feature_vector = self.combined_feature_vector.cuda() + self.combined_features_activation = self.combined_features_activation.cuda() + self.dropout, self.hidden, self.logsoftmax = self.dropout.cuda(), self.hidden.cuda(), self.logsoftmax.cuda() def forward(self, question, answer, ext_feats): diff --git a/sm_cnn/train.py b/sm_cnn/train.py index 62da27c..4af588e 100644 --- a/sm_cnn/train.py +++ b/sm_cnn/train.py @@ -22,11 +22,12 @@ logger.addHandler(ch) class Trainer(object): - def __init__(self, model, eta, mom, no_loss_reg, vec_dim): + def __init__(self, model, eta, mom, no_loss_reg, vec_dim, cuda=False): # set the random seeds for every instance of trainer. # needed to ensure reproduction of random word vectors for out of vocab terms torch.manual_seed(1234) np.random.seed(1234) + self.cuda = cuda self.unk_term = np.random.uniform(-0.25, 0.25, vec_dim) self.reg = 1e-5 @@ -56,7 +57,7 @@ class Trainer(object): utils.load_cached_embeddings(word_vectors_cache_file, vocab, self.embeddings, [] if "train" in set_folder else self.unk_term) - + def regularize_loss(self, loss): @@ -141,7 +142,7 @@ class Trainer(object): labels[batch_start:batch_end], ext_feats[batch_start:batch_end], word_vectors, vec_dim - ) + ) xq, xa, x_ext_feats = batch_inputs[0] y = batch_labels[0] @@ -189,7 +190,7 @@ class Trainer(object): labels[batch_start:batch_end], ext_feats[batch_start:batch_end], word_vectors, vec_dim - ) + ) xq, xa, x_ext_feats = batch_inputs[0] @@ -200,7 +201,7 @@ class Trainer(object): # logger.debug('batch_loss {}, batch_correct {}'.format(batch_loss, batch_correct)) train_loss += batch_loss # train_correct += batch_correct - if debug_single_batch: + if debug_single_batch: break # logger.info('train_correct {}'.format(train_correct)) @@ -225,6 +226,8 @@ class Trainer(object): input_tensor = torch.zeros(1, vec_dim, len(terms)) input_tensor[0] = torch.transpose(word_embeddings, 0, 1) + if self.cuda and torch.cuda.is_available(): + input_tensor = input_tensor.cuda() return input_tensor @@ -240,12 +243,17 @@ class Trainer(object): # - would zero endings work for other smaller sentences? y = torch.LongTensor(batch_size).type(torch.LongTensor) + if self.cuda and torch.cuda.is_available(): + y = y.cuda() tensorized_inputs = [] for i in range(len(batch_ques)): xq = Variable(self.make_input_matrix(batch_ques[i], word_vectors, vec_dim)) xs = Variable(self.make_input_matrix(batch_sents[i], word_vectors, vec_dim)) - ext_feats = Variable(torch.FloatTensor(batch_ext_feats[i])) + ext_feats = torch.FloatTensor(batch_ext_feats[i]) + if self.cuda and torch.cuda.is_available(): + ext_feats = ext_feats.cuda() + ext_feats = Variable(ext_feats) ext_feats = torch.unsqueeze(ext_feats, 0) y[i] = batch_labels[i] tensorized_inputs.append((xq, xs, ext_feats)) diff --git a/sm_cnn/utils.py b/sm_cnn/utils.py index 1fb1863..41d6f59 100644 --- a/sm_cnn/utils.py +++ b/sm_cnn/utils.py @@ -1,11 +1,12 @@ # file input output - import os -import sys +import re +import string -import numpy as np -import torch from gensim.models.keyedvectors import KeyedVectors +from nltk.corpus import stopwords +from nltk.stem import PorterStemmer +import numpy as np # logging setup import logging @@ -155,7 +156,7 @@ def read_in_dataset(dataset_folder, set_folder, stop_punct=False, dash_split=Fal for term in sentence.split(): vocab_set.add(term) vocab = list(vocab_set) - + return [questions, sentences, labels, max(len_q_list), max(len_s_list), vocab]