diff --git a/.gitignore b/.gitignore index 007c4f3..15b8746 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.ipynb_checkpoints +__pycache__ +*.pyc .DS_Store .idea/ *idfsim diff --git a/README.md b/README.md index d4e3975..94342ea 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # Castor -Pytorch deep learning models. +PyTorch deep learning models. 1. [SM model](./sm_cnn/): Similarity between question and candidate answers. -## Setting up Pytorch +## Setting up PyTorch You need Python 3.6 to use the models in this repository. -As per [pytorch.org](pytorch.org), +As per [pytorch.org](pytorch.org), > "[Anaconda](https://www.continuum.io/downloads) is our recommended package manager" ```conda install pytorch torchvision -c soumith``` @@ -21,7 +21,7 @@ We also recommend [gensim](https://radimrehurek.com/gensim/). We use some gensim ```conda install gensim``` -Pytorch has good support for GPU computations. +PyTorch has good support for GPU computations. CUDA installation guide for linux can be found [here](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/) **NOTE**: Install CUDA libraries **before** installing conda and pytorch. @@ -34,3 +34,7 @@ Sourcing and pre-processing of input data for each model is described in respect ## Baselines 1. [IDF Baseline](./idf_baseline/): IDF overlap between question and candidate answers. + +## Tutorials + +SM Model tutorial: [sm_cnn/tutorial.ipynb](sm_cnn/tutorial.ipynb) - notebook that walks through SM CNN model, good for beginnners. diff --git a/sm_cnn/.gitignore b/sm_cnn/.gitignore index 3c6f031..85d6be8 100644 --- a/sm_cnn/.gitignore +++ b/sm_cnn/.gitignore @@ -2,3 +2,4 @@ trec_eval-8.0/trec_eval data/ trained_models/ +trec_eval-8.0/trec_eval.dSYM diff --git a/sm_cnn/nn-architecture.png b/sm_cnn/nn-architecture.png new file mode 100644 index 0000000..3734c85 Binary files /dev/null and b/sm_cnn/nn-architecture.png differ diff --git a/sm_cnn/tutorial.ipynb b/sm_cnn/tutorial.ipynb new file mode 100644 index 0000000..488cfc4 --- /dev/null +++ b/sm_cnn/tutorial.ipynb @@ -0,0 +1,714 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# SM CNN Model PyTorch Walkthrough" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This purpose of this notebook is to explain how to use PyTorch to implement the SM CNN Model for new PyTorch users. Here are the recommended prerequisites before reading this walkthrough:\n", + "\n", + "* Have knowledge of Convolutional Neural Networks. If not these are helpful slides: https://cs.uwaterloo.ca/~mli/Deep-Learning-2017-Lecture5CNN.ppt.\n", + "* Read the SM Model paper: http://dl.acm.org/citation.cfm?id=2767738\n", + "\n", + "The following is a slightly modified version of the SM CNN architecture that will be implemented in this tutorial. It does not have the bilinear similarity modeling component present in the original model by Severyn and MoschiĀŠtti. The following paper found removing this component actually improved answer selection effectiveness:\n", + "\n", + "Jinfeng Rao, Hua He, and Jimmy Lin. Experiments with Convolutional Neural Network Models for Answer Selection. *Proceedings of the 40th Annual International ACM SIGIR Conference on Research and Development in Information Retrieval (SIGIR 2017)*, August 2017, Tokyo, Japan." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "![caption](files/nn-architecture.png)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the block below we define the model, with detailed explanations in comments. This model is slightly different from the model in model.py to keep the tutorial straightforward (e.g. ignore GPU code)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "class QAModel(nn.Module):\n", + " \"\"\"\n", + " All PyTorch models should subclass nn.Module, the base class for neural network modules.\n", + " \"\"\"\n", + "\n", + " def __init__(self, input_n_dim, filter_width, conv_filters=100,\n", + " no_ext_feats=False, ext_feats_size=4, n_classes=2):\n", + " \"\"\"\n", + " :param input_n_dim: the dimension of each word vector\n", + " :param filter_width: the width of each convolution filter\n", + " :param conv_filters: the number of convolution filters\n", + " :param no_ext_feats: no additional external features\n", + " :param ext_feats_size: number of external features to use\n", + " :param n_classes: number of label classes\n", + " \"\"\"\n", + " super(QAModel, self).__init__()\n", + "\n", + " self.no_ext_feats = no_ext_feats\n", + "\n", + " # self.conv_channels specify the dimension of the output of the convolution,\n", + " # i.e. the number of convolution feature maps\n", + " self.conv_channels = conv_filters\n", + " # the elements in the hidden layer consist of equal number of inputs from the query and document (hence the 2*)\n", + " # and optionally the additional features (ext_feats_size)\n", + " n_hidden = 2*self.conv_channels + (0 if no_ext_feats else ext_feats_size)\n", + "\n", + " # define the convolution for the question/query - 1D convolution followed by tanh nonlinear activation\n", + " # modules (nn.Conv1d and nn.Tanh) will be added in the order presented to the nn.Sequential container\n", + " self.conv_q = nn.Sequential(\n", + " # the first parameter specifies the input dimension, the second parameter specifies the output dimension\n", + " nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),\n", + " # tanh activation is used to allow the network to learn non-linear decision boundaries\n", + " nn.Tanh()\n", + " )\n", + "\n", + " # define the convolution for the answer/document\n", + " self.conv_a = nn.Sequential(\n", + " nn.Conv1d(input_n_dim, self.conv_channels, filter_width, padding=filter_width-1),\n", + " nn.Tanh()\n", + " )\n", + "\n", + " # combining the features from the question, answer, and external features if any into a single vector\n", + " # note PyTorch nn classes follow a similar signature - the first parameter specifies the input dimension,\n", + " # the second parameter specifies the output dimension\n", + " # nn.Linear applies a linear transformation: Ax + b, where A and b are learned parameters.\n", + " self.combined_feature_vector = nn.Linear(2*self.conv_channels + \\\n", + " (0 if no_ext_feats else ext_feats_size), n_hidden)\n", + "\n", + " # defining other layers used in the network, note they are not yet linked with each other yet\n", + " # tanh is a non-linear activation function\n", + " self.combined_features_activation = nn.Tanh()\n", + " # dropout is used to prevent overfitting and only used during training\n", + " # elements are randomly zeroed with probability 0.5 and all elements are scaled by a factor of 1/0.5 = 2\n", + " self.dropout = nn.Dropout(0.5)\n", + " # hidden layer is used to capture additional interactions between the components of the intermediate representation\n", + " self.hidden = nn.Linear(n_hidden, n_classes)\n", + " # softmax computes probability distributions\n", + " self.logsoftmax = nn.LogSoftmax()\n", + "\n", + "\n", + " def forward(self, question, answer, ext_feats):\n", + " \"\"\"\n", + " Defines the forward pass of the network. When the model is called, e.g. model(*args) the args\n", + " are actually passed to the forward method.\n", + " The question and answer tensors are 3-dimensional. The first dimension specifies the sentence - it\n", + " can be larger than 1 since multiple sentences can be batched together in one forward pass.\n", + " The second and third dimensions specify the dimension of the word vector and the number of tokens respectively.\n", + " \n", + " :param question: the sentence matrices of questions (queries). Note the plural form - this is explained above.\n", + " :param answer: the sentence matrices of answers (documents). Note the plural form - this is explained above.\n", + " :param ext_feats: the external features for the question-answer pairs.\n", + " :returns: the log-likelihood of the question-answer pairs belonging in each class.\n", + " \"\"\"\n", + " # feed the question sentence matrices through the conv_q layers.\n", + " # IMPORTANT: the second dimension of the question MUST match the the first argument\n", + " # the Conv1d instance created (input_n_dim). The first dimension of the question specifies\n", + " # the batch size (number of questions).\n", + " q = self.conv_q.forward(question)\n", + " # max pool using q.size()[2] as the window size, which is the length of each convolution feature map\n", + " q = F.max_pool1d(q, q.size()[2])\n", + " # reshape max pooled elements into a vector of length equal to the number of feature maps\n", + " # the max pooling takes one value (the max) out of each convolution feature map\n", + " q = q.view(-1, self.conv_channels)\n", + "\n", + " # feed the answer sentence matrices through the conv_a layers, similar to the previous part for the question.\n", + " a = self.conv_a.forward(answer)\n", + " a = F.max_pool1d(a, a.size()[2])\n", + " a = a.view(-1, self.conv_channels)\n", + "\n", + " # concatenate the outputs of the conv_q, conv_a layers together\n", + " # with optionally the ext_feats along the first dimension\n", + " x = None\n", + " if self.no_ext_feats:\n", + " x = torch.cat([q, a], 1)\n", + " else:\n", + " x = torch.cat([q, a, ext_feats], 1)\n", + "\n", + " # feed the concatenated feature vector through the rest of the network (starting with join layer in figure)\n", + " x = self.combined_feature_vector.forward(x)\n", + " x = self.combined_features_activation.forward(x)\n", + " x = self.dropout(x)\n", + " x = self.hidden(x)\n", + " x = self.logsoftmax(x)\n", + "\n", + " return x\n", + " \n", + " @staticmethod\n", + " def load(model_fname):\n", + " return torch.load(model_fname)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We now load a pre-trained model with one input and see what the model actually does. For this, you'll need to clone the `data` and `models` projects in https://github.com/castorini." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The cell below contains some bootstrapping code to prepare the data, load the model, etc.. It is not important to understand it just to see how the SM CNN model itself works." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING - WARNING: expecting a .gz file. Is the ../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin in the correct format?\n", + "/Users/michael/anaconda/lib/python3.6/site-packages/torch/serialization.py:284: SourceChangeWarning: source code of class 'model.QAModel' has changed. you can retrieve the original source code by accessing the object's source attribute or set `torch.nn.Module.dump_patches = True` and use the patch tool to revert the changes.\n", + " warnings.warn(msg, SourceChangeWarning)\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "\n", + "import numpy as np\n", + "\n", + "from train import Trainer\n", + "import utils\n", + "\n", + "torch.manual_seed(1234)\n", + "np.random.seed(1234)\n", + "\n", + "# cache word embeddings\n", + "word_vectors_file = '../../data/word2vec/aquaint+wiki.txt.gz.ndim=50.bin'\n", + "cache_file = os.path.splitext(word_vectors_file)[0] + '.cache'\n", + "utils.cache_word_embeddings(word_vectors_file, cache_file)\n", + "\n", + "vocab_size, vec_dim = utils.load_embedding_dimensions(cache_file)\n", + "\n", + "# loading a pre-trained model\n", + "trained_model = QAModel.load('../../models/sm_model/sm_model.TrecQA.TRAIN-ALL.2017-04-02.castor')\n", + "evaluator = Trainer(trained_model, 0.001, 0.0, False, vec_dim)\n", + "\n", + "evaluator.load_input_data('../../data/TrecQA', cache_file, None, None, 'raw-dev')\n", + "\n", + "questions, sentences, labels, maxlen_q, maxlen_s, ext_feats = evaluator.data_splits['raw-dev']\n", + "word_vectors = evaluator.embeddings\n", + "pair_idx = 100 # particular question/answer pair we are interested in\n", + "batch_inputs, batch_labels = evaluator.get_tensorized_inputs(\n", + " questions[pair_idx:pair_idx + 1],\n", + " sentences[pair_idx:pair_idx + 1],\n", + " labels[pair_idx:pair_idx + 1],\n", + " ext_feats[pair_idx:pair_idx + 1],\n", + " word_vectors, vec_dim\n", + ")\n", + "\n", + "xq, xa, x_ext_feats = batch_inputs[0]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "The question we want to compute similarity for and its sentence matrix dimension is shown below. The first dimension is the batch size, which is one in this case. Hence, the first index can be thought of as an index into the particular single sentence matrix. For each sentence matrix, each column represents the word vector for the corresponding word/token in the sentence (5 in total)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "where was durst born ?\n", + "torch.Size([1, 50, 5])\n" + ] + } + ], + "source": [ + "print(questions[pair_idx])\n", + "print(xq.size())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The answer we want to compute similarity for and its sentence matrix is:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "born in jacksonville , fla . , durst grew up in gastonia , n.c . , where his love of hip-hop music and break dancing made him an outcast .\n", + "torch.Size([1, 50, 30])\n" + ] + } + ], + "source": [ + "print(sentences[pair_idx])\n", + "print(xa.size())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll not use any external features for this example, so `x_ext_feats` is a vector of zeros." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Variable containing:\n", + " 0 0 0 0\n", + "[torch.FloatTensor of size 1x4]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x_ext_feats" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let us step through the `forward` method of the model. Normally we'll just call `trained_model(xq, xa, x_ext_feats)` but to illustrate the steps we'll copy the lines here again and see what happens underneath the hood." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we want to compute convolutional feature maps for the question. We just call `forward` to make a forward pass. We get 100 convolutional feature maps of length 9 each. We have 5 tokens with a padding of 4 on each side, for a total width of 5+2*4 = 13. Our convolution filter width is 5. Hence, we have 13 - 5 + 1 = 9 total positions for the \"sliding window\"." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 100, 9])" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "q = trained_model.conv_q.forward(xq)\n", + "q.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then, we want to max-pool the convolutional feature maps. We take max element out of every convolutional feature map of length 9, getting back 100 elements." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "q.size()[2]: 9\n" + ] + }, + { + "data": { + "text/plain": [ + "torch.Size([1, 100, 1])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('q.size()[2]:', q.size()[2])\n", + "# max pool using q.size()[2] as the window size, which is the length of each convolution feature map\n", + "q = F.max_pool1d(q, q.size()[2])\n", + "q.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we reshape `q` into a 1 x 100 vector. Using -1 automatically determines the dimension for that index." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 100])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "q = q.view(-1, trained_model.conv_channels)\n", + "q.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Similarly, we want to compute the max-pooled convolutional feature maps for the answer. This is a vector of length 100." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 100])" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a = trained_model.conv_a.forward(xa)\n", + "a = F.max_pool1d(a, a.size()[2])\n", + "a = a.view(-1, trained_model.conv_channels)\n", + "a.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we join the max-pooled results together with the external features. Note the pre-trained model was trained with external features so we must run the code path with external features, but we can use 0 as the inputs since this is only for demonstration purposes." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 204])" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = torch.cat([q, a, x_ext_feats], 1)\n", + "x.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we forward pass the features through the join layer, getting 201 inputs into the hidden layer." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 201])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = trained_model.combined_feature_vector.forward(x)\n", + "x.size()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 201])" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = trained_model.combined_features_activation.forward(x)\n", + "x.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note the activation doesn't change the dimensions. After activation we pass it through the Dropout layer, although this doesn't do anything since are not training the model." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "First 10 elements before Dropout [-0.12813647 0.01474741 -0.12794048 -0.13291343 -0.24393715 -0.00718142\n", + " -0.08802623 -0.08587593 0.23123664 0.02877411]\n", + "First 10 elements after Dropout [-0.12813647 0.01474741 -0.12794048 -0.13291343 -0.24393715 -0.00718142\n", + " -0.08802623 -0.08587593 0.23123664 0.02877411]\n" + ] + }, + { + "data": { + "text/plain": [ + "torch.Size([1, 201])" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print('First 10 elements before Dropout', x[0, :10].data.numpy())\n", + "x = trained_model.dropout(x)\n", + "print('First 10 elements after Dropout', x[0, :10].data.numpy())\n", + "x.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we pass the elements through the hidden layer, outputing just 2 elements." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 2])" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = trained_model.hidden(x)\n", + "x.size()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we find the log-probabilities." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Variable containing:\n", + "-0.0051 -5.2729\n", + "[torch.FloatTensor of size 1x2]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x = trained_model.logsoftmax(x)\n", + "x" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get the actual probabilities by using `exp`." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "Variable containing:\n", + " 0.9949 0.0051\n", + "[torch.FloatTensor of size 1x2]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "torch.exp(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hence, the probability of label 0 is 0.9949 while the probability of label 1 is 0.0051." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}