wassname/Castor
1
0
Fork 0
mirror of https://github.com/wassname/Castor.git synced 2026-09-09 11:13:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

clean up the relation prediction model for Simple QA - Ferhan's paper (#28)

Browse Source 
+ cleaned up the model code for the simple qa directory
+ created vocab objects for pre-loading word embeddings easily
This commit is contained in:
Salman Mohammed
2017-06-19 14:24:27 -04:00
committed by Jimmy Lin
parent 945b1fa6c0
commit 4c081645a7
21 changed files with 930 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
simple_qa_rnn/.gitignore
+2 -2
View File
@@ -1,4 +1,4 @@
datasets/
data/
resources/
saved_checkpoints/
__pycache__/
__pycache__/
simple_qa_rnn/README.md
+9 -29
View File
@@ -1,34 +1,14 @@
Setup:
1. Create 3 directories under "simple_qa_rnn" - "resources", "datasets", "saved_checkpoints"
2. Download the SimpleQA dataset from [here](https://github.com/castorini/data) and put it under the "datasets" directory
3. Download these files from this [Dropbox link](https://www.dropbox.com/sh/e5g12v7zu7sgzf7/AACW272AqPZJIUC7-A40LAsNa?dl=0) and paste them in the "resources" directory
4. The directory structure should look like this now:
Directions:
1. Download SimpleQuestions data from this [link](https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz) and put it in directory "data/SimpleQuestions_v2/"
2. Run this script to download GloVe word embeddings and do some preprocessing.
```
simple_qa_rnn
├── datasets
│   └── SimpleQuestions_v2
│   ├── ...
├── model.py
├── README.md
├── resources
│   ├── rel_to_ix_SQ.pkl
│   ├── w2v_map_SQ.pkl
│   └── word_to_ix_SQ.pkl
├── saved_checkpoints
│   └── [...models will be saved here later...]
├── scripts
│   ├── ...
├── train.py
└── util.py
bash fetch_and_preprocess.sh
```
5. Please take a look at the arguments in utils.py and set them accordingly to train the model.
6. Run this command to train the model. Make sure you have PyTorch and other Python dependencies installed.
3. Run this command to train the model. Make sure you have PyTorch and other Python dependencies installed.
```
python train.py
python train_relation_model.py
```
NOTE: There are pre-trained models saved in the 'finished_checkpoints' directory. They can be loaded up using PyTorch.
You can run a pre-trained model on the test dataset:
```
python train.py --not_bidirectional --resume_snapshot finished_checkpoints/lstm1/[model_filename] --test
For GPU, use:
```
python train_relation_model.py --cuda
```
simple_qa_rnn/args.py
+25
View File
@@ -0,0 +1,25 @@
from argparse import ArgumentParser
def get_args():
parser = ArgumentParser(description='Simple QA BiGRU model - Ferhan')
parser.add_argument('--epochs', type=int, default=30)
parser.add_argument('--batch_size', type=int, default=64)
parser.add_argument('--d_embedding', type=int, default=300)
parser.add_argument('--d_hidden', type=int, default=400)
parser.add_argument('--n_layers', type=int, default=2)
parser.add_argument('--lr', type=float, default=1e-4)
parser.add_argument('--test', action='store_true', dest='test', help='turn on test mode; no training.')
parser.add_argument('--not_bidirectional', action='store_false', dest='birnn')
parser.add_argument('--clip', type=float, default=0.4, help='gradient clipping')
parser.add_argument('--log_every', type=int, default=400)
parser.add_argument('--dev_every', type=int, default=1000)
parser.add_argument('--save_every', type=int, default=1000)
parser.add_argument('--dropout_prob', type=float, default=0.3)
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--seed', type=int, default=1111, help='random seed for reproducing results')
parser.add_argument('--cuda', action='store_true', help='use CUDA')
parser.add_argument('--device', type=int, default=0, help='GPU device to use')
parser.add_argument('--save_path', type=str, default='saved_checkpoints')
parser.add_argument('--resume_snapshot', type=str, default='')
args = parser.parse_args()
return args
simple_qa_rnn/data.py
-80
View File
@@ -1,80 +0,0 @@
import nltk
import string
import pickle
import numpy as np
import torch
from torch.autograd import Variable
def get_all_lines(data_filename):
all_lines = []
with open(data_filename) as fin:
for line in fin:
all_lines.append(line.rstrip())
return all_lines
def create_rp_dataset(data_file):
dataset = []
all_lines = get_all_lines(data_file)
for line in all_lines:
line_split = line.split("\t")
text = line_split[3]
relation = line_split[1]
dataset.append( (text, relation) )
return np.array(dataset)
def tokenize_text(text):
punc_remover = str.maketrans('', '', string.punctuation)
processed_text = text.lower().translate(punc_remover)
tokens = nltk.word_tokenize(processed_text)
return tokens
def add_padding_tokens(text_tokens, max_length, pad_type='both', pad_token='<pad>'):
num_pads = max_length - len(text_tokens)
right_pad = int(num_pads / 2)
left_pad = num_pads - right_pad
if pad_type == "both":
padded_tokens = [pad_token]*left_pad + text_tokens + [pad_token]*right_pad
elif pad_type == "right":
padded_tokens = text_tokens + [pad_token]*num_pads
else:
padded_tokens = [pad_token]*num_pads + text_tokens
return padded_tokens
def load_map(pname):
ret_map = None
with open(pname, 'rb') as fh:
ret_map = pickle.load(fh)
return ret_map
def text_to_vector(text, w2v_map, pad=False, max_length=None):
vec = []
tokens = tokenize_text(text)
if pad and (max_length != None):
tokens = add_padding_tokens(tokens, max_length)
for token in tokens:
vec.append( w2v_map[token] )
return np.array(vec)
def label_to_vector(label_ix, num_labels):
# create one-hot vector label representation
y_vec = np.zeros(num_labels, dtype=np.int32)
y_vec[label_ix] = 1
return y_vec
def create_tensorized_data(sentence, label, w2v_map, label_to_ix):
# x.shape: |S| X |D| - sentence length can vary between examples, dimension is fixed
x = text_to_vector(sentence, w2v_map)
y = label_to_ix[label]
inputs = Variable(torch.Tensor(x))
targets = Variable(torch.LongTensor([y]))
return inputs, targets
def create_tensorized_batch(batch, max_sent_length, w2v_map, label_to_ix):
X = []
y = []
for sent, label in batch:
X.append( text_to_vector(sent, w2v_map, pad=True, max_length=max_sent_length) )
y.append( label_to_ix[label] )
inputs = Variable(torch.Tensor(X))
targets = Variable(torch.LongTensor(y))
return inputs, targets
simple_qa_rnn/fetch_and_preprocess.sh
Executable
+14
View File
@@ -0,0 +1,14 @@
#!/bin/bash
# download GloVe word embeddings
python2 scripts/download.py
glove_dir="data/glove"
glove_pre="glove.840B"
glove_dim="300d"
if [ ! -f $glove_dir/$glove_pre.$glove_dim.pt ]; then
python scripts/convert_wordvecs.py $glove_dir/$glove_pre.$glove_dim.txt \
$glove_dir/$glove_pre.$glove_dim.pt
else
echo "The processed word embeddings file - $glove_dir/$glove_pre.$glove_dim.pt - already exists!"
fi
simple_qa_rnn/hyperparameter_tuning.py
+18
View File
@@ -0,0 +1,18 @@
from random import randint, uniform
from subprocess import call
epochs = 50
count = 20
for id in range(count):
learning_rate = 10 ** uniform(-5, -4)
d_hidden = randint(550, 600)
n_layers = randint(4, 5)
dropout = uniform(0.5, 0.6)
clip = uniform(0.6, 0.7)
command = "python train.py --cuda --device 1 --dev_every 500 --log_every 250 --batch_size 128 " \
"--epochs {} --lr {} --d_hidden {} --n_layers {} --dropout_prob {} --clip {} >> " \
"results.txt".format(epochs, learning_rate, d_hidden, n_layers, dropout, clip)
print("Running: " + command)
call(command, shell=True)
simple_qa_rnn/model.py
-37
View File
@@ -1,37 +0,0 @@
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
class BiLSTM(nn.Module):
def __init__(self, config):
super(BiLSTM, self).__init__()
self.config = config
self.lstm = nn.LSTM(input_size=config.d_embedding, hidden_size=config.d_hidden,
num_layers=config.n_layers, dropout=config.dropout_prob,
bidirectional=config.birnn, batch_first=True)
# linear layer maps from hidden state space to label space
self.hidden2label = nn.Linear(config.n_layers*config.n_directions*config.d_hidden, config.d_out)
self.hidden = self.init_hidden()
# self.dropout = nn.Dropout(p=config.dropout_prob)
# self.log_softmax = nn.LogSoftmax()
def init_hidden(self):
# axes semantics are (num_layers, batch_size, hidden_dim)
n_layers = self.config.n_layers * self.config.n_directions
return (Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden)),
Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden)))
# embeds is Variable of size - (|B|, |S|, |D|)
def forward(self, embeds):
batch_size = embeds.data.size()[0]
sequence_length = embeds.data.size()[1]
lstm_out, self.hidden = self.lstm(embeds, self.hidden)
# print("ht size: {}".format(ht.size()))
rel_space = self.hidden2label(self.hidden[0].transpose(0, 1).contiguous().view(batch_size, -1)) # size - (|B|, |K|)
scores = F.log_softmax(rel_space)
return scores
simple_qa_rnn/relation_model.py
+65
View File
@@ -0,0 +1,65 @@
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
class Encoder(nn.Module):
def __init__(self, config):
super(Encoder, self).__init__()
self.config = config
self.lstm = nn.LSTM(input_size=config.d_embedding, hidden_size=config.d_hidden,
num_layers=config.n_layers, dropout=config.dropout_prob,
bidirectional=config.birnn, batch_first=True)
self.hidden = self.init_hidden()
def init_hidden(self):
# axes semantics are (num_layers, batch_size, hidden_dim)
n_layers = self.config.n_layers * self.config.n_directions
out = (Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden)),
Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden)))
if self.config.cuda:
out = ( Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden).cuda()),
Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden).cuda() ))
return out
def forward(self, embeds):
batch_size = embeds.data.size()[0]
# sequence_length = embeds.data.size()[1]
lstm_out, self.hidden = self.lstm(embeds, self.hidden)
# print("ht size: {}".format(ht.size()))
return self.hidden[0].transpose(0, 1).contiguous().view(batch_size, -1) # size - (|B|, |K|)
class RelationPredictor(nn.Module):
def __init__(self, config):
super(RelationPredictor, self).__init__()
self.config = config
self.embed = nn.Embedding(config.vocab_size, config.d_embedding)
self.encoder = Encoder(config)
self.dropout = nn.Dropout(p=config.dropout_prob)
self.relu = nn.ReLU()
# linear layers map from hidden state space to label space
num_in_features = config.n_layers * config.n_directions * config.d_hidden
self.hidden2label = nn.Sequential (
nn.Linear(num_in_features, num_in_features),
nn.BatchNorm1d(num_in_features),
self.relu,
self.dropout,
nn.Linear(num_in_features, num_in_features),
nn.BatchNorm1d(num_in_features),
self.relu,
self.dropout,
nn.Linear(num_in_features, config.d_out)
)
# batch_input is Variable of size - (|B|, |S|)
def forward(self, batch_input):
batch_input_embed = self.embed(batch_input) # size - (|B|, |S|, |D|)
# size - (|B|, |X|) where |X| = n_layers * n_directions * d_hidden
encoded = self.encoder(batch_input_embed)
rel_space = self.hidden2label(encoded) # size - (|B|, |K|)
scores = F.log_softmax(rel_space)
return scores
simple_qa_rnn/relation_prediction_results.txt
+207
View File
@@ -0,0 +1,207 @@
vocab size = 62622
num classes = 1837
unk vocab count = 22761
loading train/val/test datasets...
train_file: data/SimpleQuestions_v2/annotated_fb_data_train.txt, num train = 75910
val_file: data/SimpleQuestions_v2/annotated_fb_data_valid.txt, num dev = 10845
test_file: data/SimpleQuestions_v2/annotated_fb_data_test.txt, num test = 21687
Namespace(batch_size=128, birnn=True, clip=0.6, cuda=True, d_embedding=300, d_hidden=500, d_out=1837, dev_every=500, device=1, dropout_prob=0.5487647674511446, epochs=25, gpu=0, log_every=250, lr=2.3368165563254845e-05, n_directions=2, n_layers=4, resume_snapshot='', save_every=1000, save_path='saved_checkpoints', seed=1111, test=False, vocab_size=62622)
Time Epoch Iteration Loss Train/Acc. Val/Acc.
1 0 1 7.519370
28 0 250 4.217004
70 0 500 3.454021 0.464165 0.460844
97 1 750 2.695011
139 1 1000 2.023616 0.582841 0.579520
167 2 1250 1.947864
209 2 1500 2.173526 0.654498 0.643415
236 2 1750 2.050010
278 3 2000 1.698692 0.691334 0.676525
305 3 2250 1.334085
347 4 2500 1.258004 0.721398 0.703776
375 4 2750 1.318608
417 5 3000 1.237201 0.745152 0.725632
445 5 3250 1.053011
488 5 3500 1.393011 0.762713 0.740141
516 6 3750 1.064027
558 6 4000 1.188779 0.780143 0.753441
586 7 4250 1.180950
629 7 4500 1.158713 0.792488 0.765811
657 8 4750 1.092308
699 8 5000 0.772544 0.804582 0.771205
728 8 5250 0.935450
770 9 5500 1.031198 0.814555 0.776786
798 9 5750 0.795584
840 10 6000 0.759640 0.824318 0.785156
868 10 6250 0.861564
910 10 6500 0.605637 0.833289 0.787388
938 11 6750 0.720791
980 11 7000 0.615107 0.841089 0.791295
1007 12 7250 0.765555
1050 12 7500 0.601661 0.847874 0.800037
1078 13 7750 0.523285
1120 13 8000 0.550137 0.855265 0.800130
1148 13 8250 0.722784
1190 14 8500 0.632320 0.860231 0.802269
1218 14 8750 0.631018
1261 15 9000 0.584924 0.867174 0.804781
1289 15 9250 0.341058
1331 16 9500 0.483922 0.872853 0.809431
1359 16 9750 0.433858
1402 16 10000 0.514899 0.878004 0.809152
1430 17 10250 0.388845
1472 17 10500 0.379268 0.884301 0.809338
1500 18 10750 0.504304
1542 18 11000 0.477527 0.887898 0.807385
1570 18 11250 0.496538
1612 19 11500 0.457043 0.894630 0.808780
1639 19 11750 0.351258
1682 20 12000 0.567525 0.900348 0.809896
1710 20 12250 0.422562
1752 21 12500 0.327270 0.904247 0.810919
1779 21 12750 0.443917
1821 21 13000 0.429144 0.907726 0.811663
1849 22 13250 0.339602
1891 22 13500 0.393765 0.913101 0.812035
1919 23 13750 0.493038
1961 23 14000 0.322024 0.916473 0.810361
1989 24 14250 0.394269
2031 24 14500 0.315911 0.920623 0.814081
2059 24 14750 0.302249
vocab size = 62622
num classes = 1837
unk vocab count = 22761
loading train/val/test datasets...
train_file: data/SimpleQuestions_v2/annotated_fb_data_train.txt, num train = 75910
val_file: data/SimpleQuestions_v2/annotated_fb_data_valid.txt, num dev = 10845
test_file: data/SimpleQuestions_v2/annotated_fb_data_test.txt, num test = 21687
Namespace(batch_size=128, birnn=True, clip=0.6, cuda=True, d_embedding=300, d_hidden=500, d_out=1837, dev_every=500, device=1, dropout_prob=0.5235083503167401, epochs=25, gpu=0, log_every=250, lr=9.543880052712213e-05, n_directions=2, n_layers=4, resume_snapshot='', save_every=1000, save_path='saved_checkpoints', seed=1111, test=False, vocab_size=62622)
Time Epoch Iteration Loss Train/Acc. Val/Acc.
1 0 1 7.562489
28 0 250 2.717191
70 0 500 1.675420 0.683917 0.673270
98 1 750 1.317849
140 1 1000 1.132524 0.757101 0.731678
168 2 1250 1.179604
210 2 1500 0.727408 0.802843 0.771484
238 2 1750 1.093024
280 3 2000 0.711856 0.829522 0.785156
308 3 2250 0.868195
351 4 2500 0.789406 0.848229 0.793062
379 4 2750 0.748050
421 5 3000 0.530056 0.864856 0.803478
449 5 3250 0.489331
492 5 3500 0.791100 0.880520 0.802362
520 6 3750 0.684948
562 6 4000 0.574934 0.893853 0.801804
590 7 4250 0.449654
632 7 4500 0.407685 0.905407 0.804781
660 8 4750 0.298767
702 8 5000 0.382648 0.917765 0.803850
730 8 5250 0.311796
772 9 5500 0.240214 0.928199 0.802083
799 9 5750 0.403507
842 10 6000 0.195193 0.942216 0.804222
869 10 6250 0.360780
911 10 6500 0.179228 0.949542 0.801060
939 11 6750 0.244037
981 11 7000 0.239074 0.957828 0.798642
1009 12 7250 0.187415
1051 12 7500 0.242319 0.964113 0.796038
1079 13 7750 0.205365
1121 13 8000 0.188374 0.970542 0.795294
1149 13 8250 0.218278
1191 14 8500 0.165913 0.973585 0.797433
1218 14 8750 0.181852
1261 15 9000 0.171079 0.977406 0.795666
1288 15 9250 0.153494
1331 16 9500 0.063608 0.982017 0.789528
1358 16 9750 0.129606
1401 16 10000 0.067068 0.983940 0.789249
1428 17 10250 0.056862
1470 17 10500 0.113310 0.985640 0.790365
1498 18 10750 0.082778
1540 18 11000 0.108898 0.986483 0.792225
1568 18 11250 0.156388
1610 19 11500 0.074055 0.988406 0.784598
1638 19 11750 0.110450
1680 20 12000 0.103423 0.988709 0.787481
1707 20 12250 0.072456
1749 21 12500 0.057874 0.991015 0.786086
1777 21 12750 0.036475
1819 21 13000 0.082274 0.991792 0.788876
1847 22 13250 0.047618
1889 22 13500 0.054384 0.992108 0.785714
1917 23 13750 0.035868
1959 23 14000 0.062694 0.991502 0.790365
1987 24 14250 0.084560
2029 24 14500 0.061790 0.994809 0.785714
2056 24 14750 0.044786
vocab size = 62622
num classes = 1837
unk vocab count = 22761
loading train/val/test datasets...
train_file: data/SimpleQuestions_v2/annotated_fb_data_train.txt, num train = 75910
val_file: data/SimpleQuestions_v2/annotated_fb_data_valid.txt, num dev = 10845
test_file: data/SimpleQuestions_v2/annotated_fb_data_test.txt, num test = 21687
Namespace(batch_size=128, birnn=True, clip=0.6, cuda=True, d_embedding=300, d_hidden=500, d_out=1837, dev_every=500, device=1, dropout_prob=0.5973369506283627, epochs=25, gpu=0, log_every=250, lr=6.789914913570565e-05, n_directions=2, n_layers=4, resume_snapshot='', save_every=1000, save_path='saved_checkpoints', seed=1111, test=False, vocab_size=62622)
Time Epoch Iteration Loss Train/Acc. Val/Acc.
1 0 1 7.538278
28 0 250 2.793929
70 0 500 2.281076 0.605212 0.600074
98 1 750 1.900491
140 1 1000 1.823353 0.709607 0.691778
167 2 1250 1.095232
209 2 1500 1.101957 0.756521 0.735026
237 2 1750 1.370564
279 3 2000 0.972925 0.788891 0.762184
307 3 2250 0.815842
349 4 2500 1.124413 0.810511 0.773996
376 4 2750 0.545115
419 5 3000 0.806257 0.829363 0.786086
446 5 3250 0.610705
489 5 3500 0.797700 0.845858 0.794829
516 6 3750 0.497445
558 6 4000 0.694232 0.855396 0.797805
586 7 4250 0.741005
629 7 4500 0.607206 0.867873 0.801897
657 8 4750 0.565121
699 8 5000 0.472905 0.879690 0.804315
727 8 5250 0.708608
769 9 5500 0.614066 0.889242 0.803757
797 9 5750 0.456040
839 10 6000 0.432193 0.899241 0.806920
866 10 6250 0.382535
908 10 6500 0.394834 0.909333 0.807292
935 11 6750 0.367661
978 11 7000 0.637153 0.918950 0.808594
1005 12 7250 0.394814
1047 12 7500 0.319768 0.927514 0.810919
1075 13 7750 0.262185
1117 13 8000 0.243790 0.932639 0.804688
1144 13 8250 0.240168
1187 14 8500 0.224746 0.939858 0.806734
1215 14 8750 0.280511
1257 15 9000 0.223731 0.948698 0.801339
1284 15 9250 0.342026
1327 16 9500 0.219288 0.954956 0.801525
1354 16 9750 0.144273
1396 16 10000 0.268068 0.960437 0.799014
1424 17 10250 0.142029
1466 17 10500 0.204999 0.964903 0.800688
1494 18 10750 0.188032
1536 18 11000 0.197725 0.968842 0.801153
1563 18 11250 0.163313
1606 19 11500 0.213809 0.970818 0.796875
1633 19 11750 0.167466
1675 20 12000 0.110523 0.974349 0.793992
1702 20 12250 0.145699
1744 21 12500 0.074068 0.977392 0.794085
1772 21 12750 0.098240
1814 21 13000 0.112673 0.983084 0.794550
1842 22 13250 0.106451
1884 22 13500 0.222595 0.982254 0.793992
1912 23 13750 0.079745
1954 23 14000 0.101426 0.985060 0.792318
1982 24 14250 0.118253
2024 24 14500 0.109737 0.986101 0.791853
2052 24 14750 0.081499
simple_qa_rnn/scripts/build_vocab.py
Executable
+43
View File
@@ -0,0 +1,43 @@
import os
import glob
import torch
import nltk
import string
# this method was copied from utils/read_data.py
def process_tokenize_text(text):
punc_remover = str.maketrans('', '', string.punctuation)
processed_text = text.lower().translate(punc_remover)
tokens = nltk.word_tokenize(processed_text)
return tokens
def build_vocab_SQ(data_dir):
filepaths = glob.glob(os.path.join(data_dir, 'annotated*.txt'))
print("reading filepaths: {}".format(filepaths))
word_vocab = set()
relation_vocab = set()
for filepath in filepaths:
with open(filepath) as f:
for line in f:
line_items = line.split("\t")
# add relation
relation = line_items[1]
relation_vocab.add(relation)
# add text
qText = line_items[3]
tokens = process_tokenize_text(qText)
word_vocab |= set(tokens)
word2index_dict = {word: i for i, word in enumerate(sorted(word_vocab))} # word to index dictionary
rel2index_dict = {relation: i for i, relation in enumerate(sorted(relation_vocab))} # relation to index dictionary
return (word2index_dict, rel2index_dict)
print("WARNING: This script is dataset specific. Please change it to fit your own dataset.")
data_dir = 'data/SimpleQuestions_v2/'
dst_path = os.path.join(data_dir, 'vocab.pt')
print("Building vocab for data in: {}".format(data_dir))
ret = build_vocab_SQ(data_dir) # ret = (word2index dict, relation2index dict)
print("saving word2index and answer2index dicts to {}".format(dst_path))
torch.save(ret, dst_path)
print("Done!")
simple_qa_rnn/scripts/convert_wordvecs.py
+57
View File
@@ -0,0 +1,57 @@
import sys
from tqdm import tqdm
import torch
import array
import six
try:
path = sys.argv[1]
outpath = sys.argv[2]
except:
print("ERROR: the command line arguments passed in were not valid.\n");
print("USAGE: python scripts/convert_wordvecs.py [input_file] [output_file]");
print("EXAMPLE: python scripts/convert_wordvecs.py glove_300d.txt glove_300d.pt");
sys.exit(1);
prefix_toks = path.split(".")
print('Converting ' + path + ' to PyTorch serialized format...')
lines = [line.rstrip('\n') for line in open(path)]
print("number of lines: {}".format(len(lines)))
wv_tokens = []
wv_arr = array.array('d')
wv_size = None # dimension of the word vectors
vocab_size = 0 # counts the number of words saved
if lines is not None:
for i in tqdm(range(len(lines)), desc="loading word vectors from {}".format(path)):
entries = lines[i].strip().split()
word, entries = entries[0], entries[1:]
if wv_size is None:
wv_size = len(entries)
else:
# safety check that the dimension is the same
if len(entries) != wv_size:
print(len(entries))
print(lines[i])
continue
try:
if isinstance(word, six.binary_type):
word = word.decode('utf-8')
except:
print('non-UTF8 token', repr(word), 'ignored')
continue
wv_arr.extend(float(x) for x in entries)
wv_tokens.append(word)
vocab_size += 1
print("vocab size: {}".format(vocab_size))
print("dim: {}".format(wv_size))
wv_dict = {word: i for i, word in enumerate(wv_tokens)} # word to index dictionary
wv_arr = torch.Tensor(wv_arr).view(vocab_size, wv_size) # word embeddings in Tensor of shape (|V|, |D|)
ret = (wv_dict, wv_arr, wv_size) # save all three info in a tuple
print("saving word vectors to {}".format(outpath))
torch.save(ret, outpath)
simple_qa_rnn/scripts/download.py
+84
View File
@@ -0,0 +1,84 @@
"""
Downloads the following:
- Glove vectors
We Thank Kai Sheng Tai for providing the preprocessing/basis codes.
Taken from: https://github.com/castorini/NCE-CNN-Torch
"""
from __future__ import print_function
import urllib2
import sys
import os
import shutil
import zipfile
import gzip
def download(url, dirpath):
filename = url.split('/')[-1]
filepath = os.path.join(dirpath, filename)
try:
u = urllib2.urlopen(url)
except:
print("URL %s failed to open" %url)
raise Exception
try:
f = open(filepath, 'wb')
except:
print("Cannot write %s" %filepath)
raise Exception
try:
filesize = int(u.info().getheaders("Content-Length")[0])
except:
print("URL %s failed to report length" %url)
raise Exception
print("Downloading: %s Bytes: %s" % (filename, filesize))
downloaded = 0
block_sz = 8192
status_width = 70
while True:
buf = u.read(block_sz)
if not buf:
print('')
break
else:
print('', end='\r')
downloaded += len(buf)
f.write(buf)
status = (("[%-" + str(status_width + 1) + "s] %3.2f%%") %
('=' * int(float(downloaded) / filesize * status_width) + '>', downloaded * 100. / filesize))
print(status, end='')
sys.stdout.flush()
f.close()
return filepath
def unzip(filepath):
print("Extracting: " + filepath)
dirpath = os.path.dirname(filepath)
with zipfile.ZipFile(filepath) as zf:
zf.extractall(dirpath)
os.remove(filepath)
def download_wordvecs(dirpath):
if os.path.exists(dirpath):
print('Found Glove vectors - skip')
return
else:
os.makedirs(dirpath)
url = 'https://nlp.stanford.edu/data/glove.840B.300d.zip'
unzip(download(url, dirpath))
if __name__ == '__main__':
base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
# data
data_dir = os.path.join(base_dir, 'data')
wordvec_dir = os.path.join(data_dir, 'glove')
# libraries
lib_dir = os.path.join(base_dir, 'lib')
# download GloVe word embeddings
download_wordvecs(wordvec_dir)
print("Finished downloading word embeddings!")
simple_qa_rnn/scripts/sq_entity_data.py
+40
View File
@@ -0,0 +1,40 @@
import glob
import os
fnames = glob.glob('../data/SimpleQuestions_v2/annotated*.txt')
print(fnames)
all_entities = set()
for fname in fnames:
with open(fname) as fin:
for line in fin:
entity = line.split("\t")[0]
# entity: www.freebase.com/m/0f3xg_ --> m.0f3xg_
if entity.startswith("www.freebase.com/"):
entity = entity[17:].replace("/", ".")
all_entities.add(entity)
print("num of entities in train/val/test: {}".format(len(all_entities)))
names_map = {}
names_file = "../data/freebase/names-map-0.ttl"
with open(names_file) as fin:
for line in fin:
id = line.split("\t")[0][3:]
name = line.split("\t")[2].rstrip()
if name.endswith("\"@en."):
name = name[1:-5]
if id in all_entities:
names_map[id] = name
print("num of FB entities in the map: {}".format(len(names_map)))
found = len(names_map)
print("found: {}".format(found))
print("notfound: {}".format(len(all_entities) - found))
outfile = open("names_map.tsv", 'w')
for id, name in names_map.items():
outfile.write("{}\t{}\n".format(id, name))
outfile.close()
print("done")
simple_qa_rnn/scripts/wq_relation_data.py
+40
View File
@@ -0,0 +1,40 @@
import json
import os
rel_dir = 'data/dataset-factoid-webquestions/d-freebase-rp/'
text_dir = 'data/dataset-factoid-webquestions/main/'
out_dir = 'data/webquestions-custom/relation-prediction'
fname = 'val.json'
rel_filename = os.path.join(rel_dir, fname)
text_filename = os.path.join(text_dir, fname)
out_filename = os.path.join(out_dir, fname)
text_entries = []
with open(text_filename) as fin:
data = json.load(fin)
text_entries.extend(data)
rel_entries = []
with open(rel_filename) as fin:
data = json.load(fin)
rel_entries.extend(data)
print("num of examples: {}".format(len(text_entries)))
assert len(text_entries) == len(rel_entries)
outfile = open(out_filename, 'w')
count = 0
for text_entry, rel_entry in zip(text_entries, rel_entries):
assert text_entry.get('qId') == rel_entry.get('qId')
qID = text_entry.get('qId')
qText = text_entry.get('qText')
relPaths = rel_entry.get('relPaths')
for relPath in relPaths:
relations = relPath[0]
for rel in relations:
count += 1
outfile.write("{}\t{}\t{}\n".format(qID, rel, qText))
print("count: {}".format(count))
outfile.close()
simple_qa_rnn/train.py
-137
View File
@@ -1,137 +0,0 @@
import os
import sys
import time
import glob
import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim
from model import BiLSTM
from util import get_args
import data
args = get_args()
# torch.cuda.set_device(args.gpu)
# ---- Helper Methods ------
def evaluate_dataset_batch(data_set, max_sent_length, model, w2v_map, label_to_ix):
n_total = len(data_set)
n_correct = 0
num_batches = len(data_set) // args.batch_size
batch_indices = np.split(range(n_total),
range(args.batch_size, n_total, args.batch_size))
model.eval()
for batch_ix in range(num_batches):
batch = data_set[batch_indices[batch_ix]]
inputs, targets = data.create_tensorized_batch(batch, max_sent_length, w2v_map, label_to_ix)
scores = model(inputs)
pred_label_ix = np.argmax(scores.data.numpy(), axis=1) # check this properly
correct_label_ix = targets.data.numpy()
n_correct += (pred_label_ix == correct_label_ix).sum()
acc = n_correct / n_total
return acc
def repackage_hidden(h):
"""Wraps hidden states in new Variables, to detach them from their history."""
if type(h) == Variable:
return Variable(h.data)
else:
return tuple(repackage_hidden(v) for v in h)
#Load Datasets ------
train_file = "datasets/SimpleQuestions_v2/annotated_fb_data_train.txt"
val_file = "datasets/SimpleQuestions_v2/annotated_fb_data_valid.txt"
test_file = "datasets/SimpleQuestions_v2/annotated_fb_data_test.txt"
train_set = data.create_rp_dataset(train_file)
val_set = data.create_rp_dataset(val_file)
test_set = data.create_rp_dataset(test_file)
# train_set = train_set[:4] # work with few examples first
# ---- Build Vocabulary ------
w2v_map = data.load_map("resources/w2v_map_SQ.pkl")
w2v_map['<pad>'] = np.zeros(300)
word_to_ix = data.load_map("resources/word_to_ix_SQ.pkl")
label_to_ix = data.load_map("resources/rel_to_ix_SQ.pkl")
vocab_size = len(word_to_ix)
num_classes = len(label_to_ix)
max_sent_length = 36 # set from the paper
# ---- Define Model, Loss, Optim ------
config = args
config.d_out = num_classes
config.n_directions = 2 if config.birnn else 1
print(config)
model = BiLSTM(config)
loss_function = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
# ---- Test Model ------
if args.test:
print("Test Mode: loading pre-trained model and testing on test set...")
# model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage.cuda(args.gpu))
model.load_state_dict(torch.load(args.resume_snapshot))
test_acc = evaluate_dataset_batch(test_set, max_sent_length, model, w2v_map, label_to_ix)
print("Accuracy: {}".format(test_acc))
sys.exit(0)
# ---- Train Model ------
start = time.time()
best_val_acc = -1
iter = 0
header = ' Time Epoch Iteration Loss Train/Acc. Val/Acc.'
print(header)
log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>9.6f}'.split(','))
dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>9.6f},{:9.6f},{:11.6f}'.split(','))
model.train()
for epoch in range(args.epochs):
# shuffle the dataset and create batches (truncate the last batch if not of equal size)
shuffled_indices = np.random.permutation(len(train_set))
num_batches = len(shuffled_indices) // args.batch_size
batch_indices = np.split(shuffled_indices,
range(args.batch_size, len(shuffled_indices), args.batch_size))
model.hidden = model.init_hidden()
for batch_ix in range(num_batches):
iter += 1
batch = train_set[batch_indices[batch_ix]]
inputs, targets = data.create_tensorized_batch(batch, max_sent_length, w2v_map, label_to_ix)
# print("inputs size: {}".format(inputs.size()))
# print("targets size: {}".format(targets.size()))
# clear out gradients and hidden states of the model
model.zero_grad()
model.hidden = repackage_hidden(model.hidden)
# prepare inputs for LSTM model and run forward pass
scores = model(inputs)
# compute the loss, gradients, and update the parameters
loss = loss_function(scores, targets)
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm(model.parameters(), args.clip)
optimizer.step()
# log at intervals
if iter % args.dev_every == 0:
train_acc = evaluate_dataset_batch(train_set[:8000], max_sent_length, model, w2v_map, label_to_ix)
val_acc = evaluate_dataset_batch(val_set, max_sent_length, model, w2v_map, label_to_ix)
print(dev_log_template.format(time.time()-start, epoch, iter, loss.data[0], train_acc, val_acc))
if val_acc > best_val_acc:
best_val_acc = val_acc
snapshot_prefix = os.path.join(args.save_path, 'best_snapshot')
snapshot_path = snapshot_prefix + '_valacc_{:6.4f}__iter_{}_model.pt'.format(val_acc, iter)
torch.save(model.state_dict(), snapshot_path)
for f in glob.glob(snapshot_prefix + '*'):
if f != snapshot_path:
os.remove(f)
elif iter == 1 or iter % args.log_every == 0:
print(log_template.format(time.time() - start, epoch, iter, loss.data[0]))
simple_qa_rnn/train_relation_model.py
+184
View File
@@ -0,0 +1,184 @@
import os
import sys
import time
import glob
import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim
from relation_model import RelationPredictor
from args import get_args
import data
from utils.vocab import Vocab
from utils.read_data import *
args = get_args()
# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
if not args.cuda:
print("WARNING: You have a CUDA device, so you should probably run with --cuda")
else:
torch.cuda.manual_seed(args.seed)
torch.cuda.set_device(args.device)
# ---- helper methods ------
def evaluate_dataset_batch(data_set, model):
n_total = data_set["size"]
n_correct = 0
num_batches = n_total // args.batch_size
batch_indices = np.split(range(n_total),
range(args.batch_size, n_total, args.batch_size))
model.eval()
for batch_ix in range(num_batches):
batch_questions = data_set["questions"][batch_indices[batch_ix]]
batch_relations = data_set["rel_labels"][batch_indices[batch_ix]]
inputs = Variable(read_text_tensor(batch_questions, word_vocab), volatile=True)
targets = Variable(read_labels_tensor(batch_relations, rel_vocab), volatile=True)
if args.cuda:
inputs.data = inputs.data.cuda()
targets.data = targets.data.cuda()
scores = model(inputs)
pred_score, pred_label_ix = torch.max(scores, dim=1) # check this properly
pred_label_ix = pred_label_ix.view(args.batch_size)
sum_correct = torch.sum(torch.eq(pred_label_ix, targets))
n_correct += sum_correct.data[0]
acc = n_correct / (num_batches * args.batch_size)
return acc
def repackage_hidden(h):
"""Wraps hidden states in new Variables, to detach them from their history."""
if type(h) == Variable:
return Variable(h.data)
else:
return tuple(repackage_hidden(v) for v in h)
# ---- dataset paths ------
data_dir = "data/SimpleQuestions_v2/"
train_file = os.path.join(data_dir, "annotated_fb_data_train.txt")
val_file = os.path.join(data_dir, "annotated_fb_data_valid.txt")
test_file = os.path.join(data_dir, "annotated_fb_data_test.txt")
# ---- load GloVe embeddings ------
embed_pt_filepath = 'data/glove/glove.840B.300d.pt'
emb_w2i, emb_vecs = read_embedding(embed_pt_filepath)
emb_vocab = Vocab(emb_w2i)
emb_dim = emb_vecs.size()[1]
# ---- create dataset vocabulary and embeddings ------
vocab_pt_filepath = os.path.join(data_dir, "vocab.pt")
word2index_dict, rel2index_dict = torch.load(vocab_pt_filepath)
word_vocab = Vocab(word2index_dict)
word_vocab.add_pad_token("<PAD>")
word_vocab.add_unk_token("<UNK>")
rel_vocab = Vocab(rel2index_dict)
vocab_size = word_vocab.size
num_classes = len(rel2index_dict)
print('vocab size = {}'.format(vocab_size))
print('num classes = {}'.format(num_classes))
num_unk = 0
vecs = torch.FloatTensor(vocab_size, emb_dim)
for i in range(vocab_size):
word = word_vocab.get_token(i)
if emb_vocab.contains(word):
vecs[i] = emb_vecs[emb_vocab.get_index(word)]
elif word == word_vocab.pad_token:
vecs[i].zero_()
else:
num_unk += 1
vecs[i].uniform_(-0.05, 0.05)
print('unk vocab count = {}'.format(num_unk))
emb_vocab = None
emb_vecs = None
# ---- load datasets ------
print("loading train/val/test datasets...")
train_dataset = read_dataset(train_file, word_vocab, rel_vocab)
val_dataset = read_dataset(val_file, word_vocab, rel_vocab)
test_dataset = read_dataset(test_file, word_vocab, rel_vocab)
print('train_file: {}, num train = {}'.format(train_file, train_dataset["size"]))
print('val_file: {}, num dev = {}'.format(val_file, val_dataset["size"]))
print('test_file: {}, num test = {}'.format(test_file, test_dataset["size"]))
# ---- Define Model, Loss, Optim ------
config = args
config.vocab_size = vocab_size
config.d_out = num_classes
config.n_directions = 2 if config.birnn else 1
print(config)
model = RelationPredictor(config)
# initialize the embedding layer with the word vectors
model.embed.weight.data = vecs
if args.cuda:
model.cuda()
loss_function = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
# ---- Train Model ------
start = time.time()
best_val_acc = -1
iter = 0
header = ' Time Epoch Iteration Loss Train/Acc. Val/Acc.'
print(header)
log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>9.6f}'.split(','))
dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>9.6f},{:9.6f},{:11.6f}'.split(','))
model.train()
for epoch in range(args.epochs):
# shuffle the dataset and create batches (truncate the last batch if not of equal size)
shuffled_indices = np.random.permutation(train_dataset["size"])
num_batches = len(shuffled_indices) // args.batch_size
batch_indices = np.split(shuffled_indices,
range(args.batch_size, len(shuffled_indices), args.batch_size))
model.encoder.hidden = model.encoder.init_hidden()
for batch_ix in range(num_batches):
iter += 1
batch_questions = train_dataset["questions"][batch_indices[batch_ix]]
batch_relations = train_dataset["rel_labels"][batch_indices[batch_ix]]
inputs = Variable( read_text_tensor(batch_questions, word_vocab) )
targets = Variable( read_labels_tensor(batch_relations, rel_vocab) )
if args.cuda:
inputs.data = inputs.data.cuda()
targets.data = targets.data.cuda()
# clear out gradients and hidden states of the model
model.zero_grad()
model.encoder.hidden = repackage_hidden(model.encoder.hidden)
# prepare inputs for LSTM model and run forward pass
scores = model(inputs)
# compute the loss, gradients, and update the parameters
loss = loss_function(scores, targets)
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm(model.parameters(), args.clip)
optimizer.step()
# log at intervals
if iter % args.dev_every == 0:
train_acc = evaluate_dataset_batch(train_dataset, model)
val_acc = evaluate_dataset_batch(val_dataset, model)
model.train()
print(dev_log_template.format(time.time()-start, epoch, iter, loss.data[0], train_acc, val_acc))
if val_acc > best_val_acc:
best_val_acc = val_acc
snapshot_prefix = os.path.join(args.save_path, 'best_snapshot')
snapshot_path = snapshot_prefix + '_valacc_{:6.4f}_trainacc{:6.4f}_iter_{}_model.pt'.format(val_acc, train_acc, iter)
torch.save(model.state_dict(), snapshot_path)
for f in glob.glob(snapshot_prefix + '*'):
if f != snapshot_path:
os.remove(f)
elif iter == 1 or iter % args.log_every == 0:
print(log_template.format(time.time() - start, epoch, iter, loss.cpu().data[0]))
simple_qa_rnn/util.py
-22
View File
@@ -1,22 +0,0 @@
from argparse import ArgumentParser
def get_args():
parser = ArgumentParser(description='Simple QA BiGRU model - Ferhan')
parser.add_argument('--epochs', type=int, default=30)
parser.add_argument('--batch_size', type=int, default=256)
parser.add_argument('--d_embedding', type=int, default=300)
parser.add_argument('--d_hidden', type=int, default=300)
parser.add_argument('--n_layers', type=int, default=1)
parser.add_argument('--test', action='store_true', dest='test', help='turn on test mode; no training.')
parser.add_argument('--not_bidirectional', action='store_false', dest='birnn')
parser.add_argument('--clip', type=float, default=0.25, help='gradient clipping')
parser.add_argument('--log_every', type=int, default=50)
parser.add_argument('--lr', type=float, default=1e-3)
parser.add_argument('--dev_every', type=int, default=300)
parser.add_argument('--save_every', type=int, default=1000)
parser.add_argument('--dropout_prob', type=int, default=0.2)
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--save_path', type=str, default='saved_checkpoints')
parser.add_argument('--resume_snapshot', type=str, default='')
args = parser.parse_args()
return args
simple_qa_rnn/utils/__init__.py
View File
simple_qa_rnn/utils/read_data.py
+71
View File
@@ -0,0 +1,71 @@
import torch
import nltk
import string
from torch.autograd import Variable
import numpy as np
## functions for loading data from disk
def process_tokenize_text(text):
punc_remover = str.maketrans('', '', string.punctuation)
processed_text = text.lower().translate(punc_remover)
tokens = nltk.word_tokenize(processed_text)
return tokens
def read_embedding(embed_pt_filepath):
embed_tuple = torch.load(embed_pt_filepath)
word2index, w2v_tensor, dim = embed_tuple
return word2index, w2v_tensor
def find_max_seq_length(text):
max_len = -1
for tokens in text:
curr_len = len(tokens)
if curr_len > max_len:
max_len = curr_len
return max_len
def read_text_tensor(batch_text, word_vocab):
out_text = []
max_len = find_max_seq_length(batch_text)
for sent_tokens in batch_text:
S = len(sent_tokens)
sent = []
for i in range(S):
token = sent_tokens[i]
sent.append( word_vocab.get_index(token) )
# pad the right end till the max length of the mini batch
for i in range(S, max_len):
sent.append( word_vocab.pad_index )
out_text.append(sent)
return torch.LongTensor(out_text)
def read_labels_tensor(rel_labels, rel_vocab, cuda=False):
N = len(rel_labels)
labels_list = []
for i in range(N):
token = rel_labels[i]
labels_list.append( rel_vocab.get_index(token) )
return torch.LongTensor(labels_list)
def read_dataset(datapath, word_vocab, rel_vocab):
questions = []
rel_labels = []
# read questions and label from the datapath - could be train, dev, testls
with open(datapath) as f:
for line in f:
line_items = line.split("\t")
# add relation
relation = line_items[1]
rel_labels.append(relation)
# add text
qText = line_items[3]
tokens = process_tokenize_text(qText)
questions.append(tokens)
dataset = {"word_vocab": word_vocab, "rel_vocab": rel_vocab, "size": len(rel_labels),
"questions": np.array(questions), "rel_labels": np.array(rel_labels)}
return dataset
simple_qa_rnn/utils/vocab.py
+68
View File
@@ -0,0 +1,68 @@
import torch
class Vocab(object):
"""
A vocabulary object. Initialized from a file with one vocabulary token per line.
Maps between vocabulary tokens and indices. If an UNK token is defined in the
vocabulary, returns the index to this token if queried for an out-of-vocabulary
token.
"""
# def __init__(self, vocabpath):
# self.size = 0
# self.index = {}
# self.tokens = {}
#
# with open(vocabpath, 'r') as f:
# for line in f:
# word = line.rstrip()
# self.tokens[self.size] = word
# self.index[word] = self.size
# self.size += 1
# # automatically add unknown token
# # self.add_unk_token("<UNK>")
def __init__(self, word2index):
self.index = word2index
self.size = len(word2index)
self.tokens = {index: word for word, index in word2index.items()}
# self.add_unk_token("<UNK>")
def contains(self, word):
return word in self.index.keys()
def add(self, word):
if not self.contains(word):
self.tokens[self.size] = word
self.index[word] = self.size
self.size += 1
def add_unk_token(self, token):
self.unk_token = token
self.add(token)
self.unk_index = self.index[token]
def add_pad_token(self, token):
self.pad_token = token
self.add(token)
self.pad_index = self.index[token]
def get_index(self, word):
if self.contains(word):
return self.index[word]
else:
print("{} - word not found in vocab. returning unk_index".format(word))
return self.unk_index
def get_token(self, index):
if index < 0 or index >= self.size:
raise IndexError('index {} out of bounds'.format(index))
return self.tokens[index]
def map(self, tokens):
N = len(tokens)
out = torch.IntTensor(N)
for i in range(N):
out[i] = self.index(tokens[i])
return out
sm_cnn/.gitignore
+3 -1
View File
@@ -1,2 +1,4 @@
*pyc
trec_eval-8.0/trec_eval
trec_eval-8.0/trec_eval
data/
trained_models/