wassname/Castor
1
0
Fork 0
mirror of https://github.com/wassname/Castor.git synced 2026-09-09 11:13:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Used torchtext to refactor relation prediction model - much cleaner! (#33)

Browse Source 
Refactored the code for Ferhan's relation prediction model using torchtext
This commit is contained in:
Salman Mohammed
2017-07-15 07:06:10 +08:00
committed by Jimmy Lin
parent a0755e6aa3
commit 061cc8dd09
26 changed files with 359 additions and 1100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
simple_qa_rnn/.gitignore
+3 -1
View File
@@ -1,4 +1,6 @@
data/
data/*
data/glove/
resources/
saved_checkpoints/
__pycache__/
relation_prediction/*cache*/*
simple_qa_rnn/README.md
+16 -9
View File
@@ -1,14 +1,21 @@
Directions:
1. Download SimpleQuestions data from this [link](https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz) and put it in directory "data/SimpleQuestions_v2/"
2. Run this script to download GloVe word embeddings and do some preprocessing.
## Relation Prediction Model
- Download and extract SimpleQuestions dataset by running the script:
```
bash fetch_and_preprocess.sh
bash fetch_dataset.sh
```
3. Run this command to train the model. Make sure you have PyTorch and other Python dependencies installed.
- You will also require the package - [torchtext](https://github.com/pytorch/text).
```
python train_relation_model.py
git clone https://github.com/pytorch/text.git
cd path/to/torchtext
python setup.py install
```
For GPU, use:
- Run the training script with the following commands. Please check out args.py file to see the different commands available:
```
cd relation_prediction
python train.py
python train.py --no_cuda
python train.py --rnn_type gru
```
python train_relation_model.py --cuda
```
simple_qa_rnn/args.py
-25
View File
@@ -1,25 +0,0 @@
from argparse import ArgumentParser
def get_args():
parser = ArgumentParser(description='Simple QA BiGRU model - Ferhan')
parser.add_argument('--epochs', type=int, default=30)
parser.add_argument('--batch_size', type=int, default=64)
parser.add_argument('--d_embedding', type=int, default=300)
parser.add_argument('--d_hidden', type=int, default=400)
parser.add_argument('--n_layers', type=int, default=2)
parser.add_argument('--lr', type=float, default=1e-4)
parser.add_argument('--test', action='store_true', dest='test', help='turn on test mode; no training.')
parser.add_argument('--not_bidirectional', action='store_false', dest='birnn')
parser.add_argument('--clip', type=float, default=0.4, help='gradient clipping')
parser.add_argument('--log_every', type=int, default=400)
parser.add_argument('--dev_every', type=int, default=1000)
parser.add_argument('--save_every', type=int, default=1000)
parser.add_argument('--dropout_prob', type=float, default=0.3)
parser.add_argument('--gpu', type=int, default=0)
parser.add_argument('--seed', type=int, default=1111, help='random seed for reproducing results')
parser.add_argument('--cuda', action='store_true', help='use CUDA')
parser.add_argument('--device', type=int, default=0, help='GPU device to use')
parser.add_argument('--save_path', type=str, default='saved_checkpoints')
parser.add_argument('--resume_snapshot', type=str, default='')
args = parser.parse_args()
return args
simple_qa_rnn/fetch_and_preprocess.sh
-14
View File
@@ -1,14 +0,0 @@
#!/bin/bash
# download GloVe word embeddings
python2 scripts/download.py
glove_dir="data/glove"
glove_pre="glove.840B"
glove_dim="300d"
if [ ! -f $glove_dir/$glove_pre.$glove_dim.pt ]; then
python scripts/convert_wordvecs.py $glove_dir/$glove_pre.$glove_dim.txt \
$glove_dir/$glove_pre.$glove_dim.pt
else
echo "The processed word embeddings file - $glove_dir/$glove_pre.$glove_dim.pt - already exists!"
fi
simple_qa_rnn/fetch_dataset.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#!/bin/bash
# download dataset and put it in data directory
mkdir data
pushd data
wget https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz
tar -xvzf SimpleQuestions_v2.tgz
popd
simple_qa_rnn/relation_model.py
-65
View File
@@ -1,65 +0,0 @@
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
class Encoder(nn.Module):
def __init__(self, config):
super(Encoder, self).__init__()
self.config = config
self.lstm = nn.LSTM(input_size=config.d_embedding, hidden_size=config.d_hidden,
num_layers=config.n_layers, dropout=config.dropout_prob,
bidirectional=config.birnn, batch_first=True)
self.hidden = self.init_hidden()
def init_hidden(self):
# axes semantics are (num_layers, batch_size, hidden_dim)
n_layers = self.config.n_layers * self.config.n_directions
out = (Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden)),
Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden)))
if self.config.cuda:
out = ( Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden).cuda()),
Variable(torch.zeros(n_layers, self.config.batch_size, self.config.d_hidden).cuda() ))
return out
def forward(self, embeds):
batch_size = embeds.data.size()[0]
# sequence_length = embeds.data.size()[1]
lstm_out, self.hidden = self.lstm(embeds, self.hidden)
# print("ht size: {}".format(ht.size()))
return self.hidden[0].transpose(0, 1).contiguous().view(batch_size, -1) # size - (|B|, |K|)
class RelationPredictor(nn.Module):
def __init__(self, config):
super(RelationPredictor, self).__init__()
self.config = config
self.embed = nn.Embedding(config.vocab_size, config.d_embedding)
self.encoder = Encoder(config)
self.dropout = nn.Dropout(p=config.dropout_prob)
self.relu = nn.ReLU()
# linear layers map from hidden state space to label space
num_in_features = config.n_layers * config.n_directions * config.d_hidden
self.hidden2label = nn.Sequential (
nn.Linear(num_in_features, num_in_features),
nn.BatchNorm1d(num_in_features),
self.relu,
self.dropout,
nn.Linear(num_in_features, num_in_features),
nn.BatchNorm1d(num_in_features),
self.relu,
self.dropout,
nn.Linear(num_in_features, config.d_out)
)
# batch_input is Variable of size - (|B|, |S|)
def forward(self, batch_input):
batch_input_embed = self.embed(batch_input) # size - (|B|, |S|, |D|)
# size - (|B|, |X|) where |X| = n_layers * n_directions * d_hidden
encoded = self.encoder(batch_input_embed)
rel_space = self.hidden2label(encoded) # size - (|B|, |K|)
scores = F.log_softmax(rel_space)
return scores
simple_qa_rnn/relation_prediction/args.py
+32
View File
@@ -0,0 +1,32 @@
import os
from argparse import ArgumentParser
def get_args():
parser = ArgumentParser(description='Simple QA model - Ferhan Ture')
parser.add_argument('--epochs', type=int, default=40)
parser.add_argument('--batch_size', type=int, default=32)
parser.add_argument('--rnn_type', type=str, default='lstm') # or use 'gru'
parser.add_argument('--d_embed', type=int, default=300)
parser.add_argument('--d_hidden', type=int, default=400)
parser.add_argument('--n_layers', type=int, default=2)
parser.add_argument('--lr', type=float, default=1e-5)
parser.add_argument('--test', action='store_true', dest='test', help='turn on test mode; no training.')
parser.add_argument('--not_bidirectional', action='store_false', dest='birnn')
parser.add_argument('--clip_gradient', type=float, default=0.5, help='gradient clipping')
parser.add_argument('--log_every', type=int, default=300)
parser.add_argument('--dev_every', type=int, default=1200)
parser.add_argument('--save_every', type=int, default=1200)
parser.add_argument('--dropout_prob', type=float, default=0.3)
parser.add_argument('--patience', type=int, default=10, help="number of epochs to wait before early stopping")
parser.add_argument('--no_cuda', action='store_false', help='do not use CUDA', dest='cuda')
parser.add_argument('--gpu', type=int, default=0, help='GPU device to use') # use -1 for CPU
parser.add_argument('--seed', type=int, default=1111, help='random seed for reproducing results')
parser.add_argument('--save_path', type=str, default='saved_checkpoints')
parser.add_argument('--data_cache', type=str, default=os.path.join(os.getcwd(), 'data_cache'))
parser.add_argument('--vector_cache', type=str, default=os.path.join(os.getcwd(), 'vector_cache/input_vectors.pt'))
parser.add_argument('--word_vectors', type=str, default='glove.42B')
parser.add_argument('--train_embed', action='store_false', dest='fix_emb') # fine-tune the word embeddings
parser.add_argument('--resume_snapshot', type=str, default='')
args = parser.parse_args()
return args
simple_qa_rnn/hyperparameter_tuning.py → simple_qa_rnn/relation_prediction/hyperparameter_tuning.py
+2 -2
View File
@@ -10,8 +10,8 @@ for id in range(count):
dropout = uniform(0.5, 0.6)
clip = uniform(0.6, 0.7)
command = "python train.py --cuda --device 1 --dev_every 500 --log_every 250 --batch_size 128 " \
"--epochs {} --lr {} --d_hidden {} --n_layers {} --dropout_prob {} --clip {} >> " \
command = "python train.py --dev_every 500 --log_every 250 --batch_size 32 " \
"--epochs {} --lr {} --d_hidden {} --n_layers {} --dropout_prob {} --clip_gradient {} >> " \
"results.txt".format(epochs, learning_rate, d_hidden, n_layers, dropout, clip)
print("Running: " + command)
simple_qa_rnn/relation_prediction/model.py
+65
View File
@@ -0,0 +1,65 @@
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
class Encoder(nn.Module):
def __init__(self, config):
super(Encoder, self).__init__()
self.config = config
if config.rnn_type.lower() == "gru":
self.rnn = nn.GRU(input_size=config.d_embed, hidden_size=config.d_hidden,
num_layers=config.n_layers, dropout=config.dropout_prob,
bidirectional=config.birnn)
else:
self.rnn = nn.LSTM(input_size=config.d_embed, hidden_size=config.d_hidden,
num_layers=config.n_layers, dropout=config.dropout_prob,
bidirectional=config.birnn)
def forward(self, inputs):
# shape of `inputs` - (sequence length, batch size, dimension of embedding)
batch_size = inputs.size()[1]
state_shape = self.config.n_cells, batch_size, self.config.d_hidden
if self.config.rnn_type.lower() == "gru":
h0 = Variable(inputs.data.new(*state_shape).zero_())
outputs, ht = self.rnn(inputs, h0)
else:
h0 = c0 = Variable(inputs.data.new(*state_shape).zero_())
outputs, (ht, ct) = self.rnn(inputs, (h0, c0))
return ht[-1] if not self.config.birnn else ht[-2:].transpose(0, 1).contiguous().view(batch_size, -1)
class RelationClassifier(nn.Module):
def __init__(self, config):
super(RelationClassifier, self).__init__()
self.config = config
self.embed = nn.Embedding(config.n_embed, config.d_embed)
self.encoder = Encoder(config)
self.dropout = nn.Dropout(p=config.dropout_prob)
self.relu = nn.ReLU()
seq_in_size = config.d_hidden
if self.config.birnn:
seq_in_size *= 2
self.out = nn.Sequential(
nn.Linear(seq_in_size, seq_in_size), # can apply batch norm after this - add later
nn.BatchNorm1d(seq_in_size),
self.relu,
self.dropout,
nn.Linear(seq_in_size, config.d_out)
)
def forward(self, batch):
# shape of `batch` - (sequence length, batch size)
question_embed = self.embed(batch.question)
if self.config.fix_emb:
question_embed = Variable(question_embed.data)
# shape of `question_embed` - (sequence length, batch size, dimension of embedding)
question_encoded = self.encoder(question_embed)
# shape of `question_encoded` - (batch size, number of cells X size of hidden)
output = self.out(question_encoded)
scores = F.log_softmax(output)
return scores
simple_qa_rnn/relation_prediction/simple_qa_relation.py
+72
View File
@@ -0,0 +1,72 @@
import os
from torchtext import data
# most basic tokenizer - split on whitespace
def my_tokenizer():
return lambda text: [tok for tok in text.split()]
class SimpleQaRelationDataset(data.ZipDataset, data.TabularDataset):
url = 'https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz'
filename = 'SimpleQuestions_v2.tgz'
dirname = 'SimpleQuestions_v2'
@staticmethod
def sort_key(ex):
return len(ex.question)
@classmethod
def splits(cls, text_field, label_field, root='../data',
train='train.txt', validation='valid.txt', test='test.txt'):
"""Create dataset objects for splits of the Simple QA dataset.
This is the most flexible way to use the dataset.
Arguments:
text_field: The field that will be used for premise and hypothesis
data.
label_field: The field that will be used for label data.
root: The root directory that the dataset's zip archive will be
expanded into; therefore the directory in which the
train/valid/test data files will be stored.
train: The filename of the train data. Default: 'annotated_fb_data_train.txt'.
validation: The filename of the validation data, or None to not
load the validation set. Default: 'annotated_fb_data_valid.txt'.
test: The filename of the test data, or None to not load the test
set. Default: 'annotated_fb_data_test.txt'.
"""
print("root path for relation dataset: {}".format(root))
path = cls.download_or_unzip(root)
prefix_fname = 'annotated_fb_data_'
return super(SimpleQaRelationDataset, cls).splits(
os.path.join(path, prefix_fname), train, validation, test,
format='TSV', fields=[('subject', None), ('relation', label_field), (object, None), ('question', text_field)]
)
@classmethod
def iters(cls, batch_size=32, device=0, root='.', wv_dir='.',
wv_type=None, wv_dim='300d', **kwargs):
"""Create iterator objects for splits of the Simple QA dataset.
This is the simplest way to use the dataset, and assumes common
defaults for field, vocabulary, and iterator parameters.
Arguments:
batch_size: Batch size.
device: Device to create batches on. Use -1 for CPU and None for
the currently active GPU device.
root: The root directory that the dataset's zip archive will be
expanded into; therefore the directory in whose wikitext-2
subdirectory the data files will be stored.
wv_dir, wv_type, wv_dim: Passed to the Vocab constructor for the
text field. The word vectors are accessible as
train.dataset.fields['text'].vocab.vectors.
Remaining keyword arguments: Passed to the splits method.
"""
TEXT = data.Field(tokenize=my_tokenizer())
LABEL = data.Field(sequential=False)
train, val, test = cls.splits(TEXT, LABEL, root=root, **kwargs)
TEXT.build_vocab(train, wv_dir=wv_dir, wv_type=wv_type, wv_dim=wv_dim)
LABEL.build_vocab(train)
return data.BucketIterator.splits(
(train, val, test), batch_size=batch_size, device=device)
simple_qa_rnn/relation_prediction/train.py
+161
View File
@@ -0,0 +1,161 @@
import os
import sys
import time
import glob
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
from torchtext import data
from model import RelationClassifier
from args import get_args
from simple_qa_relation import SimpleQaRelationDataset
# get the configuration arguments and set machine - GPU/CPU
args = get_args()
# set random seeds for reproducibility
torch.manual_seed(args.seed)
if not args.cuda:
args.gpu = -1
if torch.cuda.is_available() and not args.cuda:
print("WARNING: You have CUDA but not using it.")
if torch.cuda.is_available() and args.cuda:
torch.cuda.set_device(args.gpu)
torch.cuda.manual_seed(args.seed)
# ---- prepare the dataset with Torchtext -----
questions = data.Field(lower=True)
relations = data.Field(sequential=False)
train, dev, test = SimpleQaRelationDataset.splits(questions, relations)
# build vocab for questions
questions.build_vocab(train, dev, test)
# load word vectors if already saved or else load it from start and save it
if os.path.isfile(args.vector_cache):
questions.vocab.vectors = torch.load(args.vector_cache)
else:
questions.vocab.load_vectors(wv_dir=args.data_cache, wv_type=args.word_vectors, wv_dim=args.d_embed)
os.makedirs(os.path.dirname(args.vector_cache), exist_ok=True)
torch.save(questions.vocab.vectors, args.vector_cache)
# build vocab for relations
relations.build_vocab(train, dev, test)
# BucketIterator buckets the examples according to length so less padding is needed
train_iter, dev_iter, test_iter = data.BucketIterator.splits(
(train, dev, test), batch_size=args.batch_size, device=args.gpu)
train_iter.repeat = False # do not repeat examples after finishing an epoch
# ---- define the model, loss, optim ------
config = args
config.n_embed = len(questions.vocab) # vocab. size / number of embeddings
config.d_out = len(relations.vocab)
config.n_cells = config.n_layers
# double the number of cells for bidirectional networks
if config.birnn:
config.n_cells *= 2
print(config)
if args.resume_snapshot:
model = torch.load(args.resume_snapshot, map_location=lambda storage,location: storage.cuda(args.gpu))
else:
model = RelationClassifier(config)
if args.word_vectors:
model.embed.weight.data = questions.vocab.vectors
if args.cuda:
model.cuda()
criterion = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
# ---- train the model ------
iterations = 0
start = time.time()
best_dev_acc = -1
train_iter.repeat = False
header = ' Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss Accuracy Dev/Accuracy'
dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(','))
log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(','))
os.makedirs(args.save_path, exist_ok=True)
print(header)
for epoch in range(args.epochs):
train_iter.init_epoch()
n_correct, n_total = 0, 0
for batch_idx, batch in enumerate(train_iter):
iterations += 1
# switch model to training mode, clear gradient accumulators
model.train(); optimizer.zero_grad()
# forward pass
answer = model(batch)
# calculate accuracy of predictions in the current batch
n_correct += (torch.max(answer, 1)[1].view(batch.relation.size()).data == batch.relation.data).sum()
n_total += batch.batch_size
train_acc = 100. * n_correct/n_total
# calculate loss of the network output with respect to training labels & backpropagate to compute gradients
loss = criterion(answer, batch.relation)
loss.backward()
# clip the gradients (prevent exploding gradients) and update the weights
torch.nn.utils.clip_grad_norm(model.parameters(), args.clip_gradient)
optimizer.step()
# checkpoint model periodically
if iterations % args.save_every == 0:
snapshot_prefix = os.path.join(args.save_path, 'snapshot')
snapshot_path = snapshot_prefix + '_acc_{:.4f}_loss_{:.6f}_iter_{}_model.pt'.format(train_acc, loss.data[0], iterations)
torch.save(model, snapshot_path)
for f in glob.glob(snapshot_prefix + '*'):
if f != snapshot_path:
os.remove(f)
# evaluate performance on validation set periodically
if iterations % args.dev_every == 0:
# switch model to evaluation mode
model.eval(); dev_iter.init_epoch()
# calculate accuracy on validation set
n_dev_correct, dev_loss = 0, 0
for dev_batch_idx, dev_batch in enumerate(dev_iter):
answer = model(dev_batch)
n_dev_correct += (torch.max(answer, 1)[1].view(dev_batch.relation.size()).data == dev_batch.relation.data).sum()
dev_loss = criterion(answer, dev_batch.relation)
dev_acc = 100. * n_dev_correct / len(dev)
print(dev_log_template.format(time.time()-start,
epoch, iterations, 1+batch_idx, len(train_iter),
100. * (1+batch_idx) / len(train_iter), loss.data[0], dev_loss.data[0], train_acc, dev_acc))
# update best valiation set accuracy
if dev_acc > best_dev_acc:
# found a model with better validation set accuracy
best_dev_acc = dev_acc
snapshot_prefix = os.path.join(args.save_path, 'best_snapshot')
snapshot_path = snapshot_prefix + '_devacc_{}_devloss_{}__iter_{}_model.pt'.format(dev_acc, dev_loss.data[0], iterations)
# save model, delete previous 'best_snapshot' files
torch.save(model, snapshot_path)
for f in glob.glob(snapshot_prefix + '*'):
if f != snapshot_path:
os.remove(f)
elif iterations % args.log_every == 0:
# print progress message
print(log_template.format(time.time()-start,
epoch, iterations, 1+batch_idx, len(train_iter),
100. * (1+batch_idx) / len(train_iter), loss.data[0], ' '*8, n_correct/n_total*100, ' '*12))
simple_qa_rnn/relation_prediction_results.txt
-207
View File
@@ -1,207 +0,0 @@
vocab size = 62622
num classes = 1837
unk vocab count = 22761
loading train/val/test datasets...
train_file: data/SimpleQuestions_v2/annotated_fb_data_train.txt, num train = 75910
val_file: data/SimpleQuestions_v2/annotated_fb_data_valid.txt, num dev = 10845
test_file: data/SimpleQuestions_v2/annotated_fb_data_test.txt, num test = 21687
Namespace(batch_size=128, birnn=True, clip=0.6, cuda=True, d_embedding=300, d_hidden=500, d_out=1837, dev_every=500, device=1, dropout_prob=0.5487647674511446, epochs=25, gpu=0, log_every=250, lr=2.3368165563254845e-05, n_directions=2, n_layers=4, resume_snapshot='', save_every=1000, save_path='saved_checkpoints', seed=1111, test=False, vocab_size=62622)
Time Epoch Iteration Loss Train/Acc. Val/Acc.
1 0 1 7.519370
28 0 250 4.217004
70 0 500 3.454021 0.464165 0.460844
97 1 750 2.695011
139 1 1000 2.023616 0.582841 0.579520
167 2 1250 1.947864
209 2 1500 2.173526 0.654498 0.643415
236 2 1750 2.050010
278 3 2000 1.698692 0.691334 0.676525
305 3 2250 1.334085
347 4 2500 1.258004 0.721398 0.703776
375 4 2750 1.318608
417 5 3000 1.237201 0.745152 0.725632
445 5 3250 1.053011
488 5 3500 1.393011 0.762713 0.740141
516 6 3750 1.064027
558 6 4000 1.188779 0.780143 0.753441
586 7 4250 1.180950
629 7 4500 1.158713 0.792488 0.765811
657 8 4750 1.092308
699 8 5000 0.772544 0.804582 0.771205
728 8 5250 0.935450
770 9 5500 1.031198 0.814555 0.776786
798 9 5750 0.795584
840 10 6000 0.759640 0.824318 0.785156
868 10 6250 0.861564
910 10 6500 0.605637 0.833289 0.787388
938 11 6750 0.720791
980 11 7000 0.615107 0.841089 0.791295
1007 12 7250 0.765555
1050 12 7500 0.601661 0.847874 0.800037
1078 13 7750 0.523285
1120 13 8000 0.550137 0.855265 0.800130
1148 13 8250 0.722784
1190 14 8500 0.632320 0.860231 0.802269
1218 14 8750 0.631018
1261 15 9000 0.584924 0.867174 0.804781
1289 15 9250 0.341058
1331 16 9500 0.483922 0.872853 0.809431
1359 16 9750 0.433858
1402 16 10000 0.514899 0.878004 0.809152
1430 17 10250 0.388845
1472 17 10500 0.379268 0.884301 0.809338
1500 18 10750 0.504304
1542 18 11000 0.477527 0.887898 0.807385
1570 18 11250 0.496538
1612 19 11500 0.457043 0.894630 0.808780
1639 19 11750 0.351258
1682 20 12000 0.567525 0.900348 0.809896
1710 20 12250 0.422562
1752 21 12500 0.327270 0.904247 0.810919
1779 21 12750 0.443917
1821 21 13000 0.429144 0.907726 0.811663
1849 22 13250 0.339602
1891 22 13500 0.393765 0.913101 0.812035
1919 23 13750 0.493038
1961 23 14000 0.322024 0.916473 0.810361
1989 24 14250 0.394269
2031 24 14500 0.315911 0.920623 0.814081
2059 24 14750 0.302249
vocab size = 62622
num classes = 1837
unk vocab count = 22761
loading train/val/test datasets...
train_file: data/SimpleQuestions_v2/annotated_fb_data_train.txt, num train = 75910
val_file: data/SimpleQuestions_v2/annotated_fb_data_valid.txt, num dev = 10845
test_file: data/SimpleQuestions_v2/annotated_fb_data_test.txt, num test = 21687
Namespace(batch_size=128, birnn=True, clip=0.6, cuda=True, d_embedding=300, d_hidden=500, d_out=1837, dev_every=500, device=1, dropout_prob=0.5235083503167401, epochs=25, gpu=0, log_every=250, lr=9.543880052712213e-05, n_directions=2, n_layers=4, resume_snapshot='', save_every=1000, save_path='saved_checkpoints', seed=1111, test=False, vocab_size=62622)
Time Epoch Iteration Loss Train/Acc. Val/Acc.
1 0 1 7.562489
28 0 250 2.717191
70 0 500 1.675420 0.683917 0.673270
98 1 750 1.317849
140 1 1000 1.132524 0.757101 0.731678
168 2 1250 1.179604
210 2 1500 0.727408 0.802843 0.771484
238 2 1750 1.093024
280 3 2000 0.711856 0.829522 0.785156
308 3 2250 0.868195
351 4 2500 0.789406 0.848229 0.793062
379 4 2750 0.748050
421 5 3000 0.530056 0.864856 0.803478
449 5 3250 0.489331
492 5 3500 0.791100 0.880520 0.802362
520 6 3750 0.684948
562 6 4000 0.574934 0.893853 0.801804
590 7 4250 0.449654
632 7 4500 0.407685 0.905407 0.804781
660 8 4750 0.298767
702 8 5000 0.382648 0.917765 0.803850
730 8 5250 0.311796
772 9 5500 0.240214 0.928199 0.802083
799 9 5750 0.403507
842 10 6000 0.195193 0.942216 0.804222
869 10 6250 0.360780
911 10 6500 0.179228 0.949542 0.801060
939 11 6750 0.244037
981 11 7000 0.239074 0.957828 0.798642
1009 12 7250 0.187415
1051 12 7500 0.242319 0.964113 0.796038
1079 13 7750 0.205365
1121 13 8000 0.188374 0.970542 0.795294
1149 13 8250 0.218278
1191 14 8500 0.165913 0.973585 0.797433
1218 14 8750 0.181852
1261 15 9000 0.171079 0.977406 0.795666
1288 15 9250 0.153494
1331 16 9500 0.063608 0.982017 0.789528
1358 16 9750 0.129606
1401 16 10000 0.067068 0.983940 0.789249
1428 17 10250 0.056862
1470 17 10500 0.113310 0.985640 0.790365
1498 18 10750 0.082778
1540 18 11000 0.108898 0.986483 0.792225
1568 18 11250 0.156388
1610 19 11500 0.074055 0.988406 0.784598
1638 19 11750 0.110450
1680 20 12000 0.103423 0.988709 0.787481
1707 20 12250 0.072456
1749 21 12500 0.057874 0.991015 0.786086
1777 21 12750 0.036475
1819 21 13000 0.082274 0.991792 0.788876
1847 22 13250 0.047618
1889 22 13500 0.054384 0.992108 0.785714
1917 23 13750 0.035868
1959 23 14000 0.062694 0.991502 0.790365
1987 24 14250 0.084560
2029 24 14500 0.061790 0.994809 0.785714
2056 24 14750 0.044786
vocab size = 62622
num classes = 1837
unk vocab count = 22761
loading train/val/test datasets...
train_file: data/SimpleQuestions_v2/annotated_fb_data_train.txt, num train = 75910
val_file: data/SimpleQuestions_v2/annotated_fb_data_valid.txt, num dev = 10845
test_file: data/SimpleQuestions_v2/annotated_fb_data_test.txt, num test = 21687
Namespace(batch_size=128, birnn=True, clip=0.6, cuda=True, d_embedding=300, d_hidden=500, d_out=1837, dev_every=500, device=1, dropout_prob=0.5973369506283627, epochs=25, gpu=0, log_every=250, lr=6.789914913570565e-05, n_directions=2, n_layers=4, resume_snapshot='', save_every=1000, save_path='saved_checkpoints', seed=1111, test=False, vocab_size=62622)
Time Epoch Iteration Loss Train/Acc. Val/Acc.
1 0 1 7.538278
28 0 250 2.793929
70 0 500 2.281076 0.605212 0.600074
98 1 750 1.900491
140 1 1000 1.823353 0.709607 0.691778
167 2 1250 1.095232
209 2 1500 1.101957 0.756521 0.735026
237 2 1750 1.370564
279 3 2000 0.972925 0.788891 0.762184
307 3 2250 0.815842
349 4 2500 1.124413 0.810511 0.773996
376 4 2750 0.545115
419 5 3000 0.806257 0.829363 0.786086
446 5 3250 0.610705
489 5 3500 0.797700 0.845858 0.794829
516 6 3750 0.497445
558 6 4000 0.694232 0.855396 0.797805
586 7 4250 0.741005
629 7 4500 0.607206 0.867873 0.801897
657 8 4750 0.565121
699 8 5000 0.472905 0.879690 0.804315
727 8 5250 0.708608
769 9 5500 0.614066 0.889242 0.803757
797 9 5750 0.456040
839 10 6000 0.432193 0.899241 0.806920
866 10 6250 0.382535
908 10 6500 0.394834 0.909333 0.807292
935 11 6750 0.367661
978 11 7000 0.637153 0.918950 0.808594
1005 12 7250 0.394814
1047 12 7500 0.319768 0.927514 0.810919
1075 13 7750 0.262185
1117 13 8000 0.243790 0.932639 0.804688
1144 13 8250 0.240168
1187 14 8500 0.224746 0.939858 0.806734
1215 14 8750 0.280511
1257 15 9000 0.223731 0.948698 0.801339
1284 15 9250 0.342026
1327 16 9500 0.219288 0.954956 0.801525
1354 16 9750 0.144273
1396 16 10000 0.268068 0.960437 0.799014
1424 17 10250 0.142029
1466 17 10500 0.204999 0.964903 0.800688
1494 18 10750 0.188032
1536 18 11000 0.197725 0.968842 0.801153
1563 18 11250 0.163313
1606 19 11500 0.213809 0.970818 0.796875
1633 19 11750 0.167466
1675 20 12000 0.110523 0.974349 0.793992
1702 20 12250 0.145699
1744 21 12500 0.074068 0.977392 0.794085
1772 21 12750 0.098240
1814 21 13000 0.112673 0.983084 0.794550
1842 22 13250 0.106451
1884 22 13500 0.222595 0.982254 0.793992
1912 23 13750 0.079745
1954 23 14000 0.101426 0.985060 0.792318
1982 24 14250 0.118253
2024 24 14500 0.109737 0.986101 0.791853
2052 24 14750 0.081499
simple_qa_rnn/scripts/build_vocab.py
-43
View File
@@ -1,43 +0,0 @@
import os
import glob
import torch
import nltk
import string
# this method was copied from utils/read_data.py
def process_tokenize_text(text):
punc_remover = str.maketrans('', '', string.punctuation)
processed_text = text.lower().translate(punc_remover)
tokens = nltk.word_tokenize(processed_text)
return tokens
def build_vocab_SQ(data_dir):
filepaths = glob.glob(os.path.join(data_dir, 'annotated*.txt'))
print("reading filepaths: {}".format(filepaths))
word_vocab = set()
relation_vocab = set()
for filepath in filepaths:
with open(filepath) as f:
for line in f:
line_items = line.split("\t")
# add relation
relation = line_items[1]
relation_vocab.add(relation)
# add text
qText = line_items[3]
tokens = process_tokenize_text(qText)
word_vocab |= set(tokens)
word2index_dict = {word: i for i, word in enumerate(sorted(word_vocab))} # word to index dictionary
rel2index_dict = {relation: i for i, relation in enumerate(sorted(relation_vocab))} # relation to index dictionary
return (word2index_dict, rel2index_dict)
print("WARNING: This script is dataset specific. Please change it to fit your own dataset.")
data_dir = 'data/SimpleQuestions_v2/'
dst_path = os.path.join(data_dir, 'vocab.pt')
print("Building vocab for data in: {}".format(data_dir))
ret = build_vocab_SQ(data_dir) # ret = (word2index dict, relation2index dict)
print("saving word2index and answer2index dicts to {}".format(dst_path))
torch.save(ret, dst_path)
print("Done!")
simple_qa_rnn/scripts/convert_wordvecs.py
-57
View File
@@ -1,57 +0,0 @@
import sys
from tqdm import tqdm
import torch
import array
import six
try:
path = sys.argv[1]
outpath = sys.argv[2]
except:
print("ERROR: the command line arguments passed in were not valid.\n");
print("USAGE: python scripts/convert_wordvecs.py [input_file] [output_file]");
print("EXAMPLE: python scripts/convert_wordvecs.py glove_300d.txt glove_300d.pt");
sys.exit(1);
prefix_toks = path.split(".")
print('Converting ' + path + ' to PyTorch serialized format...')
lines = [line.rstrip('\n') for line in open(path)]
print("number of lines: {}".format(len(lines)))
wv_tokens = []
wv_arr = array.array('d')
wv_size = None # dimension of the word vectors
vocab_size = 0 # counts the number of words saved
if lines is not None:
for i in tqdm(range(len(lines)), desc="loading word vectors from {}".format(path)):
entries = lines[i].strip().split()
word, entries = entries[0], entries[1:]
if wv_size is None:
wv_size = len(entries)
else:
# safety check that the dimension is the same
if len(entries) != wv_size:
print(len(entries))
print(lines[i])
continue
try:
if isinstance(word, six.binary_type):
word = word.decode('utf-8')
except:
print('non-UTF8 token', repr(word), 'ignored')
continue
wv_arr.extend(float(x) for x in entries)
wv_tokens.append(word)
vocab_size += 1
print("vocab size: {}".format(vocab_size))
print("dim: {}".format(wv_size))
wv_dict = {word: i for i, word in enumerate(wv_tokens)} # word to index dictionary
wv_arr = torch.Tensor(wv_arr).view(vocab_size, wv_size) # word embeddings in Tensor of shape (|V|, |D|)
ret = (wv_dict, wv_arr, wv_size) # save all three info in a tuple
print("saving word vectors to {}".format(outpath))
torch.save(ret, outpath)
simple_qa_rnn/scripts/download.py
-84
View File
@@ -1,84 +0,0 @@
"""
Downloads the following:
- Glove vectors
We Thank Kai Sheng Tai for providing the preprocessing/basis codes.
Taken from: https://github.com/castorini/NCE-CNN-Torch
"""
from __future__ import print_function
import urllib2
import sys
import os
import shutil
import zipfile
import gzip
def download(url, dirpath):
filename = url.split('/')[-1]
filepath = os.path.join(dirpath, filename)
try:
u = urllib2.urlopen(url)
except:
print("URL %s failed to open" %url)
raise Exception
try:
f = open(filepath, 'wb')
except:
print("Cannot write %s" %filepath)
raise Exception
try:
filesize = int(u.info().getheaders("Content-Length")[0])
except:
print("URL %s failed to report length" %url)
raise Exception
print("Downloading: %s Bytes: %s" % (filename, filesize))
downloaded = 0
block_sz = 8192
status_width = 70
while True:
buf = u.read(block_sz)
if not buf:
print('')
break
else:
print('', end='\r')
downloaded += len(buf)
f.write(buf)
status = (("[%-" + str(status_width + 1) + "s] %3.2f%%") %
('=' * int(float(downloaded) / filesize * status_width) + '>', downloaded * 100. / filesize))
print(status, end='')
sys.stdout.flush()
f.close()
return filepath
def unzip(filepath):
print("Extracting: " + filepath)
dirpath = os.path.dirname(filepath)
with zipfile.ZipFile(filepath) as zf:
zf.extractall(dirpath)
os.remove(filepath)
def download_wordvecs(dirpath):
if os.path.exists(dirpath):
print('Found Glove vectors - skip')
return
else:
os.makedirs(dirpath)
url = 'https://nlp.stanford.edu/data/glove.840B.300d.zip'
unzip(download(url, dirpath))
if __name__ == '__main__':
base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
# data
data_dir = os.path.join(base_dir, 'data')
wordvec_dir = os.path.join(data_dir, 'glove')
# libraries
lib_dir = os.path.join(base_dir, 'lib')
# download GloVe word embeddings
download_wordvecs(wordvec_dir)
print("Finished downloading word embeddings!")
simple_qa_rnn/scripts/extract_relevant_words_SQ.py
-72
View File
@@ -1,72 +0,0 @@
import gensim
import glob
import pickle
import nltk
import string
import numpy as np
fnames = glob.glob('../datasets/SimpleQuestions_v2/annotated*.txt')
all_lines = []
for fname in fnames:
with open(fname) as fin:
for line in fin:
all_lines.append(line.rstrip())
print("num of examples in train/val/test: {}".format(len(all_lines)))
max_sent_length = -1
for line in all_lines:
qText = line.split("\t")[3]
punc_remover = str.maketrans('', '', string.punctuation)
processed_text = qText.lower().translate(punc_remover)
tokens = nltk.word_tokenize(processed_text)
if len(tokens) > max_sent_length:
max_sent_length = len(tokens)
print("max_sent_length: {}".format(max_sent_length))
# # get all the words in the train, dev, val, test set
# all_words = set()
# for line in all_lines:
# qText = line.split("\t")[3]
# # process text: remove punctuations, lowercase
# punc_remover = str.maketrans('', '', string.punctuation)
# processed_text = qText.lower().translate(punc_remover)
# tokens = nltk.word_tokenize(processed_text)
# for tok in tokens:
# all_words.add(tok)
#
# print("vocab. size: {}".format(len(all_words)))
#
# # word_to_ix = { word:i for i, word in enumerate(all_words) }
# # # dump the pickle
# # with open("../resources/word_to_ix_SQ.pkl", 'wb') as fh:
# # pickle.dump(word_to_ix, fh)
#
# word_to_vector_map = {}
# w2v_path = "../resources/GoogleNews-vectors-negative300.bin.gz"
# # get their word vectors
# print("loading word vectors...")
# word_vectors = gensim.models.KeyedVectors.load_word2vec_format(w2v_path, binary=True)
#
# # store in a dict
# found = 0
# random = 0
# for w in all_words:
# # print(w)
# try:
# word_to_vector_map[w] = word_vectors[w]
# found += 1
# except:
# word_to_vector_map[w] = np.random.uniform(low=0.0, high=1.0, size=300)
# random += 1
#
# print("found: {}".format(found))
# print("random: {}".format(random))
#
# # dump the pickle
# print("dumping the w2v map pickle...")
# with open("../resources/w2v_map_SQ.pkl", 'wb') as fh:
# pickle.dump(word_to_vector_map, fh)
simple_qa_rnn/scripts/extract_relevant_words_WQ.py
-56
View File
@@ -1,56 +0,0 @@
import gensim
import glob
import json
import pickle
import nltk
import string
import numpy as np
fnames = glob.glob('../datasets/dataset-factoid-webquestions/main/*.json')
all_entries = []
for fname in fnames:
with open(fname) as fin:
data = json.load(fin)
all_entries.extend(data)
print("num of examples in train/val/test: {}".format(len(all_entries)))
# get all the words in the train, dev, val, test set
all_words = set()
for entry in all_entries:
qText = entry.get('qText')
# process text: remove punctuations, lowercase
punc_remover = str.maketrans('', '', string.punctuation)
processed_text = qText.lower().translate(punc_remover)
tokens = nltk.word_tokenize(processed_text)
for tok in tokens:
all_words.add(tok)
print("vocab. size: {}".format(len(all_words)))
word_to_vector_map = {}
w2v_path = "../resources/GoogleNews-vectors-negative300.bin.gz"
# get their word vectors
print("loading word vectors...")
word_vectors = gensim.models.KeyedVectors.load_word2vec_format(w2v_path, binary=True)
# store in a dict
found = 0
random = 0
for w in all_words:
# print(w)
try:
word_to_vector_map[w] = word_vectors[w]
found += 1
except:
word_to_vector_map[w] = np.random.uniform(low=0.0, high=1.0, size=300)
random += 1
print("found: {}".format(found))
print("random: {}".format(random))
# dump the pickle
print("dumping the w2v map pickle...")
with open("../resources/w2v_map_WQ.pkl", 'wb') as fh:
pickle.dump(word_to_vector_map, fh)
simple_qa_rnn/scripts/load_w2v.py
-11
View File
@@ -1,11 +0,0 @@
import pickle
# load the pickled word vectors
w2v_pkl_path = "w2v_map_SQ.pkl"
print("loading word vectors from the pickle...")
w2v_map = None
with open(w2v_pkl_path, 'rb') as fh:
w2v_map = pickle.load(fh)
print(list(w2v_map.keys()))
print(len(w2v_map.keys()))
simple_qa_rnn/scripts/relation_prediction_SQ.py
-27
View File
@@ -1,27 +0,0 @@
import glob
import pickle
fnames = glob.glob('../datasets/SimpleQuestions_v2/annotated*.txt')
all_lines = []
for fname in fnames:
with open(fname) as fin:
for line in fin:
all_lines.append(line.rstrip())
print("num of examples in train/val/test: {}".format(len(all_lines)))
all_relations = set()
for line in all_lines:
rel = line.split("\t")[1]
all_relations.add(rel)
print("num of relation types: {}".format(len(all_relations)))
print(all_relations)
print( ("www.freebase.com/music/release/region") in all_relations )
rel_to_ix = { rel:i for i, rel in enumerate(all_relations) }
# print(rel_to_ix)
# # dump the pickle
# print("dumping the w2v map pickle...")
# with open("../resources/rel_to_ix_SQ.pkl", 'wb') as fh:
# pickle.dump(rel_to_ix, fh)
simple_qa_rnn/scripts/relation_prediction_WQ.py
-24
View File
@@ -1,24 +0,0 @@
import glob
import json
fnames = glob.glob('dataset-factoid-webquestions/d-freebase-rp/*.json')
all_entries = []
for fname in fnames:
with open(fname) as fin:
data = json.load(fin)
all_entries.extend(data)
print("num of examples in train/val/test: {}".format(len(all_entries)))
all_relations = set()
for entry in all_entries:
qID = entry.get('qId')
relPaths = entry.get('relPaths')
for relPath in relPaths:
relations = relPath[0]
for rel in relations:
all_relations.add(rel)
print("num of relation types: {}".format(len(all_relations)))
print(all_relations)
print( ("/people/marriage/spouse") in all_relations )
simple_qa_rnn/scripts/sq_entity_data.py
-40
View File
@@ -1,40 +0,0 @@
import glob
import os
fnames = glob.glob('../data/SimpleQuestions_v2/annotated*.txt')
print(fnames)
all_entities = set()
for fname in fnames:
with open(fname) as fin:
for line in fin:
entity = line.split("\t")[0]
# entity: www.freebase.com/m/0f3xg_ --> m.0f3xg_
if entity.startswith("www.freebase.com/"):
entity = entity[17:].replace("/", ".")
all_entities.add(entity)
print("num of entities in train/val/test: {}".format(len(all_entities)))
names_map = {}
names_file = "../data/freebase/names-map-0.ttl"
with open(names_file) as fin:
for line in fin:
id = line.split("\t")[0][3:]
name = line.split("\t")[2].rstrip()
if name.endswith("\"@en."):
name = name[1:-5]
if id in all_entities:
names_map[id] = name
print("num of FB entities in the map: {}".format(len(names_map)))
found = len(names_map)
print("found: {}".format(found))
print("notfound: {}".format(len(all_entities) - found))
outfile = open("names_map.tsv", 'w')
for id, name in names_map.items():
outfile.write("{}\t{}\n".format(id, name))
outfile.close()
print("done")
simple_qa_rnn/scripts/wq_relation_data.py
-40
View File
@@ -1,40 +0,0 @@
import json
import os
rel_dir = 'data/dataset-factoid-webquestions/d-freebase-rp/'
text_dir = 'data/dataset-factoid-webquestions/main/'
out_dir = 'data/webquestions-custom/relation-prediction'
fname = 'val.json'
rel_filename = os.path.join(rel_dir, fname)
text_filename = os.path.join(text_dir, fname)
out_filename = os.path.join(out_dir, fname)
text_entries = []
with open(text_filename) as fin:
data = json.load(fin)
text_entries.extend(data)
rel_entries = []
with open(rel_filename) as fin:
data = json.load(fin)
rel_entries.extend(data)
print("num of examples: {}".format(len(text_entries)))
assert len(text_entries) == len(rel_entries)
outfile = open(out_filename, 'w')
count = 0
for text_entry, rel_entry in zip(text_entries, rel_entries):
assert text_entry.get('qId') == rel_entry.get('qId')
qID = text_entry.get('qId')
qText = text_entry.get('qText')
relPaths = rel_entry.get('relPaths')
for relPath in relPaths:
relations = relPath[0]
for rel in relations:
count += 1
outfile.write("{}\t{}\t{}\n".format(qID, rel, qText))
print("count: {}".format(count))
outfile.close()
simple_qa_rnn/train_relation_model.py
-184
View File
@@ -1,184 +0,0 @@
import os
import sys
import time
import glob
import numpy as np
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim
from relation_model import RelationPredictor
from args import get_args
import data
from utils.vocab import Vocab
from utils.read_data import *
args = get_args()
# Set the random seed manually for reproducibility.
torch.manual_seed(args.seed)
if torch.cuda.is_available():
if not args.cuda:
print("WARNING: You have a CUDA device, so you should probably run with --cuda")
else:
torch.cuda.manual_seed(args.seed)
torch.cuda.set_device(args.device)
# ---- helper methods ------
def evaluate_dataset_batch(data_set, model):
n_total = data_set["size"]
n_correct = 0
num_batches = n_total // args.batch_size
batch_indices = np.split(range(n_total),
range(args.batch_size, n_total, args.batch_size))
model.eval()
for batch_ix in range(num_batches):
batch_questions = data_set["questions"][batch_indices[batch_ix]]
batch_relations = data_set["rel_labels"][batch_indices[batch_ix]]
inputs = Variable(read_text_tensor(batch_questions, word_vocab), volatile=True)
targets = Variable(read_labels_tensor(batch_relations, rel_vocab), volatile=True)
if args.cuda:
inputs.data = inputs.data.cuda()
targets.data = targets.data.cuda()
scores = model(inputs)
pred_score, pred_label_ix = torch.max(scores, dim=1) # check this properly
pred_label_ix = pred_label_ix.view(args.batch_size)
sum_correct = torch.sum(torch.eq(pred_label_ix, targets))
n_correct += sum_correct.data[0]
acc = n_correct / (num_batches * args.batch_size)
return acc
def repackage_hidden(h):
"""Wraps hidden states in new Variables, to detach them from their history."""
if type(h) == Variable:
return Variable(h.data)
else:
return tuple(repackage_hidden(v) for v in h)
# ---- dataset paths ------
data_dir = "data/SimpleQuestions_v2/"
train_file = os.path.join(data_dir, "annotated_fb_data_train.txt")
val_file = os.path.join(data_dir, "annotated_fb_data_valid.txt")
test_file = os.path.join(data_dir, "annotated_fb_data_test.txt")
# ---- load GloVe embeddings ------
embed_pt_filepath = 'data/glove/glove.840B.300d.pt'
emb_w2i, emb_vecs = read_embedding(embed_pt_filepath)
emb_vocab = Vocab(emb_w2i)
emb_dim = emb_vecs.size()[1]
# ---- create dataset vocabulary and embeddings ------
vocab_pt_filepath = os.path.join(data_dir, "vocab.pt")
word2index_dict, rel2index_dict = torch.load(vocab_pt_filepath)
word_vocab = Vocab(word2index_dict)
word_vocab.add_pad_token("<PAD>")
word_vocab.add_unk_token("<UNK>")
rel_vocab = Vocab(rel2index_dict)
vocab_size = word_vocab.size
num_classes = len(rel2index_dict)
print('vocab size = {}'.format(vocab_size))
print('num classes = {}'.format(num_classes))
num_unk = 0
vecs = torch.FloatTensor(vocab_size, emb_dim)
for i in range(vocab_size):
word = word_vocab.get_token(i)
if emb_vocab.contains(word):
vecs[i] = emb_vecs[emb_vocab.get_index(word)]
elif word == word_vocab.pad_token:
vecs[i].zero_()
else:
num_unk += 1
vecs[i].uniform_(-0.05, 0.05)
print('unk vocab count = {}'.format(num_unk))
emb_vocab = None
emb_vecs = None
# ---- load datasets ------
print("loading train/val/test datasets...")
train_dataset = read_dataset(train_file, word_vocab, rel_vocab)
val_dataset = read_dataset(val_file, word_vocab, rel_vocab)
test_dataset = read_dataset(test_file, word_vocab, rel_vocab)
print('train_file: {}, num train = {}'.format(train_file, train_dataset["size"]))
print('val_file: {}, num dev = {}'.format(val_file, val_dataset["size"]))
print('test_file: {}, num test = {}'.format(test_file, test_dataset["size"]))
# ---- Define Model, Loss, Optim ------
config = args
config.vocab_size = vocab_size
config.d_out = num_classes
config.n_directions = 2 if config.birnn else 1
print(config)
model = RelationPredictor(config)
# initialize the embedding layer with the word vectors
model.embed.weight.data = vecs
if args.cuda:
model.cuda()
loss_function = nn.NLLLoss()
optimizer = optim.Adam(model.parameters(), lr=args.lr)
# ---- Train Model ------
start = time.time()
best_val_acc = -1
iter = 0
header = ' Time Epoch Iteration Loss Train/Acc. Val/Acc.'
print(header)
log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>9.6f}'.split(','))
dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>9.6f},{:9.6f},{:11.6f}'.split(','))
model.train()
for epoch in range(args.epochs):
# shuffle the dataset and create batches (truncate the last batch if not of equal size)
shuffled_indices = np.random.permutation(train_dataset["size"])
num_batches = len(shuffled_indices) // args.batch_size
batch_indices = np.split(shuffled_indices,
range(args.batch_size, len(shuffled_indices), args.batch_size))
model.encoder.hidden = model.encoder.init_hidden()
for batch_ix in range(num_batches):
iter += 1
batch_questions = train_dataset["questions"][batch_indices[batch_ix]]
batch_relations = train_dataset["rel_labels"][batch_indices[batch_ix]]
inputs = Variable( read_text_tensor(batch_questions, word_vocab) )
targets = Variable( read_labels_tensor(batch_relations, rel_vocab) )
if args.cuda:
inputs.data = inputs.data.cuda()
targets.data = targets.data.cuda()
# clear out gradients and hidden states of the model
model.zero_grad()
model.encoder.hidden = repackage_hidden(model.encoder.hidden)
# prepare inputs for LSTM model and run forward pass
scores = model(inputs)
# compute the loss, gradients, and update the parameters
loss = loss_function(scores, targets)
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm(model.parameters(), args.clip)
optimizer.step()
# log at intervals
if iter % args.dev_every == 0:
train_acc = evaluate_dataset_batch(train_dataset, model)
val_acc = evaluate_dataset_batch(val_dataset, model)
model.train()
print(dev_log_template.format(time.time()-start, epoch, iter, loss.data[0], train_acc, val_acc))
if val_acc > best_val_acc:
best_val_acc = val_acc
snapshot_prefix = os.path.join(args.save_path, 'best_snapshot')
snapshot_path = snapshot_prefix + '_valacc_{:6.4f}_trainacc{:6.4f}_iter_{}_model.pt'.format(val_acc, train_acc, iter)
torch.save(model.state_dict(), snapshot_path)
for f in glob.glob(snapshot_prefix + '*'):
if f != snapshot_path:
os.remove(f)
elif iter == 1 or iter % args.log_every == 0:
print(log_template.format(time.time() - start, epoch, iter, loss.cpu().data[0]))
simple_qa_rnn/utils/__init__.py
View File
simple_qa_rnn/utils/read_data.py
-71
View File
@@ -1,71 +0,0 @@
import torch
import nltk
import string
from torch.autograd import Variable
import numpy as np
## functions for loading data from disk
def process_tokenize_text(text):
punc_remover = str.maketrans('', '', string.punctuation)
processed_text = text.lower().translate(punc_remover)
tokens = nltk.word_tokenize(processed_text)
return tokens
def read_embedding(embed_pt_filepath):
embed_tuple = torch.load(embed_pt_filepath)
word2index, w2v_tensor, dim = embed_tuple
return word2index, w2v_tensor
def find_max_seq_length(text):
max_len = -1
for tokens in text:
curr_len = len(tokens)
if curr_len > max_len:
max_len = curr_len
return max_len
def read_text_tensor(batch_text, word_vocab):
out_text = []
max_len = find_max_seq_length(batch_text)
for sent_tokens in batch_text:
S = len(sent_tokens)
sent = []
for i in range(S):
token = sent_tokens[i]
sent.append( word_vocab.get_index(token) )
# pad the right end till the max length of the mini batch
for i in range(S, max_len):
sent.append( word_vocab.pad_index )
out_text.append(sent)
return torch.LongTensor(out_text)
def read_labels_tensor(rel_labels, rel_vocab, cuda=False):
N = len(rel_labels)
labels_list = []
for i in range(N):
token = rel_labels[i]
labels_list.append( rel_vocab.get_index(token) )
return torch.LongTensor(labels_list)
def read_dataset(datapath, word_vocab, rel_vocab):
questions = []
rel_labels = []
# read questions and label from the datapath - could be train, dev, testls
with open(datapath) as f:
for line in f:
line_items = line.split("\t")
# add relation
relation = line_items[1]
rel_labels.append(relation)
# add text
qText = line_items[3]
tokens = process_tokenize_text(qText)
questions.append(tokens)
dataset = {"word_vocab": word_vocab, "rel_vocab": rel_vocab, "size": len(rel_labels),
"questions": np.array(questions), "rel_labels": np.array(rel_labels)}
return dataset
simple_qa_rnn/utils/vocab.py
-68
View File
@@ -1,68 +0,0 @@
import torch
class Vocab(object):
"""
A vocabulary object. Initialized from a file with one vocabulary token per line.
Maps between vocabulary tokens and indices. If an UNK token is defined in the
vocabulary, returns the index to this token if queried for an out-of-vocabulary
token.
"""
# def __init__(self, vocabpath):
# self.size = 0
# self.index = {}
# self.tokens = {}
#
# with open(vocabpath, 'r') as f:
# for line in f:
# word = line.rstrip()
# self.tokens[self.size] = word
# self.index[word] = self.size
# self.size += 1
# # automatically add unknown token
# # self.add_unk_token("<UNK>")
def __init__(self, word2index):
self.index = word2index
self.size = len(word2index)
self.tokens = {index: word for word, index in word2index.items()}
# self.add_unk_token("<UNK>")
def contains(self, word):
return word in self.index.keys()
def add(self, word):
if not self.contains(word):
self.tokens[self.size] = word
self.index[word] = self.size
self.size += 1
def add_unk_token(self, token):
self.unk_token = token
self.add(token)
self.unk_index = self.index[token]
def add_pad_token(self, token):
self.pad_token = token
self.add(token)
self.pad_index = self.index[token]
def get_index(self, word):
if self.contains(word):
return self.index[word]
else:
print("{} - word not found in vocab. returning unk_index".format(word))
return self.unk_index
def get_token(self, index):
if index < 0 or index >= self.size:
raise IndexError('index {} out of bounds'.format(index))
return self.tokens[index]
def map(self, tokens):
N = len(tokens)
out = torch.IntTensor(N)
for i in range(N):
out[i] = self.index(tokens[i])
return out