diff --git a/simple_qa_rnn/.gitignore b/simple_qa_rnn/.gitignore deleted file mode 100644 index 5465b04..0000000 --- a/simple_qa_rnn/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -data/* -data/glove/ -resources/ -saved_checkpoints/ -__pycache__/ -relation_prediction/*cache*/* \ No newline at end of file diff --git a/simple_qa_rnn/README.md b/simple_qa_rnn/README.md deleted file mode 100644 index b1238a6..0000000 --- a/simple_qa_rnn/README.md +++ /dev/null @@ -1,21 +0,0 @@ -## Relation Prediction Model - -- Download and extract SimpleQuestions dataset by running the script: -``` -bash fetch_dataset.sh -``` - -- You will also require the package - [torchtext](https://github.com/pytorch/text). -``` -git clone https://github.com/pytorch/text.git -cd path/to/torchtext -python setup.py install -``` - -- Run the training script with the following commands. Please check out args.py file to see the different commands available: -``` -cd relation_prediction -python train.py -python train.py --no_cuda -python train.py --rnn_type gru -``` diff --git a/simple_qa_rnn/fetch_dataset.sh b/simple_qa_rnn/fetch_dataset.sh deleted file mode 100755 index 624572a..0000000 --- a/simple_qa_rnn/fetch_dataset.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# download dataset and put it in data directory -mkdir data -pushd data -wget https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz -tar -xvzf SimpleQuestions_v2.tgz -popd \ No newline at end of file diff --git a/simple_qa_rnn/relation_prediction/args.py b/simple_qa_rnn/relation_prediction/args.py deleted file mode 100644 index b50d494..0000000 --- a/simple_qa_rnn/relation_prediction/args.py +++ /dev/null @@ -1,32 +0,0 @@ -import os - -from argparse import ArgumentParser - -def get_args(): - parser = ArgumentParser(description='Simple QA model - Ferhan Ture') - parser.add_argument('--epochs', type=int, default=40) - parser.add_argument('--batch_size', type=int, default=32) - parser.add_argument('--rnn_type', type=str, default='lstm') # or use 'gru' - parser.add_argument('--d_embed', type=int, default=300) - parser.add_argument('--d_hidden', type=int, default=400) - parser.add_argument('--n_layers', type=int, default=2) - parser.add_argument('--lr', type=float, default=1e-5) - parser.add_argument('--test', action='store_true', dest='test', help='turn on test mode; no training.') - parser.add_argument('--not_bidirectional', action='store_false', dest='birnn') - parser.add_argument('--clip_gradient', type=float, default=0.5, help='gradient clipping') - parser.add_argument('--log_every', type=int, default=300) - parser.add_argument('--dev_every', type=int, default=1200) - parser.add_argument('--save_every', type=int, default=1200) - parser.add_argument('--dropout_prob', type=float, default=0.3) - parser.add_argument('--patience', type=int, default=10, help="number of epochs to wait before early stopping") - parser.add_argument('--no_cuda', action='store_false', help='do not use CUDA', dest='cuda') - parser.add_argument('--gpu', type=int, default=0, help='GPU device to use') # use -1 for CPU - parser.add_argument('--seed', type=int, default=1111, help='random seed for reproducing results') - parser.add_argument('--save_path', type=str, default='saved_checkpoints') - parser.add_argument('--data_cache', type=str, default=os.path.join(os.getcwd(), 'data_cache')) - parser.add_argument('--vector_cache', type=str, default=os.path.join(os.getcwd(), 'vector_cache/input_vectors.pt')) - parser.add_argument('--word_vectors', type=str, default='glove.42B') - parser.add_argument('--train_embed', action='store_false', dest='fix_emb') # fine-tune the word embeddings - parser.add_argument('--resume_snapshot', type=str, default='') - args = parser.parse_args() - return args diff --git a/simple_qa_rnn/relation_prediction/hyperparameter_tuning.py b/simple_qa_rnn/relation_prediction/hyperparameter_tuning.py deleted file mode 100644 index 651e489..0000000 --- a/simple_qa_rnn/relation_prediction/hyperparameter_tuning.py +++ /dev/null @@ -1,18 +0,0 @@ -from random import randint, uniform -from subprocess import call - -epochs = 50 -count = 20 -for id in range(count): - learning_rate = 10 ** uniform(-5, -4) - d_hidden = randint(550, 600) - n_layers = randint(4, 5) - dropout = uniform(0.5, 0.6) - clip = uniform(0.6, 0.7) - - command = "python train.py --dev_every 500 --log_every 250 --batch_size 32 " \ - "--epochs {} --lr {} --d_hidden {} --n_layers {} --dropout_prob {} --clip_gradient {} >> " \ - "results.txt".format(epochs, learning_rate, d_hidden, n_layers, dropout, clip) - - print("Running: " + command) - call(command, shell=True) diff --git a/simple_qa_rnn/relation_prediction/model.py b/simple_qa_rnn/relation_prediction/model.py deleted file mode 100644 index 1b9cc6a..0000000 --- a/simple_qa_rnn/relation_prediction/model.py +++ /dev/null @@ -1,65 +0,0 @@ -import torch -import torch.nn as nn -from torch.autograd import Variable -import torch.nn.functional as F - -class Encoder(nn.Module): - - def __init__(self, config): - super(Encoder, self).__init__() - self.config = config - if config.rnn_type.lower() == "gru": - self.rnn = nn.GRU(input_size=config.d_embed, hidden_size=config.d_hidden, - num_layers=config.n_layers, dropout=config.dropout_prob, - bidirectional=config.birnn) - else: - self.rnn = nn.LSTM(input_size=config.d_embed, hidden_size=config.d_hidden, - num_layers=config.n_layers, dropout=config.dropout_prob, - bidirectional=config.birnn) - - - def forward(self, inputs): - # shape of `inputs` - (sequence length, batch size, dimension of embedding) - batch_size = inputs.size()[1] - state_shape = self.config.n_cells, batch_size, self.config.d_hidden - if self.config.rnn_type.lower() == "gru": - h0 = Variable(inputs.data.new(*state_shape).zero_()) - outputs, ht = self.rnn(inputs, h0) - else: - h0 = c0 = Variable(inputs.data.new(*state_shape).zero_()) - outputs, (ht, ct) = self.rnn(inputs, (h0, c0)) - return ht[-1] if not self.config.birnn else ht[-2:].transpose(0, 1).contiguous().view(batch_size, -1) - - -class RelationClassifier(nn.Module): - - def __init__(self, config): - super(RelationClassifier, self).__init__() - self.config = config - self.embed = nn.Embedding(config.n_embed, config.d_embed) - self.encoder = Encoder(config) - self.dropout = nn.Dropout(p=config.dropout_prob) - self.relu = nn.ReLU() - seq_in_size = config.d_hidden - if self.config.birnn: - seq_in_size *= 2 - - self.out = nn.Sequential( - nn.Linear(seq_in_size, seq_in_size), # can apply batch norm after this - add later - nn.BatchNorm1d(seq_in_size), - self.relu, - self.dropout, - nn.Linear(seq_in_size, config.d_out) - ) - - def forward(self, batch): - # shape of `batch` - (sequence length, batch size) - question_embed = self.embed(batch.question) - if self.config.fix_emb: - question_embed = Variable(question_embed.data) - # shape of `question_embed` - (sequence length, batch size, dimension of embedding) - question_encoded = self.encoder(question_embed) - # shape of `question_encoded` - (batch size, number of cells X size of hidden) - output = self.out(question_encoded) - scores = F.log_softmax(output) - return scores diff --git a/simple_qa_rnn/relation_prediction/simple_qa_relation.py b/simple_qa_rnn/relation_prediction/simple_qa_relation.py deleted file mode 100644 index 1f89b15..0000000 --- a/simple_qa_rnn/relation_prediction/simple_qa_relation.py +++ /dev/null @@ -1,72 +0,0 @@ -import os - -from torchtext import data - -# most basic tokenizer - split on whitespace -def my_tokenizer(): - return lambda text: [tok for tok in text.split()] - -class SimpleQaRelationDataset(data.ZipDataset, data.TabularDataset): - - url = 'https://www.dropbox.com/s/tohrsllcfy7rch4/SimpleQuestions_v2.tgz' - filename = 'SimpleQuestions_v2.tgz' - dirname = 'SimpleQuestions_v2' - - @staticmethod - def sort_key(ex): - return len(ex.question) - - @classmethod - def splits(cls, text_field, label_field, root='../data', - train='train.txt', validation='valid.txt', test='test.txt'): - """Create dataset objects for splits of the Simple QA dataset. - This is the most flexible way to use the dataset. - Arguments: - text_field: The field that will be used for premise and hypothesis - data. - label_field: The field that will be used for label data. - root: The root directory that the dataset's zip archive will be - expanded into; therefore the directory in which the - train/valid/test data files will be stored. - train: The filename of the train data. Default: 'annotated_fb_data_train.txt'. - validation: The filename of the validation data, or None to not - load the validation set. Default: 'annotated_fb_data_valid.txt'. - test: The filename of the test data, or None to not load the test - set. Default: 'annotated_fb_data_test.txt'. - """ - print("root path for relation dataset: {}".format(root)) - path = cls.download_or_unzip(root) - prefix_fname = 'annotated_fb_data_' - return super(SimpleQaRelationDataset, cls).splits( - os.path.join(path, prefix_fname), train, validation, test, - format='TSV', fields=[('subject', None), ('relation', label_field), (object, None), ('question', text_field)] - ) - - @classmethod - def iters(cls, batch_size=32, device=0, root='.', wv_dir='.', - wv_type=None, wv_dim='300d', **kwargs): - """Create iterator objects for splits of the Simple QA dataset. - This is the simplest way to use the dataset, and assumes common - defaults for field, vocabulary, and iterator parameters. - Arguments: - batch_size: Batch size. - device: Device to create batches on. Use -1 for CPU and None for - the currently active GPU device. - root: The root directory that the dataset's zip archive will be - expanded into; therefore the directory in whose wikitext-2 - subdirectory the data files will be stored. - wv_dir, wv_type, wv_dim: Passed to the Vocab constructor for the - text field. The word vectors are accessible as - train.dataset.fields['text'].vocab.vectors. - Remaining keyword arguments: Passed to the splits method. - """ - TEXT = data.Field(tokenize=my_tokenizer()) - LABEL = data.Field(sequential=False) - - train, val, test = cls.splits(TEXT, LABEL, root=root, **kwargs) - - TEXT.build_vocab(train, wv_dir=wv_dir, wv_type=wv_type, wv_dim=wv_dim) - LABEL.build_vocab(train) - - return data.BucketIterator.splits( - (train, val, test), batch_size=batch_size, device=device) \ No newline at end of file diff --git a/simple_qa_rnn/relation_prediction/train.py b/simple_qa_rnn/relation_prediction/train.py deleted file mode 100644 index ae59e38..0000000 --- a/simple_qa_rnn/relation_prediction/train.py +++ /dev/null @@ -1,161 +0,0 @@ -import os -import sys -import time -import glob -import numpy as np -import torch -import torch.nn as nn -import torch.optim as optim - -from torch.autograd import Variable -from torchtext import data - -from model import RelationClassifier -from args import get_args -from simple_qa_relation import SimpleQaRelationDataset - -# get the configuration arguments and set machine - GPU/CPU -args = get_args() -# set random seeds for reproducibility -torch.manual_seed(args.seed) -if not args.cuda: - args.gpu = -1 -if torch.cuda.is_available() and not args.cuda: - print("WARNING: You have CUDA but not using it.") -if torch.cuda.is_available() and args.cuda: - torch.cuda.set_device(args.gpu) - torch.cuda.manual_seed(args.seed) - -# ---- prepare the dataset with Torchtext ----- -questions = data.Field(lower=True) -relations = data.Field(sequential=False) - -train, dev, test = SimpleQaRelationDataset.splits(questions, relations) - -# build vocab for questions -questions.build_vocab(train, dev, test) - -# load word vectors if already saved or else load it from start and save it -if os.path.isfile(args.vector_cache): - questions.vocab.vectors = torch.load(args.vector_cache) -else: - questions.vocab.load_vectors(wv_dir=args.data_cache, wv_type=args.word_vectors, wv_dim=args.d_embed) - os.makedirs(os.path.dirname(args.vector_cache), exist_ok=True) - torch.save(questions.vocab.vectors, args.vector_cache) - -# build vocab for relations -relations.build_vocab(train, dev, test) - -# BucketIterator buckets the examples according to length so less padding is needed -train_iter, dev_iter, test_iter = data.BucketIterator.splits( - (train, dev, test), batch_size=args.batch_size, device=args.gpu) -train_iter.repeat = False # do not repeat examples after finishing an epoch - - -# ---- define the model, loss, optim ------ -config = args -config.n_embed = len(questions.vocab) # vocab. size / number of embeddings -config.d_out = len(relations.vocab) -config.n_cells = config.n_layers -# double the number of cells for bidirectional networks -if config.birnn: - config.n_cells *= 2 -print(config) - -if args.resume_snapshot: - model = torch.load(args.resume_snapshot, map_location=lambda storage,location: storage.cuda(args.gpu)) -else: - model = RelationClassifier(config) - if args.word_vectors: - model.embed.weight.data = questions.vocab.vectors - if args.cuda: - model.cuda() - -criterion = nn.NLLLoss() -optimizer = optim.Adam(model.parameters(), lr=args.lr) - - -# ---- train the model ------ -iterations = 0 -start = time.time() -best_dev_acc = -1 -train_iter.repeat = False -header = ' Time Epoch Iteration Progress (%Epoch) Loss Dev/Loss Accuracy Dev/Accuracy' -dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(',')) -log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(',')) -os.makedirs(args.save_path, exist_ok=True) -print(header) - -for epoch in range(args.epochs): - train_iter.init_epoch() - n_correct, n_total = 0, 0 - - for batch_idx, batch in enumerate(train_iter): - iterations += 1 - - # switch model to training mode, clear gradient accumulators - model.train(); optimizer.zero_grad() - - # forward pass - answer = model(batch) - - # calculate accuracy of predictions in the current batch - n_correct += (torch.max(answer, 1)[1].view(batch.relation.size()).data == batch.relation.data).sum() - n_total += batch.batch_size - train_acc = 100. * n_correct/n_total - - # calculate loss of the network output with respect to training labels & backpropagate to compute gradients - loss = criterion(answer, batch.relation) - loss.backward() - - # clip the gradients (prevent exploding gradients) and update the weights - torch.nn.utils.clip_grad_norm(model.parameters(), args.clip_gradient) - optimizer.step() - - # checkpoint model periodically - if iterations % args.save_every == 0: - snapshot_prefix = os.path.join(args.save_path, 'snapshot') - snapshot_path = snapshot_prefix + '_acc_{:.4f}_loss_{:.6f}_iter_{}_model.pt'.format(train_acc, loss.data[0], iterations) - torch.save(model, snapshot_path) - for f in glob.glob(snapshot_prefix + '*'): - if f != snapshot_path: - os.remove(f) - - # evaluate performance on validation set periodically - if iterations % args.dev_every == 0: - - # switch model to evaluation mode - model.eval(); dev_iter.init_epoch() - - # calculate accuracy on validation set - n_dev_correct, dev_loss = 0, 0 - for dev_batch_idx, dev_batch in enumerate(dev_iter): - answer = model(dev_batch) - n_dev_correct += (torch.max(answer, 1)[1].view(dev_batch.relation.size()).data == dev_batch.relation.data).sum() - dev_loss = criterion(answer, dev_batch.relation) - dev_acc = 100. * n_dev_correct / len(dev) - - print(dev_log_template.format(time.time()-start, - epoch, iterations, 1+batch_idx, len(train_iter), - 100. * (1+batch_idx) / len(train_iter), loss.data[0], dev_loss.data[0], train_acc, dev_acc)) - - # update best valiation set accuracy - if dev_acc > best_dev_acc: - # found a model with better validation set accuracy - best_dev_acc = dev_acc - snapshot_prefix = os.path.join(args.save_path, 'best_snapshot') - snapshot_path = snapshot_prefix + '_devacc_{}_devloss_{}__iter_{}_model.pt'.format(dev_acc, dev_loss.data[0], iterations) - - # save model, delete previous 'best_snapshot' files - torch.save(model, snapshot_path) - for f in glob.glob(snapshot_prefix + '*'): - if f != snapshot_path: - os.remove(f) - - elif iterations % args.log_every == 0: - - # print progress message - print(log_template.format(time.time()-start, - epoch, iterations, 1+batch_idx, len(train_iter), - 100. * (1+batch_idx) / len(train_iter), loss.data[0], ' '*8, n_correct/n_total*100, ' '*12)) -