From 6daa5a128fd651b6190db7c73fdffdf07262a5d7 Mon Sep 17 00:00:00 2001 From: Achyudh Ram Date: Fri, 26 Oct 2018 19:10:19 -0400 Subject: [PATCH] Replication of STOA for Reuters Dataset (#152) * Add ReutersTrainer, ReutersEvaluator options in Factory classes * Add Reuters to Kim-CNN command line arguments * Fix SST dataset path according to changes in Kim-CNN args The dataset path in args.py was made to point at the dataset folder rather than dataset/SST folder. Hence SST folder was added to paths in the SST dataset class * Add Reuters dataset class, and support in __main__ * Add Reuters dataset trainers and evaluators * Remove debug print statement in reuters_evaluator * Fix rounding bug in reuters_trainer and reuters_evaluator * Add LSTM for baseline text classification measurements * Add eval metrics for lstm_baseline * Set batch_first param in lstm_baseline * Remove onnx args from lstm_baseline * Pack padded sequences in LSTM_baseline * Add TensorBoardX support for Reuters trainer * Add Arxiv Academic Paper Dataset (AAPD) * Add Hidden Bottleneck Layer to BiLSTM * Fix packing of padded tensors in Reuters * Add cmdline args for Hidden Bottleneck Layer for BiLSTM * Include pre-padding lengths in AAPD dataset * Remove duplication of preprocessing code in AAPD * Remove batch_size condition in ReutersTrainer --- common/dataset.py | 6 +++ common/evaluation.py | 1 + common/evaluators/reuters_evaluator.py | 7 ++- common/train.py | 1 + common/trainers/reuters_trainer.py | 15 +++++-- datasets/aapd.py | 59 ++++++++++++++++++++++++++ datasets/reuters.py | 4 +- lstm_baseline/__main__.py | 10 +++++ lstm_baseline/args.py | 7 +-- lstm_baseline/model.py | 28 +++++++++--- 10 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 datasets/aapd.py diff --git a/common/dataset.py b/common/dataset.py index 076a45f..48c2b59 100644 --- a/common/dataset.py +++ b/common/dataset.py @@ -12,6 +12,7 @@ from datasets.snli import SNLI from datasets.sts2014 import STS2014 from datasets.quora import Quora from datasets.reuters import Reuters +from datasets.aapd import AAPD class UnknownWordVecCache(object): """ @@ -92,6 +93,11 @@ class DatasetFactory(object): train_loader, dev_loader, test_loader = Reuters.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding = nn.Embedding.from_pretrained(Reuters.TEXT_FIELD.vocab.vectors) return Reuters, embedding, train_loader, test_loader, dev_loader + elif dataset_name == 'aapd': + dataset_root = os.path.join(castor_dir, os.pardir, 'Castor-data', 'datasets', 'AAPD/') + train_loader, dev_loader, test_loader = AAPD.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) + embedding = nn.Embedding.from_pretrained(AAPD.TEXT_FIELD.vocab.vectors) + return AAPD, embedding, train_loader, test_loader, dev_loader else: raise ValueError('{} is not a valid dataset.'.format(dataset_name)) diff --git a/common/evaluation.py b/common/evaluation.py index d28c9d7..2451d46 100644 --- a/common/evaluation.py +++ b/common/evaluation.py @@ -26,6 +26,7 @@ class EvaluatorFactory(object): 'pit2015': PIT2015Evaluator, 'twitterurl': PIT2015Evaluator, 'Reuters': ReutersEvaluator, + 'AAPD': ReutersEvaluator, 'SNLI': SNLIEvaluator, 'sts2014': STS2014Evaluator, 'Quora': QuoraEvaluator diff --git a/common/evaluators/reuters_evaluator.py b/common/evaluators/reuters_evaluator.py index 4f84b36..9925e52 100644 --- a/common/evaluators/reuters_evaluator.py +++ b/common/evaluators/reuters_evaluator.py @@ -14,11 +14,14 @@ class ReutersEvaluator(Evaluator): total_loss = 0 for batch_idx, batch in enumerate(self.data_loader): - scores = self.model(batch.text) + scores = self.model(batch.text[0], lengths=batch.text[1]) + scores_rounded = F.sigmoid(scores).round().long() + # Using binary accuracy - for tensor1, tensor2 in zip(F.sigmoid(scores).round().long(), batch.label): + for tensor1, tensor2 in zip(scores_rounded, batch.label): if np.array_equal(tensor1, tensor2): n_dev_correct += 1 + total_loss += F.binary_cross_entropy_with_logits(scores, batch.label.float(), size_average=False).item() accuracy = 100. * n_dev_correct / len(self.data_loader.dataset.examples) diff --git a/common/train.py b/common/train.py index abe1123..d45c432 100644 --- a/common/train.py +++ b/common/train.py @@ -26,6 +26,7 @@ class TrainerFactory(object): 'pit2015': PIT2015Trainer, 'twitterurl': PIT2015Trainer, 'Reuters': ReutersTrainer, + 'AAPD': ReutersTrainer, 'snli': SNLITrainer, 'sts2014': STS2014Trainer, 'quora': QuoraTrainer diff --git a/common/trainers/reuters_trainer.py b/common/trainers/reuters_trainer.py index 1d310b2..9befafd 100644 --- a/common/trainers/reuters_trainer.py +++ b/common/trainers/reuters_trainer.py @@ -1,12 +1,13 @@ import time -import os +import datetime +import numpy as np +import os import torch import torch.nn.functional as F -import numpy as np +from tensorboardX import SummaryWriter from .trainer import Trainer -from utils.serialization import save_checkpoint class ReutersTrainer(Trainer): @@ -21,6 +22,7 @@ class ReutersTrainer(Trainer): self.log_template = ' '.join( '{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(',')) self.dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(',')) + self.writer = SummaryWriter(log_dir="tensorboard_logs/" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) def train_epoch(self, epoch): self.train_loader.init_epoch() @@ -29,7 +31,7 @@ class ReutersTrainer(Trainer): self.iterations += 1 self.model.train() self.optimizer.zero_grad() - scores = self.model(batch.text) + scores = self.model(batch.text[0], lengths=batch.text[1]) # Using binary accuracy for tensor1, tensor2 in zip(F.sigmoid(scores).round().long(), batch.label): if np.array_equal(tensor1, tensor2): @@ -44,6 +46,11 @@ class ReutersTrainer(Trainer): # Evaluate performance on validation set if self.iterations % self.dev_log_interval == 1: dev_acc, dev_loss = self.dev_evaluator.get_scores()[0] + niter = epoch * len(self.train_loader) + batch_idx + self.writer.add_scalar('Train/Loss', loss.data[0], niter) + self.writer.add_scalar('Dev/Loss', dev_loss, niter) + self.writer.add_scalar('Train/Accuracy', train_acc, niter) + self.writer.add_scalar('Dev/Accuracy', dev_acc, niter) print(self.dev_log_template.format(time.time() - self.start, epoch, self.iterations, 1 + batch_idx, len(self.train_loader), 100. * (1 + batch_idx) / len(self.train_loader), loss.item(), diff --git a/datasets/aapd.py b/datasets/aapd.py new file mode 100644 index 0000000..1db10db --- /dev/null +++ b/datasets/aapd.py @@ -0,0 +1,59 @@ +import re +import os + +import torch +from datasets.reuters import clean_string, clean_string_fl +from torchtext.data import Field, TabularDataset +from torchtext.data.iterator import BucketIterator +from torchtext.vocab import Vectors + + +def process_labels(string): + """ + Returns the label string as a list of integers + :param string: + :return: + """ + return [float(x) for x in string] + + +class AAPD(TabularDataset): + NAME = 'AAPD' + NUM_CLASSES = 54 + + TEXT_FIELD = Field(batch_first=True, tokenize=clean_string, include_lengths=True) + LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=process_labels) + + @staticmethod + def sort_key(ex): + return len(ex.text) + + @classmethod + def splits(cls, path, train=os.path.join('AAPD', 'data', 'aapd_train.tsv'), + validation=os.path.join('AAPD', 'data', 'aapd_validation.tsv'), + test=os.path.join('AAPD', 'data','aapd_test.tsv'), **kwargs): + return super(AAPD, cls).splits( + path, train=train, validation=validation, test=test, + format='tsv', fields=[('label', cls.LABEL_FIELD), ('text', cls.TEXT_FIELD)] + ) + + @classmethod + def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, + unk_init=torch.Tensor.zero_): + """ + :param path: directory containing train, test, dev files + :param vectors_name: name of word vectors file + :param vectors_cache: path to directory containing word vectors file + :param batch_size: batch size + :param device: GPU device + :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes + :param unk_init: function used to generate vector for OOV words + :return: + """ + if vectors is None: + vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init) + + train, val, test = cls.splits(path) + cls.TEXT_FIELD.build_vocab(train, val, test, vectors=vectors) + return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle, + sort_within_batch=True, device=device) \ No newline at end of file diff --git a/datasets/reuters.py b/datasets/reuters.py index 47e3c7b..ebe24f6 100644 --- a/datasets/reuters.py +++ b/datasets/reuters.py @@ -43,8 +43,8 @@ def process_labels(string): class Reuters(TabularDataset): NAME = 'Reuters' NUM_CLASSES = 90 - - TEXT_FIELD = Field(batch_first=True, tokenize=clean_string) + + TEXT_FIELD = Field(batch_first=True, tokenize=clean_string, include_lengths=True) LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=process_labels) @staticmethod diff --git a/lstm_baseline/__main__.py b/lstm_baseline/__main__.py index 02ca99b..4071249 100644 --- a/lstm_baseline/__main__.py +++ b/lstm_baseline/__main__.py @@ -12,6 +12,7 @@ from common.train import TrainerFactory from datasets.sst import SST1 from datasets.sst import SST2 from datasets.reuters import Reuters +from datasets.aapd import AAPD from lstm_baseline.args import get_args from lstm_baseline.model import LSTMBaseline @@ -81,6 +82,8 @@ if __name__ == '__main__': train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) elif args.dataset == 'Reuters': train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) + elif args.dataset == 'AAPD': + train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) else: raise ValueError('Unrecognized dataset') @@ -122,6 +125,10 @@ if __name__ == '__main__': train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu) test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu) dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu) + elif args.dataset == 'AAPD': + train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu) + test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu) + dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu) else: raise ValueError('Unrecognized dataset') @@ -153,6 +160,9 @@ if __name__ == '__main__': elif args.dataset == 'Reuters': evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu) evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu) + elif args.dataset == 'AAPD': + evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu) + evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu) else: raise ValueError('Unrecognized dataset') diff --git a/lstm_baseline/args.py b/lstm_baseline/args.py index aeee7a6..e1d3980 100644 --- a/lstm_baseline/args.py +++ b/lstm_baseline/args.py @@ -7,15 +7,16 @@ def get_args(): parser = ArgumentParser(description="Baseline LSTM for text classification") parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU - parser.add_argument('--epochs', type=int, default=30) + parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--batch_size', type=int, default=1024) - parser.add_argument('--bidirectional', type=bool, default=True), + parser.add_argument('--bidirectional', action='store_true'), + parser.add_argument('--bottleneck_layer', action='store_true'), parser.add_argument('--num_layers', type=int, default=2) parser.add_argument('--hidden_dim', type=int, default=256) parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static']) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--seed', type=int, default=3435) - parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters']) + parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD']) parser.add_argument('--resume_snapshot', type=str, default=None) parser.add_argument('--dev_every', type=int, default=30) parser.add_argument('--log_every', type=int, default=10) diff --git a/lstm_baseline/model.py b/lstm_baseline/model.py index 64206d7..4ec07c5 100644 --- a/lstm_baseline/model.py +++ b/lstm_baseline/model.py @@ -10,6 +10,7 @@ class LSTMBaseline(nn.Module): dataset = config.dataset target_class = config.target_class self.is_bidirectional = config.bidirectional + self.has_bottleneck_layer = config.bottleneck_layer self.mode = config.mode input_channel = 1 @@ -27,12 +28,20 @@ class LSTMBaseline(nn.Module): self.lstm = nn.LSTM(config.words_dim, config.hidden_dim, dropout=config.dropout, num_layers=config.num_layers, bidirectional=self.is_bidirectional, batch_first=True) self.dropout = nn.Dropout(config.dropout) - if self.is_bidirectional: - self.fc1 = nn.Linear(2 * config.hidden_dim, target_class) + if self.has_bottleneck_layer: + if self.is_bidirectional: + self.fc1 = nn.Linear(2 * config.hidden_dim, config.hidden_dim) # Hidden Bottleneck Layer + self.fc2 = nn.Linear(config.hidden_dim, target_class) + else: + self.fc1 = nn.Linear(config.hidden_dim, config.hidden_dim//2) # Hidden Bottleneck Layer + self.fc2 = nn.Linear(config.hidden_dim//2, target_class) else: - self.fc1 = nn.Linear(config.hidden_dim, target_class) + if self.is_bidirectional: + self.fc1 = nn.Linear(2 * config.hidden_dim, target_class) + else: + self.fc1 = nn.Linear(config.hidden_dim, target_class) - def forward(self, x): + def forward(self, x, lengths=None): if self.mode == 'rand': x = self.embed(x) elif self.mode == 'static': @@ -42,9 +51,16 @@ class LSTMBaseline(nn.Module): else: print("Unsupported Mode") exit() + if lengths is not None: + x = torch.nn.utils.rnn.pack_padded_sequence(x, lengths, batch_first=True) x, _ = self.lstm(x) + if lengths is not None: + x, _ = torch.nn.utils.rnn.pad_packed_sequence(x, batch_first=True) x = F.relu(torch.transpose(x, 1, 2)) x = F.max_pool1d(x, x.size(2)).squeeze(2) x = self.dropout(x) - logit = self.fc1(x) # (batch, target_size) - return logit + if self.has_bottleneck_layer: + x = F.relu(self.fc1(x)) + return self.fc2(x) + else: + return self.fc1(x)