Replication of STOA for Reuters Dataset (#152)

* Add ReutersTrainer, ReutersEvaluator options in Factory classes

* Add Reuters to Kim-CNN command line arguments

* Fix SST dataset path according to changes in Kim-CNN args

The dataset path in args.py was made to point at the dataset folder rather than dataset/SST folder. Hence SST folder was added to paths in the SST dataset class

* Add Reuters dataset class, and support in __main__

* Add Reuters dataset trainers and evaluators

* Remove debug print statement in reuters_evaluator

* Fix rounding bug in reuters_trainer and reuters_evaluator

* Add LSTM for baseline text classification measurements

* Add eval metrics for lstm_baseline

* Set batch_first param in lstm_baseline

* Remove onnx args from lstm_baseline

* Pack padded sequences in LSTM_baseline

* Add TensorBoardX support for Reuters trainer

* Add Arxiv Academic Paper Dataset (AAPD)

* Add Hidden Bottleneck Layer to BiLSTM

* Fix packing of padded tensors in Reuters

* Add cmdline args for Hidden Bottleneck Layer for BiLSTM

* Include pre-padding lengths in AAPD dataset

* Remove duplication of preprocessing code in AAPD

* Remove batch_size condition in ReutersTrainer

common/dataset.py

@@ -12,6 +12,7 @@ from datasets.snli import SNLI
 from datasets.sts2014 import STS2014
 from datasets.quora import Quora
 from datasets.reuters import Reuters
+from datasets.aapd import AAPD
 
 class UnknownWordVecCache(object):
     """
@@ -92,6 +93,11 @@ class DatasetFactory(object):
             train_loader, dev_loader, test_loader = Reuters.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
             embedding = nn.Embedding.from_pretrained(Reuters.TEXT_FIELD.vocab.vectors)
             return Reuters, embedding, train_loader, test_loader, dev_loader
+        elif dataset_name == 'aapd':
+            dataset_root = os.path.join(castor_dir, os.pardir, 'Castor-data', 'datasets', 'AAPD/')
+            train_loader, dev_loader, test_loader = AAPD.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk)
+            embedding = nn.Embedding.from_pretrained(AAPD.TEXT_FIELD.vocab.vectors)
+            return AAPD, embedding, train_loader, test_loader, dev_loader
         else:
             raise ValueError('{} is not a valid dataset.'.format(dataset_name))
 

common/evaluation.py

@@ -26,6 +26,7 @@ class EvaluatorFactory(object):
         'pit2015': PIT2015Evaluator,
         'twitterurl': PIT2015Evaluator,
         'Reuters': ReutersEvaluator,
+        'AAPD': ReutersEvaluator,
         'SNLI': SNLIEvaluator,
         'sts2014': STS2014Evaluator,
         'Quora': QuoraEvaluator

common/evaluators/reuters_evaluator.py

@@ -14,11 +14,14 @@ class ReutersEvaluator(Evaluator):
         total_loss = 0
 
         for batch_idx, batch in enumerate(self.data_loader):
-            scores = self.model(batch.text)
+            scores = self.model(batch.text[0], lengths=batch.text[1])
+            scores_rounded = F.sigmoid(scores).round().long()
+
             # Using binary accuracy
-            for tensor1, tensor2 in zip(F.sigmoid(scores).round().long(), batch.label):
+            for tensor1, tensor2 in zip(scores_rounded, batch.label):
                 if np.array_equal(tensor1, tensor2):
                     n_dev_correct += 1
+
             total_loss += F.binary_cross_entropy_with_logits(scores, batch.label.float(), size_average=False).item()
 
         accuracy = 100. * n_dev_correct / len(self.data_loader.dataset.examples)

common/train.py

@@ -26,6 +26,7 @@ class TrainerFactory(object):
         'pit2015': PIT2015Trainer,
         'twitterurl': PIT2015Trainer,
         'Reuters': ReutersTrainer,
+        'AAPD': ReutersTrainer,
         'snli': SNLITrainer,
         'sts2014': STS2014Trainer,
         'quora': QuoraTrainer

common/trainers/reuters_trainer.py

@@ -1,12 +1,13 @@
 import time
-import os
 
+import datetime
+import numpy as np
+import os
 import torch
 import torch.nn.functional as F
-import numpy as np
+from tensorboardX import SummaryWriter
 
 from .trainer import Trainer
-from utils.serialization import save_checkpoint
 
 
 class ReutersTrainer(Trainer):
@@ -21,6 +22,7 @@ class ReutersTrainer(Trainer):
         self.log_template = ' '.join(
             '{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(','))
         self.dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(','))
+        self.writer = SummaryWriter(log_dir="tensorboard_logs/" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
 
     def train_epoch(self, epoch):
         self.train_loader.init_epoch()
@@ -29,7 +31,7 @@ class ReutersTrainer(Trainer):
             self.iterations += 1
             self.model.train()
             self.optimizer.zero_grad()
-            scores = self.model(batch.text)
+            scores = self.model(batch.text[0], lengths=batch.text[1])
             # Using binary accuracy
             for tensor1, tensor2 in zip(F.sigmoid(scores).round().long(), batch.label):
                 if np.array_equal(tensor1, tensor2):
@@ -44,6 +46,11 @@ class ReutersTrainer(Trainer):
             # Evaluate performance on validation set
             if self.iterations % self.dev_log_interval == 1:
                 dev_acc, dev_loss = self.dev_evaluator.get_scores()[0]
+                niter = epoch * len(self.train_loader) + batch_idx
+                self.writer.add_scalar('Train/Loss', loss.data[0], niter)
+                self.writer.add_scalar('Dev/Loss', dev_loss, niter)
+                self.writer.add_scalar('Train/Accuracy', train_acc, niter)
+                self.writer.add_scalar('Dev/Accuracy', dev_acc, niter)
                 print(self.dev_log_template.format(time.time() - self.start,
                       epoch, self.iterations, 1 + batch_idx, len(self.train_loader),
                       100. * (1 + batch_idx) / len(self.train_loader), loss.item(),

datasets/aapd.py

@@ -0,0 +1,59 @@
+import re
+import os
+
+import torch
+from datasets.reuters import clean_string, clean_string_fl
+from torchtext.data import Field, TabularDataset
+from torchtext.data.iterator import BucketIterator
+from torchtext.vocab import Vectors
+
+
+def process_labels(string):
+    """
+    Returns the label string as a list of integers
+    :param string:
+    :return:
+    """
+    return [float(x) for x in string]
+
+
+class AAPD(TabularDataset):
+    NAME = 'AAPD'
+    NUM_CLASSES = 54
+
+    TEXT_FIELD = Field(batch_first=True, tokenize=clean_string, include_lengths=True)
+    LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=process_labels)
+
+    @staticmethod
+    def sort_key(ex):
+        return len(ex.text)
+
+    @classmethod
+    def splits(cls, path, train=os.path.join('AAPD', 'data', 'aapd_train.tsv'),
+               validation=os.path.join('AAPD', 'data', 'aapd_validation.tsv'),
+               test=os.path.join('AAPD', 'data','aapd_test.tsv'), **kwargs):
+        return super(AAPD, cls).splits(
+            path, train=train, validation=validation, test=test,
+            format='tsv', fields=[('label', cls.LABEL_FIELD), ('text', cls.TEXT_FIELD)]
+        )
+
+    @classmethod
+    def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None,
+              unk_init=torch.Tensor.zero_):
+        """
+        :param path: directory containing train, test, dev files
+        :param vectors_name: name of word vectors file
+        :param vectors_cache: path to directory containing word vectors file
+        :param batch_size: batch size
+        :param device: GPU device
+        :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes
+        :param unk_init: function used to generate vector for OOV words
+        :return:
+        """
+        if vectors is None:
+            vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init)
+
+        train, val, test = cls.splits(path)
+        cls.TEXT_FIELD.build_vocab(train, val, test, vectors=vectors)
+        return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle,
+                                     sort_within_batch=True, device=device)

datasets/reuters.py

@@ -43,8 +43,8 @@ def process_labels(string):
 class Reuters(TabularDataset):
     NAME = 'Reuters'
     NUM_CLASSES = 90
-    
-    TEXT_FIELD = Field(batch_first=True, tokenize=clean_string)
+
+    TEXT_FIELD = Field(batch_first=True, tokenize=clean_string, include_lengths=True)
     LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=process_labels)
 
     @staticmethod

lstm_baseline/__main__.py

@@ -12,6 +12,7 @@ from common.train import TrainerFactory
 from datasets.sst import SST1
 from datasets.sst import SST2
 from datasets.reuters import Reuters
+from datasets.aapd import AAPD
 from lstm_baseline.args import get_args
 from lstm_baseline.model import LSTMBaseline
 
@@ -81,6 +82,8 @@ if __name__ == '__main__':
         train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
     elif args.dataset == 'Reuters':
         train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
+    elif args.dataset == 'AAPD':
+        train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
     else:
         raise ValueError('Unrecognized dataset')
 
@@ -122,6 +125,10 @@ if __name__ == '__main__':
         train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu)
         test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu)
         dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu)
+    elif args.dataset == 'AAPD':
+        train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu)
+        test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu)
+        dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu)
     else:
         raise ValueError('Unrecognized dataset')
 
@@ -153,6 +160,9 @@ if __name__ == '__main__':
     elif args.dataset == 'Reuters':
         evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu)
         evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu)
+    elif args.dataset == 'AAPD':
+        evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu)
+        evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu)
     else:
         raise ValueError('Unrecognized dataset')
 

lstm_baseline/args.py

@@ -7,15 +7,16 @@ def get_args():
     parser = ArgumentParser(description="Baseline LSTM for text classification")
     parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda')
     parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU
-    parser.add_argument('--epochs', type=int, default=30)
+    parser.add_argument('--epochs', type=int, default=50)
     parser.add_argument('--batch_size', type=int, default=1024)
-    parser.add_argument('--bidirectional', type=bool, default=True),
+    parser.add_argument('--bidirectional', action='store_true'),
+    parser.add_argument('--bottleneck_layer', action='store_true'),
     parser.add_argument('--num_layers', type=int, default=2)
     parser.add_argument('--hidden_dim', type=int, default=256)
     parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static'])
     parser.add_argument('--lr', type=float, default=0.001)
     parser.add_argument('--seed', type=int, default=3435)
-    parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters'])
+    parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD'])
     parser.add_argument('--resume_snapshot', type=str, default=None)
     parser.add_argument('--dev_every', type=int, default=30)
     parser.add_argument('--log_every', type=int, default=10)

lstm_baseline/model.py

@@ -10,6 +10,7 @@ class LSTMBaseline(nn.Module):
         dataset = config.dataset
         target_class = config.target_class
         self.is_bidirectional = config.bidirectional
+        self.has_bottleneck_layer = config.bottleneck_layer
         self.mode = config.mode
 
         input_channel = 1
@@ -27,12 +28,20 @@ class LSTMBaseline(nn.Module):
         self.lstm = nn.LSTM(config.words_dim, config.hidden_dim, dropout=config.dropout, num_layers=config.num_layers,
                             bidirectional=self.is_bidirectional, batch_first=True)
         self.dropout = nn.Dropout(config.dropout)
-        if self.is_bidirectional:
-            self.fc1 = nn.Linear(2 * config.hidden_dim, target_class)
+        if self.has_bottleneck_layer:
+            if self.is_bidirectional:
+                self.fc1 = nn.Linear(2 * config.hidden_dim, config.hidden_dim)  # Hidden Bottleneck Layer
+                self.fc2 = nn.Linear(config.hidden_dim, target_class)
+            else:
+                self.fc1 = nn.Linear(config.hidden_dim, config.hidden_dim//2)   # Hidden Bottleneck Layer
+                self.fc2 = nn.Linear(config.hidden_dim//2, target_class)
         else:
-            self.fc1 = nn.Linear(config.hidden_dim, target_class)
+            if self.is_bidirectional:
+                self.fc1 = nn.Linear(2 * config.hidden_dim, target_class)
+            else:
+                self.fc1 = nn.Linear(config.hidden_dim, target_class)
 
-    def forward(self, x):
+    def forward(self, x, lengths=None):
         if self.mode == 'rand':
             x = self.embed(x)
         elif self.mode == 'static':
@@ -42,9 +51,16 @@ class LSTMBaseline(nn.Module):
         else:
             print("Unsupported Mode")
             exit()
+        if lengths is not None:
+            x = torch.nn.utils.rnn.pack_padded_sequence(x, lengths, batch_first=True)
         x, _ = self.lstm(x)
+        if lengths is not None:
+            x, _ = torch.nn.utils.rnn.pad_packed_sequence(x, batch_first=True)
         x = F.relu(torch.transpose(x, 1, 2))
         x = F.max_pool1d(x, x.size(2)).squeeze(2)
         x = self.dropout(x)
-        logit = self.fc1(x) # (batch, target_size)
-        return logit
+        if self.has_bottleneck_layer:
+            x = F.relu(self.fc1(x))
+            return self.fc2(x)
+        else:
+            return self.fc1(x)