Neural Document Classification (#159)

* Add ReutersTrainer, ReutersEvaluator options in Factory classes

* Add Reuters to Kim-CNN command line arguments

* Fix SST dataset path according to changes in Kim-CNN args

The dataset path in args.py was made to point at the dataset folder rather than dataset/SST folder. Hence SST folder was added to paths in the SST dataset class

* Add Reuters dataset class, and support in __main__

* Add Reuters dataset trainers and evaluators

* Remove debug print statement in reuters_evaluator

* Fix rounding bug in reuters_trainer and reuters_evaluator

* Add LSTM for baseline text classification measurements

* Add eval metrics for lstm_baseline

* Set batch_first param in lstm_baseline

* Remove onnx args from lstm_baseline

* Pack padded sequences in LSTM_baseline

* Add TensorBoardX support for Reuters trainer

* Add Arxiv Academic Paper Dataset (AAPD)

* Add Hidden Bottleneck Layer to BiLSTM

* Fix packing of padded tensors in Reuters

* Add cmdline args for Hidden Bottleneck Layer for BiLSTM

* Include pre-padding lengths in AAPD dataset

* Remove duplication of preprocessing code in AAPD

* Remove batch_size condition in ReutersTrainer

* Add ignore_lengths option to ReutersTrainer and ReutersEvaluator

* Add AAPDCharQuantized and ReutersCharQuantized

* Rename Reuters_hierarchical to ReutersHierarchical

* Add CharacterCNN for document classification

* Update README.md for CharacterCNN

* Fix table in README.md for CharacterCNN

* Add AAPDHierarchical for HAN

* Update HAN for changes in Reuters dataset endpoints

* Fix bug in CharCNN when running on CPU

* Add AAPD dataset support for KimCNN

* Fix dataset paths for SST-1

* Fix dimensions of FC1 in CharCNN

* Add model checkpointing for Reuters based on F1

* Refactor LSTM baseline __main__

* Add precision, recall and F1 to Reuters evaluator

* Checkpoint only at the end of an epoch for ReutersTrainer

Add detailed log printing for dev evaluations

* Fix log_template and dev_log_template in ReutersTrainer

* Add IMDB dataset

* Add support for single_label datasets in ReutersTrainer

* Add support for IMDB dataset in lstm_baseline and lstm_reg

lstm_regularization/__main__.py

@@ -13,6 +13,7 @@ from datasets.sst import SST1
 from datasets.sst import SST2
 from datasets.reuters import Reuters
 from datasets.aapd import AAPD
+from datasets.imdb import IMDB
 from lstm_regularization.args import get_args
 from lstm_regularization.model import LSTMBaseline
 
@@ -77,8 +78,10 @@ if __name__ == '__main__':
         'SST-1': SST1,
         'SST-2': SST2,
         'Reuters': Reuters,
-        'AAPD': AAPD
+        'AAPD': AAPD,
+        'IMDB': IMDB
     }
+
     if args.dataset not in dataset_map:
         raise ValueError('Unrecognized dataset')
     else:
@@ -116,6 +119,9 @@ if __name__ == '__main__':
         train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu)
         test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
         dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
+        train_evaluator.single_label = args.single_label
+        test_evaluator.single_label = args.single_label
+        dev_evaluator.single_label = args.single_label
     trainer_config = {
         'optimizer': optimizer,
         'batch_size': args.batch_size,
@@ -123,7 +129,8 @@ if __name__ == '__main__':
         'dev_log_interval': args.dev_every,
         'patience': args.patience,
         'model_outfile': args.save_path,   # actually a directory, using model_outfile to conform to Trainer naming convention
-        'logger': logger
+        'logger': logger,
+        'single_label': args.single_label
     }
     trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
 
@@ -135,37 +142,17 @@ if __name__ == '__main__':
         else:
             model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
 
+    # Calculate dev and test metrics
+    model.load_state_dict(torch.load(trainer.snapshot_path))
+    if model.beta_ema > 0:
+        old_params = model.get_params()
+        model.load_ema_params()
+
     if args.dataset not in dataset_map:
         raise ValueError('Unrecognized dataset')
     else:
         evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
         evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
 
-    # Calculate dev and test metrics
     if model.beta_ema > 0:
-        old_params = model.get_params()
-        model.load_ema_params()
-
-    for data_loader in [dev_iter, test_iter]:
-        predicted_labels = list()
-        target_labels = list()
-        for batch_idx, batch in enumerate(data_loader):
-            if model.TAR:
-                scores_rounded = F.sigmoid(model(batch.text[0])[0]).round().long()
-            else:
-                scores_rounded = F.sigmoid(model(batch.text[0])).round().long()
-            predicted_labels.extend(scores_rounded.cpu().detach().numpy())
-            target_labels.extend(batch.label.cpu().detach().numpy())
-        predicted_labels = np.array(predicted_labels)
-        target_labels = np.array(target_labels)
-        accuracy = metrics.accuracy_score(target_labels, predicted_labels)
-        precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
-        recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
-        f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
-        if data_loader == dev_iter:
-            print("Dev metrics:")
-        else:
-            print("Test metrics:")
-        print(accuracy, precision, recall, f1)
-    if model.beta_ema > 0:
-        model.load_params(old_params)
+        model.load_params(old_params)