diff --git a/char_cnn/__main__.py b/char_cnn/__main__.py index 0f1dac1..0bebe92 100644 --- a/char_cnn/__main__.py +++ b/char_cnn/__main__.py @@ -1,18 +1,22 @@ -from sklearn import metrics - +from copy import deepcopy import logging -import numpy as np import random + +import numpy as np +from sklearn import metrics import torch import torch.nn.functional as F -from copy import deepcopy +from char_cnn.args import get_args +from char_cnn.model import CharCNN from common.evaluation import EvaluatorFactory from common.train import TrainerFactory from datasets.aapd import AAPDCharQuantized as AAPD +from datasets.imdb import IMDBCharQuantized as IMDB from datasets.reuters import ReutersCharQuantized as Reuters -from char_cnn.args import get_args -from char_cnn.model import CharCNN +from datasets.yelp2014 import Yelp2014CharQuantized as Yelp2014 + + class UnknownWordVecCache(object): @@ -45,13 +49,14 @@ def get_logger(): return logger -def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device): +def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label): saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device) saved_model_evaluator.ignore_lengths = True + saved_model_evaluator.single_label = single_label scores, metric_names = saved_model_evaluator.get_scores() - logger.info('Evaluation metrics for {}'.format(split_name)) - logger.info('\t'.join([' '] + metric_names)) - logger.info('\t'.join([split_name] + list(map(str, scores)))) + print('Evaluation metrics for', split_name) + print(metric_names) + print(scores) if __name__ == '__main__': @@ -73,13 +78,20 @@ if __name__ == '__main__': random.seed(args.seed) logger = get_logger() - # Set up the data for training SST-1 - if args.dataset == 'Reuters': - train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - elif args.dataset == 'AAPD': - train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - else: + dataset_map = { + 'Reuters': Reuters, + 'AAPD': AAPD, + 'IMDB': IMDB, + 'Yelp2014': Yelp2014 + } + + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file, + args.word_vectors_dir, + batch_size=args.batch_size, device=args.gpu, + unk_init=UnknownWordVecCache.unk) config = deepcopy(args) config.dataset = train_iter.dataset @@ -104,19 +116,18 @@ if __name__ == '__main__': parameter = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(parameter, lr=args.lr, weight_decay=args.weight_decay) - if args.dataset == 'Reuters': - train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu) - else: + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu) + test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) + dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) + train_evaluator.single_label = args.single_label + test_evaluator.single_label = args.single_label + dev_evaluator.single_label = args.single_label + dev_evaluator.ignore_lengths = True + test_evaluator.ignore_lengths = True - dev_evaluator.ignore_lengths = True - test_evaluator.ignore_lengths = True trainer_config = { 'optimizer': optimizer, 'batch_size': args.batch_size, @@ -125,7 +136,8 @@ if __name__ == '__main__': 'patience': args.patience, 'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention 'logger': logger, - 'ignore_lengths': True + 'ignore_lengths': True, + 'single_label': args.single_label } trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator) @@ -137,31 +149,10 @@ if __name__ == '__main__': else: model = torch.load(args.trained_model, map_location=lambda storage, location: storage) - if args.dataset == 'Reuters': - evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu) - else: - raise ValueError('Unrecognized dataset') - # Calculate dev and test metrics - for data_loader in [dev_iter, test_iter]: - predicted_labels = list() - target_labels = list() - for batch_idx, batch in enumerate(data_loader): - scores_rounded = F.sigmoid(model(batch.text)).round().long() - predicted_labels.extend(scores_rounded.cpu().detach().numpy()) - target_labels.extend(batch.label.cpu().detach().numpy()) - predicted_labels = np.array(predicted_labels) - target_labels = np.array(target_labels) - accuracy = metrics.accuracy_score(target_labels, predicted_labels) - precision = metrics.precision_score(target_labels, predicted_labels, average='micro') - recall = metrics.recall_score(target_labels, predicted_labels, average='micro') - f1 = metrics.f1_score(target_labels, predicted_labels, average='micro') - if data_loader == dev_iter: - print("Dev metrics:") - else: - print("Test metrics:") - print(accuracy, precision, recall, f1) \ No newline at end of file + model = torch.load(trainer.snapshot_path) + if args.dataset not in dataset_map: + raise ValueError('Unrecognized dataset') + else: + evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label) + evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label) diff --git a/char_cnn/args.py b/char_cnn/args.py index 385c885..611444a 100644 --- a/char_cnn/args.py +++ b/char_cnn/args.py @@ -11,12 +11,13 @@ def get_args(): parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--seed', type=int, default=3435) - parser.add_argument('--dataset', type=str, default='Reuters', choices=['Reuters', 'AAPD']) + parser.add_argument('--single_label', action='store_true'), + parser.add_argument('--dataset', type=str, default='Reuters', choices=['Reuters', 'AAPD', 'IMDB', 'Yelp2014']) parser.add_argument('--resume_snapshot', type=str, default=None) parser.add_argument('--dev_every', type=int, default=30) parser.add_argument('--log_every', type=int, default=10) parser.add_argument('--patience', type=int, default=100) - parser.add_argument('--save_path', type=str, default='kim_cnn/saves') + parser.add_argument('--save_path', type=str, default='char_cnn/saves') parser.add_argument('--num_conv_filters', type=int, default=256) parser.add_argument('--num_affine_neurons', type=int, default=1024) parser.add_argument('--output_channel', type=int, default=256) diff --git a/common/evaluation.py b/common/evaluation.py index a521064..e930699 100644 --- a/common/evaluation.py +++ b/common/evaluation.py @@ -28,6 +28,7 @@ class EvaluatorFactory(object): 'Reuters': ReutersEvaluator, 'AAPD': ReutersEvaluator, 'IMDB': ReutersEvaluator, + 'Yelp2014': ReutersEvaluator, 'SNLI': SNLIEvaluator, 'sts2014': STS2014Evaluator, 'Quora': QuoraEvaluator diff --git a/common/evaluators/reuters_evaluator.py b/common/evaluators/reuters_evaluator.py index 2472d40..2761b1d 100644 --- a/common/evaluators/reuters_evaluator.py +++ b/common/evaluators/reuters_evaluator.py @@ -28,12 +28,12 @@ class ReutersEvaluator(Evaluator): for batch_idx, batch in enumerate(self.data_loader): if hasattr(self.model, 'TAR') and self.model.TAR: # TAR condition if self.ignore_lengths: - scores, rnn_outs = self.model(batch.text, lengths=batch.text) + scores, rnn_outs = self.model(batch.text) else: scores, rnn_outs = self.model(batch.text[0], lengths=batch.text[1]) else: if self.ignore_lengths: - scores = self.model(batch.text, lengths=batch.text) + scores = self.model(batch.text) else: scores = self.model(batch.text[0], lengths=batch.text[1]) @@ -62,4 +62,4 @@ class ReutersEvaluator(Evaluator): if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0: self.model.load_params(old_params) - return [accuracy, precision, recall, f1, avg_loss], ['accuracy', 'precision', 'recall', 'f1' 'cross_entropy_loss'] + return [accuracy, precision, recall, f1, avg_loss], ['accuracy', 'precision', 'recall', 'f1', 'cross_entropy_loss'] diff --git a/common/train.py b/common/train.py index 75a69c7..7f13ba9 100644 --- a/common/train.py +++ b/common/train.py @@ -28,6 +28,7 @@ class TrainerFactory(object): 'Reuters': ReutersTrainer, 'AAPD': ReutersTrainer, 'IMDB': ReutersTrainer, + 'Yelp2014': ReutersTrainer, 'snli': SNLITrainer, 'sts2014': STS2014Trainer, 'quora': QuoraTrainer diff --git a/common/trainers/reuters_trainer.py b/common/trainers/reuters_trainer.py index b843bf7..69b8e3c 100644 --- a/common/trainers/reuters_trainer.py +++ b/common/trainers/reuters_trainer.py @@ -35,12 +35,12 @@ class ReutersTrainer(Trainer): self.optimizer.zero_grad() if hasattr(self.model, 'TAR') and self.model.TAR: if 'ignore_lengths' in self.config and self.config['ignore_lengths']: - scores, rnn_outs = self.model(batch.text, lengths=batch.text) + scores, rnn_outs = self.model(batch.text) else: scores, rnn_outs = self.model(batch.text[0], lengths=batch.text[1]) else: if 'ignore_lengths' in self.config and self.config['ignore_lengths']: - scores = self.model(batch.text, lengths=batch.text) + scores = self.model(batch.text) else: scores = self.model(batch.text[0], lengths=batch.text[1]) @@ -85,9 +85,9 @@ class ReutersTrainer(Trainer): # model_outfile is actually a directory, using model_outfile to conform to Trainer naming convention os.makedirs(self.model_outfile, exist_ok=True) os.makedirs(os.path.join(self.model_outfile, self.train_loader.dataset.NAME), exist_ok=True) - print(header) for epoch in range(1, epochs + 1): + print('\n' + header) self.train_epoch(epoch) # Evaluate performance on validation set @@ -100,13 +100,12 @@ class ReutersTrainer(Trainer): print('\n' + dev_header) print(self.dev_log_template.format(time.time() - self.start, epoch, self.iterations, epoch, epochs, dev_acc, dev_precision, dev_recall, dev_f1, dev_loss)) - print('\n' + header) # Update validation results if dev_f1 > self.best_dev_f1: self.iters_not_improved = 0 self.best_dev_f1 = dev_f1 - torch.save(self.model.state_dict(), self.snapshot_path) + torch.save(self.model, self.snapshot_path) else: self.iters_not_improved += 1 if self.iters_not_improved >= self.patience: diff --git a/datasets/aapd.py b/datasets/aapd.py index ae2b7be..fb41923 100644 --- a/datasets/aapd.py +++ b/datasets/aapd.py @@ -1,12 +1,14 @@ -import re import os +import re +import numpy as np import torch -from datasets.reuters import clean_string, char_quantize, clean_string_fl, split_sents from torchtext.data import NestedField, Field, TabularDataset from torchtext.data.iterator import BucketIterator from torchtext.vocab import Vectors +from datasets.reuters import clean_string, clean_string_fl, split_sents + def process_labels(string): """ @@ -76,5 +78,5 @@ class AAPDCharQuantized(AAPD): return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device) class AAPDHierarchical(AAPD): - In_FIELD = Field(batch_first=True, tokenize=clean_string) - TEXT_FIELD = NestedField(In_FIELD, tokenize=split_sents) \ No newline at end of file + NESTING_FIELD = Field(batch_first=True, tokenize=clean_string) + TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents) diff --git a/datasets/imdb.py b/datasets/imdb.py index 64fda88..a62b95c 100644 --- a/datasets/imdb.py +++ b/datasets/imdb.py @@ -1,14 +1,16 @@ -import numpy as np import os import re + +import numpy as np import torch -from datasets.reuters import clean_string, clean_string_fl, split_sents from torchtext.data import NestedField, Field, TabularDataset from torchtext.data.iterator import BucketIterator from torchtext.vocab import Vectors +from datasets.reuters import clean_string, clean_string_fl, split_sents -def char_quantize(string, max_length=1000): + +def char_quantize(string, max_length=500): identity = np.identity(len(IMDBCharQuantized.ALPHABET)) quantized_string = np.array([identity[IMDBCharQuantized.ALPHABET[char]] for char in list(string.lower()) if char in IMDBCharQuantized.ALPHABET], dtype=np.float32) if len(quantized_string) > max_length: @@ -85,5 +87,5 @@ class IMDBCharQuantized(IMDB): class IMDBHierarchical(IMDB): - In_FIELD = Field(batch_first=True, tokenize=clean_string) - TEXT_FIELD = NestedField(In_FIELD, tokenize=split_sents) + NESTING_FIELD = Field(batch_first=True, tokenize=clean_string) + TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents) diff --git a/datasets/reuters.py b/datasets/reuters.py index f83c5d1..546ee19 100644 --- a/datasets/reuters.py +++ b/datasets/reuters.py @@ -1,6 +1,7 @@ -import numpy as np import os import re + +import numpy as np import torch from torchtext.data import NestedField, Field, TabularDataset from torchtext.data.iterator import BucketIterator @@ -109,5 +110,5 @@ class ReutersCharQuantized(Reuters): class ReutersHierarchical(Reuters): - In_FIELD = Field(batch_first=True, tokenize=clean_string) - TEXT_FIELD = NestedField(In_FIELD, tokenize=split_sents) + NESTING_FIELD = Field(batch_first=True, tokenize=clean_string) + TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents) diff --git a/datasets/yelp2014.py b/datasets/yelp2014.py new file mode 100644 index 0000000..a77f0da --- /dev/null +++ b/datasets/yelp2014.py @@ -0,0 +1,91 @@ +import os +import re + +import numpy as np +import torch +from torchtext.data import NestedField, Field, TabularDataset +from torchtext.data.iterator import BucketIterator +from torchtext.vocab import Vectors + +from datasets.reuters import clean_string, clean_string_fl, split_sents + + +def char_quantize(string, max_length=1000): + identity = np.identity(len(Yelp2014CharQuantized.ALPHABET)) + quantized_string = np.array([identity[Yelp2014CharQuantized.ALPHABET[char]] for char in list(string.lower()) if char in Yelp2014CharQuantized.ALPHABET], dtype=np.float32) + if len(quantized_string) > max_length: + return quantized_string[:max_length] + else: + return np.concatenate((quantized_string, np.zeros((max_length - len(quantized_string), len(Yelp2014CharQuantized.ALPHABET)), dtype=np.float32))) + + +def process_labels(string): + """ + Returns the label string as a list of integers + :param string: + :return: + """ + return [float(x) for x in string] + + +class Yelp2014(TabularDataset): + NAME = 'Yelp2014' + NUM_CLASSES = 5 + TEXT_FIELD = Field(batch_first=True, tokenize=clean_string, include_lengths=True) + LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=process_labels) + + @staticmethod + def sort_key(ex): + return len(ex.text) + + @classmethod + def splits(cls, path, train=os.path.join('Yelp-Reviews-2014', 'data', 'yelp2014_train.tsv'), + validation=os.path.join('Yelp-Reviews-2014', 'data', 'yelp2014_validation.tsv'), + test=os.path.join('Yelp-Reviews-2014', 'data', 'yelp2014_test.tsv'), **kwargs): + return super(Yelp2014, cls).splits( + path, train=train, validation=validation, test=test, + format='tsv', fields=[('label', cls.LABEL_FIELD), ('text', cls.TEXT_FIELD)] + ) + + @classmethod + def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, + unk_init=torch.Tensor.zero_): + """ + :param path: directory containing train, test, dev files + :param vectors_name: name of word vectors file + :param vectors_cache: path to directory containing word vectors file + :param batch_size: batch size + :param device: GPU device + :param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes + :param unk_init: function used to generate vector for OOV words + :return: + """ + if vectors is None: + vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init) + + train, val, test = cls.splits(path) + cls.TEXT_FIELD.build_vocab(train, val, test, vectors=vectors) + return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle, + sort_within_batch=True, device=device) + + +class Yelp2014CharQuantized(Yelp2014): + ALPHABET = dict(map(lambda t: (t[1], t[0]), enumerate(list("""abcdefghijklmnopqrstuvwxyz0123456789,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}""")))) + TEXT_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=char_quantize) + + @classmethod + def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None, + unk_init=torch.Tensor.zero_): + """ + :param path: directory containing train, test, dev files + :param batch_size: batch size + :param device: GPU device + :return: + """ + train, val, test = cls.splits(path) + return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device) + + +class Yelp2014Hierarchical(Yelp2014): + NESTING_FIELD = Field(batch_first=True, tokenize=clean_string) + TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents) diff --git a/han/__main__.py b/han/__main__.py index 55c70f2..3eaac2a 100644 --- a/han/__main__.py +++ b/han/__main__.py @@ -1,20 +1,25 @@ from copy import deepcopy import logging import random -from sklearn import metrics + import numpy as np +from sklearn import metrics import torch +import torch.nn.functional as F import torch.onnx from common.evaluation import EvaluatorFactory from common.train import TrainerFactory +from datasets.aapd import AAPDHierarchical as AAPD +from datasets.imdb import IMDBHierarchical as IMDB from datasets.sst import SST1 from datasets.sst import SST2 from datasets.reuters import ReutersHierarchical as Reuters -from datasets.aapd import AAPDHierarchical as AAPD +from datasets.yelp2014 import Yelp2014Hierarchical as Yelp2014 from han.args import get_args from han.model import HAN -import torch.nn.functional as F + + class UnknownWordVecCache(object): """ @@ -46,13 +51,14 @@ def get_logger(): return logger -def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device): +def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label): saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device) + saved_model_evaluator.single_label = single_label saved_model_evaluator.ignore_lengths = True scores, metric_names = saved_model_evaluator.get_scores() - logger.info('Evaluation metrics for {}'.format(split_name)) - logger.info('\t'.join([' '] + metric_names)) - logger.info('\t'.join([split_name] + list(map(str, scores)))) + print('Evaluation metrics for', split_name) + print(metric_names) + print(scores) if __name__ == '__main__': @@ -74,18 +80,19 @@ if __name__ == '__main__': random.seed(args.seed) logger = get_logger() - # Set up the data for training SST-1 - if args.dataset == 'SST-1': - train_iter, dev_iter, test_iter = SST1.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - # Set up the data for training SST-2 - elif args.dataset == 'SST-2': - train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - elif args.dataset == 'Reuters': - train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - elif args.dataset == 'AAPD': - train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - else: + dataset_map = { + 'SST-1': SST1, + 'SST-2': SST2, + 'Reuters': Reuters, + 'AAPD': AAPD, + 'IMDB': IMDB, + 'Yelp2014': Yelp2014 + } + + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) config = deepcopy(args) config.dataset = train_iter.dataset @@ -115,24 +122,16 @@ if __name__ == '__main__': #optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay) #optimizer = torch.optim.SGD(parameter, lr = args.lr, momentum = 0.9) optimizer = torch.optim.Adam(parameter, lr = args.lr) - if args.dataset == 'SST-1': - train_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'SST-2': - train_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'Reuters': - train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu) - else: + + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu) + test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) + dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) + train_evaluator.single_label = args.single_label + test_evaluator.single_label = args.single_label + dev_evaluator.single_label = args.single_label dev_evaluator.ignore_lengths = True test_evaluator.ignore_lengths = True @@ -144,7 +143,8 @@ if __name__ == '__main__': 'patience': args.patience, 'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention 'logger': logger, - 'ignore_lengths': True + 'ignore_lengths': True, + 'single_label': args.single_label } trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator) @@ -156,45 +156,13 @@ if __name__ == '__main__': else: model = torch.load(args.trained_model, map_location=lambda storage, location: storage) - if args.dataset == 'SST-1': - evaluate_dataset('dev', SST1, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', SST1, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'SST-2': - evaluate_dataset('dev', SST2, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', SST2, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'Reuters': - evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu) - else: - raise ValueError('Unrecognized dataset') - - - # Calculate dev and test metrics - for data_loader in [dev_iter, test_iter]: - predicted_labels = list() - target_labels = list() - for batch_idx, batch in enumerate(data_loader): - scores_rounded = F.sigmoid(model(batch.text)).round().long() - predicted_labels.extend(scores_rounded.cpu().detach().numpy()) - target_labels.extend(batch.label.cpu().detach().numpy()) - predicted_labels = np.array(predicted_labels) - target_labels = np.array(target_labels) - accuracy = metrics.accuracy_score(target_labels, predicted_labels) - precision = metrics.precision_score(target_labels, predicted_labels, average='micro') - recall = metrics.recall_score(target_labels, predicted_labels, average='micro') - f1 = metrics.f1_score(target_labels, predicted_labels, average='micro') - if data_loader == dev_iter: - print("Dev metrics:") - else: - print("Test metrics:") - print(accuracy, precision, recall, f1) - - + model = torch.load(trainer.snapshot_path) + if args.dataset not in dataset_map: + raise ValueError('Unrecognized dataset') + else: + evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label) + evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label) if args.onnx: device = torch.device('cuda') if torch.cuda.is_available() and args.cuda else torch.device('cpu') diff --git a/han/args.py b/han/args.py index e6a0cdc..05fad1e 100644 --- a/han/args.py +++ b/han/args.py @@ -10,15 +10,15 @@ def get_args(): parser.add_argument('--epochs', type=int, default=30) - parser.add_argument('--word_num_hidden', type = int, default = 50) - parser.add_argument('--sentence_num_hidden', type = int, default = 50) - + parser.add_argument('--word_num_hidden', type=int, default=50) + parser.add_argument('--sentence_num_hidden', type=int, default=50) + parser.add_argument('--single_label', action='store_true') parser.add_argument('--batch_size', type=int, default=64) parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static']) parser.add_argument('--lr', type=float, default=1.0) parser.add_argument('--seed', type=int, default=3435) - parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD']) + parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014']) parser.add_argument('--resume_snapshot', type=str, default=None) parser.add_argument('--dev_every', type=int, default=30) parser.add_argument('--log_every', type=int, default=10) diff --git a/han/model.py b/han/model.py index 08e87a7..325ffd9 100755 --- a/han/model.py +++ b/han/model.py @@ -10,7 +10,7 @@ from han.word_level_rnn import WordLevelRNN class HAN(nn.Module): def __init__(self, config): super(HAN, self).__init__() - self.dataset = config.dataset + dataset = config.dataset self.mode = config.mode self.word_attention_rnn = WordLevelRNN(config) self.sentence_attention_rnn = SentLevelRNN(config) diff --git a/kim_cnn/__main__.py b/kim_cnn/__main__.py index fafad7b..3e4cae8 100644 --- a/kim_cnn/__main__.py +++ b/kim_cnn/__main__.py @@ -14,9 +14,12 @@ from datasets.sst import SST1 from datasets.sst import SST2 from datasets.aapd import AAPD from datasets.reuters import Reuters +from datasets.yelp2014 import Yelp2014 +from datasets.imdb import IMDB from kim_cnn.args import get_args from kim_cnn.model import KimCNN + class UnknownWordVecCache(object): """ Caches the first randomly generated word vector for a certain size to make it is reused. @@ -47,8 +50,10 @@ def get_logger(): return logger -def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device): +def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label): saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device) + if hasattr(saved_model_evaluator, 'single_label'): + saved_model_evaluator.single_label = single_label scores, metric_names = saved_model_evaluator.get_scores() logger.info('Evaluation metrics for {}'.format(split_name)) logger.info('\t'.join([' '] + metric_names)) @@ -74,18 +79,22 @@ if __name__ == '__main__': random.seed(args.seed) logger = get_logger() - # Set up the data for training SST-1 - if args.dataset == 'SST-1': - train_iter, dev_iter, test_iter = SST1.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - # Set up the data for training SST-2 - elif args.dataset == 'SST-2': - train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - elif args.dataset == 'Reuters': - train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - elif args.dataset == 'AAPD': - train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - else: + dataset_map = { + 'SST-1': SST1, + 'SST-2': SST2, + 'Reuters': Reuters, + 'AAPD': AAPD, + 'IMDB': IMDB, + 'Yelp2014': Yelp2014 + } + + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file, + args.word_vectors_dir, + batch_size=args.batch_size, device=args.gpu, + unk_init=UnknownWordVecCache.unk) config = deepcopy(args) config.dataset = train_iter.dataset @@ -113,24 +122,18 @@ if __name__ == '__main__': parameter = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay) - if args.dataset == 'SST-1': - train_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'SST-2': - train_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'Reuters': - train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu) - else: + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu) + test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) + dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) + if hasattr(train_evaluator, 'single_label'): + train_evaluator.single_label = args.single_label + if hasattr(test_evaluator, 'single_label'): + test_evaluator.single_label = args.single_label + if hasattr(dev_evaluator, 'single_label'): + dev_evaluator.single_label = args.single_label trainer_config = { 'optimizer': optimizer, @@ -138,9 +141,11 @@ if __name__ == '__main__': 'log_interval': args.log_every, 'dev_log_interval': args.dev_every, 'patience': args.patience, - 'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention - 'logger': logger + 'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention + 'logger': logger, + 'single_label': args.single_label } + trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator) if not args.trained_model: @@ -151,20 +156,14 @@ if __name__ == '__main__': else: model = torch.load(args.trained_model, map_location=lambda storage, location: storage) - if args.dataset == 'SST-1': - evaluate_dataset('dev', SST1, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', SST1, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'SST-2': - evaluate_dataset('dev', SST2, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', SST2, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'Reuters': - evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu) - else: + # Calculate dev and test metrics + if hasattr(trainer, 'snapshot_path'): + model = torch.load(trainer.snapshot_path) + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label) + evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label) if args.onnx: device = torch.device('cuda') if torch.cuda.is_available() and args.cuda else torch.device('cpu') diff --git a/kim_cnn/args.py b/kim_cnn/args.py index b0bf4d3..1d6b510 100644 --- a/kim_cnn/args.py +++ b/kim_cnn/args.py @@ -12,8 +12,9 @@ def get_args(): parser.add_argument('--mode', type=str, default='multichannel', choices=['rand', 'static', 'non-static', 'multichannel']) parser.add_argument('--lr', type=float, default=1.0) parser.add_argument('--seed', type=int, default=3435) - parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD']) + parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014']) parser.add_argument('--resume_snapshot', type=str, default=None) + parser.add_argument('--single_label', action='store_true') parser.add_argument('--dev_every', type=int, default=30) parser.add_argument('--log_every', type=int, default=10) parser.add_argument('--patience', type=int, default=50) diff --git a/lstm_baseline/__main__.py b/lstm_baseline/__main__.py index 753c4f1..4c5e20a 100644 --- a/lstm_baseline/__main__.py +++ b/lstm_baseline/__main__.py @@ -14,6 +14,7 @@ from datasets.sst import SST2 from datasets.reuters import Reuters from datasets.imdb import IMDB from datasets.aapd import AAPD +from datasets.yelp2014 import Yelp2014 from lstm_baseline.args import get_args from lstm_baseline.model import LSTMBaseline @@ -48,12 +49,13 @@ def get_logger(): return logger -def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device): +def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label): saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device) + saved_model_evaluator.single_label = single_label scores, metric_names = saved_model_evaluator.get_scores() - logger.info('Evaluation metrics for {}'.format(split_name)) - logger.info('\t'.join([' '] + metric_names)) - logger.info('\t'.join([split_name] + list(map(str, scores)))) + print('Evaluation metrics for', split_name) + print(metric_names) + print(scores) if __name__ == '__main__': @@ -80,7 +82,8 @@ if __name__ == '__main__': 'SST-2': SST2, 'Reuters': Reuters, 'AAPD': AAPD, - 'IMDB': IMDB + 'IMDB': IMDB, + 'Yelp2014': Yelp2014 } if args.dataset not in dataset_map: @@ -145,9 +148,9 @@ if __name__ == '__main__': model = torch.load(args.trained_model, map_location=lambda storage, location: storage) # Calculate dev and test metrics - model.load_state_dict(torch.load(trainer.snapshot_path)) + model = torch.load(trainer.snapshot_path) if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') else: - evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) \ No newline at end of file + evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label) + evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label) \ No newline at end of file diff --git a/lstm_baseline/args.py b/lstm_baseline/args.py index 4b5d502..f597b97 100644 --- a/lstm_baseline/args.py +++ b/lstm_baseline/args.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser def get_args(): parser = ArgumentParser(description="Baseline LSTM for text classification") parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') - parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU + parser.add_argument('--gpu', type=int, default=0, help="Use -1 for CPU") parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--batch_size', type=int, default=1024) parser.add_argument('--bidirectional', action='store_true'), @@ -17,11 +17,11 @@ def get_args(): parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static']) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--seed', type=int, default=3435) - parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB']) + parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014']) parser.add_argument('--resume_snapshot', type=str, default=None) parser.add_argument('--dev_every', type=int, default=30) parser.add_argument('--log_every', type=int, default=10) - parser.add_argument('--patience', type=int, default=50) + parser.add_argument('--patience', type=int, default=5) parser.add_argument('--save_path', type=str, default='lstm_baseline/saves') parser.add_argument('--words_dim', type=int, default=300) parser.add_argument('--embed_dim', type=int, default=300) diff --git a/lstm_baseline/model.py b/lstm_baseline/model.py index 4ec07c5..2be3485 100644 --- a/lstm_baseline/model.py +++ b/lstm_baseline/model.py @@ -61,6 +61,7 @@ class LSTMBaseline(nn.Module): x = self.dropout(x) if self.has_bottleneck_layer: x = F.relu(self.fc1(x)) + # x = self.dropout(x) return self.fc2(x) else: return self.fc1(x) diff --git a/lstm_regularization/__main__.py b/lstm_regularization/__main__.py index f54bda3..546188b 100644 --- a/lstm_regularization/__main__.py +++ b/lstm_regularization/__main__.py @@ -14,6 +14,7 @@ from datasets.sst import SST2 from datasets.reuters import Reuters from datasets.aapd import AAPD from datasets.imdb import IMDB +from datasets.yelp2014 import Yelp2014 from lstm_regularization.args import get_args from lstm_regularization.model import LSTMBaseline @@ -47,12 +48,13 @@ def get_logger(): return logger -def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device): +def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label): saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device) + saved_model_evaluator.single_label = single_label scores, metric_names = saved_model_evaluator.get_scores() - logger.info('Evaluation metrics for {}'.format(split_name)) - logger.info('\t'.join([' '] + metric_names)) - logger.info('\t'.join([split_name] + list(map(str, scores)))) + print('Evaluation metrics for', split_name) + print(metric_names) + print(scores) if __name__ == '__main__': @@ -79,7 +81,8 @@ if __name__ == '__main__': 'SST-2': SST2, 'Reuters': Reuters, 'AAPD': AAPD, - 'IMDB': IMDB + 'IMDB': IMDB, + 'Yelp2014': Yelp2014 } if args.dataset not in dataset_map: @@ -143,7 +146,7 @@ if __name__ == '__main__': model = torch.load(args.trained_model, map_location=lambda storage, location: storage) # Calculate dev and test metrics - model.load_state_dict(torch.load(trainer.snapshot_path)) + model = torch.load(trainer.snapshot_path) if model.beta_ema > 0: old_params = model.get_params() model.load_ema_params() @@ -151,8 +154,8 @@ if __name__ == '__main__': if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') else: - evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) + evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label) + evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label) if model.beta_ema > 0: model.load_params(old_params) \ No newline at end of file diff --git a/lstm_regularization/args.py b/lstm_regularization/args.py index f73ebeb..3e823e2 100644 --- a/lstm_regularization/args.py +++ b/lstm_regularization/args.py @@ -6,22 +6,22 @@ from argparse import ArgumentParser def get_args(): parser = ArgumentParser(description="Regularized LSTM for text classification with Regularization") parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') - parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU + parser.add_argument('--gpu', type=int, default=0, help="Use -1 for CPU") parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--batch_size', type=int, default=1024) - parser.add_argument('--bidirectional', action='store_true'), - parser.add_argument('--bottleneck_layer', action='store_true'), - parser.add_argument('--single_label', action='store_true'), + parser.add_argument('--bidirectional', action='store_true') + parser.add_argument('--bottleneck_layer', action='store_true') + parser.add_argument('--single_label', action='store_true') parser.add_argument('--num_layers', type=int, default=2) parser.add_argument('--hidden_dim', type=int, default=256) parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static']) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--seed', type=int, default=3435) - parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB']) + parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014']) parser.add_argument('--resume_snapshot', type=str, default=None) parser.add_argument('--dev_every', type=int, default=30) parser.add_argument('--log_every', type=int, default=10) - parser.add_argument('--patience', type=int, default=50) + parser.add_argument('--patience', type=int, default=5) parser.add_argument('--save_path', type=str, default='lstm_regularization/saves') parser.add_argument('--words_dim', type=int, default=300) parser.add_argument('--embed_dim', type=int, default=300) diff --git a/lstm_regularization/model.py b/lstm_regularization/model.py index eadbe07..e19879b 100644 --- a/lstm_regularization/model.py +++ b/lstm_regularization/model.py @@ -18,7 +18,7 @@ class LSTMBaseline(nn.Module): self.mode = config.mode self.TAR = config.TAR self.beta_ema = config.beta_ema ## Temporal averaging - self.wdrop = config.wdrop ## WEight dropping + self.wdrop = config.wdrop ## Weight dropping self.embed_droprate = config.embed_droprate ## Embedding dropout input_channel = 1 @@ -83,6 +83,7 @@ class LSTMBaseline(nn.Module): x = self.dropout(x) if self.has_bottleneck_layer: x = F.relu(self.fc1(x)) + # x = self.dropout(x) if self.TAR: return self.fc2(x), rnn_outs.permute(1,0,2) return self.fc2(x) diff --git a/xml_cnn/__main__.py b/xml_cnn/__main__.py index 283c596..95d88b3 100644 --- a/xml_cnn/__main__.py +++ b/xml_cnn/__main__.py @@ -14,6 +14,8 @@ from datasets.sst import SST1 from datasets.sst import SST2 from datasets.reuters import Reuters from datasets.aapd import AAPD +from datasets.yelp2014 import Yelp2014 +from datasets.imdb import IMDB from xml_cnn.args import get_args from xml_cnn.model import XmlCNN @@ -48,12 +50,13 @@ def get_logger(): return logger -def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device): +def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label): saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device) + saved_model_evaluator.single_label = single_label scores, metric_names = saved_model_evaluator.get_scores() - logger.info('Evaluation metrics for {}'.format(split_name)) - logger.info('\t'.join([' '] + metric_names)) - logger.info('\t'.join([split_name] + list(map(str, scores)))) + print('Evaluation metrics for', split_name) + print(metric_names) + print(scores) if __name__ == '__main__': @@ -79,7 +82,9 @@ if __name__ == '__main__': 'SST-1': SST1, 'SST-2': SST2, 'Reuters': Reuters, - 'AAPD': AAPD + 'AAPD': AAPD, + 'IMDB': IMDB, + 'Yelp2014':Yelp2014 } if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') @@ -119,6 +124,9 @@ if __name__ == '__main__': train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu) test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) + train_evaluator.single_label = args.single_label + test_evaluator.single_label = args.single_label + dev_evaluator.single_label = args.single_label trainer_config = { 'optimizer': optimizer, 'batch_size': args.batch_size, @@ -126,7 +134,8 @@ if __name__ == '__main__': 'dev_log_interval': args.dev_every, 'patience': args.patience, 'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention - 'logger': logger + 'logger': logger, + 'single_label': args.single_label } trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator) @@ -141,28 +150,8 @@ if __name__ == '__main__': if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') else: - evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) - - # Calculate dev and test metrics - for data_loader in [dev_iter, test_iter]: - predicted_labels = list() - target_labels = list() - for batch_idx, batch in enumerate(data_loader): - scores_rounded = F.sigmoid(model(batch.text)).round().long() - predicted_labels.extend(scores_rounded.cpu().detach().numpy()) - target_labels.extend(batch.label.cpu().detach().numpy()) - predicted_labels = np.array(predicted_labels) - target_labels = np.array(target_labels) - accuracy = metrics.accuracy_score(target_labels, predicted_labels) - precision = metrics.precision_score(target_labels, predicted_labels, average='micro') - recall = metrics.recall_score(target_labels, predicted_labels, average='micro') - f1 = metrics.f1_score(target_labels, predicted_labels, average='micro') - if data_loader == dev_iter: - print("Dev metrics:") - else: - print("Test metrics:") - print(accuracy, precision, recall, f1) + evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label) + evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label) if args.onnx: device = torch.device('cuda') if torch.cuda.is_available() and args.cuda else torch.device('cpu') diff --git a/xml_cnn/args.py b/xml_cnn/args.py index a72ed4a..869076f 100644 --- a/xml_cnn/args.py +++ b/xml_cnn/args.py @@ -12,7 +12,7 @@ def get_args(): parser.add_argument('--mode', type=str, default='multichannel', choices=['rand', 'static', 'non-static', 'multichannel']) parser.add_argument('--lr', type=float, default=1.0) parser.add_argument('--seed', type=int, default=3435) - parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters','AAPD']) + parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters','AAPD', 'IMDB', 'Yelp2014']) parser.add_argument('--resume_snapshot', type=str, default=None) parser.add_argument('--dev_every', type=int, default=30) parser.add_argument('--log_every', type=int, default=10) @@ -38,6 +38,6 @@ def get_args(): parser.add_argument('--onnx', action='store_true', default=False, help='Export model in ONNX format') parser.add_argument('--onnx_batch_size', type=int, default=1024, help='Batch size for ONNX export') parser.add_argument('--onnx_sent_len', type=int, default=32, help='Sentence length for ONNX export') - + parser.add_argument('--single_label', action='store_true') args = parser.parse_args() return args