mirror of
https://github.com/wassname/Castor.git
synced 2026-08-20 12:00:37 +08:00
Add document classification models and datasets (#171)
* Add ReutersTrainer, ReutersEvaluator options in Factory classes * Add Reuters to Kim-CNN command line arguments * Fix SST dataset path according to changes in Kim-CNN args The dataset path in args.py was made to point at the dataset folder rather than dataset/SST folder. Hence SST folder was added to paths in the SST dataset class * Add Reuters dataset class, and support in __main__ * Add Reuters dataset trainers and evaluators * Remove debug print statement in reuters_evaluator * Fix rounding bug in reuters_trainer and reuters_evaluator * Add LSTM for baseline text classification measurements * Add eval metrics for lstm_baseline * Set batch_first param in lstm_baseline * Remove onnx args from lstm_baseline * Pack padded sequences in LSTM_baseline * Add TensorBoardX support for Reuters trainer * Add Arxiv Academic Paper Dataset (AAPD) * Add Hidden Bottleneck Layer to BiLSTM * Fix packing of padded tensors in Reuters * Add cmdline args for Hidden Bottleneck Layer for BiLSTM * Include pre-padding lengths in AAPD dataset * Remove duplication of preprocessing code in AAPD * Remove batch_size condition in ReutersTrainer * Add ignore_lengths option to ReutersTrainer and ReutersEvaluator * Add AAPDCharQuantized and ReutersCharQuantized * Rename Reuters_hierarchical to ReutersHierarchical * Add CharacterCNN for document classification * Update README.md for CharacterCNN * Fix table in README.md for CharacterCNN * Add AAPDHierarchical for HAN * Update HAN for changes in Reuters dataset endpoints * Fix bug in CharCNN when running on CPU * Add AAPD dataset support for KimCNN * Fix dataset paths for SST-1 * Fix dimensions of FC1 in CharCNN * Add model checkpointing for Reuters based on F1 * Refactor LSTM baseline __main__ * Add precision, recall and F1 to Reuters evaluator * Checkpoint only at the end of an epoch for ReutersTrainer Add detailed log printing for dev evaluations * Fix log_template and dev_log_template in ReutersTrainer * Add IMDB dataset * Fix duplicate printing of header in ReutersTrainer * Add support for single_label datasets in ReutersTrainer * Add support for IMDB dataset in lstm_baseline and lstm_reg * Fix evaluator call in main method of HAN * Add IMDB for HAN * Fix for single_label * Fix evaluate_dataset method for single_label datasets * Reduce default patience to 5 epochs before early stopping * Revert change to save_state rather than the entire model * Add Yelp 2018 dataset * Integrate Yelp2018 with LSTM baseline * Replace Yelp2018 with Yelp2014 dataset * Add Yelp2014 to LSTM Baseline * Integrate Yelp14 into LSTM Regularization * Remove dropout in HBL for LSTM Baseline and Reg * Add Yelp for HAN * Fix the saving issue for HAN * Fix loading for HAN * Fix typo in ReutersEvaluator * Print to STDOUT rather than logger * Print XML-CNN eval to STDOUT rather than logger * Update max_length for IMDB dataset * Add single_label support for char_cnn * Fix evaluation method for char_cnn * Remove unwanted parameters from ReutersTrainer and ReutersEval * Fix code formatting in lstm_reg/args * Add support for IMDB and Yelp in KimCNN * Fix single_label incorporation * Remove unnecessary conditions * Fix num_classes in Yelp2014 * Add single_label support for XML-CNN * Fix call to evaluator in XML-CNN * Address PEP8 issues * Address PEP8 issues * Address PEP8 issues * Address PEP8 issues
This commit is contained in:
Ashutosh-Adhikari
committed by
Ralph Tang
Ralph Tang
parent
57f53a81b5
commit
dc086e895f
+46
-55
@@ -1,18 +1,22 @@
|
||||
from sklearn import metrics
|
||||
|
||||
from copy import deepcopy
|
||||
import logging
|
||||
import numpy as np
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
from sklearn import metrics
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from copy import deepcopy
|
||||
|
||||
from char_cnn.args import get_args
|
||||
from char_cnn.model import CharCNN
|
||||
from common.evaluation import EvaluatorFactory
|
||||
from common.train import TrainerFactory
|
||||
from datasets.aapd import AAPDCharQuantized as AAPD
|
||||
from datasets.imdb import IMDBCharQuantized as IMDB
|
||||
from datasets.reuters import ReutersCharQuantized as Reuters
|
||||
from char_cnn.args import get_args
|
||||
from char_cnn.model import CharCNN
|
||||
from datasets.yelp2014 import Yelp2014CharQuantized as Yelp2014
|
||||
|
||||
|
||||
|
||||
|
||||
class UnknownWordVecCache(object):
|
||||
@@ -45,13 +49,14 @@ def get_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device):
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device)
|
||||
saved_model_evaluator.ignore_lengths = True
|
||||
saved_model_evaluator.single_label = single_label
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([split_name] + list(map(str, scores))))
|
||||
print('Evaluation metrics for', split_name)
|
||||
print(metric_names)
|
||||
print(scores)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@@ -73,13 +78,20 @@ if __name__ == '__main__':
|
||||
random.seed(args.seed)
|
||||
logger = get_logger()
|
||||
|
||||
# Set up the data for training SST-1
|
||||
if args.dataset == 'Reuters':
|
||||
train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
else:
|
||||
dataset_map = {
|
||||
'Reuters': Reuters,
|
||||
'AAPD': AAPD,
|
||||
'IMDB': IMDB,
|
||||
'Yelp2014': Yelp2014
|
||||
}
|
||||
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file,
|
||||
args.word_vectors_dir,
|
||||
batch_size=args.batch_size, device=args.gpu,
|
||||
unk_init=UnknownWordVecCache.unk)
|
||||
|
||||
config = deepcopy(args)
|
||||
config.dataset = train_iter.dataset
|
||||
@@ -104,19 +116,18 @@ if __name__ == '__main__':
|
||||
parameter = filter(lambda p: p.requires_grad, model.parameters())
|
||||
optimizer = torch.optim.Adam(parameter, lr=args.lr, weight_decay=args.weight_decay)
|
||||
|
||||
if args.dataset == 'Reuters':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
train_evaluator.single_label = args.single_label
|
||||
test_evaluator.single_label = args.single_label
|
||||
dev_evaluator.single_label = args.single_label
|
||||
dev_evaluator.ignore_lengths = True
|
||||
test_evaluator.ignore_lengths = True
|
||||
|
||||
dev_evaluator.ignore_lengths = True
|
||||
test_evaluator.ignore_lengths = True
|
||||
trainer_config = {
|
||||
'optimizer': optimizer,
|
||||
'batch_size': args.batch_size,
|
||||
@@ -125,7 +136,8 @@ if __name__ == '__main__':
|
||||
'patience': args.patience,
|
||||
'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention
|
||||
'logger': logger,
|
||||
'ignore_lengths': True
|
||||
'ignore_lengths': True,
|
||||
'single_label': args.single_label
|
||||
}
|
||||
trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
@@ -137,31 +149,10 @@ if __name__ == '__main__':
|
||||
else:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
|
||||
|
||||
if args.dataset == 'Reuters':
|
||||
evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
|
||||
# Calculate dev and test metrics
|
||||
for data_loader in [dev_iter, test_iter]:
|
||||
predicted_labels = list()
|
||||
target_labels = list()
|
||||
for batch_idx, batch in enumerate(data_loader):
|
||||
scores_rounded = F.sigmoid(model(batch.text)).round().long()
|
||||
predicted_labels.extend(scores_rounded.cpu().detach().numpy())
|
||||
target_labels.extend(batch.label.cpu().detach().numpy())
|
||||
predicted_labels = np.array(predicted_labels)
|
||||
target_labels = np.array(target_labels)
|
||||
accuracy = metrics.accuracy_score(target_labels, predicted_labels)
|
||||
precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
|
||||
recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
|
||||
f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
|
||||
if data_loader == dev_iter:
|
||||
print("Dev metrics:")
|
||||
else:
|
||||
print("Test metrics:")
|
||||
print(accuracy, precision, recall, f1)
|
||||
model = torch.load(trainer.snapshot_path)
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label)
|
||||
|
||||
+3
-2
@@ -11,12 +11,13 @@ def get_args():
|
||||
parser.add_argument('--batch_size', type=int, default=128)
|
||||
parser.add_argument('--lr', type=float, default=0.001)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='Reuters', choices=['Reuters', 'AAPD'])
|
||||
parser.add_argument('--single_label', action='store_true'),
|
||||
parser.add_argument('--dataset', type=str, default='Reuters', choices=['Reuters', 'AAPD', 'IMDB', 'Yelp2014'])
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
parser.add_argument('--patience', type=int, default=100)
|
||||
parser.add_argument('--save_path', type=str, default='kim_cnn/saves')
|
||||
parser.add_argument('--save_path', type=str, default='char_cnn/saves')
|
||||
parser.add_argument('--num_conv_filters', type=int, default=256)
|
||||
parser.add_argument('--num_affine_neurons', type=int, default=1024)
|
||||
parser.add_argument('--output_channel', type=int, default=256)
|
||||
|
||||
@@ -28,6 +28,7 @@ class EvaluatorFactory(object):
|
||||
'Reuters': ReutersEvaluator,
|
||||
'AAPD': ReutersEvaluator,
|
||||
'IMDB': ReutersEvaluator,
|
||||
'Yelp2014': ReutersEvaluator,
|
||||
'SNLI': SNLIEvaluator,
|
||||
'sts2014': STS2014Evaluator,
|
||||
'Quora': QuoraEvaluator
|
||||
|
||||
@@ -28,12 +28,12 @@ class ReutersEvaluator(Evaluator):
|
||||
for batch_idx, batch in enumerate(self.data_loader):
|
||||
if hasattr(self.model, 'TAR') and self.model.TAR: # TAR condition
|
||||
if self.ignore_lengths:
|
||||
scores, rnn_outs = self.model(batch.text, lengths=batch.text)
|
||||
scores, rnn_outs = self.model(batch.text)
|
||||
else:
|
||||
scores, rnn_outs = self.model(batch.text[0], lengths=batch.text[1])
|
||||
else:
|
||||
if self.ignore_lengths:
|
||||
scores = self.model(batch.text, lengths=batch.text)
|
||||
scores = self.model(batch.text)
|
||||
else:
|
||||
scores = self.model(batch.text[0], lengths=batch.text[1])
|
||||
|
||||
@@ -62,4 +62,4 @@ class ReutersEvaluator(Evaluator):
|
||||
if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0:
|
||||
self.model.load_params(old_params)
|
||||
|
||||
return [accuracy, precision, recall, f1, avg_loss], ['accuracy', 'precision', 'recall', 'f1' 'cross_entropy_loss']
|
||||
return [accuracy, precision, recall, f1, avg_loss], ['accuracy', 'precision', 'recall', 'f1', 'cross_entropy_loss']
|
||||
|
||||
@@ -28,6 +28,7 @@ class TrainerFactory(object):
|
||||
'Reuters': ReutersTrainer,
|
||||
'AAPD': ReutersTrainer,
|
||||
'IMDB': ReutersTrainer,
|
||||
'Yelp2014': ReutersTrainer,
|
||||
'snli': SNLITrainer,
|
||||
'sts2014': STS2014Trainer,
|
||||
'quora': QuoraTrainer
|
||||
|
||||
@@ -35,12 +35,12 @@ class ReutersTrainer(Trainer):
|
||||
self.optimizer.zero_grad()
|
||||
if hasattr(self.model, 'TAR') and self.model.TAR:
|
||||
if 'ignore_lengths' in self.config and self.config['ignore_lengths']:
|
||||
scores, rnn_outs = self.model(batch.text, lengths=batch.text)
|
||||
scores, rnn_outs = self.model(batch.text)
|
||||
else:
|
||||
scores, rnn_outs = self.model(batch.text[0], lengths=batch.text[1])
|
||||
else:
|
||||
if 'ignore_lengths' in self.config and self.config['ignore_lengths']:
|
||||
scores = self.model(batch.text, lengths=batch.text)
|
||||
scores = self.model(batch.text)
|
||||
else:
|
||||
scores = self.model(batch.text[0], lengths=batch.text[1])
|
||||
|
||||
@@ -85,9 +85,9 @@ class ReutersTrainer(Trainer):
|
||||
# model_outfile is actually a directory, using model_outfile to conform to Trainer naming convention
|
||||
os.makedirs(self.model_outfile, exist_ok=True)
|
||||
os.makedirs(os.path.join(self.model_outfile, self.train_loader.dataset.NAME), exist_ok=True)
|
||||
print(header)
|
||||
|
||||
for epoch in range(1, epochs + 1):
|
||||
print('\n' + header)
|
||||
self.train_epoch(epoch)
|
||||
|
||||
# Evaluate performance on validation set
|
||||
@@ -100,13 +100,12 @@ class ReutersTrainer(Trainer):
|
||||
print('\n' + dev_header)
|
||||
print(self.dev_log_template.format(time.time() - self.start, epoch, self.iterations, epoch, epochs,
|
||||
dev_acc, dev_precision, dev_recall, dev_f1, dev_loss))
|
||||
print('\n' + header)
|
||||
|
||||
# Update validation results
|
||||
if dev_f1 > self.best_dev_f1:
|
||||
self.iters_not_improved = 0
|
||||
self.best_dev_f1 = dev_f1
|
||||
torch.save(self.model.state_dict(), self.snapshot_path)
|
||||
torch.save(self.model, self.snapshot_path)
|
||||
else:
|
||||
self.iters_not_improved += 1
|
||||
if self.iters_not_improved >= self.patience:
|
||||
|
||||
+6
-4
@@ -1,12 +1,14 @@
|
||||
import re
|
||||
import os
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from datasets.reuters import clean_string, char_quantize, clean_string_fl, split_sents
|
||||
from torchtext.data import NestedField, Field, TabularDataset
|
||||
from torchtext.data.iterator import BucketIterator
|
||||
from torchtext.vocab import Vectors
|
||||
|
||||
from datasets.reuters import clean_string, clean_string_fl, split_sents
|
||||
|
||||
|
||||
def process_labels(string):
|
||||
"""
|
||||
@@ -76,5 +78,5 @@ class AAPDCharQuantized(AAPD):
|
||||
return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device)
|
||||
|
||||
class AAPDHierarchical(AAPD):
|
||||
In_FIELD = Field(batch_first=True, tokenize=clean_string)
|
||||
TEXT_FIELD = NestedField(In_FIELD, tokenize=split_sents)
|
||||
NESTING_FIELD = Field(batch_first=True, tokenize=clean_string)
|
||||
TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents)
|
||||
|
||||
+7
-5
@@ -1,14 +1,16 @@
|
||||
import numpy as np
|
||||
import os
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from datasets.reuters import clean_string, clean_string_fl, split_sents
|
||||
from torchtext.data import NestedField, Field, TabularDataset
|
||||
from torchtext.data.iterator import BucketIterator
|
||||
from torchtext.vocab import Vectors
|
||||
|
||||
from datasets.reuters import clean_string, clean_string_fl, split_sents
|
||||
|
||||
def char_quantize(string, max_length=1000):
|
||||
|
||||
def char_quantize(string, max_length=500):
|
||||
identity = np.identity(len(IMDBCharQuantized.ALPHABET))
|
||||
quantized_string = np.array([identity[IMDBCharQuantized.ALPHABET[char]] for char in list(string.lower()) if char in IMDBCharQuantized.ALPHABET], dtype=np.float32)
|
||||
if len(quantized_string) > max_length:
|
||||
@@ -85,5 +87,5 @@ class IMDBCharQuantized(IMDB):
|
||||
|
||||
|
||||
class IMDBHierarchical(IMDB):
|
||||
In_FIELD = Field(batch_first=True, tokenize=clean_string)
|
||||
TEXT_FIELD = NestedField(In_FIELD, tokenize=split_sents)
|
||||
NESTING_FIELD = Field(batch_first=True, tokenize=clean_string)
|
||||
TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents)
|
||||
|
||||
+4
-3
@@ -1,6 +1,7 @@
|
||||
import numpy as np
|
||||
import os
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torchtext.data import NestedField, Field, TabularDataset
|
||||
from torchtext.data.iterator import BucketIterator
|
||||
@@ -109,5 +110,5 @@ class ReutersCharQuantized(Reuters):
|
||||
|
||||
|
||||
class ReutersHierarchical(Reuters):
|
||||
In_FIELD = Field(batch_first=True, tokenize=clean_string)
|
||||
TEXT_FIELD = NestedField(In_FIELD, tokenize=split_sents)
|
||||
NESTING_FIELD = Field(batch_first=True, tokenize=clean_string)
|
||||
TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents)
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import os
|
||||
import re
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torchtext.data import NestedField, Field, TabularDataset
|
||||
from torchtext.data.iterator import BucketIterator
|
||||
from torchtext.vocab import Vectors
|
||||
|
||||
from datasets.reuters import clean_string, clean_string_fl, split_sents
|
||||
|
||||
|
||||
def char_quantize(string, max_length=1000):
|
||||
identity = np.identity(len(Yelp2014CharQuantized.ALPHABET))
|
||||
quantized_string = np.array([identity[Yelp2014CharQuantized.ALPHABET[char]] for char in list(string.lower()) if char in Yelp2014CharQuantized.ALPHABET], dtype=np.float32)
|
||||
if len(quantized_string) > max_length:
|
||||
return quantized_string[:max_length]
|
||||
else:
|
||||
return np.concatenate((quantized_string, np.zeros((max_length - len(quantized_string), len(Yelp2014CharQuantized.ALPHABET)), dtype=np.float32)))
|
||||
|
||||
|
||||
def process_labels(string):
|
||||
"""
|
||||
Returns the label string as a list of integers
|
||||
:param string:
|
||||
:return:
|
||||
"""
|
||||
return [float(x) for x in string]
|
||||
|
||||
|
||||
class Yelp2014(TabularDataset):
|
||||
NAME = 'Yelp2014'
|
||||
NUM_CLASSES = 5
|
||||
TEXT_FIELD = Field(batch_first=True, tokenize=clean_string, include_lengths=True)
|
||||
LABEL_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=process_labels)
|
||||
|
||||
@staticmethod
|
||||
def sort_key(ex):
|
||||
return len(ex.text)
|
||||
|
||||
@classmethod
|
||||
def splits(cls, path, train=os.path.join('Yelp-Reviews-2014', 'data', 'yelp2014_train.tsv'),
|
||||
validation=os.path.join('Yelp-Reviews-2014', 'data', 'yelp2014_validation.tsv'),
|
||||
test=os.path.join('Yelp-Reviews-2014', 'data', 'yelp2014_test.tsv'), **kwargs):
|
||||
return super(Yelp2014, cls).splits(
|
||||
path, train=train, validation=validation, test=test,
|
||||
format='tsv', fields=[('label', cls.LABEL_FIELD), ('text', cls.TEXT_FIELD)]
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None,
|
||||
unk_init=torch.Tensor.zero_):
|
||||
"""
|
||||
:param path: directory containing train, test, dev files
|
||||
:param vectors_name: name of word vectors file
|
||||
:param vectors_cache: path to directory containing word vectors file
|
||||
:param batch_size: batch size
|
||||
:param device: GPU device
|
||||
:param vectors: custom vectors - either predefined torchtext vectors or your own custom Vector classes
|
||||
:param unk_init: function used to generate vector for OOV words
|
||||
:return:
|
||||
"""
|
||||
if vectors is None:
|
||||
vectors = Vectors(name=vectors_name, cache=vectors_cache, unk_init=unk_init)
|
||||
|
||||
train, val, test = cls.splits(path)
|
||||
cls.TEXT_FIELD.build_vocab(train, val, test, vectors=vectors)
|
||||
return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle,
|
||||
sort_within_batch=True, device=device)
|
||||
|
||||
|
||||
class Yelp2014CharQuantized(Yelp2014):
|
||||
ALPHABET = dict(map(lambda t: (t[1], t[0]), enumerate(list("""abcdefghijklmnopqrstuvwxyz0123456789,;.!?:'\"/\\|_@#$%^&*~`+-=<>()[]{}"""))))
|
||||
TEXT_FIELD = Field(sequential=False, use_vocab=False, batch_first=True, preprocessing=char_quantize)
|
||||
|
||||
@classmethod
|
||||
def iters(cls, path, vectors_name, vectors_cache, batch_size=64, shuffle=True, device=0, vectors=None,
|
||||
unk_init=torch.Tensor.zero_):
|
||||
"""
|
||||
:param path: directory containing train, test, dev files
|
||||
:param batch_size: batch size
|
||||
:param device: GPU device
|
||||
:return:
|
||||
"""
|
||||
train, val, test = cls.splits(path)
|
||||
return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device)
|
||||
|
||||
|
||||
class Yelp2014Hierarchical(Yelp2014):
|
||||
NESTING_FIELD = Field(batch_first=True, tokenize=clean_string)
|
||||
TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents)
|
||||
+42
-74
@@ -1,20 +1,25 @@
|
||||
from copy import deepcopy
|
||||
import logging
|
||||
import random
|
||||
from sklearn import metrics
|
||||
|
||||
import numpy as np
|
||||
from sklearn import metrics
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import torch.onnx
|
||||
|
||||
from common.evaluation import EvaluatorFactory
|
||||
from common.train import TrainerFactory
|
||||
from datasets.aapd import AAPDHierarchical as AAPD
|
||||
from datasets.imdb import IMDBHierarchical as IMDB
|
||||
from datasets.sst import SST1
|
||||
from datasets.sst import SST2
|
||||
from datasets.reuters import ReutersHierarchical as Reuters
|
||||
from datasets.aapd import AAPDHierarchical as AAPD
|
||||
from datasets.yelp2014 import Yelp2014Hierarchical as Yelp2014
|
||||
from han.args import get_args
|
||||
from han.model import HAN
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
|
||||
class UnknownWordVecCache(object):
|
||||
"""
|
||||
@@ -46,13 +51,14 @@ def get_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device):
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device)
|
||||
saved_model_evaluator.single_label = single_label
|
||||
saved_model_evaluator.ignore_lengths = True
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([split_name] + list(map(str, scores))))
|
||||
print('Evaluation metrics for', split_name)
|
||||
print(metric_names)
|
||||
print(scores)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@@ -74,18 +80,19 @@ if __name__ == '__main__':
|
||||
random.seed(args.seed)
|
||||
logger = get_logger()
|
||||
|
||||
# Set up the data for training SST-1
|
||||
if args.dataset == 'SST-1':
|
||||
train_iter, dev_iter, test_iter = SST1.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
# Set up the data for training SST-2
|
||||
elif args.dataset == 'SST-2':
|
||||
train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
else:
|
||||
dataset_map = {
|
||||
'SST-1': SST1,
|
||||
'SST-2': SST2,
|
||||
'Reuters': Reuters,
|
||||
'AAPD': AAPD,
|
||||
'IMDB': IMDB,
|
||||
'Yelp2014': Yelp2014
|
||||
}
|
||||
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
|
||||
config = deepcopy(args)
|
||||
config.dataset = train_iter.dataset
|
||||
@@ -115,24 +122,16 @@ if __name__ == '__main__':
|
||||
#optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay)
|
||||
#optimizer = torch.optim.SGD(parameter, lr = args.lr, momentum = 0.9)
|
||||
optimizer = torch.optim.Adam(parameter, lr = args.lr)
|
||||
if args.dataset == 'SST-1':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
train_evaluator.single_label = args.single_label
|
||||
test_evaluator.single_label = args.single_label
|
||||
dev_evaluator.single_label = args.single_label
|
||||
|
||||
dev_evaluator.ignore_lengths = True
|
||||
test_evaluator.ignore_lengths = True
|
||||
@@ -144,7 +143,8 @@ if __name__ == '__main__':
|
||||
'patience': args.patience,
|
||||
'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention
|
||||
'logger': logger,
|
||||
'ignore_lengths': True
|
||||
'ignore_lengths': True,
|
||||
'single_label': args.single_label
|
||||
}
|
||||
trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
@@ -156,45 +156,13 @@ if __name__ == '__main__':
|
||||
else:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
|
||||
|
||||
if args.dataset == 'SST-1':
|
||||
evaluate_dataset('dev', SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
evaluate_dataset('dev', SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
|
||||
|
||||
|
||||
# Calculate dev and test metrics
|
||||
for data_loader in [dev_iter, test_iter]:
|
||||
predicted_labels = list()
|
||||
target_labels = list()
|
||||
for batch_idx, batch in enumerate(data_loader):
|
||||
scores_rounded = F.sigmoid(model(batch.text)).round().long()
|
||||
predicted_labels.extend(scores_rounded.cpu().detach().numpy())
|
||||
target_labels.extend(batch.label.cpu().detach().numpy())
|
||||
predicted_labels = np.array(predicted_labels)
|
||||
target_labels = np.array(target_labels)
|
||||
accuracy = metrics.accuracy_score(target_labels, predicted_labels)
|
||||
precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
|
||||
recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
|
||||
f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
|
||||
if data_loader == dev_iter:
|
||||
print("Dev metrics:")
|
||||
else:
|
||||
print("Test metrics:")
|
||||
print(accuracy, precision, recall, f1)
|
||||
|
||||
|
||||
model = torch.load(trainer.snapshot_path)
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label)
|
||||
|
||||
if args.onnx:
|
||||
device = torch.device('cuda') if torch.cuda.is_available() and args.cuda else torch.device('cpu')
|
||||
|
||||
+4
-4
@@ -10,15 +10,15 @@ def get_args():
|
||||
parser.add_argument('--epochs', type=int, default=30)
|
||||
|
||||
|
||||
parser.add_argument('--word_num_hidden', type = int, default = 50)
|
||||
parser.add_argument('--sentence_num_hidden', type = int, default = 50)
|
||||
|
||||
parser.add_argument('--word_num_hidden', type=int, default=50)
|
||||
parser.add_argument('--sentence_num_hidden', type=int, default=50)
|
||||
|
||||
parser.add_argument('--single_label', action='store_true')
|
||||
parser.add_argument('--batch_size', type=int, default=64)
|
||||
parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static'])
|
||||
parser.add_argument('--lr', type=float, default=1.0)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD'])
|
||||
parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014'])
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ from han.word_level_rnn import WordLevelRNN
|
||||
class HAN(nn.Module):
|
||||
def __init__(self, config):
|
||||
super(HAN, self).__init__()
|
||||
self.dataset = config.dataset
|
||||
dataset = config.dataset
|
||||
self.mode = config.mode
|
||||
self.word_attention_rnn = WordLevelRNN(config)
|
||||
self.sentence_attention_rnn = SentLevelRNN(config)
|
||||
|
||||
+43
-44
@@ -14,9 +14,12 @@ from datasets.sst import SST1
|
||||
from datasets.sst import SST2
|
||||
from datasets.aapd import AAPD
|
||||
from datasets.reuters import Reuters
|
||||
from datasets.yelp2014 import Yelp2014
|
||||
from datasets.imdb import IMDB
|
||||
from kim_cnn.args import get_args
|
||||
from kim_cnn.model import KimCNN
|
||||
|
||||
|
||||
class UnknownWordVecCache(object):
|
||||
"""
|
||||
Caches the first randomly generated word vector for a certain size to make it is reused.
|
||||
@@ -47,8 +50,10 @@ def get_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device):
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device)
|
||||
if hasattr(saved_model_evaluator, 'single_label'):
|
||||
saved_model_evaluator.single_label = single_label
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
@@ -74,18 +79,22 @@ if __name__ == '__main__':
|
||||
random.seed(args.seed)
|
||||
logger = get_logger()
|
||||
|
||||
# Set up the data for training SST-1
|
||||
if args.dataset == 'SST-1':
|
||||
train_iter, dev_iter, test_iter = SST1.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
# Set up the data for training SST-2
|
||||
elif args.dataset == 'SST-2':
|
||||
train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
else:
|
||||
dataset_map = {
|
||||
'SST-1': SST1,
|
||||
'SST-2': SST2,
|
||||
'Reuters': Reuters,
|
||||
'AAPD': AAPD,
|
||||
'IMDB': IMDB,
|
||||
'Yelp2014': Yelp2014
|
||||
}
|
||||
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file,
|
||||
args.word_vectors_dir,
|
||||
batch_size=args.batch_size, device=args.gpu,
|
||||
unk_init=UnknownWordVecCache.unk)
|
||||
|
||||
config = deepcopy(args)
|
||||
config.dataset = train_iter.dataset
|
||||
@@ -113,24 +122,18 @@ if __name__ == '__main__':
|
||||
parameter = filter(lambda p: p.requires_grad, model.parameters())
|
||||
optimizer = torch.optim.Adadelta(parameter, lr=args.lr, weight_decay=args.weight_decay)
|
||||
|
||||
if args.dataset == 'SST-1':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
if hasattr(train_evaluator, 'single_label'):
|
||||
train_evaluator.single_label = args.single_label
|
||||
if hasattr(test_evaluator, 'single_label'):
|
||||
test_evaluator.single_label = args.single_label
|
||||
if hasattr(dev_evaluator, 'single_label'):
|
||||
dev_evaluator.single_label = args.single_label
|
||||
|
||||
trainer_config = {
|
||||
'optimizer': optimizer,
|
||||
@@ -138,9 +141,11 @@ if __name__ == '__main__':
|
||||
'log_interval': args.log_every,
|
||||
'dev_log_interval': args.dev_every,
|
||||
'patience': args.patience,
|
||||
'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention
|
||||
'logger': logger
|
||||
'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention
|
||||
'logger': logger,
|
||||
'single_label': args.single_label
|
||||
}
|
||||
|
||||
trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
if not args.trained_model:
|
||||
@@ -151,20 +156,14 @@ if __name__ == '__main__':
|
||||
else:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
|
||||
|
||||
if args.dataset == 'SST-1':
|
||||
evaluate_dataset('dev', SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
evaluate_dataset('dev', SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
# Calculate dev and test metrics
|
||||
if hasattr(trainer, 'snapshot_path'):
|
||||
model = torch.load(trainer.snapshot_path)
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label)
|
||||
|
||||
if args.onnx:
|
||||
device = torch.device('cuda') if torch.cuda.is_available() and args.cuda else torch.device('cpu')
|
||||
|
||||
+2
-1
@@ -12,8 +12,9 @@ def get_args():
|
||||
parser.add_argument('--mode', type=str, default='multichannel', choices=['rand', 'static', 'non-static', 'multichannel'])
|
||||
parser.add_argument('--lr', type=float, default=1.0)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD'])
|
||||
parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014'])
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--single_label', action='store_true')
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
parser.add_argument('--patience', type=int, default=50)
|
||||
|
||||
@@ -14,6 +14,7 @@ from datasets.sst import SST2
|
||||
from datasets.reuters import Reuters
|
||||
from datasets.imdb import IMDB
|
||||
from datasets.aapd import AAPD
|
||||
from datasets.yelp2014 import Yelp2014
|
||||
from lstm_baseline.args import get_args
|
||||
from lstm_baseline.model import LSTMBaseline
|
||||
|
||||
@@ -48,12 +49,13 @@ def get_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device):
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device)
|
||||
saved_model_evaluator.single_label = single_label
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([split_name] + list(map(str, scores))))
|
||||
print('Evaluation metrics for', split_name)
|
||||
print(metric_names)
|
||||
print(scores)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@@ -80,7 +82,8 @@ if __name__ == '__main__':
|
||||
'SST-2': SST2,
|
||||
'Reuters': Reuters,
|
||||
'AAPD': AAPD,
|
||||
'IMDB': IMDB
|
||||
'IMDB': IMDB,
|
||||
'Yelp2014': Yelp2014
|
||||
}
|
||||
|
||||
if args.dataset not in dataset_map:
|
||||
@@ -145,9 +148,9 @@ if __name__ == '__main__':
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
|
||||
|
||||
# Calculate dev and test metrics
|
||||
model.load_state_dict(torch.load(trainer.snapshot_path))
|
||||
model = torch.load(trainer.snapshot_path)
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label)
|
||||
@@ -6,7 +6,7 @@ from argparse import ArgumentParser
|
||||
def get_args():
|
||||
parser = ArgumentParser(description="Baseline LSTM for text classification")
|
||||
parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda')
|
||||
parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU
|
||||
parser.add_argument('--gpu', type=int, default=0, help="Use -1 for CPU")
|
||||
parser.add_argument('--epochs', type=int, default=50)
|
||||
parser.add_argument('--batch_size', type=int, default=1024)
|
||||
parser.add_argument('--bidirectional', action='store_true'),
|
||||
@@ -17,11 +17,11 @@ def get_args():
|
||||
parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static'])
|
||||
parser.add_argument('--lr', type=float, default=0.001)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB'])
|
||||
parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014'])
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
parser.add_argument('--patience', type=int, default=50)
|
||||
parser.add_argument('--patience', type=int, default=5)
|
||||
parser.add_argument('--save_path', type=str, default='lstm_baseline/saves')
|
||||
parser.add_argument('--words_dim', type=int, default=300)
|
||||
parser.add_argument('--embed_dim', type=int, default=300)
|
||||
|
||||
@@ -61,6 +61,7 @@ class LSTMBaseline(nn.Module):
|
||||
x = self.dropout(x)
|
||||
if self.has_bottleneck_layer:
|
||||
x = F.relu(self.fc1(x))
|
||||
# x = self.dropout(x)
|
||||
return self.fc2(x)
|
||||
else:
|
||||
return self.fc1(x)
|
||||
|
||||
@@ -14,6 +14,7 @@ from datasets.sst import SST2
|
||||
from datasets.reuters import Reuters
|
||||
from datasets.aapd import AAPD
|
||||
from datasets.imdb import IMDB
|
||||
from datasets.yelp2014 import Yelp2014
|
||||
from lstm_regularization.args import get_args
|
||||
from lstm_regularization.model import LSTMBaseline
|
||||
|
||||
@@ -47,12 +48,13 @@ def get_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device):
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device)
|
||||
saved_model_evaluator.single_label = single_label
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([split_name] + list(map(str, scores))))
|
||||
print('Evaluation metrics for', split_name)
|
||||
print(metric_names)
|
||||
print(scores)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@@ -79,7 +81,8 @@ if __name__ == '__main__':
|
||||
'SST-2': SST2,
|
||||
'Reuters': Reuters,
|
||||
'AAPD': AAPD,
|
||||
'IMDB': IMDB
|
||||
'IMDB': IMDB,
|
||||
'Yelp2014': Yelp2014
|
||||
}
|
||||
|
||||
if args.dataset not in dataset_map:
|
||||
@@ -143,7 +146,7 @@ if __name__ == '__main__':
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
|
||||
|
||||
# Calculate dev and test metrics
|
||||
model.load_state_dict(torch.load(trainer.snapshot_path))
|
||||
model = torch.load(trainer.snapshot_path)
|
||||
if model.beta_ema > 0:
|
||||
old_params = model.get_params()
|
||||
model.load_ema_params()
|
||||
@@ -151,8 +154,8 @@ if __name__ == '__main__':
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label)
|
||||
|
||||
if model.beta_ema > 0:
|
||||
model.load_params(old_params)
|
||||
@@ -6,22 +6,22 @@ from argparse import ArgumentParser
|
||||
def get_args():
|
||||
parser = ArgumentParser(description="Regularized LSTM for text classification with Regularization")
|
||||
parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda')
|
||||
parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU
|
||||
parser.add_argument('--gpu', type=int, default=0, help="Use -1 for CPU")
|
||||
parser.add_argument('--epochs', type=int, default=50)
|
||||
parser.add_argument('--batch_size', type=int, default=1024)
|
||||
parser.add_argument('--bidirectional', action='store_true'),
|
||||
parser.add_argument('--bottleneck_layer', action='store_true'),
|
||||
parser.add_argument('--single_label', action='store_true'),
|
||||
parser.add_argument('--bidirectional', action='store_true')
|
||||
parser.add_argument('--bottleneck_layer', action='store_true')
|
||||
parser.add_argument('--single_label', action='store_true')
|
||||
parser.add_argument('--num_layers', type=int, default=2)
|
||||
parser.add_argument('--hidden_dim', type=int, default=256)
|
||||
parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static'])
|
||||
parser.add_argument('--lr', type=float, default=0.001)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB'])
|
||||
parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters', 'AAPD', 'IMDB', 'Yelp2014'])
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
parser.add_argument('--patience', type=int, default=50)
|
||||
parser.add_argument('--patience', type=int, default=5)
|
||||
parser.add_argument('--save_path', type=str, default='lstm_regularization/saves')
|
||||
parser.add_argument('--words_dim', type=int, default=300)
|
||||
parser.add_argument('--embed_dim', type=int, default=300)
|
||||
|
||||
@@ -18,7 +18,7 @@ class LSTMBaseline(nn.Module):
|
||||
self.mode = config.mode
|
||||
self.TAR = config.TAR
|
||||
self.beta_ema = config.beta_ema ## Temporal averaging
|
||||
self.wdrop = config.wdrop ## WEight dropping
|
||||
self.wdrop = config.wdrop ## Weight dropping
|
||||
self.embed_droprate = config.embed_droprate ## Embedding dropout
|
||||
|
||||
input_channel = 1
|
||||
@@ -83,6 +83,7 @@ class LSTMBaseline(nn.Module):
|
||||
x = self.dropout(x)
|
||||
if self.has_bottleneck_layer:
|
||||
x = F.relu(self.fc1(x))
|
||||
# x = self.dropout(x)
|
||||
if self.TAR:
|
||||
return self.fc2(x), rnn_outs.permute(1,0,2)
|
||||
return self.fc2(x)
|
||||
|
||||
+17
-28
@@ -14,6 +14,8 @@ from datasets.sst import SST1
|
||||
from datasets.sst import SST2
|
||||
from datasets.reuters import Reuters
|
||||
from datasets.aapd import AAPD
|
||||
from datasets.yelp2014 import Yelp2014
|
||||
from datasets.imdb import IMDB
|
||||
from xml_cnn.args import get_args
|
||||
from xml_cnn.model import XmlCNN
|
||||
|
||||
@@ -48,12 +50,13 @@ def get_logger():
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device):
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device, single_label):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device)
|
||||
saved_model_evaluator.single_label = single_label
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([split_name] + list(map(str, scores))))
|
||||
print('Evaluation metrics for', split_name)
|
||||
print(metric_names)
|
||||
print(scores)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@@ -79,7 +82,9 @@ if __name__ == '__main__':
|
||||
'SST-1': SST1,
|
||||
'SST-2': SST2,
|
||||
'Reuters': Reuters,
|
||||
'AAPD': AAPD
|
||||
'AAPD': AAPD,
|
||||
'IMDB': IMDB,
|
||||
'Yelp2014':Yelp2014
|
||||
}
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
@@ -119,6 +124,9 @@ if __name__ == '__main__':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
train_evaluator.single_label = args.single_label
|
||||
test_evaluator.single_label = args.single_label
|
||||
dev_evaluator.single_label = args.single_label
|
||||
trainer_config = {
|
||||
'optimizer': optimizer,
|
||||
'batch_size': args.batch_size,
|
||||
@@ -126,7 +134,8 @@ if __name__ == '__main__':
|
||||
'dev_log_interval': args.dev_every,
|
||||
'patience': args.patience,
|
||||
'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention
|
||||
'logger': logger
|
||||
'logger': logger,
|
||||
'single_label': args.single_label
|
||||
}
|
||||
trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
@@ -141,28 +150,8 @@ if __name__ == '__main__':
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
|
||||
# Calculate dev and test metrics
|
||||
for data_loader in [dev_iter, test_iter]:
|
||||
predicted_labels = list()
|
||||
target_labels = list()
|
||||
for batch_idx, batch in enumerate(data_loader):
|
||||
scores_rounded = F.sigmoid(model(batch.text)).round().long()
|
||||
predicted_labels.extend(scores_rounded.cpu().detach().numpy())
|
||||
target_labels.extend(batch.label.cpu().detach().numpy())
|
||||
predicted_labels = np.array(predicted_labels)
|
||||
target_labels = np.array(target_labels)
|
||||
accuracy = metrics.accuracy_score(target_labels, predicted_labels)
|
||||
precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
|
||||
recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
|
||||
f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
|
||||
if data_loader == dev_iter:
|
||||
print("Dev metrics:")
|
||||
else:
|
||||
print("Test metrics:")
|
||||
print(accuracy, precision, recall, f1)
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu, args.single_label)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu, args.single_label)
|
||||
|
||||
if args.onnx:
|
||||
device = torch.device('cuda') if torch.cuda.is_available() and args.cuda else torch.device('cpu')
|
||||
|
||||
+2
-2
@@ -12,7 +12,7 @@ def get_args():
|
||||
parser.add_argument('--mode', type=str, default='multichannel', choices=['rand', 'static', 'non-static', 'multichannel'])
|
||||
parser.add_argument('--lr', type=float, default=1.0)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters','AAPD'])
|
||||
parser.add_argument('--dataset', type=str, default='SST-1', choices=['SST-1', 'SST-2', 'Reuters','AAPD', 'IMDB', 'Yelp2014'])
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
@@ -38,6 +38,6 @@ def get_args():
|
||||
parser.add_argument('--onnx', action='store_true', default=False, help='Export model in ONNX format')
|
||||
parser.add_argument('--onnx_batch_size', type=int, default=1024, help='Batch size for ONNX export')
|
||||
parser.add_argument('--onnx_sent_len', type=int, default=32, help='Sentence length for ONNX export')
|
||||
|
||||
parser.add_argument('--single_label', action='store_true')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
Reference in New Issue
Block a user