mirror of
https://github.com/wassname/Castor.git
synced 2026-09-09 11:13:20 +08:00
Add model checkpointing to ReutersTrainer (#158)
* Add ReutersTrainer, ReutersEvaluator options in Factory classes * Add Reuters to Kim-CNN command line arguments * Fix SST dataset path according to changes in Kim-CNN args The dataset path in args.py was made to point at the dataset folder rather than dataset/SST folder. Hence SST folder was added to paths in the SST dataset class * Add Reuters dataset class, and support in __main__ * Add Reuters dataset trainers and evaluators * Remove debug print statement in reuters_evaluator * Fix rounding bug in reuters_trainer and reuters_evaluator * Add LSTM for baseline text classification measurements * Add eval metrics for lstm_baseline * Set batch_first param in lstm_baseline * Remove onnx args from lstm_baseline * Pack padded sequences in LSTM_baseline * Add TensorBoardX support for Reuters trainer * Add Arxiv Academic Paper Dataset (AAPD) * Add Hidden Bottleneck Layer to BiLSTM * Fix packing of padded tensors in Reuters * Add cmdline args for Hidden Bottleneck Layer for BiLSTM * Include pre-padding lengths in AAPD dataset * Remove duplication of preprocessing code in AAPD * Remove batch_size condition in ReutersTrainer * Add ignore_lengths option to ReutersTrainer and ReutersEvaluator * Add AAPDCharQuantized and ReutersCharQuantized * Rename Reuters_hierarchical to ReutersHierarchical * Add CharacterCNN for document classification * Update README.md for CharacterCNN * Fix table in README.md for CharacterCNN * Add AAPDHierarchical for HAN * Update HAN for changes in Reuters dataset endpoints * Fix bug in CharCNN when running on CPU * Add AAPD dataset support for KimCNN * Fix dataset paths for SST-1 * Fix dimensions of FC1 in CharCNN * Add model checkpointing for Reuters based on F1 * Refactor LSTM baseline __main__ * Add precision, recall and F1 to Reuters evaluator
This commit is contained in:
Achyudh Ram
+1
-1
@@ -22,7 +22,7 @@ class CharCNN(nn.Module):
|
||||
self.conv5 = nn.Conv1d(num_conv_filters, num_conv_filters, kernel_size=3)
|
||||
self.conv6 = nn.Conv1d(num_conv_filters, output_channel, kernel_size=3)
|
||||
self.dropout = nn.Dropout(config.dropout)
|
||||
self.fc1 = nn.Linear(num_conv_filters, num_affine_neurons)
|
||||
self.fc1 = nn.Linear(output_channel, num_affine_neurons)
|
||||
self.fc2 = nn.Linear(num_affine_neurons, num_affine_neurons)
|
||||
self.fc3 = nn.Linear(num_affine_neurons, target_class)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
import numpy as np
|
||||
|
||||
from sklearn import metrics
|
||||
from .evaluator import Evaluator
|
||||
|
||||
|
||||
@@ -16,14 +17,15 @@ class ReutersEvaluator(Evaluator):
|
||||
self.data_loader.init_epoch()
|
||||
n_dev_correct = 0
|
||||
total_loss = 0
|
||||
############
|
||||
## Temp Ave
|
||||
|
||||
# Temp Ave
|
||||
if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0:
|
||||
old_params = self.model.get_params()
|
||||
self.model.load_ema_params()
|
||||
############
|
||||
|
||||
predicted_labels, target_labels = list(), list()
|
||||
for batch_idx, batch in enumerate(self.data_loader):
|
||||
if hasattr(self.model, 'TAR') and self.model.TAR: ## TAR Condition
|
||||
if hasattr(self.model, 'TAR') and self.model.TAR: # TAR condition
|
||||
if self.ignore_lengths:
|
||||
scores, rnn_outs = self.model(batch.text, lengths=batch.text)
|
||||
else:
|
||||
@@ -33,22 +35,25 @@ class ReutersEvaluator(Evaluator):
|
||||
scores = self.model(batch.text, lengths=batch.text)
|
||||
else:
|
||||
scores = self.model(batch.text[0], lengths=batch.text[1])
|
||||
scores_rounded = F.sigmoid(scores).round().long()
|
||||
|
||||
# Using binary accuracy
|
||||
for tensor1, tensor2 in zip(scores_rounded, batch.label):
|
||||
if np.array_equal(tensor1, tensor2):
|
||||
n_dev_correct += 1
|
||||
|
||||
total_loss += F.binary_cross_entropy_with_logits(scores, batch.label.float(), size_average=False).item()
|
||||
if hasattr(self.model, 'TAR') and self.model.TAR: ### TAR condition
|
||||
total_loss += (rnn_outs[1:]-rnn_outs[:-1]).pow(2).mean()
|
||||
accuracy = 100. * n_dev_correct / len(self.data_loader.dataset.examples)
|
||||
if hasattr(self.model, 'TAR') and self.model.TAR: # TAR condition
|
||||
total_loss += (rnn_outs[1:]-rnn_outs[:-1]).pow(2).mean()
|
||||
|
||||
scores_rounded = F.sigmoid(scores).round().long()
|
||||
predicted_labels.extend(scores_rounded.cpu().detach().numpy())
|
||||
target_labels.extend(batch.label.cpu().detach().numpy())
|
||||
|
||||
predicted_labels = np.array(predicted_labels)
|
||||
target_labels = np.array(target_labels)
|
||||
accuracy = metrics.accuracy_score(target_labels, predicted_labels)
|
||||
precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
|
||||
recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
|
||||
f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
|
||||
avg_loss = total_loss / len(self.data_loader.dataset.examples)
|
||||
#############
|
||||
## Temp Ave
|
||||
|
||||
# Temp Ave
|
||||
if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0:
|
||||
self.model.load_params(old_params)
|
||||
#############
|
||||
|
||||
return [accuracy, avg_loss], ['accuracy', 'cross_entropy_loss']
|
||||
return [accuracy, precision, recall, f1, avg_loss], ['accuracy', 'precision', 'recall', 'f1' 'cross_entropy_loss']
|
||||
|
||||
@@ -16,7 +16,7 @@ class ReutersTrainer(Trainer):
|
||||
super(ReutersTrainer, self).__init__(model, embedding, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
|
||||
self.config = trainer_config
|
||||
self.early_stop = False
|
||||
self.best_dev_acc = 0
|
||||
self.best_dev_f1 = 0
|
||||
self.iterations = 0
|
||||
self.iters_not_improved = 0
|
||||
self.start = None
|
||||
@@ -24,6 +24,7 @@ class ReutersTrainer(Trainer):
|
||||
'{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(','))
|
||||
self.dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(','))
|
||||
self.writer = SummaryWriter(log_dir="tensorboard_logs/" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
|
||||
self.snapshot_path = os.path.join(self.model_outfile, self.train_loader.dataset.NAME, 'best_model.pt')
|
||||
|
||||
def train_epoch(self, epoch):
|
||||
self.train_loader.init_epoch()
|
||||
@@ -54,31 +55,32 @@ class ReutersTrainer(Trainer):
|
||||
loss.backward()
|
||||
|
||||
self.optimizer.step()
|
||||
#############
|
||||
## Temp Ave
|
||||
|
||||
# Temp Ave
|
||||
if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0:
|
||||
self.model.update_ema()
|
||||
#############
|
||||
|
||||
# Evaluate performance on validation set
|
||||
if self.iterations % self.dev_log_interval == 1:
|
||||
dev_acc, dev_loss = self.dev_evaluator.get_scores()[0]
|
||||
dev_acc, dev_precision, dev_recall, dev_f1, dev_loss = self.dev_evaluator.get_scores()[0]
|
||||
niter = epoch * len(self.train_loader) + batch_idx
|
||||
self.writer.add_scalar('Train/Loss', loss.data[0], niter)
|
||||
self.writer.add_scalar('Dev/Loss', dev_loss, niter)
|
||||
self.writer.add_scalar('Train/Accuracy', train_acc, niter)
|
||||
self.writer.add_scalar('Dev/Accuracy', dev_acc, niter)
|
||||
self.writer.add_scalar('Dev/Precision', dev_precision, niter)
|
||||
self.writer.add_scalar('Dev/Recall', dev_recall, niter)
|
||||
self.writer.add_scalar('Dev/F-measure', dev_f1, niter)
|
||||
print(self.dev_log_template.format(time.time() - self.start,
|
||||
epoch, self.iterations, 1 + batch_idx, len(self.train_loader),
|
||||
100. * (1 + batch_idx) / len(self.train_loader), loss.item(),
|
||||
dev_loss, train_acc, dev_acc))
|
||||
|
||||
# Update validation results
|
||||
if dev_acc > self.best_dev_acc:
|
||||
if dev_f1 > self.best_dev_f1:
|
||||
self.iters_not_improved = 0
|
||||
self.best_dev_acc = dev_acc
|
||||
snapshot_path = os.path.join(self.model_outfile, self.train_loader.dataset.NAME, datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + '_reuters_best_model.pt')
|
||||
torch.save(self.model, snapshot_path)
|
||||
self.best_dev_f1 = dev_f1
|
||||
torch.save(self.model.state_dict(), self.snapshot_path)
|
||||
else:
|
||||
self.iters_not_improved += 1
|
||||
if self.iters_not_improved >= self.patience:
|
||||
@@ -102,6 +104,6 @@ class ReutersTrainer(Trainer):
|
||||
|
||||
for epoch in range(1, epochs + 1):
|
||||
if self.early_stop:
|
||||
print("Early Stopping. Epoch: {}, Best Dev Acc: {}".format(epoch, self.best_dev_acc))
|
||||
print("Early Stopping. Epoch: {}, Best Dev F1: {}".format(epoch, self.best_dev_f1))
|
||||
break
|
||||
self.train_epoch(epoch)
|
||||
|
||||
+21
-61
@@ -74,18 +74,17 @@ if __name__ == '__main__':
|
||||
random.seed(args.seed)
|
||||
logger = get_logger()
|
||||
|
||||
# Set up the data for training SST-1
|
||||
if args.dataset == 'SST-1':
|
||||
train_iter, dev_iter, test_iter = SST1.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
# Set up the data for training SST-2
|
||||
elif args.dataset == 'SST-2':
|
||||
train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
else:
|
||||
dataset_map = {
|
||||
'SST-1': SST1,
|
||||
'SST-2': SST2,
|
||||
'Reuters': Reuters,
|
||||
'AAPD': AAPD
|
||||
}
|
||||
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
|
||||
config = deepcopy(args)
|
||||
config.dataset = train_iter.dataset
|
||||
@@ -113,24 +112,12 @@ if __name__ == '__main__':
|
||||
parameter = filter(lambda p: p.requires_grad, model.parameters())
|
||||
optimizer = torch.optim.Adam(parameter, lr=args.lr, weight_decay=args.weight_decay)
|
||||
|
||||
if args.dataset == 'SST-1':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
|
||||
trainer_config = {
|
||||
'optimizer': optimizer,
|
||||
@@ -151,37 +138,10 @@ if __name__ == '__main__':
|
||||
else:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
|
||||
|
||||
if args.dataset == 'SST-1':
|
||||
evaluate_dataset('dev', SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
evaluate_dataset('dev', SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'AAPD':
|
||||
evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
|
||||
# Calculate dev and test metrics
|
||||
for data_loader in [dev_iter, test_iter]:
|
||||
predicted_labels = list()
|
||||
target_labels = list()
|
||||
for batch_idx, batch in enumerate(data_loader):
|
||||
scores_rounded = F.sigmoid(model(batch.text[0])).round().long()
|
||||
predicted_labels.extend(scores_rounded.cpu().detach().numpy())
|
||||
target_labels.extend(batch.label.cpu().detach().numpy())
|
||||
predicted_labels = np.array(predicted_labels)
|
||||
target_labels = np.array(target_labels)
|
||||
accuracy = metrics.accuracy_score(target_labels, predicted_labels)
|
||||
precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
|
||||
recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
|
||||
f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
|
||||
if data_loader == dev_iter:
|
||||
print("Dev metrics:")
|
||||
else:
|
||||
print("Test metrics:")
|
||||
print(accuracy, precision, recall, f1)
|
||||
model.load_state_dict(torch.load(trainer.snapshot_path))
|
||||
if args.dataset not in dataset_map:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
else:
|
||||
evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu)
|
||||
Reference in New Issue
Block a user