diff --git a/char_cnn/model.py b/char_cnn/model.py index 7ddb0a0..af5b3c4 100644 --- a/char_cnn/model.py +++ b/char_cnn/model.py @@ -22,7 +22,7 @@ class CharCNN(nn.Module): self.conv5 = nn.Conv1d(num_conv_filters, num_conv_filters, kernel_size=3) self.conv6 = nn.Conv1d(num_conv_filters, output_channel, kernel_size=3) self.dropout = nn.Dropout(config.dropout) - self.fc1 = nn.Linear(num_conv_filters, num_affine_neurons) + self.fc1 = nn.Linear(output_channel, num_affine_neurons) self.fc2 = nn.Linear(num_affine_neurons, num_affine_neurons) self.fc3 = nn.Linear(num_affine_neurons, target_class) diff --git a/common/evaluators/reuters_evaluator.py b/common/evaluators/reuters_evaluator.py index dad5063..90bd0a8 100644 --- a/common/evaluators/reuters_evaluator.py +++ b/common/evaluators/reuters_evaluator.py @@ -2,6 +2,7 @@ import torch import torch.nn.functional as F import numpy as np +from sklearn import metrics from .evaluator import Evaluator @@ -16,14 +17,15 @@ class ReutersEvaluator(Evaluator): self.data_loader.init_epoch() n_dev_correct = 0 total_loss = 0 - ############ - ## Temp Ave + + # Temp Ave if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0: old_params = self.model.get_params() self.model.load_ema_params() - ############ + + predicted_labels, target_labels = list(), list() for batch_idx, batch in enumerate(self.data_loader): - if hasattr(self.model, 'TAR') and self.model.TAR: ## TAR Condition + if hasattr(self.model, 'TAR') and self.model.TAR: # TAR condition if self.ignore_lengths: scores, rnn_outs = self.model(batch.text, lengths=batch.text) else: @@ -33,22 +35,25 @@ class ReutersEvaluator(Evaluator): scores = self.model(batch.text, lengths=batch.text) else: scores = self.model(batch.text[0], lengths=batch.text[1]) - scores_rounded = F.sigmoid(scores).round().long() - - # Using binary accuracy - for tensor1, tensor2 in zip(scores_rounded, batch.label): - if np.array_equal(tensor1, tensor2): - n_dev_correct += 1 total_loss += F.binary_cross_entropy_with_logits(scores, batch.label.float(), size_average=False).item() - if hasattr(self.model, 'TAR') and self.model.TAR: ### TAR condition - total_loss += (rnn_outs[1:]-rnn_outs[:-1]).pow(2).mean() - accuracy = 100. * n_dev_correct / len(self.data_loader.dataset.examples) + if hasattr(self.model, 'TAR') and self.model.TAR: # TAR condition + total_loss += (rnn_outs[1:]-rnn_outs[:-1]).pow(2).mean() + + scores_rounded = F.sigmoid(scores).round().long() + predicted_labels.extend(scores_rounded.cpu().detach().numpy()) + target_labels.extend(batch.label.cpu().detach().numpy()) + + predicted_labels = np.array(predicted_labels) + target_labels = np.array(target_labels) + accuracy = metrics.accuracy_score(target_labels, predicted_labels) + precision = metrics.precision_score(target_labels, predicted_labels, average='micro') + recall = metrics.recall_score(target_labels, predicted_labels, average='micro') + f1 = metrics.f1_score(target_labels, predicted_labels, average='micro') avg_loss = total_loss / len(self.data_loader.dataset.examples) - ############# - ## Temp Ave + + # Temp Ave if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0: self.model.load_params(old_params) - ############# - return [accuracy, avg_loss], ['accuracy', 'cross_entropy_loss'] + return [accuracy, precision, recall, f1, avg_loss], ['accuracy', 'precision', 'recall', 'f1' 'cross_entropy_loss'] diff --git a/common/trainers/reuters_trainer.py b/common/trainers/reuters_trainer.py index 00e0368..f46f382 100644 --- a/common/trainers/reuters_trainer.py +++ b/common/trainers/reuters_trainer.py @@ -16,7 +16,7 @@ class ReutersTrainer(Trainer): super(ReutersTrainer, self).__init__(model, embedding, train_loader, trainer_config, train_evaluator, test_evaluator, dev_evaluator) self.config = trainer_config self.early_stop = False - self.best_dev_acc = 0 + self.best_dev_f1 = 0 self.iterations = 0 self.iters_not_improved = 0 self.start = None @@ -24,6 +24,7 @@ class ReutersTrainer(Trainer): '{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{},{:12.4f},{}'.split(',')) self.dev_log_template = ' '.join('{:>6.0f},{:>5.0f},{:>9.0f},{:>5.0f}/{:<5.0f} {:>7.0f}%,{:>8.6f},{:8.6f},{:12.4f},{:12.4f}'.split(',')) self.writer = SummaryWriter(log_dir="tensorboard_logs/" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")) + self.snapshot_path = os.path.join(self.model_outfile, self.train_loader.dataset.NAME, 'best_model.pt') def train_epoch(self, epoch): self.train_loader.init_epoch() @@ -54,31 +55,32 @@ class ReutersTrainer(Trainer): loss.backward() self.optimizer.step() - ############# - ## Temp Ave + + # Temp Ave if hasattr(self.model, 'beta_ema') and self.model.beta_ema > 0: self.model.update_ema() - ############# # Evaluate performance on validation set if self.iterations % self.dev_log_interval == 1: - dev_acc, dev_loss = self.dev_evaluator.get_scores()[0] + dev_acc, dev_precision, dev_recall, dev_f1, dev_loss = self.dev_evaluator.get_scores()[0] niter = epoch * len(self.train_loader) + batch_idx self.writer.add_scalar('Train/Loss', loss.data[0], niter) self.writer.add_scalar('Dev/Loss', dev_loss, niter) self.writer.add_scalar('Train/Accuracy', train_acc, niter) self.writer.add_scalar('Dev/Accuracy', dev_acc, niter) + self.writer.add_scalar('Dev/Precision', dev_precision, niter) + self.writer.add_scalar('Dev/Recall', dev_recall, niter) + self.writer.add_scalar('Dev/F-measure', dev_f1, niter) print(self.dev_log_template.format(time.time() - self.start, epoch, self.iterations, 1 + batch_idx, len(self.train_loader), 100. * (1 + batch_idx) / len(self.train_loader), loss.item(), dev_loss, train_acc, dev_acc)) # Update validation results - if dev_acc > self.best_dev_acc: + if dev_f1 > self.best_dev_f1: self.iters_not_improved = 0 - self.best_dev_acc = dev_acc - snapshot_path = os.path.join(self.model_outfile, self.train_loader.dataset.NAME, datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + '_reuters_best_model.pt') - torch.save(self.model, snapshot_path) + self.best_dev_f1 = dev_f1 + torch.save(self.model.state_dict(), self.snapshot_path) else: self.iters_not_improved += 1 if self.iters_not_improved >= self.patience: @@ -102,6 +104,6 @@ class ReutersTrainer(Trainer): for epoch in range(1, epochs + 1): if self.early_stop: - print("Early Stopping. Epoch: {}, Best Dev Acc: {}".format(epoch, self.best_dev_acc)) + print("Early Stopping. Epoch: {}, Best Dev F1: {}".format(epoch, self.best_dev_f1)) break self.train_epoch(epoch) diff --git a/lstm_baseline/__main__.py b/lstm_baseline/__main__.py index 2b2eea7..625da15 100644 --- a/lstm_baseline/__main__.py +++ b/lstm_baseline/__main__.py @@ -74,18 +74,17 @@ if __name__ == '__main__': random.seed(args.seed) logger = get_logger() - # Set up the data for training SST-1 - if args.dataset == 'SST-1': - train_iter, dev_iter, test_iter = SST1.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - # Set up the data for training SST-2 - elif args.dataset == 'SST-2': - train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - elif args.dataset == 'Reuters': - train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - elif args.dataset == 'AAPD': - train_iter, dev_iter, test_iter = AAPD.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) - else: + dataset_map = { + 'SST-1': SST1, + 'SST-2': SST2, + 'Reuters': Reuters, + 'AAPD': AAPD + } + + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_iter, dev_iter, test_iter = dataset_map[args.dataset].iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk) config = deepcopy(args) config.dataset = train_iter.dataset @@ -113,24 +112,12 @@ if __name__ == '__main__': parameter = filter(lambda p: p.requires_grad, model.parameters()) optimizer = torch.optim.Adam(parameter, lr=args.lr, weight_decay=args.weight_decay) - if args.dataset == 'SST-1': - train_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'SST-2': - train_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'Reuters': - train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - train_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, train_iter, args.batch_size, args.gpu) - test_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, test_iter, args.batch_size, args.gpu) - dev_evaluator = EvaluatorFactory.get_evaluator(AAPD, model, None, dev_iter, args.batch_size, args.gpu) - else: + if args.dataset not in dataset_map: raise ValueError('Unrecognized dataset') + else: + train_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, train_iter, args.batch_size, args.gpu) + test_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) + dev_evaluator = EvaluatorFactory.get_evaluator(dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) trainer_config = { 'optimizer': optimizer, @@ -151,37 +138,10 @@ if __name__ == '__main__': else: model = torch.load(args.trained_model, map_location=lambda storage, location: storage) - if args.dataset == 'SST-1': - evaluate_dataset('dev', SST1, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', SST1, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'SST-2': - evaluate_dataset('dev', SST2, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', SST2, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'Reuters': - evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu) - elif args.dataset == 'AAPD': - evaluate_dataset('dev', AAPD, model, None, dev_iter, args.batch_size, args.gpu) - evaluate_dataset('test', AAPD, model, None, test_iter, args.batch_size, args.gpu) - else: - raise ValueError('Unrecognized dataset') - # Calculate dev and test metrics - for data_loader in [dev_iter, test_iter]: - predicted_labels = list() - target_labels = list() - for batch_idx, batch in enumerate(data_loader): - scores_rounded = F.sigmoid(model(batch.text[0])).round().long() - predicted_labels.extend(scores_rounded.cpu().detach().numpy()) - target_labels.extend(batch.label.cpu().detach().numpy()) - predicted_labels = np.array(predicted_labels) - target_labels = np.array(target_labels) - accuracy = metrics.accuracy_score(target_labels, predicted_labels) - precision = metrics.precision_score(target_labels, predicted_labels, average='micro') - recall = metrics.recall_score(target_labels, predicted_labels, average='micro') - f1 = metrics.f1_score(target_labels, predicted_labels, average='micro') - if data_loader == dev_iter: - print("Dev metrics:") - else: - print("Test metrics:") - print(accuracy, precision, recall, f1) + model.load_state_dict(torch.load(trainer.snapshot_path)) + if args.dataset not in dataset_map: + raise ValueError('Unrecognized dataset') + else: + evaluate_dataset('dev', dataset_map[args.dataset], model, None, dev_iter, args.batch_size, args.gpu) + evaluate_dataset('test', dataset_map[args.dataset], model, None, test_iter, args.batch_size, args.gpu) \ No newline at end of file