mirror of
https://github.com/wassname/Castor.git
synced 2026-08-20 12:00:37 +08:00
Baseline LSTM implementation (#150)
* Add ReutersTrainer, ReutersEvaluator options in Factory classes * Add Reuters to Kim-CNN command line arguments * Fix SST dataset path according to changes in Kim-CNN args The dataset path in args.py was made to point at the dataset folder rather than dataset/SST folder. Hence SST folder was added to paths in the SST dataset class * Add Reuters dataset class, and support in __main__ * Add Reuters dataset trainers and evaluators * Remove debug print statement in reuters_evaluator * Fix rounding bug in reuters_trainer and reuters_evaluator * Add LSTM for baseline text classification measurements * Add eval metrics for lstm_baseline * Set batch_first param in lstm_baseline * Remove onnx args from lstm_baseline
This commit is contained in:
Achyudh Ram
@@ -5,6 +5,8 @@ import random
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.onnx
|
||||
import torch.nn.functional as F
|
||||
from sklearn import metrics
|
||||
|
||||
from common.evaluation import EvaluatorFactory
|
||||
from common.train import TrainerFactory
|
||||
@@ -155,6 +157,26 @@ if __name__ == '__main__':
|
||||
else:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
|
||||
# Calculate dev and test metrics
|
||||
for data_loader in [dev_iter, test_iter]:
|
||||
predicted_labels = list()
|
||||
target_labels = list()
|
||||
for batch_idx, batch in enumerate(data_loader):
|
||||
scores_rounded = F.sigmoid(model(batch.text)).round().long()
|
||||
predicted_labels.extend(scores_rounded.cpu().detach().numpy())
|
||||
target_labels.extend(batch.label.cpu().detach().numpy())
|
||||
predicted_labels = np.array(predicted_labels)
|
||||
target_labels = np.array(target_labels)
|
||||
accuracy = metrics.accuracy_score(target_labels, predicted_labels)
|
||||
precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
|
||||
recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
|
||||
f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
|
||||
if data_loader == dev_iter:
|
||||
print("Dev metrics:")
|
||||
else:
|
||||
print("Test metrics:")
|
||||
print(accuracy, precision, recall, f1)
|
||||
|
||||
if args.onnx:
|
||||
device = torch.device('cuda') if torch.cuda.is_available() and args.cuda else torch.device('cpu')
|
||||
dummy_input = torch.zeros(args.onnx_batch_size, args.onnx_sent_len, dtype=torch.long, device=device)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# lstm_baseline
|
||||
|
||||
Implementation of a standard LSTM using PyTorch and Torchtext for text classification baseline measurements.
|
||||
|
||||
## Model Type
|
||||
|
||||
- rand: All words are randomly initialized and then modified during training.
|
||||
- static: A model with pre-trained vectors from [word2vec](https://code.google.com/archive/p/word2vec/). All words -- including the unknown ones that are initialized with zero -- are kept static and only the other parameters of the model are learned.
|
||||
- non-static: Same as above but the pretrained vectors are fine-tuned for each task.
|
||||
|
||||
## Quick Start
|
||||
|
||||
To run the model on Reuters dataset on static, just run the following from the Castor working directory.
|
||||
|
||||
```
|
||||
python -m lstm_baseline --mode static
|
||||
```
|
||||
|
||||
## Dataset
|
||||
|
||||
We experiment the model on the following datasets.
|
||||
|
||||
- Reuters dataset - ModApte splits
|
||||
|
||||
## Settings
|
||||
|
||||
Adadelta is used for training.
|
||||
|
||||
## TODO
|
||||
- Support ONNX export. Currently throws a ONNX export failed (Couldn't export Python operator forward_flattened_wrapper) exception.
|
||||
- Add dataset results with different hyperparameters
|
||||
- Parameters tuning
|
||||
@@ -0,0 +1,177 @@
|
||||
from copy import deepcopy
|
||||
import logging
|
||||
import random
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from sklearn import metrics
|
||||
|
||||
from common.evaluation import EvaluatorFactory
|
||||
from common.train import TrainerFactory
|
||||
from datasets.sst import SST1
|
||||
from datasets.sst import SST2
|
||||
from datasets.reuters import Reuters
|
||||
from lstm_baseline.args import get_args
|
||||
from lstm_baseline.model import LSTMBaseline
|
||||
|
||||
|
||||
class UnknownWordVecCache(object):
|
||||
"""
|
||||
Caches the first randomly generated word vector for a certain size to make it is reused.
|
||||
"""
|
||||
cache = {}
|
||||
|
||||
@classmethod
|
||||
def unk(cls, tensor):
|
||||
size_tup = tuple(tensor.size())
|
||||
if size_tup not in cls.cache:
|
||||
cls.cache[size_tup] = torch.Tensor(tensor.size())
|
||||
# choose 0.25 so unknown vectors have approximately same variance as pre-trained ones
|
||||
# same as original implementation: https://github.com/yoonkim/CNN_sentence/blob/0a626a048757d5272a7e8ccede256a434a6529be/process_data.py#L95
|
||||
cls.cache[size_tup].uniform_(-0.25, 0.25)
|
||||
return cls.cache[size_tup]
|
||||
|
||||
|
||||
def get_logger():
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
ch = logging.StreamHandler()
|
||||
ch.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter('%(levelname)s - %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.addHandler(ch)
|
||||
|
||||
return logger
|
||||
|
||||
|
||||
def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_size, device):
|
||||
saved_model_evaluator = EvaluatorFactory.get_evaluator(dataset_cls, model, embedding, loader, batch_size, device)
|
||||
scores, metric_names = saved_model_evaluator.get_scores()
|
||||
logger.info('Evaluation metrics for {}'.format(split_name))
|
||||
logger.info('\t'.join([' '] + metric_names))
|
||||
logger.info('\t'.join([split_name] + list(map(str, scores))))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Set default configuration in : args.py
|
||||
args = get_args()
|
||||
|
||||
# Set random seed for reproducibility
|
||||
torch.manual_seed(args.seed)
|
||||
torch.backends.cudnn.deterministic = True
|
||||
if not args.cuda:
|
||||
args.gpu = -1
|
||||
if torch.cuda.is_available() and args.cuda:
|
||||
print('Note: You are using GPU for training')
|
||||
torch.cuda.set_device(args.gpu)
|
||||
torch.cuda.manual_seed(args.seed)
|
||||
if torch.cuda.is_available() and not args.cuda:
|
||||
print('Warning: You have Cuda but not use it. You are using CPU for training.')
|
||||
np.random.seed(args.seed)
|
||||
random.seed(args.seed)
|
||||
logger = get_logger()
|
||||
|
||||
# Set up the data for training SST-1
|
||||
if args.dataset == 'SST-1':
|
||||
train_iter, dev_iter, test_iter = SST1.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
# Set up the data for training SST-2
|
||||
elif args.dataset == 'SST-2':
|
||||
train_iter, dev_iter, test_iter = SST2.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_iter, dev_iter, test_iter = Reuters.iters(args.data_dir, args.word_vectors_file, args.word_vectors_dir, batch_size=args.batch_size, device=args.gpu, unk_init=UnknownWordVecCache.unk)
|
||||
else:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
|
||||
config = deepcopy(args)
|
||||
config.dataset = train_iter.dataset
|
||||
config.target_class = train_iter.dataset.NUM_CLASSES
|
||||
config.words_num = len(train_iter.dataset.TEXT_FIELD.vocab)
|
||||
|
||||
print('Dataset {} Mode {}'.format(args.dataset, args.mode))
|
||||
print('VOCAB num',len(train_iter.dataset.TEXT_FIELD.vocab))
|
||||
print('LABEL.target_class:', train_iter.dataset.NUM_CLASSES)
|
||||
print('Train instance', len(train_iter.dataset))
|
||||
print('Dev instance', len(dev_iter.dataset))
|
||||
print('Test instance', len(test_iter.dataset))
|
||||
|
||||
if args.resume_snapshot:
|
||||
if args.cuda:
|
||||
model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage.cuda(args.gpu))
|
||||
else:
|
||||
model = torch.load(args.resume_snapshot, map_location=lambda storage, location: storage)
|
||||
else:
|
||||
model = LSTMBaseline(config)
|
||||
if args.cuda:
|
||||
model.cuda()
|
||||
print('Shift model to GPU')
|
||||
|
||||
parameter = filter(lambda p: p.requires_grad, model.parameters())
|
||||
optimizer = torch.optim.Adam(parameter, lr=args.lr, weight_decay=args.weight_decay)
|
||||
|
||||
if args.dataset == 'SST-1':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
train_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, train_iter, args.batch_size, args.gpu)
|
||||
test_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
dev_evaluator = EvaluatorFactory.get_evaluator(Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
|
||||
trainer_config = {
|
||||
'optimizer': optimizer,
|
||||
'batch_size': args.batch_size,
|
||||
'log_interval': args.log_every,
|
||||
'dev_log_interval': args.dev_every,
|
||||
'patience': args.patience,
|
||||
'model_outfile': args.save_path, # actually a directory, using model_outfile to conform to Trainer naming convention
|
||||
'logger': logger
|
||||
}
|
||||
trainer = TrainerFactory.get_trainer(args.dataset, model, None, train_iter, trainer_config, train_evaluator, test_evaluator, dev_evaluator)
|
||||
|
||||
if not args.trained_model:
|
||||
trainer.train(args.epochs)
|
||||
else:
|
||||
if args.cuda:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage.cuda(args.gpu))
|
||||
else:
|
||||
model = torch.load(args.trained_model, map_location=lambda storage, location: storage)
|
||||
|
||||
if args.dataset == 'SST-1':
|
||||
evaluate_dataset('dev', SST1, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST1, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'SST-2':
|
||||
evaluate_dataset('dev', SST2, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', SST2, model, None, test_iter, args.batch_size, args.gpu)
|
||||
elif args.dataset == 'Reuters':
|
||||
evaluate_dataset('dev', Reuters, model, None, dev_iter, args.batch_size, args.gpu)
|
||||
evaluate_dataset('test', Reuters, model, None, test_iter, args.batch_size, args.gpu)
|
||||
else:
|
||||
raise ValueError('Unrecognized dataset')
|
||||
|
||||
# Calculate dev and test metrics
|
||||
for data_loader in [dev_iter, test_iter]:
|
||||
predicted_labels = list()
|
||||
target_labels = list()
|
||||
for batch_idx, batch in enumerate(data_loader):
|
||||
scores_rounded = F.sigmoid(model(batch.text)).round().long()
|
||||
predicted_labels.extend(scores_rounded.cpu().detach().numpy())
|
||||
target_labels.extend(batch.label.cpu().detach().numpy())
|
||||
predicted_labels = np.array(predicted_labels)
|
||||
target_labels = np.array(target_labels)
|
||||
accuracy = metrics.accuracy_score(target_labels, predicted_labels)
|
||||
precision = metrics.precision_score(target_labels, predicted_labels, average='micro')
|
||||
recall = metrics.recall_score(target_labels, predicted_labels, average='micro')
|
||||
f1 = metrics.f1_score(target_labels, predicted_labels, average='micro')
|
||||
if data_loader == dev_iter:
|
||||
print("Dev metrics:")
|
||||
else:
|
||||
print("Test metrics:")
|
||||
print(accuracy, precision, recall, f1)
|
||||
@@ -0,0 +1,37 @@
|
||||
import os
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
|
||||
def get_args():
|
||||
parser = ArgumentParser(description="Baseline LSTM for text classification")
|
||||
parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda')
|
||||
parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU
|
||||
parser.add_argument('--epochs', type=int, default=30)
|
||||
parser.add_argument('--batch_size', type=int, default=1024)
|
||||
parser.add_argument('--bidirectional', type=bool, default=True),
|
||||
parser.add_argument('--num_layers', type=int, default=2)
|
||||
parser.add_argument('--hidden_dim', type=int, default=256)
|
||||
parser.add_argument('--mode', type=str, default='static', choices=['rand', 'static', 'non-static'])
|
||||
parser.add_argument('--lr', type=float, default=0.001)
|
||||
parser.add_argument('--seed', type=int, default=3435)
|
||||
parser.add_argument('--dataset', type=str, default='Reuters', choices=['SST-1', 'SST-2', 'Reuters'])
|
||||
parser.add_argument('--resume_snapshot', type=str, default=None)
|
||||
parser.add_argument('--dev_every', type=int, default=30)
|
||||
parser.add_argument('--log_every', type=int, default=10)
|
||||
parser.add_argument('--patience', type=int, default=50)
|
||||
parser.add_argument('--save_path', type=str, default='lstm_baseline/saves')
|
||||
parser.add_argument('--words_dim', type=int, default=300)
|
||||
parser.add_argument('--embed_dim', type=int, default=300)
|
||||
parser.add_argument('--dropout', type=float, default=0.5)
|
||||
parser.add_argument('--epoch_decay', type=int, default=15)
|
||||
parser.add_argument('--data_dir', help='word vectors directory',
|
||||
default=os.path.join(os.pardir, 'Castor-data', 'datasets'))
|
||||
parser.add_argument('--word_vectors_dir', help='word vectors directory',
|
||||
default=os.path.join(os.pardir, 'Castor-data', 'embeddings', 'word2vec'))
|
||||
parser.add_argument('--word_vectors_file', help='word vectors filename', default='GoogleNews-vectors-negative300.txt')
|
||||
parser.add_argument('--trained_model', type=str, default="")
|
||||
parser.add_argument('--weight_decay', type=float, default=0)
|
||||
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
@@ -0,0 +1,50 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class LSTMBaseline(nn.Module):
|
||||
def __init__(self, config):
|
||||
super(LSTMBaseline, self).__init__()
|
||||
dataset = config.dataset
|
||||
target_class = config.target_class
|
||||
self.is_bidirectional = config.bidirectional
|
||||
self.mode = config.mode
|
||||
|
||||
input_channel = 1
|
||||
if config.mode == 'rand':
|
||||
rand_embed_init = torch.Tensor(config.words_num, config.words_dim).uniform_(-0.25, 0.25)
|
||||
self.embed = nn.Embedding.from_pretrained(rand_embed_init, freeze=False)
|
||||
elif config.mode == 'static':
|
||||
self.static_embed = nn.Embedding.from_pretrained(dataset.TEXT_FIELD.vocab.vectors, freeze=True)
|
||||
elif config.mode == 'non-static':
|
||||
self.non_static_embed = nn.Embedding.from_pretrained(dataset.TEXT_FIELD.vocab.vectors, freeze=False)
|
||||
else:
|
||||
print("Unsupported Mode")
|
||||
exit()
|
||||
|
||||
self.lstm = nn.LSTM(config.words_dim, config.hidden_dim, dropout=config.dropout, num_layers=config.num_layers,
|
||||
bidirectional=self.is_bidirectional, batch_first=True)
|
||||
self.dropout = nn.Dropout(config.dropout)
|
||||
if self.is_bidirectional:
|
||||
self.fc1 = nn.Linear(2 * config.hidden_dim, target_class)
|
||||
else:
|
||||
self.fc1 = nn.Linear(config.hidden_dim, target_class)
|
||||
|
||||
def forward(self, x):
|
||||
if self.mode == 'rand':
|
||||
x = self.embed(x)
|
||||
elif self.mode == 'static':
|
||||
x = self.static_embed(x)
|
||||
elif self.mode == 'non-static':
|
||||
x = self.non_static_embed(x)
|
||||
else:
|
||||
print("Unsupported Mode")
|
||||
exit()
|
||||
x, _ = self.lstm(x)
|
||||
x = F.relu(torch.transpose(x, 1, 2))
|
||||
x = F.max_pool1d(x, x.size(2)).squeeze(2)
|
||||
x = self.dropout(x)
|
||||
logit = self.fc1(x) # (batch, target_size)
|
||||
return logit
|
||||
Reference in New Issue
Block a user