diff --git a/conv_rnn/README.md b/conv_rnn/README.md index 0224078..fa1e43a 100644 --- a/conv_rnn/README.md +++ b/conv_rnn/README.md @@ -17,7 +17,7 @@ You may then run `python train.py` and `python test.py` for training and testing ### Empirical results Best dev | Test -- | -- -51.1 | 50.7 +52.04359673024523 | 50.85972850678733 ### References [1] Chenglong Wang, Feijun Jiang, and Hongxia Yang. 2017. A Hybrid Framework for Text Modeling with Convolutional RNN. In Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD '17). diff --git a/conv_rnn/data.py b/conv_rnn/data.py index 868e32d..4482536 100644 --- a/conv_rnn/data.py +++ b/conv_rnn/data.py @@ -5,13 +5,7 @@ import numpy as np import torch.utils.data as data def sst_tokenize(sentence): - extraneous_pattern = re.compile(r"^(--lrb--|--rrb--|``|''|--|\.)$") - words = [] - for word in sentence.split(): - if re.match(extraneous_pattern, word): - continue - words.append(word) - return words + return sentence.split() class SSTEmbeddingLoader(object): def __init__(self, dirname, fmt="stsa.fine.{}", word2vec_file="word2vec.sst-1"): @@ -26,7 +20,6 @@ class SSTEmbeddingLoader(object): with open(os.path.join(self.dirname, self.word2vec_file)) as f: for i, line in enumerate(f.readlines()): word, vec = line.replace("\n", "").split(" ", 1) - word = word.replace("#", "") vec = np.array([float(v) for v in vec.split(" ")]) weights.append(vec) id_dict[word] = i diff --git a/conv_rnn/model.py b/conv_rnn/model.py index 0e16146..e2c0c90 100644 --- a/conv_rnn/model.py +++ b/conv_rnn/model.py @@ -15,7 +15,6 @@ class ConvRNNModel(nn.Module): self.hidden_size = config["hidden_size"] fc_size = config["fc_size"] self.batch_size = config["mbatch_size"] - dropout = config["dropout_prob"] n_fmaps = config["n_feature_maps"] self.rnn_type = config["rnn_type"] @@ -34,8 +33,6 @@ class ConvRNNModel(nn.Module): else: raise ValueError("RNN type must be one of LSTM or GRU") self.conv = nn.Conv2d(1, n_fmaps, (1, self.hidden_size * 2)) - if dropout: - self.dropout = nn.Dropout(dropout) self.fc1 = nn.Linear(n_fmaps + 2 * self.hidden_size, fc_size) self.fc2 = nn.Linear(fc_size, config["n_labels"]) @@ -77,8 +74,6 @@ class ConvRNNModel(nn.Module): out = [t.squeeze(1) for t in rnn_out.chunk(2, 1)] out.append(x) x = torch.cat(out, 1).squeeze(2) - if hasattr(self, "dropout"): - x = self.dropout(x) x = nn_func.relu(self.fc1(x)) return self.fc2(x) diff --git a/conv_rnn/test.py b/conv_rnn/test.py index 53f66ee..14bb08a 100644 --- a/conv_rnn/test.py +++ b/conv_rnn/test.py @@ -5,6 +5,7 @@ import random import numpy as np import torch import torch.nn as nn +import torch.utils as utils import data import model @@ -17,19 +18,18 @@ def main(): parser.add_argument("--gpu_number", default=0, type=int) args = parser.parse_args() - model.set_seed(5, no_cuda=args.no_cuda) - data_loader = data.SSTDataLoader(args.data_dir) conv_rnn = torch.load(args.input_file) if not args.no_cuda: torch.cuda.set_device(args.gpu_number) conv_rnn.cuda() - _, _, test_set = data_loader.load_sst_sets() + _, _, test_set = data.SSTDataset.load_sst_sets("data") + test_loader = utils.data.DataLoader(test_set, batch_size=len(test_set), collate_fn=conv_rnn.convert_dataset) conv_rnn.eval() - test_in, test_out = conv_rnn.convert_dataset(test_set) - scores = conv_rnn(test_in) - n_correct = (torch.max(scores, 1)[1].view(len(test_set)).data == test_out.data).sum() - accuracy = n_correct / len(test_set) + for test_in, test_out in test_loader: + scores = conv_rnn(test_in) + n_correct = (torch.max(scores, 1)[1].view(-1).data == test_out.data).sum() + accuracy = n_correct / len(test_set) print("Test set accuracy: {}".format(accuracy)) if __name__ == "__main__": diff --git a/conv_rnn/train.py b/conv_rnn/train.py index 675529d..981f671 100644 --- a/conv_rnn/train.py +++ b/conv_rnn/train.py @@ -2,10 +2,11 @@ import argparse import os import random +from torch import utils +from torch.optim.lr_scheduler import ReduceLROnPlateau import numpy as np import torch import torch.nn as nn -from torch import utils import data import model @@ -58,7 +59,6 @@ def train(**kwargs): verbose = not kwargs["quiet"] lr = kwargs["lr"] weight_decay = kwargs["weight_decay"] - gradient_clip = kwargs["gradient_clip"] seed = kwargs["seed"] if not kwargs["no_cuda"]: @@ -79,7 +79,8 @@ def train(**kwargs): conv_rnn.train() criterion = nn.CrossEntropyLoss() parameters = list(filter(lambda p: p.requires_grad, conv_rnn.parameters())) - optimizer = torch.optim.Adadelta(parameters, lr=lr, weight_decay=weight_decay) + optimizer = torch.optim.SGD(parameters, lr=lr, weight_decay=weight_decay, momentum=0.9) + scheduler = ReduceLROnPlateau(optimizer, patience=kwargs["dev_per_epoch"] * 4) train_set, dev_set, test_set = data.SSTDataset.load_sst_sets("data") collate_fn = conv_rnn.convert_dataset @@ -92,26 +93,26 @@ def train(**kwargs): conv_rnn.eval() for m_in, m_out in loader: scores = conv_rnn(m_in) - loss = criterion(scores, m_out) + loss = criterion(scores, m_out).cpu().data[0] n_correct = (torch.max(scores, 1)[1].view(m_in.size(0)).data == m_out.data).sum() accuracy = n_correct / m_in.size(0) - if dev and accuracy > evaluate.best_dev: + scheduler.step(accuracy) + if dev and accuracy >= evaluate.best_dev: evaluate.best_dev = accuracy + print("Saving best model ({})...".format(accuracy)) torch.save(conv_rnn, kwargs["output_file"]) if verbose: - print("{} set accuracy: {}, loss: {}".format("dev" if dev else "test", accuracy, loss.cpu().data[0])) + print("{} set accuracy: {}, loss: {}".format("dev" if dev else "test", accuracy, loss)) conv_rnn.train() evaluate.best_dev = 0 for epoch in range(n_epochs): - optimizer.zero_grad() print("Epoch number: {}".format(epoch), end="\r") if verbose: print() i = 0 for j, (train_in, train_out) in enumerate(train_loader): - if verbose and i % (mbatch_size * 10) == 0: - print("{} / {}".format(j * mbatch_size, len(train_set)), end="\r") + optimizer.zero_grad() if not kwargs["no_cuda"]: train_in.cuda() @@ -120,10 +121,12 @@ def train(**kwargs): scores = conv_rnn(train_in) loss = criterion(scores, train_out) loss.backward() - torch.nn.utils.clip_grad_norm(parameters, gradient_clip) optimizer.step() + accuracy = (torch.max(scores, 1)[1].view(-1).data == train_out.data).sum() / mbatch_size + if verbose and i % (mbatch_size * 10) == 0: + print("accuracy: {}, {} / {}".format(accuracy, j * mbatch_size, len(train_set))) i += mbatch_size - if i % (mbatch_size * 256) == 0: + if i % (len(train_set) // kwargs["dev_per_epoch"]) < mbatch_size: evaluate(dev_loader) evaluate(test_loader, dev=False) return evaluate.best_dev @@ -147,13 +150,12 @@ def do_random_search(given_params): def main(): parser = argparse.ArgumentParser() - parser.add_argument("--dropout_prob", default=0.5, type=float) + parser.add_argument("--dev_per_epoch", default=9, type=int) parser.add_argument("--fc_size", default=200, type=int) parser.add_argument("--gpu_number", default=0, type=int) - parser.add_argument("--gradient_clip", default=5, type=float) parser.add_argument("--hidden_size", default=200, type=int) parser.add_argument("--input_file", default="saves/model.pt", type=str) - parser.add_argument("--lr", default=5E-2, type=float) + parser.add_argument("--lr", default=1E-1, type=float) parser.add_argument("--mbatch_size", default=64, type=int) parser.add_argument("--n_epochs", default=30, type=int) parser.add_argument("--n_feature_maps", default=200, type=float) @@ -165,7 +167,7 @@ def main(): parser.add_argument("--rnn_type", choices=["lstm", "gru"], default="lstm", type=str) parser.add_argument("--seed", default=3, type=int) parser.add_argument("--quiet", action="store_true", default=False) - parser.add_argument("--weight_decay", default=1E-3, type=float) + parser.add_argument("--weight_decay", default=1E-4, type=float) args = parser.parse_args() if args.random_search: do_random_search(vars(args))