wassname/Castor
1
0
Fork 0
mirror of https://github.com/wassname/Castor.git synced 2026-09-09 11:13:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Tweak conv-rnn model (#75)

Browse Source 
* Tweak conv-rnn model

- Fix misplaced zero_grad()
- Tweak model hyperparams and optimization algorithm

* Fix typo

* Add new results

* Clean up extraneous code
This commit is contained in:
Ralph Tang
2017-10-29 15:55:36 -04:00
committed by Michael Tu
parent 8fd1ddcb4c
commit 7957dc7638
5 changed files with 26 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conv_rnn/README.md
+1 -1
View File
@@ -17,7 +17,7 @@ You may then run `python train.py` and `python test.py` for training and testing
### Empirical results
Best dev | Test
-- | --
51.1 | 50.7
52.04359673024523 | 50.85972850678733
### References
[1] Chenglong Wang, Feijun Jiang, and Hongxia Yang. 2017. A Hybrid Framework for Text Modeling with Convolutional RNN. In Proceedings of the 23rd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining (KDD '17).
conv_rnn/data.py
+1 -8
View File
@@ -5,13 +5,7 @@ import numpy as np
import torch.utils.data as data
def sst_tokenize(sentence):
extraneous_pattern = re.compile(r"^(--lrb--|--rrb--|``|''|--|\.)$")
words = []
for word in sentence.split():
if re.match(extraneous_pattern, word):
continue
words.append(word)
return words
return sentence.split()
class SSTEmbeddingLoader(object):
def __init__(self, dirname, fmt="stsa.fine.{}", word2vec_file="word2vec.sst-1"):
@@ -26,7 +20,6 @@ class SSTEmbeddingLoader(object):
with open(os.path.join(self.dirname, self.word2vec_file)) as f:
for i, line in enumerate(f.readlines()):
word, vec = line.replace("\n", "").split(" ", 1)
word = word.replace("#", "")
vec = np.array([float(v) for v in vec.split(" ")])
weights.append(vec)
id_dict[word] = i
conv_rnn/model.py
-5
View File
@@ -15,7 +15,6 @@ class ConvRNNModel(nn.Module):
self.hidden_size = config["hidden_size"]
fc_size = config["fc_size"]
self.batch_size = config["mbatch_size"]
dropout = config["dropout_prob"]
n_fmaps = config["n_feature_maps"]
self.rnn_type = config["rnn_type"]
@@ -34,8 +33,6 @@ class ConvRNNModel(nn.Module):
else:
raise ValueError("RNN type must be one of LSTM or GRU")
self.conv = nn.Conv2d(1, n_fmaps, (1, self.hidden_size * 2))
if dropout:
self.dropout = nn.Dropout(dropout)
self.fc1 = nn.Linear(n_fmaps + 2 * self.hidden_size, fc_size)
self.fc2 = nn.Linear(fc_size, config["n_labels"])
@@ -77,8 +74,6 @@ class ConvRNNModel(nn.Module):
out = [t.squeeze(1) for t in rnn_out.chunk(2, 1)]
out.append(x)
x = torch.cat(out, 1).squeeze(2)
if hasattr(self, "dropout"):
x = self.dropout(x)
x = nn_func.relu(self.fc1(x))
return self.fc2(x)
conv_rnn/test.py
+7 -7
View File
@@ -5,6 +5,7 @@ import random
import numpy as np
import torch
import torch.nn as nn
import torch.utils as utils
import data
import model
@@ -17,19 +18,18 @@ def main():
parser.add_argument("--gpu_number", default=0, type=int)
args = parser.parse_args()
model.set_seed(5, no_cuda=args.no_cuda)
data_loader = data.SSTDataLoader(args.data_dir)
conv_rnn = torch.load(args.input_file)
if not args.no_cuda:
torch.cuda.set_device(args.gpu_number)
conv_rnn.cuda()
_, _, test_set = data_loader.load_sst_sets()
_, _, test_set = data.SSTDataset.load_sst_sets("data")
test_loader = utils.data.DataLoader(test_set, batch_size=len(test_set), collate_fn=conv_rnn.convert_dataset)
conv_rnn.eval()
test_in, test_out = conv_rnn.convert_dataset(test_set)
scores = conv_rnn(test_in)
n_correct = (torch.max(scores, 1)[1].view(len(test_set)).data == test_out.data).sum()
accuracy = n_correct / len(test_set)
for test_in, test_out in test_loader:
scores = conv_rnn(test_in)
n_correct = (torch.max(scores, 1)[1].view(-1).data == test_out.data).sum()
accuracy = n_correct / len(test_set)
print("Test set accuracy: {}".format(accuracy))
if __name__ == "__main__":
conv_rnn/train.py
+17 -15
View File
@@ -2,10 +2,11 @@ import argparse
import os
import random
from torch import utils
from torch.optim.lr_scheduler import ReduceLROnPlateau
import numpy as np
import torch
import torch.nn as nn
from torch import utils
import data
import model
@@ -58,7 +59,6 @@ def train(**kwargs):
verbose = not kwargs["quiet"]
lr = kwargs["lr"]
weight_decay = kwargs["weight_decay"]
gradient_clip = kwargs["gradient_clip"]
seed = kwargs["seed"]
if not kwargs["no_cuda"]:
@@ -79,7 +79,8 @@ def train(**kwargs):
conv_rnn.train()
criterion = nn.CrossEntropyLoss()
parameters = list(filter(lambda p: p.requires_grad, conv_rnn.parameters()))
optimizer = torch.optim.Adadelta(parameters, lr=lr, weight_decay=weight_decay)
optimizer = torch.optim.SGD(parameters, lr=lr, weight_decay=weight_decay, momentum=0.9)
scheduler = ReduceLROnPlateau(optimizer, patience=kwargs["dev_per_epoch"] * 4)
train_set, dev_set, test_set = data.SSTDataset.load_sst_sets("data")
collate_fn = conv_rnn.convert_dataset
@@ -92,26 +93,26 @@ def train(**kwargs):
conv_rnn.eval()
for m_in, m_out in loader:
scores = conv_rnn(m_in)
loss = criterion(scores, m_out)
loss = criterion(scores, m_out).cpu().data[0]
n_correct = (torch.max(scores, 1)[1].view(m_in.size(0)).data == m_out.data).sum()
accuracy = n_correct / m_in.size(0)
if dev and accuracy > evaluate.best_dev:
scheduler.step(accuracy)
if dev and accuracy >= evaluate.best_dev:
evaluate.best_dev = accuracy
print("Saving best model ({})...".format(accuracy))
torch.save(conv_rnn, kwargs["output_file"])
if verbose:
print("{} set accuracy: {}, loss: {}".format("dev" if dev else "test", accuracy, loss.cpu().data[0]))
print("{} set accuracy: {}, loss: {}".format("dev" if dev else "test", accuracy, loss))
conv_rnn.train()
evaluate.best_dev = 0
for epoch in range(n_epochs):
optimizer.zero_grad()
print("Epoch number: {}".format(epoch), end="\r")
if verbose:
print()
i = 0
for j, (train_in, train_out) in enumerate(train_loader):
if verbose and i % (mbatch_size * 10) == 0:
print("{} / {}".format(j * mbatch_size, len(train_set)), end="\r")
optimizer.zero_grad()
if not kwargs["no_cuda"]:
train_in.cuda()
@@ -120,10 +121,12 @@ def train(**kwargs):
scores = conv_rnn(train_in)
loss = criterion(scores, train_out)
loss.backward()
torch.nn.utils.clip_grad_norm(parameters, gradient_clip)
optimizer.step()
accuracy = (torch.max(scores, 1)[1].view(-1).data == train_out.data).sum() / mbatch_size
if verbose and i % (mbatch_size * 10) == 0:
print("accuracy: {}, {} / {}".format(accuracy, j * mbatch_size, len(train_set)))
i += mbatch_size
if i % (mbatch_size * 256) == 0:
if i % (len(train_set) // kwargs["dev_per_epoch"]) < mbatch_size:
evaluate(dev_loader)
evaluate(test_loader, dev=False)
return evaluate.best_dev
@@ -147,13 +150,12 @@ def do_random_search(given_params):
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--dropout_prob", default=0.5, type=float)
parser.add_argument("--dev_per_epoch", default=9, type=int)
parser.add_argument("--fc_size", default=200, type=int)
parser.add_argument("--gpu_number", default=0, type=int)
parser.add_argument("--gradient_clip", default=5, type=float)
parser.add_argument("--hidden_size", default=200, type=int)
parser.add_argument("--input_file", default="saves/model.pt", type=str)
parser.add_argument("--lr", default=5E-2, type=float)
parser.add_argument("--lr", default=1E-1, type=float)
parser.add_argument("--mbatch_size", default=64, type=int)
parser.add_argument("--n_epochs", default=30, type=int)
parser.add_argument("--n_feature_maps", default=200, type=float)
@@ -165,7 +167,7 @@ def main():
parser.add_argument("--rnn_type", choices=["lstm", "gru"], default="lstm", type=str)
parser.add_argument("--seed", default=3, type=int)
parser.add_argument("--quiet", action="store_true", default=False)
parser.add_argument("--weight_decay", default=1E-3, type=float)
parser.add_argument("--weight_decay", default=1E-4, type=float)
args = parser.parse_args()
if args.random_search:
do_random_search(vars(args))