wassname/Castor
1
0
Fork 0
mirror of https://github.com/wassname/Castor.git synced 2026-09-09 11:13:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add training code

Browse Source
This commit is contained in:
Ralph Tang
2018-02-04 21:39:25 -05:00
parent 579d187d8e
commit 5df6123aba
4 changed files with 150 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vdpwi/__main__.py
+95
View File
@@ -0,0 +1,95 @@
from collections import namedtuple
from tqdm import tqdm
import numpy as np
import scipy.stats as stats
import torch
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import torch.utils as utils
import data
import model as mod
Context = namedtuple("Context", "model, train_loader, dev_loader, test_loader, optimizer, criterion")
EvaluateResult = namedtuple("EvaluateResult", "pearsonr, spearmanr")
def create_context(config):
def collate_fn(batch):
emb1 = []
emb2 = []
labels = []
for s1, s2, l in batch:
emb1.append(s1)
emb2.append(s2)
labels.append(l)
emb1 = torch.LongTensor(emb1)
emb2 = torch.LongTensor(emb2)
labels = torch.Tensor(labels)
emb1 = torch.autograd.Variable(emb1, requires_grad=False)
emb2 = torch.autograd.Variable(emb2, requires_grad=False)
labels = torch.autograd.Variable(labels, requires_grad=False)
if not config.cpu:
emb1 = emb1.cuda()
emb2 = emb2.cuda()
labels = labels.cuda()
return emb1, emb2, labels
embedding, (train_set, dev_set, test_set) = data.load_dataset(config.dataset)
model = mod.VDPWIModel(embedding, config)
if config.restore:
model.load(config.input_file)
if not config.cpu:
model = model.cuda()
train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=1, collate_fn=collate_fn)
dev_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn)
test_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn)
params = list(filter(lambda x: x.requires_grad, model.parameters()))
optimizer = optim.RMSprop(params, lr=config.lr, alpha=config.decay, momentum=config.momentum)
criterion = nn.KLDivLoss()
return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion)
def test(config):
pass
def evaluate(model, data_loader):
model.eval()
predictions = []
true_labels = []
for sent1, sent2, label_pmf in data_loader:
scores = model(sent1, sent2)
scores = F.softmax(scores).cpu().data.numpy()
prediction = np.dot(np.arange(1, len(scores) + 1), scores)
truth = np.dot(np.arange(1, len(scores) + 1), label_pmf.cpu().data.numpy())
predictions.append(prediction); true_labels.append(truth)
return EvaluateResult(stats.pearsonr(predictions, truth)[0], stats.spearmanr(predictions, truth)[0])
def train(config):
context = create_context(config)
for epoch_no in range(config.n_epochs):
print("Epoch number: {}".format(epoch_no + 1))
loader_wrapper = tqdm(enumerate(context.train_loader), total=len(context.train_loader), desc="Loss")
for i, (sent1, sent2, label_pmf) in loader_wrapper:
context.model.train()
context.optimizer.zero_grad()
scores = context.model(sent1, sent2)
loss = context.criterion(scores, label_pmf)
loss.backward()
loader_wrapper.set_description("Loss = {}".format(loss.cpu().data[0]))
context.optimizer.step()
result = evaluate(context.model, context.dev_loader)
print(result)
def main():
config = data.Configs.base_config()
if config.mode == "train":
train(config)
elif config.mode == "test":
test(config)
if __name__ == "__main__":
main()
vdpwi/data.py
+14 -6
View File
@@ -2,25 +2,33 @@ import argparse
import os
import torch
import torch.nn as nn
import torch.utils.data as data
class Configs(object):
@staticmethod
def base_config():
parser = argparse.ArgumentParser()
parser.add_argument("--cpu", action="store_true", default=False)
parser.add_argument("--dataset", type=str, default="sick", choices=["sick"])
parser.add_argument("--decay", type=float, default=0.95)
parser.add_argument("--input_model", type=str, default="local_saves/model.pt")
parser.add_argument("--lr", type=float, default=1E-4)
parser.add_argument("--mbatch_size", type=int, default=40)
parser.add_argument("--mode", type=str, default="train", choices=["train", "test"])
parser.add_argument("--momentum", type=float, default=0.9)
parser.add_argument("--n_epochs", type=int, default=40)
parser.add_argument("--n_labels", type=int, default=5)
parser.add_argument("--output_model", type=str, default="local_saves/model.pt")
parser.add_argument("--restore", action="store_true", default=False)
parser.add_argument("--rnn_hidden_dim", type=int, default=300)
parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt")
return parser.parse_known_args()[0]
@staticmethod
def sick_config():
parser = argparse.ArgumentParser()
parser.add_argument("--n_labels", type=int, default=5)
parser.add_argument("--sick_cache", type=str, default="local_data/sick/.vec-cache")
parser.add_argument("--sick_data", type=str, default="local_data/sick")
return parser.parse_known_args()[0]
@@ -68,12 +76,12 @@ def load_sick():
indices1 = fetch_indices("a.toks")
indices2 = fetch_indices("b.toks")
sets.append(LabeledEmbeddedDataset(indices1, indices2, labels))
return embeddings, sets
embedding = nn.Embedding(len(embeddings), 300, -1)
embedding.weight.data.copy_(torch.Tensor(embeddings))
embedding.weight.requires_grad = False
return embedding, sets
def load_dataset():
config = Configs.base_config()
return _loaders[config.dataset]()
def load_dataset(dataset):
return _loaders[dataset]()
_loaders = dict(sick=load_sick)
load_dataset()
vdpwi/model.py
+33 -13
View File
@@ -16,6 +16,7 @@ class SerializableModule(nn.Module):
class VDPWIConvNet(SerializableModule):
def __init__(self, n_labels):
super().__init__()
self.conv1 = nn.Conv2d(13, 128, 3, padding=1)
self.conv2 = nn.Conv2d(128, 164, 3, padding=1)
self.conv3 = nn.Conv2d(164, 192, 3, padding=1)
@@ -26,6 +27,19 @@ class VDPWIConvNet(SerializableModule):
self.output = nn.Linear(128, n_labels)
def forward(self, x):
def pad_side(idx, max_size):
if max_size <= 32:
pad_len = 32 - x.size(idx)
elif max_size <= 48:
pad_len = 48 - x.size(idx)
else:
pad_len = 0
return [0, pad_len]
padding = pad_side(3, max(x.size()[2:]))
padding.extend(pad_side(2, max(x.size()[2:])))
x = F.pad(x, padding)
x = x[:, :, :48, :48]
pool_final = nn.MaxPool2d(2, ceil_mode=True) if x.size(2) == 32 else nn.MaxPool2d(3, 1, ceil_mode=True)
x = self.maxpool2(F.relu(self.conv1(x)))
x = self.maxpool2(F.relu(self.conv2(x)))
@@ -38,9 +52,10 @@ class VDPWIConvNet(SerializableModule):
class VDPWIModel(SerializableModule):
def __init__(self, embedding, config, classifier_net=None):
super().__init__()
self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True)
self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True, batch_first=True)
self.embedding = embedding
self.classifier_net = VDPWIConvNet(config.n_labels) if classifier is None else classifier_net
self.use_cuda = not config.cpu
self.classifier_net = VDPWIConvNet(config.n_labels) if classifier_net is None else classifier_net
def compute_sim_cube(self, seq1, seq2):
def compute_sim(h1, h2):
@@ -50,9 +65,11 @@ class VDPWIModel(SerializableModule):
dot_prod = torch.dot(h1, h2)
cos_dist = dot_prod / (h1_len * h2_len + 1E-8)
l2_dist = torch.sqrt(torch.sum((h1 - h2)**2))
return dot_prod, cos_dist, l2_dist
return torch.cat([dot_prod, cos_dist, l2_dist])
sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0)).cuda())
sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0)))
if self.use_cuda:
sim_cube = sim_cube.cuda()
seq1_f = seq1[:, 0]
seq1_b = seq1[:, 1]
seq2_f = seq2[:, 0]
@@ -66,18 +83,21 @@ class VDPWIModel(SerializableModule):
return sim_cube
def compute_focus_cube(self, sim_cube):
mask = Variable(torch.Tensor(*sim_cube.size()).cuda())
mask = Variable(torch.Tensor(*sim_cube.size()))
if self.use_cuda:
mask = mask.cuda()
mask[:, :, :] = 0.1
def build_mask(index):
s1tag = np.zeros(sim_cube.size(1))
s2tag = np.zeros(sim_cube.size(2))
_, indices = torch.sort(sim_cube[index].view(-1), descending=True)
for i, index in enumerate(indices):
for i, index in enumerate(indices.cpu().data.numpy()):
if i >= len(s1tag) + len(s2tag):
break
pos1, pos2 = index // len(s1tag), index % len(s2tag)
pos1, pos2 = index // len(s2tag), index % len(s2tag)
if s1tag[pos1] + s2tag[pos2] == 0:
s1tag[pos1] = s2tag[pos2] = 1
mask[:, pos1, pos2] = 1
mask[:, int(pos1), int(pos2)] = 1
build_mask(10)
build_mask(11)
mask[12, :, :] = 1
@@ -86,11 +106,11 @@ class VDPWIModel(SerializableModule):
def forward(self, x1, x2):
x1 = self.embedding(x1)
x2 = self.embedding(x2)
seq1, _ = self.rnn(x1, batch_first=True)
seq2, _ = self.rnn(x2, batch_first=True)
seq1 = seq1.squeeze(1) # batch size assumed to be 1
seq2 = seq2.squeeze(1)
seq1, _ = self.rnn(x1)
seq2, _ = self.rnn(x2)
seq1 = seq1.squeeze(0) # batch size assumed to be 1
seq2 = seq2.squeeze(0)
sim_cube = self.compute_sim_cube(seq1, seq2)
focus_cube = self.compute_focus_cube(sim_cube)
logits = self.classifier_net(focus_cube.unsqueeze(0))
return torch.log(F.softmax(logits))
return F.log_softmax(logits)
vdpwi/utils/preprocess.py
+8 -8
View File
@@ -35,20 +35,20 @@ def discrete_tnorm(a, b, tgt_loc, sigma=1, n_steps=100):
x = tgt_loc - sigma * tgt_loc_update(x)
tn = truncnorm((a - x) / sigma, (b - x) / sigma, loc=x, scale=sigma)
rrange = np.arange(a, b + 1)
pdf = tn.pdf(rrange)
pdf /= np.sum(pdf)
return pdf
pmf = tn.pdf(rrange)
pmf /= np.sum(pmf)
return pmf
def discrete_lerp(a, b, ground_truth):
pdf = np.zeros(b - a + 1)
pmf = np.zeros(b - a + 1)
c = int(np.ceil(ground_truth + 1E-8))
f = int(np.floor(ground_truth))
pdf[min(c - a, b - a)] = ground_truth - f
pdf[f - a] = c - ground_truth
return pdf
pmf[min(c - a, b - a)] = ground_truth - f
pmf[f - a] = c - ground_truth
return pmf
def smoothed_labels(truth, n_labels):
return discrete_tnorm(1, n_labels, truth, sigma=0.35, n_steps=0)
return discrete_lerp(1, n_labels, truth)
def preprocess(filename, output_name="sim_sparse.txt"):
print("Preprocessing {}...".format(filename))