From 5df6123abaeba7c34190015c30fc8560c68a8a4a Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sun, 4 Feb 2018 21:39:25 -0500 Subject: [PATCH] Add training code --- vdpwi/__main__.py | 95 +++++++++++++++++++++++++++++++++++++++ vdpwi/data.py | 20 ++++++--- vdpwi/model.py | 46 +++++++++++++------ vdpwi/utils/preprocess.py | 16 +++---- 4 files changed, 150 insertions(+), 27 deletions(-) create mode 100644 vdpwi/__main__.py diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py new file mode 100644 index 0000000..74b65d6 --- /dev/null +++ b/vdpwi/__main__.py @@ -0,0 +1,95 @@ +from collections import namedtuple + +from tqdm import tqdm +import numpy as np +import scipy.stats as stats +import torch +import torch.optim as optim +import torch.nn as nn +import torch.nn.functional as F +import torch.utils as utils + +import data +import model as mod + +Context = namedtuple("Context", "model, train_loader, dev_loader, test_loader, optimizer, criterion") +EvaluateResult = namedtuple("EvaluateResult", "pearsonr, spearmanr") + +def create_context(config): + def collate_fn(batch): + emb1 = [] + emb2 = [] + labels = [] + for s1, s2, l in batch: + emb1.append(s1) + emb2.append(s2) + labels.append(l) + emb1 = torch.LongTensor(emb1) + emb2 = torch.LongTensor(emb2) + labels = torch.Tensor(labels) + emb1 = torch.autograd.Variable(emb1, requires_grad=False) + emb2 = torch.autograd.Variable(emb2, requires_grad=False) + labels = torch.autograd.Variable(labels, requires_grad=False) + if not config.cpu: + emb1 = emb1.cuda() + emb2 = emb2.cuda() + labels = labels.cuda() + return emb1, emb2, labels + + embedding, (train_set, dev_set, test_set) = data.load_dataset(config.dataset) + model = mod.VDPWIModel(embedding, config) + if config.restore: + model.load(config.input_file) + if not config.cpu: + model = model.cuda() + + train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=1, collate_fn=collate_fn) + dev_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn) + test_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn) + + params = list(filter(lambda x: x.requires_grad, model.parameters())) + optimizer = optim.RMSprop(params, lr=config.lr, alpha=config.decay, momentum=config.momentum) + criterion = nn.KLDivLoss() + return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion) + +def test(config): + pass + +def evaluate(model, data_loader): + model.eval() + predictions = [] + true_labels = [] + for sent1, sent2, label_pmf in data_loader: + scores = model(sent1, sent2) + scores = F.softmax(scores).cpu().data.numpy() + prediction = np.dot(np.arange(1, len(scores) + 1), scores) + truth = np.dot(np.arange(1, len(scores) + 1), label_pmf.cpu().data.numpy()) + predictions.append(prediction); true_labels.append(truth) + return EvaluateResult(stats.pearsonr(predictions, truth)[0], stats.spearmanr(predictions, truth)[0]) + +def train(config): + context = create_context(config) + for epoch_no in range(config.n_epochs): + print("Epoch number: {}".format(epoch_no + 1)) + loader_wrapper = tqdm(enumerate(context.train_loader), total=len(context.train_loader), desc="Loss") + for i, (sent1, sent2, label_pmf) in loader_wrapper: + context.model.train() + context.optimizer.zero_grad() + scores = context.model(sent1, sent2) + + loss = context.criterion(scores, label_pmf) + loss.backward() + loader_wrapper.set_description("Loss = {}".format(loss.cpu().data[0])) + context.optimizer.step() + result = evaluate(context.model, context.dev_loader) + print(result) + +def main(): + config = data.Configs.base_config() + if config.mode == "train": + train(config) + elif config.mode == "test": + test(config) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/vdpwi/data.py b/vdpwi/data.py index 6e98ba0..7bf2c9c 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -2,25 +2,33 @@ import argparse import os import torch +import torch.nn as nn import torch.utils.data as data class Configs(object): @staticmethod def base_config(): parser = argparse.ArgumentParser() + parser.add_argument("--cpu", action="store_true", default=False) parser.add_argument("--dataset", type=str, default="sick", choices=["sick"]) + parser.add_argument("--decay", type=float, default=0.95) parser.add_argument("--input_model", type=str, default="local_saves/model.pt") parser.add_argument("--lr", type=float, default=1E-4) parser.add_argument("--mbatch_size", type=int, default=40) + parser.add_argument("--mode", type=str, default="train", choices=["train", "test"]) + parser.add_argument("--momentum", type=float, default=0.9) + parser.add_argument("--n_epochs", type=int, default=40) parser.add_argument("--n_labels", type=int, default=5) parser.add_argument("--output_model", type=str, default="local_saves/model.pt") parser.add_argument("--restore", action="store_true", default=False) + parser.add_argument("--rnn_hidden_dim", type=int, default=300) parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt") return parser.parse_known_args()[0] @staticmethod def sick_config(): parser = argparse.ArgumentParser() + parser.add_argument("--n_labels", type=int, default=5) parser.add_argument("--sick_cache", type=str, default="local_data/sick/.vec-cache") parser.add_argument("--sick_data", type=str, default="local_data/sick") return parser.parse_known_args()[0] @@ -68,12 +76,12 @@ def load_sick(): indices1 = fetch_indices("a.toks") indices2 = fetch_indices("b.toks") sets.append(LabeledEmbeddedDataset(indices1, indices2, labels)) - return embeddings, sets + embedding = nn.Embedding(len(embeddings), 300, -1) + embedding.weight.data.copy_(torch.Tensor(embeddings)) + embedding.weight.requires_grad = False + return embedding, sets -def load_dataset(): - config = Configs.base_config() - return _loaders[config.dataset]() +def load_dataset(dataset): + return _loaders[dataset]() _loaders = dict(sick=load_sick) - -load_dataset() \ No newline at end of file diff --git a/vdpwi/model.py b/vdpwi/model.py index 06b71a2..666dab9 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -16,6 +16,7 @@ class SerializableModule(nn.Module): class VDPWIConvNet(SerializableModule): def __init__(self, n_labels): + super().__init__() self.conv1 = nn.Conv2d(13, 128, 3, padding=1) self.conv2 = nn.Conv2d(128, 164, 3, padding=1) self.conv3 = nn.Conv2d(164, 192, 3, padding=1) @@ -26,6 +27,19 @@ class VDPWIConvNet(SerializableModule): self.output = nn.Linear(128, n_labels) def forward(self, x): + def pad_side(idx, max_size): + if max_size <= 32: + pad_len = 32 - x.size(idx) + elif max_size <= 48: + pad_len = 48 - x.size(idx) + else: + pad_len = 0 + return [0, pad_len] + padding = pad_side(3, max(x.size()[2:])) + padding.extend(pad_side(2, max(x.size()[2:]))) + x = F.pad(x, padding) + x = x[:, :, :48, :48] + pool_final = nn.MaxPool2d(2, ceil_mode=True) if x.size(2) == 32 else nn.MaxPool2d(3, 1, ceil_mode=True) x = self.maxpool2(F.relu(self.conv1(x))) x = self.maxpool2(F.relu(self.conv2(x))) @@ -38,9 +52,10 @@ class VDPWIConvNet(SerializableModule): class VDPWIModel(SerializableModule): def __init__(self, embedding, config, classifier_net=None): super().__init__() - self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True) + self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True, batch_first=True) self.embedding = embedding - self.classifier_net = VDPWIConvNet(config.n_labels) if classifier is None else classifier_net + self.use_cuda = not config.cpu + self.classifier_net = VDPWIConvNet(config.n_labels) if classifier_net is None else classifier_net def compute_sim_cube(self, seq1, seq2): def compute_sim(h1, h2): @@ -50,9 +65,11 @@ class VDPWIModel(SerializableModule): dot_prod = torch.dot(h1, h2) cos_dist = dot_prod / (h1_len * h2_len + 1E-8) l2_dist = torch.sqrt(torch.sum((h1 - h2)**2)) - return dot_prod, cos_dist, l2_dist + return torch.cat([dot_prod, cos_dist, l2_dist]) - sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0)).cuda()) + sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0))) + if self.use_cuda: + sim_cube = sim_cube.cuda() seq1_f = seq1[:, 0] seq1_b = seq1[:, 1] seq2_f = seq2[:, 0] @@ -66,18 +83,21 @@ class VDPWIModel(SerializableModule): return sim_cube def compute_focus_cube(self, sim_cube): - mask = Variable(torch.Tensor(*sim_cube.size()).cuda()) + mask = Variable(torch.Tensor(*sim_cube.size())) + if self.use_cuda: + mask = mask.cuda() + mask[:, :, :] = 0.1 def build_mask(index): s1tag = np.zeros(sim_cube.size(1)) s2tag = np.zeros(sim_cube.size(2)) _, indices = torch.sort(sim_cube[index].view(-1), descending=True) - for i, index in enumerate(indices): + for i, index in enumerate(indices.cpu().data.numpy()): if i >= len(s1tag) + len(s2tag): break - pos1, pos2 = index // len(s1tag), index % len(s2tag) + pos1, pos2 = index // len(s2tag), index % len(s2tag) if s1tag[pos1] + s2tag[pos2] == 0: s1tag[pos1] = s2tag[pos2] = 1 - mask[:, pos1, pos2] = 1 + mask[:, int(pos1), int(pos2)] = 1 build_mask(10) build_mask(11) mask[12, :, :] = 1 @@ -86,11 +106,11 @@ class VDPWIModel(SerializableModule): def forward(self, x1, x2): x1 = self.embedding(x1) x2 = self.embedding(x2) - seq1, _ = self.rnn(x1, batch_first=True) - seq2, _ = self.rnn(x2, batch_first=True) - seq1 = seq1.squeeze(1) # batch size assumed to be 1 - seq2 = seq2.squeeze(1) + seq1, _ = self.rnn(x1) + seq2, _ = self.rnn(x2) + seq1 = seq1.squeeze(0) # batch size assumed to be 1 + seq2 = seq2.squeeze(0) sim_cube = self.compute_sim_cube(seq1, seq2) focus_cube = self.compute_focus_cube(sim_cube) logits = self.classifier_net(focus_cube.unsqueeze(0)) - return torch.log(F.softmax(logits)) + return F.log_softmax(logits) diff --git a/vdpwi/utils/preprocess.py b/vdpwi/utils/preprocess.py index c28f4c5..c0b4e77 100644 --- a/vdpwi/utils/preprocess.py +++ b/vdpwi/utils/preprocess.py @@ -35,20 +35,20 @@ def discrete_tnorm(a, b, tgt_loc, sigma=1, n_steps=100): x = tgt_loc - sigma * tgt_loc_update(x) tn = truncnorm((a - x) / sigma, (b - x) / sigma, loc=x, scale=sigma) rrange = np.arange(a, b + 1) - pdf = tn.pdf(rrange) - pdf /= np.sum(pdf) - return pdf + pmf = tn.pdf(rrange) + pmf /= np.sum(pmf) + return pmf def discrete_lerp(a, b, ground_truth): - pdf = np.zeros(b - a + 1) + pmf = np.zeros(b - a + 1) c = int(np.ceil(ground_truth + 1E-8)) f = int(np.floor(ground_truth)) - pdf[min(c - a, b - a)] = ground_truth - f - pdf[f - a] = c - ground_truth - return pdf + pmf[min(c - a, b - a)] = ground_truth - f + pmf[f - a] = c - ground_truth + return pmf def smoothed_labels(truth, n_labels): - return discrete_tnorm(1, n_labels, truth, sigma=0.35, n_steps=0) + return discrete_lerp(1, n_labels, truth) def preprocess(filename, output_name="sim_sparse.txt"): print("Preprocessing {}...".format(filename))