From cc7471a595c8aee58446e5f3da2a348da1458ff9 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sat, 3 Feb 2018 00:10:57 -0500 Subject: [PATCH 01/12] Initial commit --- .gitignore | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++ LICENSE | 21 +++++++++++ README.md | 2 ++ 3 files changed, 124 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7bbc71c --- /dev/null +++ b/.gitignore @@ -0,0 +1,101 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0f6d923 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Ralph Tang + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..640a7ef --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# vdpwi-nn-pytorch +PyTorch implementation of VDPWI-NN From 9e2aaf278881b328009005ef766f2c173608b4c1 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sat, 3 Feb 2018 22:54:58 -0500 Subject: [PATCH 02/12] Add preprocessing scripts --- .gitignore | 3 ++ vdpwi.sublime-project | 8 ++++ vdpwi/data.py | 46 +++++++++++++++++++++++ vdpwi/utils/preprocess.py | 78 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 vdpwi.sublime-project create mode 100644 vdpwi/data.py create mode 100644 vdpwi/utils/preprocess.py diff --git a/.gitignore b/.gitignore index 7bbc71c..31104a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +*.sublime-workspace +local_* + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/vdpwi.sublime-project b/vdpwi.sublime-project new file mode 100644 index 0000000..aefb988 --- /dev/null +++ b/vdpwi.sublime-project @@ -0,0 +1,8 @@ +{ + "folders": + [ + { + "path": "vdpwi" + } + ] +} diff --git a/vdpwi/data.py b/vdpwi/data.py new file mode 100644 index 0000000..9f9b20b --- /dev/null +++ b/vdpwi/data.py @@ -0,0 +1,46 @@ +import argparse +import os + +import torch +import torch.utils.data as data + +class Configs(object): + @staticmethod + def base_config(): + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", type=str, default="sick", choices=["sick"]) + parser.add_argument("--input_model", type=str, default="local_saves/model.pt") + parser.add_argument("--lr", type=float, default=1E-4) + parser.add_argument("--mbatch_size", type=int, default=40) + parser.add_argument("--output_model", type=str, default="local_saves/model.pt") + parser.add_argument("--restore", action="store_true", default=False) + parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt") + return parser.parse_known_args()[0] + + @staticmethod + def sick_config(): + parser = argparse.ArgumentParser() + parser.add_argument("--sick_cache", type=str, default="local_data/sick/.vec-cache") + parser.add_argument("--sick_data", type=str, default="local_data/sick") + return parser.parse_known_args()[0] + +class LabeledEmbeddedDataset(data.Dataset): + def __init__(self, sentence_indices, labels): + assert len(sentence_indices) == len(labels) + self.sentence_indices = sentence_indices + self.labels = labels + + def __getitem__(self, idx): + return self.sentence_indices[idx], self.labels[idx] + + def __len__(self): + return len(self.labels) + +def load_sick(config): + pass + +def load_dataset(): + config = Configs.base_config() + return _loaders[config.dataset](config) + +_loaders = dict(sick=load_sick) \ No newline at end of file diff --git a/vdpwi/utils/preprocess.py b/vdpwi/utils/preprocess.py new file mode 100644 index 0000000..bb6a57f --- /dev/null +++ b/vdpwi/utils/preprocess.py @@ -0,0 +1,78 @@ +import argparse +import os + +from scipy.special import erf +from scipy.stats import truncnorm +import numpy as np + +import data + +def build_vector_cache(glove_filename, vec_cache_filename, vocab): + print("Building vector cache...") + with open(glove_filename) as f, open(vec_cache_filename, "w") as f2: + for line in f: + tok, vec = line.split(" ", 1) + if tok in vocab: + vocab.remove(tok) + f2.write("{} {}".format(tok, vec)) + +def discrete_tnorm(a, b, tgt_loc, sigma=1, n_steps=100): + def phi(zeta): + return 1 / (np.sqrt(2 * np.pi)) * np.exp(-0.5 * zeta**2) + def Phi(x): + return 0.5 * (1 + erf(x / np.sqrt(2))) + def tgt_loc_update(x): + y1 = phi(a - x) / sigma + y2 = phi(b - x) / sigma + x1 = Phi(b - x) / sigma + x2 = Phi(a - x) / sigma + denom = x1 - x2 + 1E-4 + return y1 / denom - y2 / denom + + x = tgt_loc + direction = np.sign(tgt_loc - (b - a)) + for _ in range(n_steps): + x = tgt_loc - sigma* tgt_loc_update(x) + tn = truncnorm((a - x) / sigma, (b - x) / sigma, loc=x, scale=sigma) + rrange = np.arange(a, b + 1) + pdf = tn.pdf(rrange) + pdf /= np.sum(pdf) + return pdf + +def discrete_lerp(a, b, ground_truth): + pdf = np.zeros(b - a + 1) + c = int(np.ceil(ground_truth + 1E-8)) + f = int(np.floor(ground_truth)) + pdf[min(c - a, b - a)] = ground_truth - f + pdf[f - a] = c - ground_truth + return pdf + +def smoothed_labels(truth, n_labels): + return discrete_tnorm(1, n_labels, truth, sigma=0.35, n_steps=0) + +def preprocess(filename, output_name="sim_sparse.txt"): + print("Preprocessing {}...".format(filename)) + with open(filename) as f: + values = [float(l.strip()) for l in f.readlines()] + values = [" ".join([str(l) for l in smoothed_labels(v, 5)]) for v in values] + with open(os.path.join(os.path.dirname(filename), output_name), "w") as f: + f.write("\n".join(values)) + +def add_vocab(tok_filename, vocab): + with open(tok_filename) as f: + for line in f: + vocab.update(line.strip().split()) + +def main(): + base_conf = data.Configs.base_config() + sick_conf = data.Configs.sick_config() + sick_folder = sick_conf.sick_data + vocab = set() + for name in ("train", "dev", "test"): + preprocess(os.path.join(sick_folder, name, "sim.txt")) + add_vocab(os.path.join(sick_folder, name, "a.toks"), vocab) + add_vocab(os.path.join(sick_folder, name, "b.toks"), vocab) + build_vector_cache(base_conf.wordvecs_file, sick_conf.sick_cache, vocab) + +if __name__ == "__main__": + main() From e08f85dc0951ed17beedcc1279dce1a7a7e1ff23 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sun, 4 Feb 2018 14:17:51 -0500 Subject: [PATCH 03/12] Fix sigma bug --- vdpwi/data.py | 48 ++++++++++++++++++++++++++++++++------- vdpwi/model.py | 0 vdpwi/utils/preprocess.py | 10 ++++---- 3 files changed, 45 insertions(+), 13 deletions(-) create mode 100644 vdpwi/model.py diff --git a/vdpwi/data.py b/vdpwi/data.py index 9f9b20b..017e29d 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -25,22 +25,54 @@ class Configs(object): return parser.parse_known_args()[0] class LabeledEmbeddedDataset(data.Dataset): - def __init__(self, sentence_indices, labels): - assert len(sentence_indices) == len(labels) - self.sentence_indices = sentence_indices + def __init__(self, sentence_indices1, sentence_indices2, labels): + assert len(sentence_indices1) == len(labels) == len(sentence_indices2) + self.sentence_indices1 = sentence_indices1 + self.sentence_indices2 = sentence_indices2 self.labels = labels def __getitem__(self, idx): - return self.sentence_indices[idx], self.labels[idx] + return self.sentence_indices1[idx], self.sentence_indices2[idx], self.labels[idx] def __len__(self): return len(self.labels) -def load_sick(config): - pass +def load_sick(): + config = Configs.sick_config() + def fetch_indices(name): + sentence_indices = [] + filename = os.path.join(config.sick_data, dataset, name) + with open(filename) as f: + for line in f: + indices = [embed_ids.get(word, -1) for word in line.strip().split()] + sentence_indices.append(indices) + return sentence_indices + + sets = [] + embeddings = [] + embed_ids = {} + with open(os.path.join(config.sick_cache)) as f: + for i, line in enumerate(f): + word, vec = line.split(" ", 1) + vec = list(map(float, vec.strip().split())) + embed_ids[word] = i + embeddings.append(vec) + + for dataset in ("train", "dev", "test"): + filename = os.path.join(config.sick_data, dataset, "sim_sparse.txt") + labels = [] + with open(filename) as f: + for line in f: + labels.append([float(val) for val in line.split()]) + indices1 = fetch_indices("a.toks") + indices2 = fetch_indices("b.toks") + sets.append(LabeledEmbeddedDataset(indices1, indices2, labels)) + return embeddings, sets def load_dataset(): config = Configs.base_config() - return _loaders[config.dataset](config) + return _loaders[config.dataset]() -_loaders = dict(sick=load_sick) \ No newline at end of file +_loaders = dict(sick=load_sick) + +load_dataset() \ No newline at end of file diff --git a/vdpwi/model.py b/vdpwi/model.py new file mode 100644 index 0000000..e69de29 diff --git a/vdpwi/utils/preprocess.py b/vdpwi/utils/preprocess.py index bb6a57f..c28f4c5 100644 --- a/vdpwi/utils/preprocess.py +++ b/vdpwi/utils/preprocess.py @@ -22,17 +22,17 @@ def discrete_tnorm(a, b, tgt_loc, sigma=1, n_steps=100): def Phi(x): return 0.5 * (1 + erf(x / np.sqrt(2))) def tgt_loc_update(x): - y1 = phi(a - x) / sigma - y2 = phi(b - x) / sigma - x1 = Phi(b - x) / sigma - x2 = Phi(a - x) / sigma + y1 = phi((a - x) / sigma) + y2 = phi((b - x) / sigma) + x1 = Phi((b - x) / sigma) + x2 = Phi((a - x) / sigma) denom = x1 - x2 + 1E-4 return y1 / denom - y2 / denom x = tgt_loc direction = np.sign(tgt_loc - (b - a)) for _ in range(n_steps): - x = tgt_loc - sigma* tgt_loc_update(x) + x = tgt_loc - sigma * tgt_loc_update(x) tn = truncnorm((a - x) / sigma, (b - x) / sigma, loc=x, scale=sigma) rrange = np.arange(a, b + 1) pdf = tn.pdf(rrange) From 579d187d8e028f628fe05bf0260bef949d01a08a Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sun, 4 Feb 2018 16:10:38 -0500 Subject: [PATCH 04/12] Add VDPWI core models --- vdpwi/data.py | 1 + vdpwi/model.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/vdpwi/data.py b/vdpwi/data.py index 017e29d..6e98ba0 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -12,6 +12,7 @@ class Configs(object): parser.add_argument("--input_model", type=str, default="local_saves/model.pt") parser.add_argument("--lr", type=float, default=1E-4) parser.add_argument("--mbatch_size", type=int, default=40) + parser.add_argument("--n_labels", type=int, default=5) parser.add_argument("--output_model", type=str, default="local_saves/model.pt") parser.add_argument("--restore", action="store_true", default=False) parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt") diff --git a/vdpwi/model.py b/vdpwi/model.py index e69de29..06b71a2 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -0,0 +1,96 @@ +from torch.autograd import Variable +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np + +class SerializableModule(nn.Module): + def __init__(self): + super().__init__() + + def save(self, filename): + torch.save(self.state_dict(), filename) + + def load(self, filename): + self.load_state_dict(torch.load(filename, map_location=lambda storage, loc: storage)) + +class VDPWIConvNet(SerializableModule): + def __init__(self, n_labels): + self.conv1 = nn.Conv2d(13, 128, 3, padding=1) + self.conv2 = nn.Conv2d(128, 164, 3, padding=1) + self.conv3 = nn.Conv2d(164, 192, 3, padding=1) + self.conv4 = nn.Conv2d(192, 192, 3, padding=1) + self.conv5 = nn.Conv2d(192, 128, 3, padding=1) + self.maxpool2 = nn.MaxPool2d(2, ceil_mode=True) + self.dnn = nn.Linear(128, 128) + self.output = nn.Linear(128, n_labels) + + def forward(self, x): + pool_final = nn.MaxPool2d(2, ceil_mode=True) if x.size(2) == 32 else nn.MaxPool2d(3, 1, ceil_mode=True) + x = self.maxpool2(F.relu(self.conv1(x))) + x = self.maxpool2(F.relu(self.conv2(x))) + x = self.maxpool2(F.relu(self.conv3(x))) + x = self.maxpool2(F.relu(self.conv4(x))) + x = pool_final(F.relu(self.conv5(x))) + x = F.relu(self.dnn(x.view(x.size(0), -1))) + return self.output(x) + +class VDPWIModel(SerializableModule): + def __init__(self, embedding, config, classifier_net=None): + super().__init__() + self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True) + self.embedding = embedding + self.classifier_net = VDPWIConvNet(config.n_labels) if classifier is None else classifier_net + + def compute_sim_cube(self, seq1, seq2): + def compute_sim(h1, h2): + h1_len = torch.sqrt(torch.sum(h1**2)) + h2_len = torch.sqrt(torch.sum(h2**2)) + + dot_prod = torch.dot(h1, h2) + cos_dist = dot_prod / (h1_len * h2_len + 1E-8) + l2_dist = torch.sqrt(torch.sum((h1 - h2)**2)) + return dot_prod, cos_dist, l2_dist + + sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0)).cuda()) + seq1_f = seq1[:, 0] + seq1_b = seq1[:, 1] + seq2_f = seq2[:, 0] + seq2_b = seq2[:, 1] + for t, (h1f, h1b) in enumerate(zip(seq1_f, seq1_b)): + for s, (h2f, h2b) in enumerate(zip(seq2_f, seq2_b)): + sim_cube[0:3, t, s] = compute_sim(torch.cat([h1f, h1b]), torch.cat([h2f, h2b])) + sim_cube[3:6, t, s] = compute_sim(h1f, h2f) + sim_cube[6:9, t, s] = compute_sim(h1b, h2b) + sim_cube[9:12, t, s] = compute_sim(h1f + h1b, h2f + h2b) + return sim_cube + + def compute_focus_cube(self, sim_cube): + mask = Variable(torch.Tensor(*sim_cube.size()).cuda()) + def build_mask(index): + s1tag = np.zeros(sim_cube.size(1)) + s2tag = np.zeros(sim_cube.size(2)) + _, indices = torch.sort(sim_cube[index].view(-1), descending=True) + for i, index in enumerate(indices): + if i >= len(s1tag) + len(s2tag): + break + pos1, pos2 = index // len(s1tag), index % len(s2tag) + if s1tag[pos1] + s2tag[pos2] == 0: + s1tag[pos1] = s2tag[pos2] = 1 + mask[:, pos1, pos2] = 1 + build_mask(10) + build_mask(11) + mask[12, :, :] = 1 + return mask * sim_cube + + def forward(self, x1, x2): + x1 = self.embedding(x1) + x2 = self.embedding(x2) + seq1, _ = self.rnn(x1, batch_first=True) + seq2, _ = self.rnn(x2, batch_first=True) + seq1 = seq1.squeeze(1) # batch size assumed to be 1 + seq2 = seq2.squeeze(1) + sim_cube = self.compute_sim_cube(seq1, seq2) + focus_cube = self.compute_focus_cube(sim_cube) + logits = self.classifier_net(focus_cube.unsqueeze(0)) + return torch.log(F.softmax(logits)) From 5df6123abaeba7c34190015c30fc8560c68a8a4a Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sun, 4 Feb 2018 21:39:25 -0500 Subject: [PATCH 05/12] Add training code --- vdpwi/__main__.py | 95 +++++++++++++++++++++++++++++++++++++++ vdpwi/data.py | 20 ++++++--- vdpwi/model.py | 46 +++++++++++++------ vdpwi/utils/preprocess.py | 16 +++---- 4 files changed, 150 insertions(+), 27 deletions(-) create mode 100644 vdpwi/__main__.py diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py new file mode 100644 index 0000000..74b65d6 --- /dev/null +++ b/vdpwi/__main__.py @@ -0,0 +1,95 @@ +from collections import namedtuple + +from tqdm import tqdm +import numpy as np +import scipy.stats as stats +import torch +import torch.optim as optim +import torch.nn as nn +import torch.nn.functional as F +import torch.utils as utils + +import data +import model as mod + +Context = namedtuple("Context", "model, train_loader, dev_loader, test_loader, optimizer, criterion") +EvaluateResult = namedtuple("EvaluateResult", "pearsonr, spearmanr") + +def create_context(config): + def collate_fn(batch): + emb1 = [] + emb2 = [] + labels = [] + for s1, s2, l in batch: + emb1.append(s1) + emb2.append(s2) + labels.append(l) + emb1 = torch.LongTensor(emb1) + emb2 = torch.LongTensor(emb2) + labels = torch.Tensor(labels) + emb1 = torch.autograd.Variable(emb1, requires_grad=False) + emb2 = torch.autograd.Variable(emb2, requires_grad=False) + labels = torch.autograd.Variable(labels, requires_grad=False) + if not config.cpu: + emb1 = emb1.cuda() + emb2 = emb2.cuda() + labels = labels.cuda() + return emb1, emb2, labels + + embedding, (train_set, dev_set, test_set) = data.load_dataset(config.dataset) + model = mod.VDPWIModel(embedding, config) + if config.restore: + model.load(config.input_file) + if not config.cpu: + model = model.cuda() + + train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=1, collate_fn=collate_fn) + dev_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn) + test_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn) + + params = list(filter(lambda x: x.requires_grad, model.parameters())) + optimizer = optim.RMSprop(params, lr=config.lr, alpha=config.decay, momentum=config.momentum) + criterion = nn.KLDivLoss() + return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion) + +def test(config): + pass + +def evaluate(model, data_loader): + model.eval() + predictions = [] + true_labels = [] + for sent1, sent2, label_pmf in data_loader: + scores = model(sent1, sent2) + scores = F.softmax(scores).cpu().data.numpy() + prediction = np.dot(np.arange(1, len(scores) + 1), scores) + truth = np.dot(np.arange(1, len(scores) + 1), label_pmf.cpu().data.numpy()) + predictions.append(prediction); true_labels.append(truth) + return EvaluateResult(stats.pearsonr(predictions, truth)[0], stats.spearmanr(predictions, truth)[0]) + +def train(config): + context = create_context(config) + for epoch_no in range(config.n_epochs): + print("Epoch number: {}".format(epoch_no + 1)) + loader_wrapper = tqdm(enumerate(context.train_loader), total=len(context.train_loader), desc="Loss") + for i, (sent1, sent2, label_pmf) in loader_wrapper: + context.model.train() + context.optimizer.zero_grad() + scores = context.model(sent1, sent2) + + loss = context.criterion(scores, label_pmf) + loss.backward() + loader_wrapper.set_description("Loss = {}".format(loss.cpu().data[0])) + context.optimizer.step() + result = evaluate(context.model, context.dev_loader) + print(result) + +def main(): + config = data.Configs.base_config() + if config.mode == "train": + train(config) + elif config.mode == "test": + test(config) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/vdpwi/data.py b/vdpwi/data.py index 6e98ba0..7bf2c9c 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -2,25 +2,33 @@ import argparse import os import torch +import torch.nn as nn import torch.utils.data as data class Configs(object): @staticmethod def base_config(): parser = argparse.ArgumentParser() + parser.add_argument("--cpu", action="store_true", default=False) parser.add_argument("--dataset", type=str, default="sick", choices=["sick"]) + parser.add_argument("--decay", type=float, default=0.95) parser.add_argument("--input_model", type=str, default="local_saves/model.pt") parser.add_argument("--lr", type=float, default=1E-4) parser.add_argument("--mbatch_size", type=int, default=40) + parser.add_argument("--mode", type=str, default="train", choices=["train", "test"]) + parser.add_argument("--momentum", type=float, default=0.9) + parser.add_argument("--n_epochs", type=int, default=40) parser.add_argument("--n_labels", type=int, default=5) parser.add_argument("--output_model", type=str, default="local_saves/model.pt") parser.add_argument("--restore", action="store_true", default=False) + parser.add_argument("--rnn_hidden_dim", type=int, default=300) parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt") return parser.parse_known_args()[0] @staticmethod def sick_config(): parser = argparse.ArgumentParser() + parser.add_argument("--n_labels", type=int, default=5) parser.add_argument("--sick_cache", type=str, default="local_data/sick/.vec-cache") parser.add_argument("--sick_data", type=str, default="local_data/sick") return parser.parse_known_args()[0] @@ -68,12 +76,12 @@ def load_sick(): indices1 = fetch_indices("a.toks") indices2 = fetch_indices("b.toks") sets.append(LabeledEmbeddedDataset(indices1, indices2, labels)) - return embeddings, sets + embedding = nn.Embedding(len(embeddings), 300, -1) + embedding.weight.data.copy_(torch.Tensor(embeddings)) + embedding.weight.requires_grad = False + return embedding, sets -def load_dataset(): - config = Configs.base_config() - return _loaders[config.dataset]() +def load_dataset(dataset): + return _loaders[dataset]() _loaders = dict(sick=load_sick) - -load_dataset() \ No newline at end of file diff --git a/vdpwi/model.py b/vdpwi/model.py index 06b71a2..666dab9 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -16,6 +16,7 @@ class SerializableModule(nn.Module): class VDPWIConvNet(SerializableModule): def __init__(self, n_labels): + super().__init__() self.conv1 = nn.Conv2d(13, 128, 3, padding=1) self.conv2 = nn.Conv2d(128, 164, 3, padding=1) self.conv3 = nn.Conv2d(164, 192, 3, padding=1) @@ -26,6 +27,19 @@ class VDPWIConvNet(SerializableModule): self.output = nn.Linear(128, n_labels) def forward(self, x): + def pad_side(idx, max_size): + if max_size <= 32: + pad_len = 32 - x.size(idx) + elif max_size <= 48: + pad_len = 48 - x.size(idx) + else: + pad_len = 0 + return [0, pad_len] + padding = pad_side(3, max(x.size()[2:])) + padding.extend(pad_side(2, max(x.size()[2:]))) + x = F.pad(x, padding) + x = x[:, :, :48, :48] + pool_final = nn.MaxPool2d(2, ceil_mode=True) if x.size(2) == 32 else nn.MaxPool2d(3, 1, ceil_mode=True) x = self.maxpool2(F.relu(self.conv1(x))) x = self.maxpool2(F.relu(self.conv2(x))) @@ -38,9 +52,10 @@ class VDPWIConvNet(SerializableModule): class VDPWIModel(SerializableModule): def __init__(self, embedding, config, classifier_net=None): super().__init__() - self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True) + self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True, batch_first=True) self.embedding = embedding - self.classifier_net = VDPWIConvNet(config.n_labels) if classifier is None else classifier_net + self.use_cuda = not config.cpu + self.classifier_net = VDPWIConvNet(config.n_labels) if classifier_net is None else classifier_net def compute_sim_cube(self, seq1, seq2): def compute_sim(h1, h2): @@ -50,9 +65,11 @@ class VDPWIModel(SerializableModule): dot_prod = torch.dot(h1, h2) cos_dist = dot_prod / (h1_len * h2_len + 1E-8) l2_dist = torch.sqrt(torch.sum((h1 - h2)**2)) - return dot_prod, cos_dist, l2_dist + return torch.cat([dot_prod, cos_dist, l2_dist]) - sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0)).cuda()) + sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0))) + if self.use_cuda: + sim_cube = sim_cube.cuda() seq1_f = seq1[:, 0] seq1_b = seq1[:, 1] seq2_f = seq2[:, 0] @@ -66,18 +83,21 @@ class VDPWIModel(SerializableModule): return sim_cube def compute_focus_cube(self, sim_cube): - mask = Variable(torch.Tensor(*sim_cube.size()).cuda()) + mask = Variable(torch.Tensor(*sim_cube.size())) + if self.use_cuda: + mask = mask.cuda() + mask[:, :, :] = 0.1 def build_mask(index): s1tag = np.zeros(sim_cube.size(1)) s2tag = np.zeros(sim_cube.size(2)) _, indices = torch.sort(sim_cube[index].view(-1), descending=True) - for i, index in enumerate(indices): + for i, index in enumerate(indices.cpu().data.numpy()): if i >= len(s1tag) + len(s2tag): break - pos1, pos2 = index // len(s1tag), index % len(s2tag) + pos1, pos2 = index // len(s2tag), index % len(s2tag) if s1tag[pos1] + s2tag[pos2] == 0: s1tag[pos1] = s2tag[pos2] = 1 - mask[:, pos1, pos2] = 1 + mask[:, int(pos1), int(pos2)] = 1 build_mask(10) build_mask(11) mask[12, :, :] = 1 @@ -86,11 +106,11 @@ class VDPWIModel(SerializableModule): def forward(self, x1, x2): x1 = self.embedding(x1) x2 = self.embedding(x2) - seq1, _ = self.rnn(x1, batch_first=True) - seq2, _ = self.rnn(x2, batch_first=True) - seq1 = seq1.squeeze(1) # batch size assumed to be 1 - seq2 = seq2.squeeze(1) + seq1, _ = self.rnn(x1) + seq2, _ = self.rnn(x2) + seq1 = seq1.squeeze(0) # batch size assumed to be 1 + seq2 = seq2.squeeze(0) sim_cube = self.compute_sim_cube(seq1, seq2) focus_cube = self.compute_focus_cube(sim_cube) logits = self.classifier_net(focus_cube.unsqueeze(0)) - return torch.log(F.softmax(logits)) + return F.log_softmax(logits) diff --git a/vdpwi/utils/preprocess.py b/vdpwi/utils/preprocess.py index c28f4c5..c0b4e77 100644 --- a/vdpwi/utils/preprocess.py +++ b/vdpwi/utils/preprocess.py @@ -35,20 +35,20 @@ def discrete_tnorm(a, b, tgt_loc, sigma=1, n_steps=100): x = tgt_loc - sigma * tgt_loc_update(x) tn = truncnorm((a - x) / sigma, (b - x) / sigma, loc=x, scale=sigma) rrange = np.arange(a, b + 1) - pdf = tn.pdf(rrange) - pdf /= np.sum(pdf) - return pdf + pmf = tn.pdf(rrange) + pmf /= np.sum(pmf) + return pmf def discrete_lerp(a, b, ground_truth): - pdf = np.zeros(b - a + 1) + pmf = np.zeros(b - a + 1) c = int(np.ceil(ground_truth + 1E-8)) f = int(np.floor(ground_truth)) - pdf[min(c - a, b - a)] = ground_truth - f - pdf[f - a] = c - ground_truth - return pdf + pmf[min(c - a, b - a)] = ground_truth - f + pmf[f - a] = c - ground_truth + return pmf def smoothed_labels(truth, n_labels): - return discrete_tnorm(1, n_labels, truth, sigma=0.35, n_steps=0) + return discrete_lerp(1, n_labels, truth) def preprocess(filename, output_name="sim_sparse.txt"): print("Preprocessing {}...".format(filename)) From 28f62623be5c4ec444805a956e4b72066b2f3db8 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sun, 4 Feb 2018 22:00:55 -0500 Subject: [PATCH 06/12] Workaround PT padding_idx bug --- vdpwi/data.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vdpwi/data.py b/vdpwi/data.py index 7bf2c9c..85cb1dd 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -53,7 +53,7 @@ def load_sick(): filename = os.path.join(config.sick_data, dataset, name) with open(filename) as f: for line in f: - indices = [embed_ids.get(word, -1) for word in line.strip().split()] + indices = [embed_ids.get(word, padding_idx) for word in line.strip().split()] sentence_indices.append(indices) return sentence_indices @@ -66,6 +66,8 @@ def load_sick(): vec = list(map(float, vec.strip().split())) embed_ids[word] = i embeddings.append(vec) + padding_idx = len(embeddings) + embeddings.append([0.0] * 300) for dataset in ("train", "dev", "test"): filename = os.path.join(config.sick_data, dataset, "sim_sparse.txt") @@ -76,7 +78,7 @@ def load_sick(): indices1 = fetch_indices("a.toks") indices2 = fetch_indices("b.toks") sets.append(LabeledEmbeddedDataset(indices1, indices2, labels)) - embedding = nn.Embedding(len(embeddings), 300, -1) + embedding = nn.Embedding(len(embeddings), 300) embedding.weight.data.copy_(torch.Tensor(embeddings)) embedding.weight.requires_grad = False return embedding, sets From a110b031645342214dde9de7f8c5c1c868ab60a4 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Sun, 4 Feb 2018 23:20:22 -0500 Subject: [PATCH 07/12] Add GPU loop unrolling for SimCube computation --- vdpwi/__main__.py | 8 ++++---- vdpwi/model.py | 41 +++++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py index 74b65d6..7318065 100644 --- a/vdpwi/__main__.py +++ b/vdpwi/__main__.py @@ -61,11 +61,11 @@ def evaluate(model, data_loader): true_labels = [] for sent1, sent2, label_pmf in data_loader: scores = model(sent1, sent2) - scores = F.softmax(scores).cpu().data.numpy() + scores = F.softmax(scores).cpu().data.numpy()[0] prediction = np.dot(np.arange(1, len(scores) + 1), scores) - truth = np.dot(np.arange(1, len(scores) + 1), label_pmf.cpu().data.numpy()) + truth = np.dot(np.arange(1, len(scores) + 1), label_pmf.cpu().data.numpy()[0]) predictions.append(prediction); true_labels.append(truth) - return EvaluateResult(stats.pearsonr(predictions, truth)[0], stats.spearmanr(predictions, truth)[0]) + return EvaluateResult(stats.pearsonr(predictions, true_labels)[0], stats.spearmanr(predictions, true_labels)[0]) def train(config): context = create_context(config) @@ -75,7 +75,7 @@ def train(config): for i, (sent1, sent2, label_pmf) in loader_wrapper: context.model.train() context.optimizer.zero_grad() - scores = context.model(sent1, sent2) + scores = F.log_softmax(context.model(sent1, sent2)) loss = context.criterion(scores, label_pmf) loss.backward() diff --git a/vdpwi/model.py b/vdpwi/model.py index 666dab9..384af83 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -58,28 +58,33 @@ class VDPWIModel(SerializableModule): self.classifier_net = VDPWIConvNet(config.n_labels) if classifier_net is None else classifier_net def compute_sim_cube(self, seq1, seq2): - def compute_sim(h1, h2): - h1_len = torch.sqrt(torch.sum(h1**2)) - h2_len = torch.sqrt(torch.sum(h2**2)) + def compute_sim(prism1, prism2): + prism1_len = torch.sqrt(torch.sum(prism1**2, 2)) + prism2_len = torch.sqrt(torch.sum(prism2**2, 2)) - dot_prod = torch.dot(h1, h2) - cos_dist = dot_prod / (h1_len * h2_len + 1E-8) - l2_dist = torch.sqrt(torch.sum((h1 - h2)**2)) - return torch.cat([dot_prod, cos_dist, l2_dist]) + dot_prod = torch.matmul(prism1.unsqueeze(2), prism2.unsqueeze(3)) + dot_prod = dot_prod.squeeze(2).squeeze(2) + cos_dist = dot_prod / (prism1_len * prism2_len + 1E-8) + l2_dist = torch.sqrt(torch.sum((prism1 - prism2)**2, 2)) + return torch.stack([dot_prod, cos_dist, l2_dist], 0) + def compute_prism(seq1, seq2): + prism1 = seq1.repeat(seq2.size(0), 1, 1) + prism2 = seq2.repeat(seq1.size(0), 1, 1) + prism1 = prism1.permute(1, 0, 2).contiguous() + prism2 = prism2.permute(0, 1, 2).contiguous() + return compute_sim(prism1, prism2) sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0))) if self.use_cuda: sim_cube = sim_cube.cuda() - seq1_f = seq1[:, 0] - seq1_b = seq1[:, 1] - seq2_f = seq2[:, 0] - seq2_b = seq2[:, 1] - for t, (h1f, h1b) in enumerate(zip(seq1_f, seq1_b)): - for s, (h2f, h2b) in enumerate(zip(seq2_f, seq2_b)): - sim_cube[0:3, t, s] = compute_sim(torch.cat([h1f, h1b]), torch.cat([h2f, h2b])) - sim_cube[3:6, t, s] = compute_sim(h1f, h2f) - sim_cube[6:9, t, s] = compute_sim(h1b, h2b) - sim_cube[9:12, t, s] = compute_sim(h1f + h1b, h2f + h2b) + seq1_f = seq1[:, :300] + seq1_b = seq1[:, 300:] + seq2_f = seq2[:, :300] + seq2_b = seq2[:, 300:] + sim_cube[0:3] = compute_prism(seq1, seq2) + sim_cube[3:6] = compute_prism(seq1_f, seq2_f) + sim_cube[6:9] = compute_prism(seq1_b, seq2_b) + sim_cube[9:12] = compute_prism(seq1_f + seq1_b, seq2_f + seq2_b) return sim_cube def compute_focus_cube(self, sim_cube): @@ -113,4 +118,4 @@ class VDPWIModel(SerializableModule): sim_cube = self.compute_sim_cube(seq1, seq2) focus_cube = self.compute_focus_cube(sim_cube) logits = self.classifier_net(focus_cube.unsqueeze(0)) - return F.log_softmax(logits) + return logits From b06547b36eb1839e6e0ce5ad48fb7cd93f189f63 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Mon, 5 Feb 2018 02:48:34 -0500 Subject: [PATCH 08/12] Fix norm gradient explosion --- vdpwi/__main__.py | 9 +++++---- vdpwi/data.py | 4 ++-- vdpwi/model.py | 29 ++++++++++++++++++----------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py index 7318065..f343d44 100644 --- a/vdpwi/__main__.py +++ b/vdpwi/__main__.py @@ -12,7 +12,7 @@ import torch.utils as utils import data import model as mod -Context = namedtuple("Context", "model, train_loader, dev_loader, test_loader, optimizer, criterion") +Context = namedtuple("Context", "model, train_loader, dev_loader, test_loader, optimizer, criterion, params") EvaluateResult = namedtuple("EvaluateResult", "pearsonr, spearmanr") def create_context(config): @@ -45,12 +45,12 @@ def create_context(config): train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=1, collate_fn=collate_fn) dev_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn) - test_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn) + test_loader = utils.data.DataLoader(test_set, batch_size=1, collate_fn=collate_fn) params = list(filter(lambda x: x.requires_grad, model.parameters())) optimizer = optim.RMSprop(params, lr=config.lr, alpha=config.decay, momentum=config.momentum) criterion = nn.KLDivLoss() - return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion) + return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion, params) def test(config): pass @@ -72,13 +72,14 @@ def train(config): for epoch_no in range(config.n_epochs): print("Epoch number: {}".format(epoch_no + 1)) loader_wrapper = tqdm(enumerate(context.train_loader), total=len(context.train_loader), desc="Loss") + context.model.train() for i, (sent1, sent2, label_pmf) in loader_wrapper: - context.model.train() context.optimizer.zero_grad() scores = F.log_softmax(context.model(sent1, sent2)) loss = context.criterion(scores, label_pmf) loss.backward() + nn.utils.clip_grad_norm(context.params, 50) loader_wrapper.set_description("Loss = {}".format(loss.cpu().data[0])) context.optimizer.step() result = evaluate(context.model, context.dev_loader) diff --git a/vdpwi/data.py b/vdpwi/data.py index 85cb1dd..a67aa8f 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -14,14 +14,14 @@ class Configs(object): parser.add_argument("--decay", type=float, default=0.95) parser.add_argument("--input_model", type=str, default="local_saves/model.pt") parser.add_argument("--lr", type=float, default=1E-4) - parser.add_argument("--mbatch_size", type=int, default=40) + parser.add_argument("--mbatch_size", type=int, default=1) parser.add_argument("--mode", type=str, default="train", choices=["train", "test"]) parser.add_argument("--momentum", type=float, default=0.9) parser.add_argument("--n_epochs", type=int, default=40) parser.add_argument("--n_labels", type=int, default=5) parser.add_argument("--output_model", type=str, default="local_saves/model.pt") parser.add_argument("--restore", action="store_true", default=False) - parser.add_argument("--rnn_hidden_dim", type=int, default=300) + parser.add_argument("--rnn_hidden_dim", type=int, default=250) parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt") return parser.parse_known_args()[0] diff --git a/vdpwi/model.py b/vdpwi/model.py index 384af83..6b2f1ba 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -52,21 +52,23 @@ class VDPWIConvNet(SerializableModule): class VDPWIModel(SerializableModule): def __init__(self, embedding, config, classifier_net=None): super().__init__() - self.rnn = nn.LSTM(300, config.rnn_hidden_dim, 1, bidirectional=True, batch_first=True) + self.hidden_dim = config.rnn_hidden_dim + self.rnn = nn.LSTM(300, self.hidden_dim, 1, batch_first=True) self.embedding = embedding self.use_cuda = not config.cpu self.classifier_net = VDPWIConvNet(config.n_labels) if classifier_net is None else classifier_net def compute_sim_cube(self, seq1, seq2): def compute_sim(prism1, prism2): - prism1_len = torch.sqrt(torch.sum(prism1**2, 2)) - prism2_len = torch.sqrt(torch.sum(prism2**2, 2)) + prism1_len = prism1.norm(dim=2) + prism2_len = prism2.norm(dim=2) dot_prod = torch.matmul(prism1.unsqueeze(2), prism2.unsqueeze(3)) dot_prod = dot_prod.squeeze(2).squeeze(2) cos_dist = dot_prod / (prism1_len * prism2_len + 1E-8) - l2_dist = torch.sqrt(torch.sum((prism1 - prism2)**2, 2)) + l2_dist = (prism1 - prism2).norm(dim=2) return torch.stack([dot_prod, cos_dist, l2_dist], 0) + def compute_prism(seq1, seq2): prism1 = seq1.repeat(seq2.size(0), 1, 1) prism2 = seq2.repeat(seq1.size(0), 1, 1) @@ -75,12 +77,13 @@ class VDPWIModel(SerializableModule): return compute_sim(prism1, prism2) sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0))) + sim_cube[12] = 0 if self.use_cuda: sim_cube = sim_cube.cuda() - seq1_f = seq1[:, :300] - seq1_b = seq1[:, 300:] - seq2_f = seq2[:, :300] - seq2_b = seq2[:, 300:] + seq1_f = seq1[:, :self.hidden_dim] + seq1_b = seq1[:, self.hidden_dim:] + seq2_f = seq2[:, :self.hidden_dim] + seq2_b = seq2[:, self.hidden_dim:] sim_cube[0:3] = compute_prism(seq1, seq2) sim_cube[3:6] = compute_prism(seq1_f, seq2_f) sim_cube[6:9] = compute_prism(seq1_b, seq2_b) @@ -103,16 +106,20 @@ class VDPWIModel(SerializableModule): if s1tag[pos1] + s2tag[pos2] == 0: s1tag[pos1] = s2tag[pos2] = 1 mask[:, int(pos1), int(pos2)] = 1 + build_mask(9) build_mask(10) - build_mask(11) mask[12, :, :] = 1 return mask * sim_cube def forward(self, x1, x2): x1 = self.embedding(x1) x2 = self.embedding(x2) - seq1, _ = self.rnn(x1) - seq2, _ = self.rnn(x2) + seq1f, _ = self.rnn(x1) + seq2f, _ = self.rnn(x2) + seq1b, _ = self.rnn(torch.cat(x1.split(1, 1)[::-1], 1)) + seq2b, _ = self.rnn(torch.cat(x2.split(1, 1)[::-1], 1)) + seq1 = torch.cat([seq1f, seq1b], 2) + seq2 = torch.cat([seq2f, seq2b], 2) seq1 = seq1.squeeze(0) # batch size assumed to be 1 seq2 = seq2.squeeze(0) sim_cube = self.compute_sim_cube(seq1, seq2) From 901ce4d6a4aaaf76a40864461752fdbd0c622533 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Mon, 5 Feb 2018 15:25:59 -0500 Subject: [PATCH 09/12] Add tensorboard visualization --- .gitignore | 1 + vdpwi/__main__.py | 64 +++++++++++++++++++++++++++++------------ vdpwi/data.py | 37 +++++++++++++++--------- vdpwi/model.py | 28 ++++++++---------- vdpwi/utils/__init__.py | 0 vdpwi/utils/log.py | 26 +++++++++++++++++ 6 files changed, 108 insertions(+), 48 deletions(-) create mode 100644 vdpwi/utils/__init__.py create mode 100644 vdpwi/utils/log.py diff --git a/.gitignore b/.gitignore index 31104a5..96fe215 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.sublime-workspace local_* +runs # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py index f343d44..99a8e03 100644 --- a/vdpwi/__main__.py +++ b/vdpwi/__main__.py @@ -9,10 +9,11 @@ import torch.nn as nn import torch.nn.functional as F import torch.utils as utils +from utils.log import LogWriter import data import model as mod -Context = namedtuple("Context", "model, train_loader, dev_loader, test_loader, optimizer, criterion, params") +Context = namedtuple("Context", "model, train_loader, dev_loader, test_loader, optimizer, criterion, params, log_writer") EvaluateResult = namedtuple("EvaluateResult", "pearsonr, spearmanr") def create_context(config): @@ -20,10 +21,12 @@ def create_context(config): emb1 = [] emb2 = [] labels = [] - for s1, s2, l in batch: + cmp_labels = [] + for s1, s2, l, cl in batch: emb1.append(s1) emb2.append(s2) labels.append(l) + cmp_labels.append(cl) emb1 = torch.LongTensor(emb1) emb2 = torch.LongTensor(emb2) labels = torch.Tensor(labels) @@ -34,7 +37,7 @@ def create_context(config): emb1 = emb1.cuda() emb2 = emb2.cuda() labels = labels.cuda() - return emb1, emb2, labels + return emb1, emb2, labels, cmp_labels embedding, (train_set, dev_set, test_set) = data.load_dataset(config.dataset) model = mod.VDPWIModel(embedding, config) @@ -48,42 +51,65 @@ def create_context(config): test_loader = utils.data.DataLoader(test_set, batch_size=1, collate_fn=collate_fn) params = list(filter(lambda x: x.requires_grad, model.parameters())) - optimizer = optim.RMSprop(params, lr=config.lr, alpha=config.decay, momentum=config.momentum) + optimizer = optim.Adam(params, lr=config.lr, weight_decay=config.weight_decay) + # optimizer = optim.SGD(params, lr=config.lr, momentum=config.momentum, weight_decay=config.weight_decay) criterion = nn.KLDivLoss() - return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion, params) + log_writer = LogWriter() + return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion, params, log_writer) def test(config): - pass + context = create_context(config) + result = evaluate(context, context.test_loader) + print("Final test result: {}".format(result)) -def evaluate(model, data_loader): +def evaluate(context, data_loader): + model = context.model model.eval() predictions = [] true_labels = [] - for sent1, sent2, label_pmf in data_loader: + for sent1, sent2, _, truth in data_loader: scores = model(sent1, sent2) scores = F.softmax(scores).cpu().data.numpy()[0] prediction = np.dot(np.arange(1, len(scores) + 1), scores) - truth = np.dot(np.arange(1, len(scores) + 1), label_pmf.cpu().data.numpy()[0]) - predictions.append(prediction); true_labels.append(truth) - return EvaluateResult(stats.pearsonr(predictions, true_labels)[0], stats.spearmanr(predictions, true_labels)[0]) + predictions.append(prediction); true_labels.append(truth[0][0]) + + pearsonr = stats.pearsonr(predictions, true_labels)[0] + spearmanr = stats.spearmanr(predictions, true_labels)[0] + context.log_writer.log_dev_metrics(pearsonr, spearmanr) + return EvaluateResult(pearsonr, spearmanr) def train(config): context = create_context(config) + context.log_writer.log_hyperparams() + best_dev_pr = 0 for epoch_no in range(config.n_epochs): print("Epoch number: {}".format(epoch_no + 1)) loader_wrapper = tqdm(enumerate(context.train_loader), total=len(context.train_loader), desc="Loss") context.model.train() - for i, (sent1, sent2, label_pmf) in loader_wrapper: + loss = 0 + for i, (sent1, sent2, label_pmf, _) in loader_wrapper: context.optimizer.zero_grad() scores = F.log_softmax(context.model(sent1, sent2)) - loss = context.criterion(scores, label_pmf) - loss.backward() - nn.utils.clip_grad_norm(context.params, 50) - loader_wrapper.set_description("Loss = {}".format(loss.cpu().data[0])) - context.optimizer.step() - result = evaluate(context.model, context.dev_loader) - print(result) + loss = context.criterion(scores, label_pmf) + loss + if i % config.mbatch_size == (config.mbatch_size - 1): + loss /= config.mbatch_size + loss.backward() + nn.utils.clip_grad_norm(context.params, 5) + context.optimizer.step() + + loss = loss.cpu().data[0] + loader_wrapper.set_description("Loss: {:<8}".format(round(loss, 5))) + context.log_writer.log_train_loss(loss) + loss = 0 + result = evaluate(context, context.dev_loader) + print("Dev result: {}".format(result)) + if best_dev_pr < result.pearsonr: + best_dev_pr = result.pearsonr + print("Saving best model...") + context.model.save(config.output_file) + test_result = evaluate(context, context.test_loader) + print("Final test result: {}".format(test_result)) def main(): config = data.Configs.base_config() diff --git a/vdpwi/data.py b/vdpwi/data.py index a67aa8f..4180f03 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -12,16 +12,18 @@ class Configs(object): parser.add_argument("--cpu", action="store_true", default=False) parser.add_argument("--dataset", type=str, default="sick", choices=["sick"]) parser.add_argument("--decay", type=float, default=0.95) - parser.add_argument("--input_model", type=str, default="local_saves/model.pt") - parser.add_argument("--lr", type=float, default=1E-4) - parser.add_argument("--mbatch_size", type=int, default=1) + parser.add_argument("--input_file", type=str, default="local_saves/model.pt") + parser.add_argument("--lr", type=float, default=1E-3) + parser.add_argument("--mbatch_size", type=int, default=16) parser.add_argument("--mode", type=str, default="train", choices=["train", "test"]) parser.add_argument("--momentum", type=float, default=0.9) parser.add_argument("--n_epochs", type=int, default=40) parser.add_argument("--n_labels", type=int, default=5) - parser.add_argument("--output_model", type=str, default="local_saves/model.pt") + parser.add_argument("--optimizer", type=str, default="adam", choices=["adam", "sgd", "rmsprop"]) + parser.add_argument("--output_file", type=str, default="local_saves/model.pt") parser.add_argument("--restore", action="store_true", default=False) parser.add_argument("--rnn_hidden_dim", type=int, default=250) + parser.add_argument("--weight_decay", type=float, default=5E-4) parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt") return parser.parse_known_args()[0] @@ -34,14 +36,16 @@ class Configs(object): return parser.parse_known_args()[0] class LabeledEmbeddedDataset(data.Dataset): - def __init__(self, sentence_indices1, sentence_indices2, labels): + def __init__(self, sentence_indices1, sentence_indices2, labels, compare_labels=None): assert len(sentence_indices1) == len(labels) == len(sentence_indices2) self.sentence_indices1 = sentence_indices1 self.sentence_indices2 = sentence_indices2 self.labels = labels + self.compare_labels = compare_labels def __getitem__(self, idx): - return self.sentence_indices1[idx], self.sentence_indices2[idx], self.labels[idx] + cmp_lbl = None if self.compare_labels is None else self.compare_labels[idx] + return self.sentence_indices1[idx], self.sentence_indices2[idx], self.labels[idx], cmp_lbl def __len__(self): return len(self.labels) @@ -53,10 +57,18 @@ def load_sick(): filename = os.path.join(config.sick_data, dataset, name) with open(filename) as f: for line in f: - indices = [embed_ids.get(word, padding_idx) for word in line.strip().split()] + indices = [embed_ids.get(word, -1) for word in line.strip().split()] + indices = list(filter(lambda x: x >= 0, indices)) sentence_indices.append(indices) return sentence_indices + def read_labels(filename): + labels = [] + with open(filename) as f: + for line in f: + labels.append([float(val) for val in line.split()]) + return labels + sets = [] embeddings = [] embed_ids = {} @@ -70,14 +82,13 @@ def load_sick(): embeddings.append([0.0] * 300) for dataset in ("train", "dev", "test"): - filename = os.path.join(config.sick_data, dataset, "sim_sparse.txt") - labels = [] - with open(filename) as f: - for line in f: - labels.append([float(val) for val in line.split()]) + sparse_filename = os.path.join(config.sick_data, dataset, "sim_sparse.txt") + truth_filename = os.path.join(config.sick_data, dataset, "sim.txt") + sparse_labels = read_labels(sparse_filename) + cmp_labels = read_labels(truth_filename) indices1 = fetch_indices("a.toks") indices2 = fetch_indices("b.toks") - sets.append(LabeledEmbeddedDataset(indices1, indices2, labels)) + sets.append(LabeledEmbeddedDataset(indices1, indices2, sparse_labels, cmp_labels)) embedding = nn.Embedding(len(embeddings), 300) embedding.weight.data.copy_(torch.Tensor(embeddings)) embedding.weight.requires_grad = False diff --git a/vdpwi/model.py b/vdpwi/model.py index 6b2f1ba..9d54e9b 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -17,7 +17,7 @@ class SerializableModule(nn.Module): class VDPWIConvNet(SerializableModule): def __init__(self, n_labels): super().__init__() - self.conv1 = nn.Conv2d(13, 128, 3, padding=1) + self.conv1 = nn.Conv2d(12, 128, 3, padding=1) self.conv2 = nn.Conv2d(128, 164, 3, padding=1) self.conv3 = nn.Conv2d(164, 192, 3, padding=1) self.conv4 = nn.Conv2d(192, 192, 3, padding=1) @@ -25,20 +25,16 @@ class VDPWIConvNet(SerializableModule): self.maxpool2 = nn.MaxPool2d(2, ceil_mode=True) self.dnn = nn.Linear(128, 128) self.output = nn.Linear(128, n_labels) + self.input_len = 32 def forward(self, x): - def pad_side(idx, max_size): - if max_size <= 32: - pad_len = 32 - x.size(idx) - elif max_size <= 48: - pad_len = 48 - x.size(idx) - else: - pad_len = 0 + def pad_side(idx): + pad_len = max(32 - x.size(idx), 0) return [0, pad_len] - padding = pad_side(3, max(x.size()[2:])) - padding.extend(pad_side(2, max(x.size()[2:]))) + padding = pad_side(3) + padding.extend(pad_side(2)) x = F.pad(x, padding) - x = x[:, :, :48, :48] + x = x[:, :, :32, :32] pool_final = nn.MaxPool2d(2, ceil_mode=True) if x.size(2) == 32 else nn.MaxPool2d(3, 1, ceil_mode=True) x = self.maxpool2(F.relu(self.conv1(x))) @@ -58,7 +54,7 @@ class VDPWIModel(SerializableModule): self.use_cuda = not config.cpu self.classifier_net = VDPWIConvNet(config.n_labels) if classifier_net is None else classifier_net - def compute_sim_cube(self, seq1, seq2): + def compute_sim_cube(self, seq1, seq2, truncate=None): def compute_sim(prism1, prism2): prism1_len = prism1.norm(dim=2) prism2_len = prism2.norm(dim=2) @@ -76,8 +72,7 @@ class VDPWIModel(SerializableModule): prism2 = prism2.permute(0, 1, 2).contiguous() return compute_sim(prism1, prism2) - sim_cube = Variable(torch.Tensor(13, seq1.size(0), seq2.size(0))) - sim_cube[12] = 0 + sim_cube = Variable(torch.Tensor(12, seq1.size(0), seq2.size(0))) if self.use_cuda: sim_cube = sim_cube.cuda() seq1_f = seq1[:, :self.hidden_dim] @@ -88,6 +83,8 @@ class VDPWIModel(SerializableModule): sim_cube[3:6] = compute_prism(seq1_f, seq2_f) sim_cube[6:9] = compute_prism(seq1_b, seq2_b) sim_cube[9:12] = compute_prism(seq1_f + seq1_b, seq2_f + seq2_b) + if truncate is not None: + sim_cube = sim_cube[:, :truncate, :truncate].contiguous() return sim_cube def compute_focus_cube(self, sim_cube): @@ -108,7 +105,6 @@ class VDPWIModel(SerializableModule): mask[:, int(pos1), int(pos2)] = 1 build_mask(9) build_mask(10) - mask[12, :, :] = 1 return mask * sim_cube def forward(self, x1, x2): @@ -122,7 +118,7 @@ class VDPWIModel(SerializableModule): seq2 = torch.cat([seq2f, seq2b], 2) seq1 = seq1.squeeze(0) # batch size assumed to be 1 seq2 = seq2.squeeze(0) - sim_cube = self.compute_sim_cube(seq1, seq2) + sim_cube = self.compute_sim_cube(seq1, seq2, truncate=self.classifier_net.input_len) focus_cube = self.compute_focus_cube(sim_cube) logits = self.classifier_net(focus_cube.unsqueeze(0)) return logits diff --git a/vdpwi/utils/__init__.py b/vdpwi/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vdpwi/utils/log.py b/vdpwi/utils/log.py new file mode 100644 index 0000000..82f5eef --- /dev/null +++ b/vdpwi/utils/log.py @@ -0,0 +1,26 @@ +import datetime +import sys + +from tensorboardX import SummaryWriter + +class LogWriter(object): + def __init__(self, run_name_fmt="run_{}"): + self.writer = SummaryWriter() + self.run_name = run_name_fmt.format(datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S")) + self.train_idx = 0 + self.dev_idx = 0 + + def log_hyperparams(self): + self.writer.add_text("{}/hyperparams".format(self.run_name), " ".join(sys.argv)) + + def log_train_loss(self, loss): + self.writer.add_scalar("{}/train_loss".format(self.run_name), loss, self.train_idx) + self.train_idx += 1 + + def log_dev_metrics(self, pearsonr, spearmanr): + results = dict(pearsonr=pearsonr, spearmanr=spearmanr) + self.writer.add_scalars("{}/dev_metrics".format(self.run_name), results, self.dev_idx) + self.dev_idx += 1 + + def next(self): + self.i += 1 From 4b266431ffb077ed50be619cb6142030faf70337 Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Mon, 5 Feb 2018 17:57:52 -0500 Subject: [PATCH 10/12] Add hyperparameter tuning script --- vdpwi/__main__.py | 12 +++++----- vdpwi/data.py | 6 ++++- vdpwi/model.py | 54 +++++++++++++++++++++++++++++++++++---------- vdpwi/utils/tune.py | 38 +++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 18 deletions(-) create mode 100644 vdpwi/utils/tune.py diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py index 99a8e03..d174bb0 100644 --- a/vdpwi/__main__.py +++ b/vdpwi/__main__.py @@ -51,8 +51,12 @@ def create_context(config): test_loader = utils.data.DataLoader(test_set, batch_size=1, collate_fn=collate_fn) params = list(filter(lambda x: x.requires_grad, model.parameters())) - optimizer = optim.Adam(params, lr=config.lr, weight_decay=config.weight_decay) - # optimizer = optim.SGD(params, lr=config.lr, momentum=config.momentum, weight_decay=config.weight_decay) + if config.optimizer == "adam": + optimizer = optim.Adam(params, lr=config.lr, weight_decay=config.weight_decay) + elif config.optimizer == "sgd": + optimizer = optim.SGD(params, lr=config.lr, momentum=config.momentum, weight_decay=config.weight_decay) + elif config.optimizer == "rmsprop": + optimizer = optim.RMSprop(params, lr=config.lr, alpha=config.decay, momentum=config.momentum, weight_decay=config.weight_decay) criterion = nn.KLDivLoss() log_writer = LogWriter() return Context(model, train_loader, dev_loader, test_loader, optimizer, criterion, params, log_writer) @@ -95,7 +99,7 @@ def train(config): if i % config.mbatch_size == (config.mbatch_size - 1): loss /= config.mbatch_size loss.backward() - nn.utils.clip_grad_norm(context.params, 5) + nn.utils.clip_grad_norm(context.params, config.clip_norm) context.optimizer.step() loss = loss.cpu().data[0] @@ -108,8 +112,6 @@ def train(config): best_dev_pr = result.pearsonr print("Saving best model...") context.model.save(config.output_file) - test_result = evaluate(context, context.test_loader) - print("Final test result: {}".format(test_result)) def main(): config = data.Configs.base_config() diff --git a/vdpwi/data.py b/vdpwi/data.py index 4180f03..b9c4ed3 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -9,6 +9,8 @@ class Configs(object): @staticmethod def base_config(): parser = argparse.ArgumentParser() + parser.add_argument("--classifier", type=str, default="vdpwi", choices=["vdpwi", "resnet"]) + parser.add_argument("--clip_norm", type=float, default=5) parser.add_argument("--cpu", action="store_true", default=False) parser.add_argument("--dataset", type=str, default="sick", choices=["sick"]) parser.add_argument("--decay", type=float, default=0.95) @@ -17,10 +19,12 @@ class Configs(object): parser.add_argument("--mbatch_size", type=int, default=16) parser.add_argument("--mode", type=str, default="train", choices=["train", "test"]) parser.add_argument("--momentum", type=float, default=0.9) - parser.add_argument("--n_epochs", type=int, default=40) + parser.add_argument("--n_epochs", type=int, default=35) parser.add_argument("--n_labels", type=int, default=5) parser.add_argument("--optimizer", type=str, default="adam", choices=["adam", "sgd", "rmsprop"]) parser.add_argument("--output_file", type=str, default="local_saves/model.pt") + parser.add_argument("--res_fmaps", type=int, default=32) + parser.add_argument("--res_layers", type=int, default=16) parser.add_argument("--restore", action="store_true", default=False) parser.add_argument("--rnn_hidden_dim", type=int, default=250) parser.add_argument("--weight_decay", type=float, default=5E-4) diff --git a/vdpwi/model.py b/vdpwi/model.py index 9d54e9b..7d11d12 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -2,6 +2,7 @@ from torch.autograd import Variable import torch import torch.nn as nn import torch.nn.functional as F +import torchvision.models as models import numpy as np class SerializableModule(nn.Module): @@ -14,8 +15,41 @@ class SerializableModule(nn.Module): def load(self, filename): self.load_state_dict(torch.load(filename, map_location=lambda storage, loc: storage)) +def hard_pad2d(x, pad): + def pad_side(idx): + pad_len = max(pad - x.size(idx), 0) + return [0, pad_len] + padding = pad_side(3) + padding.extend(pad_side(2)) + x = F.pad(x, padding) + return x[:, :, :pad, :pad] + +class ResNet(SerializableModule): + def __init__(self, config): + super().__init__() + n_layers = config.res_layers + n_maps = config.res_fmaps + n_labels = config.n_labels + self.conv0 = nn.Conv2d(12, n_maps, (3, 3), padding=1) + self.convs = [nn.Conv2d(n_maps, n_maps, (3, 3), padding=1) for _ in range(n_layers)] + self.output = nn.Linear(n_maps, n_labels) + self.input_len = None + for i, conv in enumerate(self.convs): + self.add_module("conv{}".format(i + 1), conv) + + def forward(self, x): + x = F.relu(self.conv0(x)) + old_x = x + for i, conv in enumerate(self.convs): + x = F.relu(conv(x)) + if i % 2 == 1: + x += old_x + old_x = x + x = torch.mean(x.view(x.size(0), x.size(1), -1), 2) + return self.output(x) + class VDPWIConvNet(SerializableModule): - def __init__(self, n_labels): + def __init__(self, config): super().__init__() self.conv1 = nn.Conv2d(12, 128, 3, padding=1) self.conv2 = nn.Conv2d(128, 164, 3, padding=1) @@ -24,18 +58,11 @@ class VDPWIConvNet(SerializableModule): self.conv5 = nn.Conv2d(192, 128, 3, padding=1) self.maxpool2 = nn.MaxPool2d(2, ceil_mode=True) self.dnn = nn.Linear(128, 128) - self.output = nn.Linear(128, n_labels) + self.output = nn.Linear(128, config.n_labels) self.input_len = 32 def forward(self, x): - def pad_side(idx): - pad_len = max(32 - x.size(idx), 0) - return [0, pad_len] - padding = pad_side(3) - padding.extend(pad_side(2)) - x = F.pad(x, padding) - x = x[:, :, :32, :32] - + x = hard_pad2d(x, self.input_len) pool_final = nn.MaxPool2d(2, ceil_mode=True) if x.size(2) == 32 else nn.MaxPool2d(3, 1, ceil_mode=True) x = self.maxpool2(F.relu(self.conv1(x))) x = self.maxpool2(F.relu(self.conv2(x))) @@ -46,13 +73,16 @@ class VDPWIConvNet(SerializableModule): return self.output(x) class VDPWIModel(SerializableModule): - def __init__(self, embedding, config, classifier_net=None): + def __init__(self, embedding, config): super().__init__() self.hidden_dim = config.rnn_hidden_dim self.rnn = nn.LSTM(300, self.hidden_dim, 1, batch_first=True) self.embedding = embedding self.use_cuda = not config.cpu - self.classifier_net = VDPWIConvNet(config.n_labels) if classifier_net is None else classifier_net + if config.classifier == "vdpwi": + self.classifier_net = VDPWIConvNet(config) + elif config.classifier == "resnet": + self.classifier_net = ResNet(config) def compute_sim_cube(self, seq1, seq2, truncate=None): def compute_sim(prism1, prism2): diff --git a/vdpwi/utils/tune.py b/vdpwi/utils/tune.py new file mode 100644 index 0000000..12e4437 --- /dev/null +++ b/vdpwi/utils/tune.py @@ -0,0 +1,38 @@ +import os +import random + +class RandomParamIterator(object): + def __init__(self, param_sets): + self.param_sets = param_sets + + def random_param_set(self): + param_set = {} + for param_key, param_values in self.param_sets.items(): + param_set[param_key] = random.choice(param_values) + return param_set + +class Tuner(object): + def __init__(self, *iterators, limit=100): + self.iterators = iterators + self.limit = limit + + def start(self): + for i in range(self.limit): + iterator = random.choice(self.iterators) + params = iterator.random_param_set() + print(params) + arg_str = " ".join("--{}={}".format(k, v) for k, v in params.items()) + os.system("python . {} --output_file local_saves/model{}.pt".format(arg_str, i)) + +def main(): + vgg_param_sets = dict(classifer=["vdpwi"], clip_norm=[3, 5, 7], decay=[0.9, 0.95], lr=[5E-3, 1E-3, 5E-4], + mbatch_size=[8, 16, 32], optimizer=["adam", "rmsprop"], rnn_hidden_dim=[150, 250, 300], + weight_decay=[0, 5E-4, 1E-3]) + res_param_sets = dict(classifier=["resnet"], clip_norm=[3, 5, 7], decay=[0.9, 0.95], lr=[5E-3, 1E-3, 5E-4], + mbatch_size=[8, 16, 32], rnn_hidden_dim=[150, 250, 300], res_fmaps=[16, 24, 32], res_layers=[4, 8, 16, 24]) + vgg_iterator = RandomParamIterator(vgg_param_sets) + res_iterator = RandomParamIterator(res_param_sets) + Tuner(vgg_iterator, res_iterator).start() + +if __name__ == "__main__": + main() \ No newline at end of file From 73823fcc32922649906ba11bbbc8bc2a0bdfb82a Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Tue, 6 Feb 2018 19:49:12 -0500 Subject: [PATCH 11/12] Make model parallel wrt batch size --- vdpwi/__main__.py | 49 ++++++++++++------- vdpwi/data.py | 10 ++-- vdpwi/model.py | 106 +++++++++++++++++++++++------------------ vdpwi/train_default.sh | 2 + vdpwi/utils/tune.py | 11 ++--- 5 files changed, 104 insertions(+), 74 deletions(-) create mode 100755 vdpwi/train_default.sh diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py index d174bb0..05a56be 100644 --- a/vdpwi/__main__.py +++ b/vdpwi/__main__.py @@ -22,22 +22,40 @@ def create_context(config): emb2 = [] labels = [] cmp_labels = [] + pad_cube = [] + max_len1 = 0; max_len2 = 0 + for s1, s2, l, cl in batch: emb1.append(s1) emb2.append(s2) + max_len1 = max(max_len1, len(s1)) + max_len2 = max(max_len2, len(s2)) labels.append(l) cmp_labels.append(cl) + + for s1, s2 in zip(emb1, emb2): + pad1 = (max_len1 - len(s1)) + pad2 = (max_len2 - len(s2)) + pad_mask = np.ones((max_len1, max_len2)) + pad_mask[:len(s1), :len(s2)] = 0 + pad_cube.append(pad_mask) + s1.extend([embedding.weight.size(0) - 1] * pad1) + s2.extend([embedding.weight.size(0) - 1] * pad2) + + pad_cube = np.array(pad_cube) emb1 = torch.LongTensor(emb1) emb2 = torch.LongTensor(emb2) labels = torch.Tensor(labels) emb1 = torch.autograd.Variable(emb1, requires_grad=False) emb2 = torch.autograd.Variable(emb2, requires_grad=False) labels = torch.autograd.Variable(labels, requires_grad=False) + pad_cube = torch.autograd.Variable(torch.from_numpy(pad_cube).float(), requires_grad=False) if not config.cpu: emb1 = emb1.cuda() emb2 = emb2.cuda() labels = labels.cuda() - return emb1, emb2, labels, cmp_labels + pad_cube = pad_cube.cuda() + return emb1, emb2, labels, pad_cube, cmp_labels embedding, (train_set, dev_set, test_set) = data.load_dataset(config.dataset) model = mod.VDPWIModel(embedding, config) @@ -46,7 +64,7 @@ def create_context(config): if not config.cpu: model = model.cuda() - train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=1, collate_fn=collate_fn) + train_loader = utils.data.DataLoader(train_set, shuffle=True, batch_size=config.mbatch_size, collate_fn=collate_fn) dev_loader = utils.data.DataLoader(dev_set, batch_size=1, collate_fn=collate_fn) test_loader = utils.data.DataLoader(test_set, batch_size=1, collate_fn=collate_fn) @@ -71,8 +89,8 @@ def evaluate(context, data_loader): model.eval() predictions = [] true_labels = [] - for sent1, sent2, _, truth in data_loader: - scores = model(sent1, sent2) + for sent1, sent2, _, pad_cube, truth in data_loader: + scores = model(sent1, sent2, pad_cube) scores = F.softmax(scores).cpu().data.numpy()[0] prediction = np.dot(np.arange(1, len(scores) + 1), scores) predictions.append(prediction); true_labels.append(truth[0][0]) @@ -88,24 +106,21 @@ def train(config): best_dev_pr = 0 for epoch_no in range(config.n_epochs): print("Epoch number: {}".format(epoch_no + 1)) - loader_wrapper = tqdm(enumerate(context.train_loader), total=len(context.train_loader), desc="Loss") + loader_wrapper = tqdm(context.train_loader, total=len(context.train_loader), desc="Loss") context.model.train() loss = 0 - for i, (sent1, sent2, label_pmf, _) in loader_wrapper: + for sent1, sent2, label_pmf, pad_cube, _ in loader_wrapper: context.optimizer.zero_grad() - scores = F.log_softmax(context.model(sent1, sent2)) + scores = F.log_softmax(context.model(sent1, sent2, pad_cube)) - loss = context.criterion(scores, label_pmf) + loss - if i % config.mbatch_size == (config.mbatch_size - 1): - loss /= config.mbatch_size - loss.backward() - nn.utils.clip_grad_norm(context.params, config.clip_norm) - context.optimizer.step() + loss = context.criterion(scores, label_pmf) + loss.backward() + nn.utils.clip_grad_norm(context.params, config.clip_norm) + context.optimizer.step() - loss = loss.cpu().data[0] - loader_wrapper.set_description("Loss: {:<8}".format(round(loss, 5))) - context.log_writer.log_train_loss(loss) - loss = 0 + loss = loss.cpu().data[0] + loader_wrapper.set_description("Loss: {:<8}".format(round(loss, 5))) + context.log_writer.log_train_loss(loss) result = evaluate(context, context.dev_loader) print("Dev result: {}".format(result)) if best_dev_pr < result.pearsonr: diff --git a/vdpwi/data.py b/vdpwi/data.py index b9c4ed3..04cd560 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -10,24 +10,24 @@ class Configs(object): def base_config(): parser = argparse.ArgumentParser() parser.add_argument("--classifier", type=str, default="vdpwi", choices=["vdpwi", "resnet"]) - parser.add_argument("--clip_norm", type=float, default=5) + parser.add_argument("--clip_norm", type=float, default=50) parser.add_argument("--cpu", action="store_true", default=False) parser.add_argument("--dataset", type=str, default="sick", choices=["sick"]) parser.add_argument("--decay", type=float, default=0.95) parser.add_argument("--input_file", type=str, default="local_saves/model.pt") - parser.add_argument("--lr", type=float, default=1E-3) + parser.add_argument("--lr", type=float, default=5E-4) parser.add_argument("--mbatch_size", type=int, default=16) parser.add_argument("--mode", type=str, default="train", choices=["train", "test"]) - parser.add_argument("--momentum", type=float, default=0.9) + parser.add_argument("--momentum", type=float, default=0.1) parser.add_argument("--n_epochs", type=int, default=35) parser.add_argument("--n_labels", type=int, default=5) - parser.add_argument("--optimizer", type=str, default="adam", choices=["adam", "sgd", "rmsprop"]) + parser.add_argument("--optimizer", type=str, default="rmsprop", choices=["adam", "sgd", "rmsprop"]) parser.add_argument("--output_file", type=str, default="local_saves/model.pt") parser.add_argument("--res_fmaps", type=int, default=32) parser.add_argument("--res_layers", type=int, default=16) parser.add_argument("--restore", action="store_true", default=False) parser.add_argument("--rnn_hidden_dim", type=int, default=250) - parser.add_argument("--weight_decay", type=float, default=5E-4) + parser.add_argument("--weight_decay", type=float, default=1E-5) parser.add_argument("--wordvecs_file", type=str, default="local_data/glove/glove.840B.300d.txt") return parser.parse_known_args()[0] diff --git a/vdpwi/model.py b/vdpwi/model.py index 7d11d12..c31bcbc 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -51,11 +51,16 @@ class ResNet(SerializableModule): class VDPWIConvNet(SerializableModule): def __init__(self, config): super().__init__() - self.conv1 = nn.Conv2d(12, 128, 3, padding=1) - self.conv2 = nn.Conv2d(128, 164, 3, padding=1) - self.conv3 = nn.Conv2d(164, 192, 3, padding=1) - self.conv4 = nn.Conv2d(192, 192, 3, padding=1) - self.conv5 = nn.Conv2d(192, 128, 3, padding=1) + def make_conv(n_in, n_out): + conv = nn.Conv2d(n_in, n_out, 3, padding=1) + conv.bias.data.zero_() + nn.init.xavier_normal(conv.weight) + return conv + self.conv1 = make_conv(12, 128) + self.conv2 = make_conv(128, 164) + self.conv3 = make_conv(164, 192) + self.conv4 = make_conv(192, 192) + self.conv5 = make_conv(192, 128) self.maxpool2 = nn.MaxPool2d(2, ceil_mode=True) self.dnn = nn.Linear(128, 128) self.output = nn.Linear(128, config.n_labels) @@ -84,60 +89,67 @@ class VDPWIModel(SerializableModule): elif config.classifier == "resnet": self.classifier_net = ResNet(config) - def compute_sim_cube(self, seq1, seq2, truncate=None): + def compute_sim_cube(self, seq1, seq2): def compute_sim(prism1, prism2): - prism1_len = prism1.norm(dim=2) - prism2_len = prism2.norm(dim=2) + prism1_len = prism1.norm(dim=3) + prism2_len = prism2.norm(dim=3) - dot_prod = torch.matmul(prism1.unsqueeze(2), prism2.unsqueeze(3)) - dot_prod = dot_prod.squeeze(2).squeeze(2) + dot_prod = torch.matmul(prism1.unsqueeze(3), prism2.unsqueeze(4)) + dot_prod = dot_prod.squeeze(3).squeeze(3) cos_dist = dot_prod / (prism1_len * prism2_len + 1E-8) - l2_dist = (prism1 - prism2).norm(dim=2) - return torch.stack([dot_prod, cos_dist, l2_dist], 0) + l2_dist = -((prism1 - prism2).norm(dim=3)) + return torch.stack([dot_prod, cos_dist, l2_dist], 1) def compute_prism(seq1, seq2): - prism1 = seq1.repeat(seq2.size(0), 1, 1) - prism2 = seq2.repeat(seq1.size(0), 1, 1) - prism1 = prism1.permute(1, 0, 2).contiguous() - prism2 = prism2.permute(0, 1, 2).contiguous() + prism1 = seq1.repeat(seq2.size(1), 1, 1, 1) + prism2 = seq2.repeat(seq1.size(1), 1, 1, 1) + prism1 = prism1.permute(1, 2, 0, 3).contiguous() + prism2 = prism2.permute(1, 0, 2, 3).contiguous() return compute_sim(prism1, prism2) - sim_cube = Variable(torch.Tensor(12, seq1.size(0), seq2.size(0))) + sim_cube = Variable(torch.Tensor(seq1.size(0), 12, seq1.size(1), seq2.size(1))) if self.use_cuda: sim_cube = sim_cube.cuda() - seq1_f = seq1[:, :self.hidden_dim] - seq1_b = seq1[:, self.hidden_dim:] - seq2_f = seq2[:, :self.hidden_dim] - seq2_b = seq2[:, self.hidden_dim:] - sim_cube[0:3] = compute_prism(seq1, seq2) - sim_cube[3:6] = compute_prism(seq1_f, seq2_f) - sim_cube[6:9] = compute_prism(seq1_b, seq2_b) - sim_cube[9:12] = compute_prism(seq1_f + seq1_b, seq2_f + seq2_b) - if truncate is not None: - sim_cube = sim_cube[:, :truncate, :truncate].contiguous() + seq1_f = seq1[:, :, :self.hidden_dim] + seq1_b = seq1[:, :, self.hidden_dim:] + seq2_f = seq2[:, :, :self.hidden_dim] + seq2_b = seq2[:, :, self.hidden_dim:] + sim_cube[:, 0:3] = compute_prism(seq1, seq2) + sim_cube[:, 3:6] = compute_prism(seq1_f, seq2_f) + sim_cube[:, 6:9] = compute_prism(seq1_b, seq2_b) + sim_cube[:, 9:12] = compute_prism(seq1_f + seq1_b, seq2_f + seq2_b) return sim_cube - def compute_focus_cube(self, sim_cube): + def compute_focus_cube(self, sim_cube, pad_cube): + neg_magic = -10000 + pad_cube = pad_cube.repeat(12, 1, 1, 1) + pad_cube = pad_cube.permute(1, 0, 2, 3).contiguous() + sim_cube = neg_magic * pad_cube + sim_cube mask = Variable(torch.Tensor(*sim_cube.size())) if self.use_cuda: mask = mask.cuda() - mask[:, :, :] = 0.1 + mask[:, :, :, :] = 0.1 + def build_mask(index): - s1tag = np.zeros(sim_cube.size(1)) - s2tag = np.zeros(sim_cube.size(2)) - _, indices = torch.sort(sim_cube[index].view(-1), descending=True) - for i, index in enumerate(indices.cpu().data.numpy()): - if i >= len(s1tag) + len(s2tag): - break - pos1, pos2 = index // len(s2tag), index % len(s2tag) - if s1tag[pos1] + s2tag[pos2] == 0: - s1tag[pos1] = s2tag[pos2] = 1 - mask[:, int(pos1), int(pos2)] = 1 + max_mask = sim_cube[:, index].clone() + for _ in range(min(sim_cube.size(2), sim_cube.size(3))): + values, indices = torch.max(max_mask.view(sim_cube.size(0), -1), 1) + row_indices = indices / sim_cube.size(3) + col_indices = indices % sim_cube.size(3) + row_indices = row_indices.unsqueeze(1) + col_indices = col_indices.unsqueeze(1).unsqueeze(1) + for i, (row_i, col_i, val) in enumerate(zip(row_indices, col_indices, values)): + if val < neg_magic / 2: + continue + mask[i, :, row_i, col_i] = 1 + max_mask[i, row_i, :] = neg_magic + max_mask[i, :, col_i] = neg_magic build_mask(9) build_mask(10) - return mask * sim_cube + focus_cube = mask * sim_cube * (1 - pad_cube) + return focus_cube - def forward(self, x1, x2): + def forward(self, x1, x2, pad_cube): x1 = self.embedding(x1) x2 = self.embedding(x2) seq1f, _ = self.rnn(x1) @@ -146,9 +158,11 @@ class VDPWIModel(SerializableModule): seq2b, _ = self.rnn(torch.cat(x2.split(1, 1)[::-1], 1)) seq1 = torch.cat([seq1f, seq1b], 2) seq2 = torch.cat([seq2f, seq2b], 2) - seq1 = seq1.squeeze(0) # batch size assumed to be 1 - seq2 = seq2.squeeze(0) - sim_cube = self.compute_sim_cube(seq1, seq2, truncate=self.classifier_net.input_len) - focus_cube = self.compute_focus_cube(sim_cube) - logits = self.classifier_net(focus_cube.unsqueeze(0)) + sim_cube = self.compute_sim_cube(seq1, seq2) + truncate = self.classifier_net.input_len + if truncate is not None: + sim_cube = sim_cube[:, :, :truncate, :truncate].contiguous() + pad_cube = pad_cube[:, :truncate, :truncate].contiguous() + focus_cube = self.compute_focus_cube(sim_cube, pad_cube) + logits = self.classifier_net(focus_cube) return logits diff --git a/vdpwi/train_default.sh b/vdpwi/train_default.sh new file mode 100755 index 0000000..230a237 --- /dev/null +++ b/vdpwi/train_default.sh @@ -0,0 +1,2 @@ +#!/bin/sh +python . --clip_norm 50 --decay 0.95 --lr 1E-4 --mbatch_size 1 --momentum 0 --optimizer rmsprop --weight_decay 0 diff --git a/vdpwi/utils/tune.py b/vdpwi/utils/tune.py index 12e4437..bd4df6c 100644 --- a/vdpwi/utils/tune.py +++ b/vdpwi/utils/tune.py @@ -12,7 +12,7 @@ class RandomParamIterator(object): return param_set class Tuner(object): - def __init__(self, *iterators, limit=100): + def __init__(self, *iterators, limit=500): self.iterators = iterators self.limit = limit @@ -25,14 +25,13 @@ class Tuner(object): os.system("python . {} --output_file local_saves/model{}.pt".format(arg_str, i)) def main(): - vgg_param_sets = dict(classifer=["vdpwi"], clip_norm=[3, 5, 7], decay=[0.9, 0.95], lr=[5E-3, 1E-3, 5E-4], - mbatch_size=[8, 16, 32], optimizer=["adam", "rmsprop"], rnn_hidden_dim=[150, 250, 300], - weight_decay=[0, 5E-4, 1E-3]) - res_param_sets = dict(classifier=["resnet"], clip_norm=[3, 5, 7], decay=[0.9, 0.95], lr=[5E-3, 1E-3, 5E-4], + vgg_param_sets = dict(classifer=["vdpwi"], decay=[0.99, 0.95], lr=[5E-4, 1E-4], mbatch_size=[8, 16], + optimizer=["adam", "rmsprop"], weight_decay=[0, 1E-5, 5E-4], momentum=[0, 0.15, 0.05]) + res_param_sets = dict(classifier=["resnet"], clip_norm=[5, 7, 9], decay=[0.9, 0.95], lr=[5E-3, 1E-3, 5E-4], mbatch_size=[8, 16, 32], rnn_hidden_dim=[150, 250, 300], res_fmaps=[16, 24, 32], res_layers=[4, 8, 16, 24]) vgg_iterator = RandomParamIterator(vgg_param_sets) res_iterator = RandomParamIterator(res_param_sets) - Tuner(vgg_iterator, res_iterator).start() + Tuner(vgg_iterator).start() if __name__ == "__main__": main() \ No newline at end of file From 76a99398acd9a4771e22a0265d6cb8b8fd7403ab Mon Sep 17 00:00:00 2001 From: Ralph Tang Date: Wed, 23 May 2018 17:55:34 -0400 Subject: [PATCH 12/12] Remove extraneous files from VDPWI --- LICENSE | 21 --------------------- vdpwi.sublime-project | 8 -------- 2 files changed, 29 deletions(-) delete mode 100644 LICENSE delete mode 100644 vdpwi.sublime-project diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 0f6d923..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2018 Ralph Tang - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vdpwi.sublime-project b/vdpwi.sublime-project deleted file mode 100644 index aefb988..0000000 --- a/vdpwi.sublime-project +++ /dev/null @@ -1,8 +0,0 @@ -{ - "folders": - [ - { - "path": "vdpwi" - } - ] -}