diff --git a/char_cnn/__main__.py b/char_cnn/__main__.py index 0bebe92..da2a3eb 100644 --- a/char_cnn/__main__.py +++ b/char_cnn/__main__.py @@ -17,8 +17,6 @@ from datasets.reuters import ReutersCharQuantized as Reuters from datasets.yelp2014 import Yelp2014CharQuantized as Yelp2014 - - class UnknownWordVecCache(object): """ Caches the first randomly generated word vector for a certain size to make it is reused. diff --git a/char_cnn/args.py b/char_cnn/args.py index 611444a..4e65dae 100644 --- a/char_cnn/args.py +++ b/char_cnn/args.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser def get_args(): parser = ArgumentParser(description="Kim CNN") parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') - parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU + parser.add_argument('--gpu', type=int, default=0, help='Use -1 for CPU') parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--batch_size', type=int, default=128) parser.add_argument('--lr', type=float, default=0.001) diff --git a/char_cnn/model.py b/char_cnn/model.py index af5b3c4..0550a90 100644 --- a/char_cnn/model.py +++ b/char_cnn/model.py @@ -5,8 +5,9 @@ import torch.nn.functional as F class CharCNN(nn.Module): + def __init__(self, config): - super(CharCNN, self).__init__() + super().__init__() self.is_cuda_enabled = config.cuda dataset = config.dataset num_conv_filters = config.num_conv_filters @@ -15,7 +16,7 @@ class CharCNN(nn.Module): target_class = config.target_class input_channel = 68 - self.conv1 = nn.Conv1d(input_channel, num_conv_filters, kernel_size=7) # Default padding=0 + self.conv1 = nn.Conv1d(input_channel, num_conv_filters, kernel_size=7) self.conv2 = nn.Conv1d(num_conv_filters, num_conv_filters, kernel_size=7) self.conv3 = nn.Conv1d(num_conv_filters, num_conv_filters, kernel_size=3) self.conv4 = nn.Conv1d(num_conv_filters, num_conv_filters, kernel_size=3) diff --git a/common/dataset.py b/common/dataset.py index d95408a..cf758d1 100644 --- a/common/dataset.py +++ b/common/dataset.py @@ -72,7 +72,6 @@ class DatasetFactory(object): train_loader, dev_loader, test_loader = PIT2015.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) embedding = nn.Embedding.from_pretrained(PIT2015.TEXT_FIELD.vocab.vectors) return PIT2015, embedding, train_loader, test_loader, dev_loader - elif dataset_name == 'snli': dataset_root = os.path.join(castor_dir, os.pardir, 'Castor-data', 'datasets', 'snli_1.0/') train_loader, dev_loader, test_loader = SNLI.iters(dataset_root, word_vectors_file, word_vectors_dir, batch_size, device=device, unk_init=UnknownWordVecCache.unk) diff --git a/common/evaluation.py b/common/evaluation.py index e930699..866e011 100644 --- a/common/evaluation.py +++ b/common/evaluation.py @@ -44,11 +44,7 @@ class EvaluatorFactory(object): if data_loader is None: return None - if nce: - evaluator_map = EvaluatorFactory.evaluator_map_nce - else: - evaluator_map = EvaluatorFactory.evaluator_map - + evaluator_map = EvaluatorFactory.evaluator_map_nce if nce else EvaluatorFactory.evaluator_map if not hasattr(dataset_cls, 'NAME'): raise ValueError('Invalid dataset. Dataset should have NAME attribute.') diff --git a/conv_rnn/data.py b/conv_rnn/data.py index 4482536..ef33ddd 100644 --- a/conv_rnn/data.py +++ b/conv_rnn/data.py @@ -4,10 +4,13 @@ import re import numpy as np import torch.utils.data as data + def sst_tokenize(sentence): return sentence.split() + class SSTEmbeddingLoader(object): + def __init__(self, dirname, fmt="stsa.fine.{}", word2vec_file="word2vec.sst-1"): self.dirname = dirname self.fmt = fmt @@ -30,7 +33,9 @@ class SSTEmbeddingLoader(object): unk_vocab_set.add(word) return (id_dict, np.array(weights), list(unk_vocab_set)) + class SSTDataset(data.Dataset): + def __init__(self, sentences): super().__init__() self.sentences = sentences diff --git a/conv_rnn/model.py b/conv_rnn/model.py index e5fe4f4..c4bccea 100644 --- a/conv_rnn/model.py +++ b/conv_rnn/model.py @@ -10,6 +10,7 @@ import data class ConvRNNModel(nn.Module): + def __init__(self, word_model, **config): super().__init__() embedding_dim = word_model.dim @@ -97,7 +98,9 @@ class WordEmbeddingModel(nn.Module): def lookup(self, sentences): raise NotImplementedError + class SSTWordEmbeddingModel(WordEmbeddingModel): + def __init__(self, id_dict, weights, unknown_vocab=[]): super().__init__(id_dict, weights, unknown_vocab, padding_idx=16259) @@ -120,6 +123,7 @@ class SSTWordEmbeddingModel(WordEmbeddingModel): indices.extend([self.padding_idx] * (max_len - len(indices))) return indices_list, lengths + def set_seed(seed=0, no_cuda=False): np.random.seed(seed) if not no_cuda: diff --git a/datasets/aapd.py b/datasets/aapd.py index 88ab18d..6c4a6dd 100644 --- a/datasets/aapd.py +++ b/datasets/aapd.py @@ -84,6 +84,7 @@ class AAPDCharQuantized(AAPD): train, val, test = cls.splits(path) return BucketIterator.splits((train, val, test), batch_size=batch_size, repeat=False, shuffle=shuffle, device=device) + class AAPDHierarchical(AAPD): NESTING_FIELD = Field(batch_first=True, tokenize=clean_string) TEXT_FIELD = NestedField(NESTING_FIELD, tokenize=split_sents) diff --git a/datasets/castor_dataset.py b/datasets/castor_dataset.py index d192608..bad9785 100644 --- a/datasets/castor_dataset.py +++ b/datasets/castor_dataset.py @@ -53,7 +53,7 @@ class CastorPairDataset(Dataset, metaclass=ABCMeta): example = Example.fromlist(example_list, fields) examples.append(example) - super(CastorPairDataset, self).__init__(examples, fields) + super().__init__(examples, fields) @classmethod def set_vectors(cls, field, vector_path): diff --git a/datasets/quora.py b/datasets/quora.py index 82ad988..e443370 100644 --- a/datasets/quora.py +++ b/datasets/quora.py @@ -7,6 +7,7 @@ from torchtext.data.pipeline import Pipeline from datasets.castor_dataset import CastorPairDataset + def get_class_probs(sim, *args): """ Convert a single label into class probabilities. diff --git a/datasets/sick.py b/datasets/sick.py index 5daf8b9..95a9eaf 100644 --- a/datasets/sick.py +++ b/datasets/sick.py @@ -42,7 +42,7 @@ class SICK(CastorPairDataset): """ Create a SICK dataset instance """ - super(SICK, self).__init__(path) + super().__init__(path) @classmethod def splits(cls, path, train='train', validation='dev', test='test', **kwargs): diff --git a/datasets/snli.py b/datasets/snli.py index 8ba5c1e..431b2b3 100644 --- a/datasets/snli.py +++ b/datasets/snli.py @@ -7,6 +7,7 @@ from torchtext.data.pipeline import Pipeline from datasets.castor_dataset import CastorPairDataset + def get_class_probs(sim, *args): """ Convert a single label into class probabilities. diff --git a/datasets/trecqa.py b/datasets/trecqa.py index f26ab67..1e8963c 100644 --- a/datasets/trecqa.py +++ b/datasets/trecqa.py @@ -27,11 +27,11 @@ class TRECQA(CastorPairDataset): """ Create a TRECQA dataset instance """ - super(TRECQA, self).__init__(path, load_ext_feats=True) + super().__init__(path, load_ext_feats=True) @classmethod def splits(cls, path, train='train-all', validation='raw-dev', test='raw-test', **kwargs): - return super(TRECQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs) + return super().splits(path, train=train, validation=validation, test=test, **kwargs) @classmethod def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file=False, vectors=None, unk_init=torch.Tensor.zero_): diff --git a/datasets/wikiqa.py b/datasets/wikiqa.py index 359a1dd..fee7a6a 100644 --- a/datasets/wikiqa.py +++ b/datasets/wikiqa.py @@ -31,7 +31,7 @@ class WikiQA(CastorPairDataset): @classmethod def splits(cls, path, train='train', validation='dev', test='test', **kwargs): - return super(WikiQA, cls).splits(path, train=train, validation=validation, test=test, **kwargs) + return super().splits(path, train=train, validation=validation, test=test, **kwargs) @classmethod def iters(cls, path, vectors_name, vectors_dir, batch_size=64, shuffle=True, device=0, pt_file=False, vectors=None, diff --git a/decatt/model.py b/decatt/model.py index 35ee9c5..91c49a7 100644 --- a/decatt/model.py +++ b/decatt/model.py @@ -10,6 +10,7 @@ from torch.autograd import Variable class DecAtt(nn.Module): + def __init__(self, num_units, num_classes, embedding_size, dropout, device=0, training=True, project_input=True, use_intra_attention=False, distance_biases=10, max_sentence_length=30): diff --git a/han/__main__.py b/han/__main__.py index 3eaac2a..5001d23 100644 --- a/han/__main__.py +++ b/han/__main__.py @@ -20,7 +20,6 @@ from han.args import get_args from han.model import HAN - class UnknownWordVecCache(object): """ Caches the first randomly generated word vector for a certain size to make it is reused. diff --git a/han/args.py b/han/args.py index 05fad1e..9e20659 100644 --- a/han/args.py +++ b/han/args.py @@ -1,6 +1,5 @@ -import os - from argparse import ArgumentParser +import os def get_args(): diff --git a/han/model.py b/han/model.py index 325ffd9..ec04abc 100755 --- a/han/model.py +++ b/han/model.py @@ -1,28 +1,29 @@ import torch import torch.nn as nn -from torch.autograd import Variable -#from utils import import torch.nn.functional as F + from han.sent_level_rnn import SentLevelRNN from han.word_level_rnn import WordLevelRNN - + class HAN(nn.Module): - def __init__(self, config): - super(HAN, self).__init__() - dataset = config.dataset - self.mode = config.mode - self.word_attention_rnn = WordLevelRNN(config) - self.sentence_attention_rnn = SentLevelRNN(config) - def forward(self, x, **kwargs): - x = x.permute(1,2,0) ## Expected : #sentences, #words, batch size - num_sentences = x.size()[0] - word_attentions = None - for i in range(num_sentences): - _word_attention = self.word_attention_rnn(x[i,:,:]) - if word_attentions is None: - word_attentions = _word_attention - else: - word_attentions = torch.cat((word_attentions, _word_attention),0) - return self.sentence_attention_rnn(word_attentions) + + def __init__(self, config): + super().__init__() + dataset = config.dataset + self.mode = config.mode + self.word_attention_rnn = WordLevelRNN(config) + self.sentence_attention_rnn = SentLevelRNN(config) + + def forward(self, x, **kwargs): + x = x.permute(1, 2, 0) # Expected : # sentences, # words, batch size + num_sentences = x.size(0) + word_attentions = None + for i in range(num_sentences): + word_attn = self.word_attention_rnn(x[i, :, :]) + if word_attentions is None: + word_attentions = word_attn + else: + word_attentions = torch.cat((word_attentions, word_attn), 0) + return self.sentence_attention_rnn(word_attentions) diff --git a/han/sent_level_rnn.py b/han/sent_level_rnn.py index 01ad004..8f2080c 100644 --- a/han/sent_level_rnn.py +++ b/han/sent_level_rnn.py @@ -1,32 +1,29 @@ import torch import torch.nn as nn -from torch.autograd import Variable -import torch.nn.functional as F class SentLevelRNN(nn.Module): + def __init__(self, config): - super(SentLevelRNN, self).__init__() + super().__init__() dataset = config.dataset sentence_num_hidden = config.sentence_num_hidden word_num_hidden = config.word_num_hidden target_class = config.target_class - self.sentence_context_wghts = nn.Parameter(torch.rand(2*sentence_num_hidden, 1)) - self.sentence_context_wghts.data.uniform_(-0.1, 0.1) - self.sentence_GRU = nn.GRU(2*word_num_hidden, sentence_num_hidden, bidirectional = True) - self.sentence_linear = nn.Linear(2*sentence_num_hidden, 2*sentence_num_hidden, bias = True) - self.fc = nn.Linear(2*sentence_num_hidden , target_class) + self.sentence_context_weights = nn.Parameter(torch.rand(2 * sentence_num_hidden, 1)) + self.sentence_context_weights.data.uniform_(-0.1, 0.1) + self.sentence_gru = nn.GRU(2 * word_num_hidden, sentence_num_hidden, bidirectional=True) + self.sentence_linear = nn.Linear(2 * sentence_num_hidden, 2 * sentence_num_hidden, bias=True) + self.fc = nn.Linear(2 * sentence_num_hidden , target_class) self.soft_sent = nn.Softmax() - self.final_log_soft = F.log_softmax def forward(self,x): - sentence_h,_ = self.sentence_GRU(x) - x = torch.tanh(self.sentence_linear(sentence_h)) - x = torch.matmul(x, self.sentence_context_wghts) - x = x.squeeze(dim=2) - x = self.soft_sent(x.transpose(1,0)) - x = torch.mul(sentence_h.permute(2,0,1), x.transpose(1,0)) - x = torch.sum(x,dim = 1).transpose(1,0).unsqueeze(0) - #x = self.final_log_soft(self.fc(x.squeeze(0))) - x = self.fc(x.squeeze(0)) - return x + sentence_h,_ = self.sentence_gru(x) + x = torch.tanh(self.sentence_linear(sentence_h)) + x = torch.matmul(x, self.sentence_context_weights) + x = x.squeeze(dim=2) + x = self.soft_sent(x.transpose(1,0)) + x = torch.mul(sentence_h.permute(2, 0, 1), x.transpose(1, 0)) + x = torch.sum(x, dim=1).transpose(1, 0).unsqueeze(0) + x = self.fc(x.squeeze(0)) + return x diff --git a/han/word_level_rnn.py b/han/word_level_rnn.py index 0f98f0a..68e786d 100644 --- a/han/word_level_rnn.py +++ b/han/word_level_rnn.py @@ -1,49 +1,48 @@ import torch import torch.nn as nn -from torch.autograd import Variable -import torch.nn.functional as F + class WordLevelRNN(nn.Module): + def __init__(self, config): - super(WordLevelRNN, self).__init__() + super().__init__() dataset = config.dataset word_num_hidden = config.word_num_hidden words_num = config.words_num words_dim = config.words_dim self.mode = config.mode if self.mode == 'rand': - rand_embed_init = torch.Tensor(words_num, words_dim).uniform(-0.25, 0.25) - self.embed = nn.Embedding.from_pretrained(rand_embed_init, freeze = False) + rand_embed_init = torch.Tensor(words_num, words_dim).uniform(-0.25, 0.25) + self.embed = nn.Embedding.from_pretrained(rand_embed_init, freeze=False) elif self.mode == 'static': - self.static_embed = nn.Embedding.from_pretrained(dataset.TEXT_FIELD.vocab.vectors, freeze = True) + self.static_embed = nn.Embedding.from_pretrained(dataset.TEXT_FIELD.vocab.vectors, freeze=True) elif self.mode == 'non-static': - self.non_static_embed = nn.Embedding.from_pretrained(dataset.TEXT_FIELD.vocab.vectors, freeze = False) + self.non_static_embed = nn.Embedding.from_pretrained(dataset.TEXT_FIELD.vocab.vectors, freeze=False) else: - print("Unsupported order") - exit() - self.word_context_wghts = nn.Parameter(torch.rand(2*word_num_hidden,1)) - self.GRU = nn.GRU(words_dim, word_num_hidden, bidirectional = True) - self.linear = nn.Linear(2*word_num_hidden, 2*word_num_hidden, bias = True) - self.word_context_wghts.data.uniform_(-0.25, 0.25) + print("Unsupported order") + exit() + self.word_context_weights = nn.Parameter(torch.rand(2 * word_num_hidden, 1)) + self.GRU = nn.GRU(words_dim, word_num_hidden, bidirectional=True) + self.linear = nn.Linear(2 * word_num_hidden, 2 * word_num_hidden, bias=True) + self.word_context_weights.data.uniform_(-0.25, 0.25) self.soft_word = nn.Softmax() - def forward(self, x): - ################## - ## x expected to be of dimensions--> (num_words, batch_size) - if self.mode == 'rand': - x = self.embed(x) - elif self.mode == 'static': - x = self.static_embed(x) - elif self.mode == 'non-static': - x = self.non_static_embed(x) - else : - print("Unsuported mode") - exit() - h,_ = self.GRU(x) - x = torch.tanh(self.linear(h)) - x = torch.matmul(x, self.word_context_wghts) - x = x.squeeze(dim=2) - x = self.soft_word(x.transpose(1,0)) - x = torch.mul(h.permute(2,0,1), x.transpose(1,0)) - x = torch.sum(x, dim = 1).transpose(1,0).unsqueeze(0) + def forward(self, x): + # x expected to be of dimensions--> (num_words, batch_size) + if self.mode == 'rand': + x = self.embed(x) + elif self.mode == 'static': + x = self.static_embed(x) + elif self.mode == 'non-static': + x = self.non_static_embed(x) + else : + print("Unsupported mode") + exit() + h, _ = self.GRU(x) + x = torch.tanh(self.linear(h)) + x = torch.matmul(x, self.word_context_weights) + x = x.squeeze(dim=2) + x = self.soft_word(x.transpose(1, 0)) + x = torch.mul(h.permute(2, 0, 1), x.transpose(1, 0)) + x = torch.sum(x, dim=1).transpose(1, 0).unsqueeze(0) return x diff --git a/idf_baseline/experimental_settings.py b/idf_baseline/experimental_settings.py index cd3c8ba..9ea0834 100644 --- a/idf_baseline/experimental_settings.py +++ b/idf_baseline/experimental_settings.py @@ -5,6 +5,7 @@ import itertools import shlex import subprocess + class Setting(object): def __init__(self, label, value_flag_map): self.label = label @@ -22,6 +23,7 @@ class Setting(object): options.append("{}:{}".format(self.label, key)) return options + class Experiments(object): def __init__(self, qa_dataset): self.settings = {} @@ -51,7 +53,7 @@ class Experiments(object): bufsize=1, universal_newlines=True) pout, perr = p.communicate() return pout, perr - + def _run_eval(self): for split in ['train-all', 'raw-dev', 'raw-test']: cmd = '{} {}/{}.qrel run.{}.idfsim'.format(self.eval_cmd_root, @@ -67,7 +69,6 @@ class Experiments(object): fields = line.strip().split() metrics.append(fields[0]) scores.append(fields[-1]) - # rbp_eval scores cmd = '{} {}/{}.qrel run.{}.idfsim'.format(self.rbp_cmd_root, self.qa_data, split, split) @@ -81,7 +82,6 @@ class Experiments(object): print('\t'.join(metrics)) print('\t'.join(scores)) - def run(self, indices): """ runs a particular combination of settings diff --git a/kim_cnn/args.py b/kim_cnn/args.py index 1d6b510..fa343ca 100644 --- a/kim_cnn/args.py +++ b/kim_cnn/args.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser def get_args(): parser = ArgumentParser(description="Kim CNN") parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') - parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU + parser.add_argument('--gpu', type=int, default=0, help='Use -1 for CPU') parser.add_argument('--epochs', type=int, default=30) parser.add_argument('--batch_size', type=int, default=1024) parser.add_argument('--mode', type=str, default='multichannel', choices=['rand', 'static', 'non-static', 'multichannel']) diff --git a/kim_cnn/model.py b/kim_cnn/model.py index 0af3169..68bfb6a 100644 --- a/kim_cnn/model.py +++ b/kim_cnn/model.py @@ -5,15 +5,16 @@ import torch.nn.functional as F class KimCNN(nn.Module): + def __init__(self, config): - super(KimCNN, self).__init__() + super().__init__() dataset = config.dataset output_channel = config.output_channel target_class = config.target_class words_num = config.words_num words_dim = config.words_dim self.mode = config.mode - Ks = 3 # There are three conv nets here + ks = 3 # There are three conv nets here input_channel = 1 if config.mode == 'rand': @@ -36,7 +37,7 @@ class KimCNN(nn.Module): self.conv3 = nn.Conv2d(input_channel, output_channel, (5, words_dim), padding=(4,0)) self.dropout = nn.Dropout(config.dropout) - self.fc1 = nn.Linear(Ks * output_channel, target_class) + self.fc1 = nn.Linear(ks * output_channel, target_class) def forward(self, x, **kwargs): if self.mode == 'rand': @@ -56,10 +57,10 @@ class KimCNN(nn.Module): print("Unsupported Mode") exit() x = [F.relu(self.conv1(x)).squeeze(3), F.relu(self.conv2(x)).squeeze(3), F.relu(self.conv3(x)).squeeze(3)] - # (batch, channel_output, ~=sent_len) * Ks + # (batch, channel_output, ~=sent_len) * ks x = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in x] # max-over-time pooling - # (batch, channel_output) * Ks - x = torch.cat(x, 1) # (batch, channel_output * Ks) + # (batch, channel_output) * ks + x = torch.cat(x, 1) # (batch, channel_output * ks) x = self.dropout(x) logit = self.fc1(x) # (batch, target_size) return logit diff --git a/lstm_baseline/args.py b/lstm_baseline/args.py index f597b97..0678259 100644 --- a/lstm_baseline/args.py +++ b/lstm_baseline/args.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser def get_args(): parser = ArgumentParser(description="Baseline LSTM for text classification") parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') - parser.add_argument('--gpu', type=int, default=0, help="Use -1 for CPU") + parser.add_argument('--gpu', type=int, default=0, help='Use -1 for CPU') parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--batch_size', type=int, default=1024) parser.add_argument('--bidirectional', action='store_true'), diff --git a/lstm_baseline/model.py b/lstm_baseline/model.py index 2be3485..7feda13 100644 --- a/lstm_baseline/model.py +++ b/lstm_baseline/model.py @@ -5,8 +5,9 @@ import torch.nn.functional as F class LSTMBaseline(nn.Module): + def __init__(self, config): - super(LSTMBaseline, self).__init__() + super().__init__() dataset = config.dataset target_class = config.target_class self.is_bidirectional = config.bidirectional @@ -30,11 +31,11 @@ class LSTMBaseline(nn.Module): self.dropout = nn.Dropout(config.dropout) if self.has_bottleneck_layer: if self.is_bidirectional: - self.fc1 = nn.Linear(2 * config.hidden_dim, config.hidden_dim) # Hidden Bottleneck Layer + self.fc1 = nn.Linear(2 * config.hidden_dim, config.hidden_dim) # Hidden Bottleneck Layer self.fc2 = nn.Linear(config.hidden_dim, target_class) else: - self.fc1 = nn.Linear(config.hidden_dim, config.hidden_dim//2) # Hidden Bottleneck Layer - self.fc2 = nn.Linear(config.hidden_dim//2, target_class) + self.fc1 = nn.Linear(config.hidden_dim, config.hidden_dim // 2) # Hidden Bottleneck Layer + self.fc2 = nn.Linear(config.hidden_dim // 2, target_class) else: if self.is_bidirectional: self.fc1 = nn.Linear(2 * config.hidden_dim, target_class) @@ -61,7 +62,6 @@ class LSTMBaseline(nn.Module): x = self.dropout(x) if self.has_bottleneck_layer: x = F.relu(self.fc1(x)) - # x = self.dropout(x) return self.fc2(x) else: return self.fc1(x) diff --git a/lstm_regularization/locked_dropout.py b/lstm_regularization/locked_dropout.py index 65a2416..af86c6c 100644 --- a/lstm_regularization/locked_dropout.py +++ b/lstm_regularization/locked_dropout.py @@ -1,8 +1,9 @@ import torch import torch.nn as nn -from torch.autograd import Variable + class LockedDropout(nn.Module): + def __init__(self): super().__init__() diff --git a/lstm_regularization/model.py b/lstm_regularization/model.py index 2117091..9dc4f2e 100644 --- a/lstm_regularization/model.py +++ b/lstm_regularization/model.py @@ -9,8 +9,9 @@ from lstm_regularization.embed_regularize import embedded_dropout class LSTMBaseline(nn.Module): + def __init__(self, config): - super(LSTMBaseline, self).__init__() + super().__init__() dataset = config.dataset target_class = config.target_class self.is_bidirectional = config.bidirectional diff --git a/lstm_regularization/weight_drop.py b/lstm_regularization/weight_drop.py index d1c473d..5a08e85 100644 --- a/lstm_regularization/weight_drop.py +++ b/lstm_regularization/weight_drop.py @@ -33,26 +33,25 @@ import torch from torch.nn import Parameter from functools import wraps + class WeightDrop(torch.nn.Module): + def __init__(self, module, weights, dropout=0, variational=False): - super(WeightDrop, self).__init__() + super().__init__() self.module = module self.weights = weights self.dropout = dropout self.variational = variational self._setup() - def widget_demagnetizer_y2k_edition(*args, **kwargs): + def null_function(*args, **kwargs): # We need to replace flatten_parameters with a nothing function - # It must be a function rather than a lambda as otherwise pickling explodes - # We can't write boring code though, so ... WIDGET DEMAGNETIZER Y2K EDITION! - return def _setup(self): # Terrible temporary solution to an issue regarding compacting weights re: CUDNN RNN if issubclass(type(self.module), torch.nn.RNNBase): - self.module.flatten_parameters = self.widget_demagnetizer_y2k_edition + self.module.flatten_parameters = self.null_function for name_w in self.weights: print('Applying weight drop of {} to {}'.format(self.dropout, name_w)) diff --git a/mp_cnn/model.py b/mp_cnn/model.py index 96fc658..b008f6d 100644 --- a/mp_cnn/model.py +++ b/mp_cnn/model.py @@ -7,7 +7,7 @@ import torch.nn.functional as F class MPCNN(nn.Module): def __init__(self, n_word_dim, n_holistic_filters, n_per_dim_filters, filter_widths, hidden_layer_units, num_classes, dropout, ext_feats, attention, wide_conv): - super(MPCNN, self).__init__() + super().__init__() self.arch = 'mpcnn' self.n_word_dim = n_word_dim self.n_holistic_filters = n_holistic_filters diff --git a/sse/model.py b/sse/model.py index 74ad0c4..80e3f44 100644 --- a/sse/model.py +++ b/sse/model.py @@ -1,5 +1,4 @@ import numpy as np - import torch import torch.nn as nn import torch.nn.functional as F @@ -8,6 +7,7 @@ from utils import torch_util class StackBiLSTMMaxout(nn.Module): + def __init__(self, h_size=[512, 1024, 2048], d=300, mlp_d=1600, dropout_r=0.1, max_l=60, num_classes=3): super().__init__() @@ -84,4 +84,3 @@ class StackBiLSTMMaxout(nn.Module): out = self.classifier(features) out = F.log_softmax(out, dim=1) return out - diff --git a/vdpwi/__main__.py b/vdpwi/__main__.py index e1befcc..5e3abe6 100644 --- a/vdpwi/__main__.py +++ b/vdpwi/__main__.py @@ -22,6 +22,7 @@ def evaluate_dataset(split_name, dataset_cls, model, embedding, loader, batch_si logger.info('\t'.join([' '] + metric_names)) logger.info('\t'.join([split_name] + list(map(str, scores)))) + if __name__ == '__main__': parser = argparse.ArgumentParser(description='PyTorch implementation of VDPWI') parser.add_argument('model_outfile', help='file to save final model') diff --git a/vdpwi/data.py b/vdpwi/data.py index 04cd560..cdbbee6 100644 --- a/vdpwi/data.py +++ b/vdpwi/data.py @@ -6,6 +6,7 @@ import torch.nn as nn import torch.utils.data as data class Configs(object): + @staticmethod def base_config(): parser = argparse.ArgumentParser() @@ -39,6 +40,7 @@ class Configs(object): parser.add_argument("--sick_data", type=str, default="local_data/sick") return parser.parse_known_args()[0] + class LabeledEmbeddedDataset(data.Dataset): def __init__(self, sentence_indices1, sentence_indices2, labels, compare_labels=None): assert len(sentence_indices1) == len(labels) == len(sentence_indices2) @@ -54,6 +56,7 @@ class LabeledEmbeddedDataset(data.Dataset): def __len__(self): return len(self.labels) + def load_sick(): config = Configs.sick_config() def fetch_indices(name): @@ -98,7 +101,9 @@ def load_sick(): embedding.weight.requires_grad = False return embedding, sets + def load_dataset(dataset): return _loaders[dataset]() + _loaders = dict(sick=load_sick) diff --git a/vdpwi/model.py b/vdpwi/model.py index 42fc1f9..3a62a0a 100644 --- a/vdpwi/model.py +++ b/vdpwi/model.py @@ -1,7 +1,8 @@ +import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -import numpy as np + def hard_pad2d(x, pad): def pad_side(idx): @@ -12,7 +13,9 @@ def hard_pad2d(x, pad): x = F.pad(x, padding) return x[:, :, :pad, :pad] + class ResNet(nn.Module): + def __init__(self, config): super().__init__() n_layers = config['res_layers'] @@ -34,7 +37,9 @@ class ResNet(nn.Module): x = torch.mean(x.view(x.size(0), x.size(1), -1), 2) return F.log_softmax(self.output(x), 1) + class VDPWIConvNet(nn.Module): + def __init__(self, config): super().__init__() def make_conv(n_in, n_out): @@ -63,7 +68,9 @@ class VDPWIConvNet(nn.Module): x = F.relu(self.dnn(x.view(x.size(0), -1))) return F.log_softmax(self.output(x), 1) + class VDPWIModel(nn.Module): + def __init__(self, dim, config): super().__init__() self.arch = 'vdpwi' diff --git a/xml_cnn/args.py b/xml_cnn/args.py index 869076f..5785f8a 100644 --- a/xml_cnn/args.py +++ b/xml_cnn/args.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser def get_args(): parser = ArgumentParser(description="XML CNN") parser.add_argument('--no_cuda', action='store_false', help='do not use cuda', dest='cuda') - parser.add_argument('--gpu', type=int, default=0) # Use -1 for CPU + parser.add_argument('--gpu', type=int, default=0, help='Use -1 for CPU') parser.add_argument('--epochs', type=int, default=30) parser.add_argument('--batch_size', type=int, default=1024) parser.add_argument('--mode', type=str, default='multichannel', choices=['rand', 'static', 'non-static', 'multichannel']) @@ -24,9 +24,8 @@ def get_args(): parser.add_argument('--dropout', type=float, default=0.5) parser.add_argument('--epoch_decay', type=int, default=15) - parser.add_argument('--num_bottleneck_hidden', type=int, default=512) #bottleneck layer - parser.add_argument('--dynamic_pool_length', type=int, default=32) #dynamic pool length - + parser.add_argument('--num_bottleneck_hidden', type=int, default=512) # bottleneck layer + parser.add_argument('--dynamic_pool_length', type=int, default=32) # dynamic pool length parser.add_argument('--data_dir', help='word vectors directory', default=os.path.join(os.pardir, 'Castor-data', 'datasets')) diff --git a/xml_cnn/model.py b/xml_cnn/model.py index 6b9c3dc..d2d6032 100644 --- a/xml_cnn/model.py +++ b/xml_cnn/model.py @@ -1,12 +1,12 @@ import torch import torch.nn as nn - import torch.nn.functional as F class XmlCNN(nn.Module): + def __init__(self, config): - super(XmlCNN, self).__init__() + super().__init__() dataset = config.dataset self.output_channel = config.output_channel target_class = config.target_class @@ -15,7 +15,7 @@ class XmlCNN(nn.Module): self.mode = config.mode self.num_bottleneck_hidden = config.num_bottleneck_hidden self.dynamic_pool_length = config.dynamic_pool_length - self.Ks = 3 # There are three conv nets here + self.ks = 3 # There are three conv nets here input_channel = 1 if config.mode == 'rand': @@ -34,20 +34,16 @@ class XmlCNN(nn.Module): exit() ## Different filter sizes in xml_cnn than kim_cnn - self.conv1 = nn.Conv2d(input_channel, self.output_channel, (2, words_dim), padding=(1,0)) self.conv2 = nn.Conv2d(input_channel, self.output_channel, (4, words_dim), padding=(3,0)) self.conv3 = nn.Conv2d(input_channel, self.output_channel, (8, words_dim), padding=(7,0)) - self.dropout = nn.Dropout(config.dropout) - self.bottleneck = nn.Linear(self.Ks*self.output_channel*self.dynamic_pool_length, self.num_bottleneck_hidden) + self.bottleneck = nn.Linear(self.ks * self.output_channel * self.dynamic_pool_length, self.num_bottleneck_hidden) self.fc1 = nn.Linear(self.num_bottleneck_hidden, target_class) self.pool = nn.AdaptiveMaxPool1d(self.dynamic_pool_length) #Adaptive pooling - - def forward(self, x, **kwargs): if self.mode == 'rand': word_input = self.embed(x) # (batch, sent_len, embed_dim) @@ -67,10 +63,10 @@ class XmlCNN(nn.Module): exit() x = [F.relu(self.conv1(x)).squeeze(3), F.relu(self.conv2(x)).squeeze(3), F.relu(self.conv3(x)).squeeze(3)] x = [self.pool(i).squeeze(2) for i in x] - - # (batch, channel_output) * Ks - x = torch.cat(x, 1) # (batch, channel_output * Ks) - x = F.relu(self.bottleneck(x.view(-1, self.Ks*self.output_channel*self.dynamic_pool_length))) + + # (batch, channel_output) * ks + x = torch.cat(x, 1) # (batch, channel_output * ks) + x = F.relu(self.bottleneck(x.view(-1, self.ks * self.output_channel * self.dynamic_pool_length))) x = self.dropout(x) logit = self.fc1(x) # (batch, target_size) return logit