From cba86ab5d475e7ae93ea599ab3268d6e4bcc2887 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 13 Nov 2018 17:46:28 +0000 Subject: [PATCH 01/22] Added initial classes and changes for BiLM implementation --- fastai_contrib/learner.py | 34 ++++++++++++++++ fastai_contrib/models.py | 85 +++++++++++++++++++++++++++++++++++++++ ulmfit/pretrain_lm.py | 13 ++++-- 3 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 fastai_contrib/learner.py create mode 100644 fastai_contrib/models.py diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py new file mode 100644 index 0000000..59995fe --- /dev/null +++ b/fastai_contrib/learner.py @@ -0,0 +1,34 @@ +from fastai.callbacks import * +from fastai.basic_data import * +from fastai.datasets import untar_data +from fastai_contrib.models import get_bilm +from fastai.text.learner import * + + +def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:int=3, pad_token:int=1, + drop_mult:float=1., tie_weights:bool=True, bias:bool=True, qrnn:bool=False, pretrained_model=None, + pretrained_fnames:OptStrTuple=None, **kwargs) -> 'LanguageLearner': + "Create a `Learner` with a language model." + dps = default_dropout['language'] * drop_mult + vocab_size = data.train_ds.vocab_size + model = get_bilm(vocab_size, emb_sz, nh, nl, pad_token, input_p=dps[0], output_p=dps[1], + weight_p=dps[2], embed_p=dps[3], hidden_p=dps[4], tie_weights=tie_weights, bias=bias, qrnn=qrnn) + learn = LanguageLearner(data, model, bptt, split_func=bilm_split, **kwargs) + if pretrained_model is not None: + model_path = untar_data(pretrained_model, data=False) + fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']] + learn.load_pretrained(*fnames) + learn.freeze() + if pretrained_fnames is not None: + fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])] + learn.load_pretrained(*fnames) + learn.freeze() + return learn + + +def bilm_split(model:nn.Module) -> List[nn.Module]: + "Split a RNN `model` in groups for differential learning rates." + groups = [[rnn, dp] for rnn, dp in zip(model[0].forward_rnns, model[0].hidden_dps)] + groups += [[rnn, dp] for rnn, dp in zip(model[0].backward_rnns, model[0].hidden_dps)] + groups.append([model[0].encoder, model[0].encoder_dp, model[1]]) + return groups diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py new file mode 100644 index 0000000..7a39771 --- /dev/null +++ b/fastai_contrib/models.py @@ -0,0 +1,85 @@ +from fastai.torch_core import * +from fastai.layers import * +from fastai.text.models import * + + +class BiLMCore(nn.Module): + """ + AWD-LSTM/QRNN inspired by https://arxiv.org/abs/1708.02182. + Inspired by https://github.com/allenai/allennlp/blob/master/allennlp/models/bidirectional_lm.py#L65 + """ + initrange=0.1 + + def __init__(self, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, bidir:bool=False, + hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, qrnn:bool=False): + + super().__init__() + self.bs,self.qrnn,self.ndir = 1, qrnn,(2 if bidir else 1) + self.emb_sz,self.n_hid,self.n_layers = emb_sz,n_hid,n_layers + # embeddings are shared between forward and backward LMs + self.encoder = nn.Embedding(vocab_sz, emb_sz, padding_idx=pad_token) + self.encoder_dp = EmbeddingDropout(self.encoder, embed_p) + if self.qrnn: + #Using QRNN requires cupy: https://github.com/cupy/cupy + from fastai.text.qrnn.qrnn import QRNNLayer + + def create_qrnn_layers(): + return [QRNNLayer(emb_sz if l == 0 else n_hid, (n_hid if l != n_layers - 1 else emb_sz)//self.ndir, + save_prev_x=True, zoneout=0, window=2 if l == 0 else 1, output_gate=True, + use_cuda=torch.cuda.is_available()) for l in range(n_layers)] + self.forward_rnns = create_qrnn_layers() + self.backward_rnns = create_qrnn_layers() + for rnn in self.forward_rnns + self.backward_rnns: + rnn.linear = WeightDropout(rnn.linear, weight_p, layer_names=['weight']) + else: + def create_lstm_layers(): + return [nn.LSTM(emb_sz if l == 0 else n_hid, (n_hid if l != n_layers - 1 else emb_sz)//self.ndir, + 1, bidirectional=False) for l in range(n_layers)] + self.forward_rnns = [WeightDropout(rnn, weight_p) for rnn in create_lstm_layers()] + self.backward_rnns = [WeightDropout(rnn, weight_p) for rnn in create_lstm_layers()] + self.forward_rnns = torch.nn.ModuleList(self.forward_rnns) + self.backward_rnns = torch.nn.ModuleList(self.backward_rnns) + self.encoder.weight.data.uniform_(-self.initrange, self.initrange) + self.input_dp = RNNDropout(input_p) + self.hidden_dps = nn.ModuleList([RNNDropout(hidden_p) for l in range(n_layers)]) + + def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]: + sl,bs = input.size() + if bs!=self.bs: + self.bs=bs + self.reset() + raw_output = self.input_dp(self.encoder_dp(input)) + + # TODO get reverse input and compute backward representation + new_hidden,raw_outputs,outputs = [],[],[] + for l, (rnn,hid_dp) in enumerate(zip(self.forward_rnns, self.hidden_dps)): + raw_output, new_h = rnn(raw_output, self.hidden[l]) + new_hidden.append(new_h) + raw_outputs.append(raw_output) + if l != self.n_layers - 1: raw_output = hid_dp(raw_output) + outputs.append(raw_output) + self.hidden = to_detach(new_hidden) + return raw_outputs, outputs + + def _one_hidden(self, l:int)->Tensor: + "Return one hidden state." + nh = (self.n_hid if l != self.n_layers - 1 else self.emb_sz)//self.ndir + return self.weights.new(self.ndir, self.bs, nh).zero_() + + def reset(self): + "Reset the hidden states." + [r.reset() for r in self.forward_rnns if hasattr(r, 'reset')] + [r.reset() for r in self.backward_rnns if hasattr(r, 'reset')] + self.weights = next(self.parameters()).data + if self.qrnn: self.hidden = [self._one_hidden(l) for l in range(self.n_layers)] + else: self.hidden = [(self._one_hidden(l), self._one_hidden(l)) for l in range(self.n_layers)] + + +def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True, + qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6, + embed_p:float=0.1, weight_p:float=0.5)->nn.Module: + "Create a full AWD-LSTM." + rnn_enc = BiLMCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, + hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) + enc = rnn_enc.encoder if tie_weights else None + return SequentialRNN(rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 1836708..fe631b1 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -14,6 +14,7 @@ from fastai.text import LanguageModelLoader, get_language_model, RNNLearner, Tex import torch from fastai_contrib.utils import read_file, read_whitespace_file,\ DataStump, validate, PAD, UNK +from fastai_contrib.learner import bilm_learner import pickle @@ -27,7 +28,8 @@ from collections import Counter def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, - bs=70, bptt=70, name='wt-103', model_dir='models', num_epochs=10): + bs=70, bptt=70, name='wt-103', model_dir='models', num_epochs=10, + bidir=True): """ :param dir_path: The path to the directory of the file. :param cuda_id: The id of the GPU. Uses GPU 0 by default or no GPU when @@ -39,6 +41,7 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, :param bptt: The back-propagation-through-time sequence length. :param name: The name used for both the model and the vocabulary. :param model_dir: The path to the directory where the models should be saved + :param bidir: whether the language model is bidirectional """ if not torch.cuda.is_available(): print('CUDA not available. Setting device=-1.') @@ -109,9 +112,11 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, drop_mult = 0.1 fastai.text.learner.default_dropout['language'] = dps - learn = language_model_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, - drop_mult=drop_mult, tie_weights=True, - bias=True, qrnn=True, clip=0.12) + + lm_learner = bilm_learner if bidir else language_model_learner + learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, + drop_mult=drop_mult, tie_weights=True, + bias=True, qrnn=qrnn, clip=0.12) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.true_wd = False From 6ee1a2b27df1b5aba30ac9095af4000dab864069 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 14 Nov 2018 12:48:29 +0100 Subject: [PATCH 02/22] Add BiLM LanguageModelLoader with tests --- fastai_contrib/data.py | 60 +++++++++++++++++++++++++++++++++++++++++ tests/test_text_data.py | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 fastai_contrib/data.py create mode 100644 tests/test_text_data.py diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py new file mode 100644 index 0000000..74c6b6c --- /dev/null +++ b/fastai_contrib/data.py @@ -0,0 +1,60 @@ +"NLP data loading pipeline. Supports csv, folders, and preprocessed data." +from fastai.text import * +from fastai.torch_core import * +from fastai.text.transform import * +from fastai.basic_data import * +from fastai.data_block import * + + +###################### UPDATED CODE + +LanguageModelType=Enum('LanguageModelType', 'FwdLM BwdLM BiLM') + +class LanguageModelLoader(): # copy of the original LanguageModelLoader + "Create a dataloader with bptt slightly changing." + def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70, + lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False, + max_len:int=25): + self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle = dataset,bs,bptt,lm_type,shuffle + self.first,self.i,self.iter = True,0,0 + self.n = len(np.concatenate(dataset.x.items)) // self.bs + self.max_len,self.num_workers = max_len,0 + + def __iter__(self): + if getattr(self.dataset, 'item', None) is not None: + yield LongTensor(getattr(self.dataset, 'item')).unsqueeze(1),LongTensor([0]) + idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset)) + data = self.batchify(np.concatenate([self.dataset.x[i] for i in idx])) + + pos, itr = 0,0 + while pos < self.n-1 and itr int: return (self.n-1) // self.bptt + def __getattr__(self,k:str)->Any: return getattr(self.dataset, k) + + def batchify(self, data:np.ndarray) -> LongTensor: + "Split the corpus `data` in batches." + nb = data.shape[0] // self.bs + data = np.array(data[:nb*self.bs]).reshape(self.bs, -1).T + if self.lm_type == LanguageModelType.BwdLM: data=data[::-1].copy() + elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[::-1].copy()], axis=2) + return LongTensor(data) + + def get_batch(self, data:LongTensor, i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]: + "Create a batch at `i` of a given `seq_len`." + seq_len = min(seq_len, len(self.data) - 1 - i) + x = data[i:i+seq_len] + y = data[i+1:i+1+seq_len].contiguous() # x & y has 2 elements on the last dimension + y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1) + return x,y + +###################### NEW CODE diff --git a/tests/test_text_data.py b/tests/test_text_data.py new file mode 100644 index 0000000..6b3dabf --- /dev/null +++ b/tests/test_text_data.py @@ -0,0 +1,59 @@ +import pytest +import fastai.text + +from fastai import * +from fastai.text import * + +import fastai_contrib.data as contrib_data + +def text_df(labels): + data = [] + texts = ["fast ai is a cool project", "hello world"] * 20 + for ind, text in enumerate(texts): + sample = {} + sample["label"] = labels[ind%len(labels)] + sample["text"] = text + data.append(sample) + return pd.DataFrame(data) + +###################### UPDATED CODE +def test_should_load_backwards_lm(): + path = untar_data(URLs.IMDB_SAMPLE) + df = text_df(['neg','pos']) + + data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, + lm_type=contrib_data.LanguageModelType.BwdLM, + ld_cls=contrib_data.LanguageModelLoader) + lml = data.train_dl.dl + lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) + batch = lml.get_batch(lml.data, 0, 70) + + assert batch[0].shape == (70, lml.bs) + assert batch[1].shape == (70*lml.bs,) + + + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + +def test_should_load_bi_lm(): + path = untar_data(URLs.IMDB_SAMPLE) + df = text_df(['neg', 'pos']) + + data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, + lm_type=contrib_data.LanguageModelType.BiLM, + ld_cls=contrib_data.LanguageModelLoader) + lml = data.train_dl.dl + lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) + batch = lml.get_batch(lml.data, 0, 70) + + assert batch[0].shape == (70, lml.bs, 2) + assert batch[1].shape == (70*lml.bs, 2) + + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:, 0, 0]] + np.testing.assert_array_equal(as_text[:7], "xxfld 1 fast ai is a cool".split()) + + as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0,1]] + np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',]) + +###################### NEW CODE + From a85800610b0792b793b29f0ffd15190ba5e5a0a3 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 14 Nov 2018 13:22:03 +0100 Subject: [PATCH 03/22] WIP Working Backward LM using our new LangaugeModelLoader --- fastai_contrib/data.py | 4 +-- fastai_contrib/learner.py | 2 +- fastai_contrib/models.py | 7 ++++- tests/test_text_train.py | 66 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 4 deletions(-) create mode 100644 tests/test_text_train.py diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py index 74c6b6c..fb1ea20 100644 --- a/fastai_contrib/data.py +++ b/fastai_contrib/data.py @@ -24,7 +24,7 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader if getattr(self.dataset, 'item', None) is not None: yield LongTensor(getattr(self.dataset, 'item')).unsqueeze(1),LongTensor([0]) idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset)) - data = self.batchify(np.concatenate([self.dataset.x[i] for i in idx])) + data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx])) pos, itr = 0,0 while pos < self.n-1 and itr Tuple[LongTensor, LongTensor]: "Create a batch at `i` of a given `seq_len`." - seq_len = min(seq_len, len(self.data) - 1 - i) + seq_len = min(seq_len, len(data) - 1 - i) x = data[i:i+seq_len] y = data[i+1:i+1+seq_len].contiguous() # x & y has 2 elements on the last dimension y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index 59995fe..af72f95 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -10,7 +10,7 @@ def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:in pretrained_fnames:OptStrTuple=None, **kwargs) -> 'LanguageLearner': "Create a `Learner` with a language model." dps = default_dropout['language'] * drop_mult - vocab_size = data.train_ds.vocab_size + vocab_size = len(data.vocab.itos) model = get_bilm(vocab_size, emb_sz, nh, nl, pad_token, input_p=dps[0], output_p=dps[1], weight_p=dps[2], embed_p=dps[3], hidden_p=dps[4], tie_weights=tie_weights, bias=bias, qrnn=qrnn) learn = LanguageLearner(data, model, bptt, split_func=bilm_split, **kwargs) diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index 7a39771..c7924e3 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -44,7 +44,10 @@ class BiLMCore(nn.Module): self.hidden_dps = nn.ModuleList([RNNDropout(hidden_p) for l in range(n_layers)]) def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]: - sl,bs = input.size() + sl,bs,tracks = input.size() + assert tracks == 2, "It should have two tracks for forward and backward pass" + + input = input[...,0] # Select forward pass only if bs!=self.bs: self.bs=bs self.reset() @@ -59,6 +62,8 @@ class BiLMCore(nn.Module): if l != self.n_layers - 1: raw_output = hid_dp(raw_output) outputs.append(raw_output) self.hidden = to_detach(new_hidden) + + #bi_raw_outputs = torch.stack((outputs, outputs), dim=2) return raw_outputs, outputs def _one_hidden(self, l:int)->Tensor: diff --git a/tests/test_text_train.py b/tests/test_text_train.py new file mode 100644 index 0000000..cd3f394 --- /dev/null +++ b/tests/test_text_train.py @@ -0,0 +1,66 @@ +import pytest +from fastai import * +from fastai.text import * + +pytestmark = pytest.mark.integration + +import fastai_contrib.data as contrib_data + +from fastai_contrib.learner import bilm_learner + +def read_file(fname): + texts = [] + with open(fname, 'r') as f: + texts = f.readlines() + labels = [0] * len(texts) + df = pd.DataFrame({'labels':labels, 'texts':texts}, columns = ['labels', 'texts']) + return df + +def prep_human_numbers(): + path = untar_data(URLs.HUMAN_NUMBERS) + df_trn = read_file(path/'train.txt') + df_val = read_file(path/'valid.txt') + return path, df_trn, df_val + +def manual_seed(seed=42): + torch.manual_seed(seed) + np.random.seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + +@pytest.fixture(scope="module") +def learn(): + path, df_trn, df_val = prep_human_numbers() + data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer)) + learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1) + learn.fit_one_cycle(4, 5e-3) + return learn + +###################### NEW CODE + +def test_val_loss(learn): + assert learn.validate()[1] > 0.5 + +def test_bwdlm_lstm_can_be_trained(): + manual_seed() + path, df_trn, df_val = prep_human_numbers() + data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), + lm_type = contrib_data.LanguageModelType.BiLM, + ld_cls = contrib_data.LanguageModelLoader) + + learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + learn.fit_one_cycle(4, 5e-3) + assert learn.validate()[1] > 0.5 + +def test_bilm_lstm_can_be_trained(): + manual_seed() + path, df_trn, df_val = prep_human_numbers() + data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), + lm_type = contrib_data.LanguageModelType.BwdLM, + ld_cls = contrib_data.LanguageModelLoader) + + learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + learn.fit_one_cycle(4, 5e-3) + assert learn.validate()[1] > 0.5 From 5ba83b1d4e89a4eff5b139fb69a76d40a129e19a Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 14 Nov 2018 14:55:56 +0100 Subject: [PATCH 04/22] Working version of BILM - probably won't train well yet --- fastai_contrib/data.py | 4 +++ fastai_contrib/models.py | 72 ++++++++++++++++++++++++++++++++-------- tests/test_text_train.py | 25 +++++++------- 3 files changed, 74 insertions(+), 27 deletions(-) diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py index fb1ea20..3db59c5 100644 --- a/fastai_contrib/data.py +++ b/fastai_contrib/data.py @@ -58,3 +58,7 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader return x,y ###################### NEW CODE + + +import fastai.text.data +fastai.text.data.LanguageModelLoader = LanguageModelLoader # Replace original LanguageModelLoader with new verion diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index c7924e3..dbb8be4 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -46,25 +46,37 @@ class BiLMCore(nn.Module): def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]: sl,bs,tracks = input.size() assert tracks == 2, "It should have two tracks for forward and backward pass" - - input = input[...,0] # Select forward pass only - if bs!=self.bs: - self.bs=bs + if bs != self.bs: + self.bs = bs self.reset() - raw_output = self.input_dp(self.encoder_dp(input)) - # TODO get reverse input and compute backward representation + return [self.fwdlm_forwad(input[..., 0]), self.bwdlm_forwad(input[..., 1])] + + def bwdlm_forwad(self, input): + raw_output = self.input_dp(self.encoder_dp(input)) new_hidden,raw_outputs,outputs = [],[],[] - for l, (rnn,hid_dp) in enumerate(zip(self.forward_rnns, self.hidden_dps)): - raw_output, new_h = rnn(raw_output, self.hidden[l]) + for l, (rnn,hid_dp) in enumerate(zip(self.backward_rnns, self.hidden_dps)): + raw_output, new_h = rnn(raw_output, self.bwdlm_hidden[l]) new_hidden.append(new_h) raw_outputs.append(raw_output) if l != self.n_layers - 1: raw_output = hid_dp(raw_output) outputs.append(raw_output) - self.hidden = to_detach(new_hidden) + self.bwdlm_hidden = to_detach(new_hidden) - #bi_raw_outputs = torch.stack((outputs, outputs), dim=2) - return raw_outputs, outputs + return (raw_outputs, outputs) + + def fwdlm_forwad(self, input): + raw_output = self.input_dp(self.encoder_dp(input)) + new_hidden,raw_outputs,outputs = [],[],[] + for l, (rnn,hid_dp) in enumerate(zip(self.forward_rnns, self.hidden_dps)): + raw_output, new_h = rnn(raw_output, self.fwdlm_hidden[l]) + new_hidden.append(new_h) + raw_outputs.append(raw_output) + if l != self.n_layers - 1: raw_output = hid_dp(raw_output) + outputs.append(raw_output) + self.fwdlm_hidden = to_detach(new_hidden) + + return (raw_outputs, outputs) def _one_hidden(self, l:int)->Tensor: "Return one hidden state." @@ -76,8 +88,40 @@ class BiLMCore(nn.Module): [r.reset() for r in self.forward_rnns if hasattr(r, 'reset')] [r.reset() for r in self.backward_rnns if hasattr(r, 'reset')] self.weights = next(self.parameters()).data - if self.qrnn: self.hidden = [self._one_hidden(l) for l in range(self.n_layers)] - else: self.hidden = [(self._one_hidden(l), self._one_hidden(l)) for l in range(self.n_layers)] + if self.qrnn: self.fwdlm_hidden = [self._one_hidden(l) for l in range(self.n_layers)] + else: self.fwdlm_hidden = [(self._one_hidden(l), self._one_hidden(l)) for l in range(self.n_layers)] + if self.qrnn: self.bwdlm_hidden = [self._one_hidden(l) for l in range(self.n_layers)] + else: self.bwdlm_hidden = [(self._one_hidden(l), self._one_hidden(l)) for l in range(self.n_layers)] + +class BiLinearDecoder(nn.Module): + "To go on top of a RNNCore module and create a Language Model." + + initrange=0.1 + + def __init__(self, n_out:int, n_hid:int, output_p:float, tie_encoder:nn.Module=None, bias:bool=True): + super().__init__() + self.decoder = nn.Linear(n_hid, n_out, bias=bias) + self.decoder.weight.data.uniform_(-self.initrange, self.initrange) + self.output_dp = RNNDropout(output_p) + if bias: self.decoder.bias.data.zero_() + if tie_encoder: self.decoder.weight = tie_encoder.weight + + def forward(self, input:List[Tuple[Tensor,Tensor]])->Tuple[Tensor,Tensor,Tensor]: + decoded=[] + raw_outputs=[] + outputs=[] + for lm_input in input: + d, ro, o = self.one_forward(lm_input) + decoded.append(d) + raw_outputs += ro + outputs += o + return torch.stack(decoded, dim=2), raw_outputs, outputs + + def one_forward(self, input): + raw_outputs, outputs = input + output = self.output_dp(outputs[-1]) + decoded = self.decoder(output.view(output.size(0)*output.size(1), output.size(2))) + return decoded, raw_outputs, outputs def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True, @@ -87,4 +131,4 @@ def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, t rnn_enc = BiLMCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) enc = rnn_enc.encoder if tie_weights else None - return SequentialRNN(rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)) + return SequentialRNN(rnn_enc, BiLinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)) diff --git a/tests/test_text_train.py b/tests/test_text_train.py index cd3f394..5116cfa 100644 --- a/tests/test_text_train.py +++ b/tests/test_text_train.py @@ -43,23 +43,22 @@ def learn(): def test_val_loss(learn): assert learn.validate()[1] > 0.5 -def test_bwdlm_lstm_can_be_trained(): - manual_seed() - path, df_trn, df_val = prep_human_numbers() - data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), - lm_type = contrib_data.LanguageModelType.BiLM, - ld_cls = contrib_data.LanguageModelLoader) - - learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) - learn.fit_one_cycle(4, 5e-3) - assert learn.validate()[1] > 0.5 - def test_bilm_lstm_can_be_trained(): manual_seed() path, df_trn, df_val = prep_human_numbers() data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), - lm_type = contrib_data.LanguageModelType.BwdLM, - ld_cls = contrib_data.LanguageModelLoader) + lm_type = contrib_data.LanguageModelType.BiLM) + + learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + learn.metrics = [] + learn.fit_one_cycle(4, 5e-3) + assert learn.validate()[0] < 2 #TODO Change to accuracy once it is fixed + +def test_bwdlm_lstm_can_be_trained(): + manual_seed() + path, df_trn, df_val = prep_human_numbers() + data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), + lm_type = contrib_data.LanguageModelType.BwdLM) learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) learn.fit_one_cycle(4, 5e-3) From cf7d93c3784fe20de0ab9c4131be71113ee5c502 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 14 Nov 2018 14:59:23 +0100 Subject: [PATCH 05/22] Changes to pretrain_lm so that it works with bilm --- ulmfit/pretrain_lm.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index b86a811..34455a9 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -20,6 +20,7 @@ import pickle from pathlib import Path from collections import Counter +import fastai_contrib.data as contrib_data # to install, do: # conda install -c pytorch -c fastai fastai pytorch-nightly [cuda92] @@ -28,7 +29,7 @@ from collections import Counter def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, bs=70, bptt=70, name='wt-103', model_dir='models', num_epochs=10, - bidir=True): + bidir=False): """ :param dir_path: The path to the directory of the file. :param cuda_id: The id of the GPU. Uses GPU 0 by default or no GPU when @@ -57,7 +58,8 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, if qrnn: print('Using QRNNs...') - trn_path = dir_path / 'wiki.train.tokens' + #trn_path = dir_path / 'wiki.train.tokens' + trn_path = dir_path / 'wiki.valid.tokens' val_path = dir_path / 'wiki.valid.tokens' tst_path = dir_path / 'wiki.test.tokens' for path_ in [trn_path, val_path, tst_path]: @@ -80,9 +82,13 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) + lm_type = contrib_data.LanguageModelType.BiLM if bidir else contrib_data.LanguageModelType.FwdLM + # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) data_lm = TextLMDataBunch.from_ids(path=dir_path, vocab=vocab, train_ids=trn_ids, - valid_ids=val_ids, bs=bs, bptt=bptt) + valid_ids=val_ids, bs=bs, bptt=bptt, + lm_type=lm_type) + else: # apply fastai preprocessing and tokenization read_file(trn_path, 'train') @@ -117,13 +123,13 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.true_wd = False - + learn.metrics=[] # accuracy does not work when we have multiple dimensions at the end. # save vocabulary print('Saving vocabulary...') with open(model_dir / f'itos_{name}.pkl', 'wb') as f: pickle.dump(itos, f) - fit_one_cycle(learn, num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) + learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) if clean and max_vocab is None: # only if we use the unpreprocessed version and the full vocabulary From 33f9eb2cc7ea320506b97f9dbde8fe71e5feca77 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 14 Nov 2018 21:16:16 +0100 Subject: [PATCH 06/22] Reuse RNNCore in implementation of BiLM, add accuracy --- fastai_contrib/learner.py | 6 +- fastai_contrib/models.py | 147 ++++++++------------------------------ ulmfit/pretrain_lm.py | 19 +++-- 3 files changed, 46 insertions(+), 126 deletions(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index af72f95..d787867 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -28,7 +28,5 @@ def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:in def bilm_split(model:nn.Module) -> List[nn.Module]: "Split a RNN `model` in groups for differential learning rates." - groups = [[rnn, dp] for rnn, dp in zip(model[0].forward_rnns, model[0].hidden_dps)] - groups += [[rnn, dp] for rnn, dp in zip(model[0].backward_rnns, model[0].hidden_dps)] - groups.append([model[0].encoder, model[0].encoder_dp, model[1]]) - return groups + + return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))] diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index dbb8be4..df323af 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -2,133 +2,44 @@ from fastai.torch_core import * from fastai.layers import * from fastai.text.models import * +class BiLMModel(nn.Module): -class BiLMCore(nn.Module): - """ - AWD-LSTM/QRNN inspired by https://arxiv.org/abs/1708.02182. - Inspired by https://github.com/allenai/allennlp/blob/master/allennlp/models/bidirectional_lm.py#L65 - """ - initrange=0.1 - - def __init__(self, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, bidir:bool=False, - hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, qrnn:bool=False): - + def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module): super().__init__() - self.bs,self.qrnn,self.ndir = 1, qrnn,(2 if bidir else 1) - self.emb_sz,self.n_hid,self.n_layers = emb_sz,n_hid,n_layers - # embeddings are shared between forward and backward LMs - self.encoder = nn.Embedding(vocab_sz, emb_sz, padding_idx=pad_token) - self.encoder_dp = EmbeddingDropout(self.encoder, embed_p) - if self.qrnn: - #Using QRNN requires cupy: https://github.com/cupy/cupy - from fastai.text.qrnn.qrnn import QRNNLayer + self.fwd_lm = fwd_lm + self.bwd_lm = bwd_lm - def create_qrnn_layers(): - return [QRNNLayer(emb_sz if l == 0 else n_hid, (n_hid if l != n_layers - 1 else emb_sz)//self.ndir, - save_prev_x=True, zoneout=0, window=2 if l == 0 else 1, output_gate=True, - use_cuda=torch.cuda.is_available()) for l in range(n_layers)] - self.forward_rnns = create_qrnn_layers() - self.backward_rnns = create_qrnn_layers() - for rnn in self.forward_rnns + self.backward_rnns: - rnn.linear = WeightDropout(rnn.linear, weight_p, layer_names=['weight']) - else: - def create_lstm_layers(): - return [nn.LSTM(emb_sz if l == 0 else n_hid, (n_hid if l != n_layers - 1 else emb_sz)//self.ndir, - 1, bidirectional=False) for l in range(n_layers)] - self.forward_rnns = [WeightDropout(rnn, weight_p) for rnn in create_lstm_layers()] - self.backward_rnns = [WeightDropout(rnn, weight_p) for rnn in create_lstm_layers()] - self.forward_rnns = torch.nn.ModuleList(self.forward_rnns) - self.backward_rnns = torch.nn.ModuleList(self.backward_rnns) - self.encoder.weight.data.uniform_(-self.initrange, self.initrange) - self.input_dp = RNNDropout(input_p) - self.hidden_dps = nn.ModuleList([RNNDropout(hidden_p) for l in range(n_layers)]) + def forward(self, input): + sl, bs, tracks = input.size() - def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]: - sl,bs,tracks = input.size() - assert tracks == 2, "It should have two tracks for forward and backward pass" - if bs != self.bs: - self.bs = bs - self.reset() + decoded = [] + raw_outputs = [] + outputs = [] - return [self.fwdlm_forwad(input[..., 0]), self.bwdlm_forwad(input[..., 1])] + fwd_o = self.fwd_lm(input[..., 0]) + bwd_o = self.bwd_lm(input[..., 1]) - def bwdlm_forwad(self, input): - raw_output = self.input_dp(self.encoder_dp(input)) - new_hidden,raw_outputs,outputs = [],[],[] - for l, (rnn,hid_dp) in enumerate(zip(self.backward_rnns, self.hidden_dps)): - raw_output, new_h = rnn(raw_output, self.bwdlm_hidden[l]) - new_hidden.append(new_h) - raw_outputs.append(raw_output) - if l != self.n_layers - 1: raw_output = hid_dp(raw_output) - outputs.append(raw_output) - self.bwdlm_hidden = to_detach(new_hidden) - - return (raw_outputs, outputs) - - def fwdlm_forwad(self, input): - raw_output = self.input_dp(self.encoder_dp(input)) - new_hidden,raw_outputs,outputs = [],[],[] - for l, (rnn,hid_dp) in enumerate(zip(self.forward_rnns, self.hidden_dps)): - raw_output, new_h = rnn(raw_output, self.fwdlm_hidden[l]) - new_hidden.append(new_h) - raw_outputs.append(raw_output) - if l != self.n_layers - 1: raw_output = hid_dp(raw_output) - outputs.append(raw_output) - self.fwdlm_hidden = to_detach(new_hidden) - - return (raw_outputs, outputs) - - def _one_hidden(self, l:int)->Tensor: - "Return one hidden state." - nh = (self.n_hid if l != self.n_layers - 1 else self.emb_sz)//self.ndir - return self.weights.new(self.ndir, self.bs, nh).zero_() + return torch.stack([fwd_o[0], bwd_o[0]], dim=2), (fwd_o[1]+bwd_o[1]), (fwd_o[2] + bwd_o[2]) def reset(self): - "Reset the hidden states." - [r.reset() for r in self.forward_rnns if hasattr(r, 'reset')] - [r.reset() for r in self.backward_rnns if hasattr(r, 'reset')] - self.weights = next(self.parameters()).data - if self.qrnn: self.fwdlm_hidden = [self._one_hidden(l) for l in range(self.n_layers)] - else: self.fwdlm_hidden = [(self._one_hidden(l), self._one_hidden(l)) for l in range(self.n_layers)] - if self.qrnn: self.bwdlm_hidden = [self._one_hidden(l) for l in range(self.n_layers)] - else: self.bwdlm_hidden = [(self._one_hidden(l), self._one_hidden(l)) for l in range(self.n_layers)] - -class BiLinearDecoder(nn.Module): - "To go on top of a RNNCore module and create a Language Model." - - initrange=0.1 - - def __init__(self, n_out:int, n_hid:int, output_p:float, tie_encoder:nn.Module=None, bias:bool=True): - super().__init__() - self.decoder = nn.Linear(n_hid, n_out, bias=bias) - self.decoder.weight.data.uniform_(-self.initrange, self.initrange) - self.output_dp = RNNDropout(output_p) - if bias: self.decoder.bias.data.zero_() - if tie_encoder: self.decoder.weight = tie_encoder.weight - - def forward(self, input:List[Tuple[Tensor,Tensor]])->Tuple[Tensor,Tensor,Tensor]: - decoded=[] - raw_outputs=[] - outputs=[] - for lm_input in input: - d, ro, o = self.one_forward(lm_input) - decoded.append(d) - raw_outputs += ro - outputs += o - return torch.stack(decoded, dim=2), raw_outputs, outputs - - def one_forward(self, input): - raw_outputs, outputs = input - output = self.output_dp(outputs[-1]) - decoded = self.decoder(output.view(output.size(0)*output.size(1), output.size(2))) - return decoded, raw_outputs, outputs - + "Reset the hidden states of underlaying lms." + self.fwd_lm.reset() + self.bwd_lm.reset() def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True, qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5)->nn.Module: - "Create a full AWD-LSTM." - rnn_enc = BiLMCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, - hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - enc = rnn_enc.encoder if tie_weights else None - return SequentialRNN(rnn_enc, BiLinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)) + "Create a two AWD-LSTM one for each direction " + fwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, + hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) + bwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir, + hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) + enc = None + if tie_weights: + enc = fwd_rnn_enc.encoder + fwd_rnn_enc.encoder.weight = enc.weight + bwd_rnn_enc.encoder.weight = enc.weight + + return BiLMModel( + fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), + bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias))) \ No newline at end of file diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 34455a9..55a64db 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -26,10 +26,15 @@ import fastai_contrib.data as contrib_data # conda install -c pytorch -c fastai fastai pytorch-nightly [cuda92] # cupy needs to be installed for QRNN +def accuracy_fwd(input, targs): + return accuracy(input[...,0], targs[...,0]) +def accuracy_bwd(input, targs): + return accuracy(input[...,1], targs[...,1]) + def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, bs=70, bptt=70, name='wt-103', model_dir='models', num_epochs=10, - bidir=False): + bidir=False, ds_pct=1.0): """ :param dir_path: The path to the directory of the file. :param cuda_id: The id of the GPU. Uses GPU 0 by default or no GPU when @@ -58,8 +63,7 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, if qrnn: print('Using QRNNs...') - #trn_path = dir_path / 'wiki.train.tokens' - trn_path = dir_path / 'wiki.valid.tokens' + trn_path = dir_path / 'wiki.train.tokens' val_path = dir_path / 'wiki.valid.tokens' tst_path = dir_path / 'wiki.test.tokens' for path_ in [trn_path, val_path, tst_path]: @@ -69,6 +73,9 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, # read the already whitespace separated data without any preprocessing trn_tok = read_whitespace_file(trn_path) val_tok = read_whitespace_file(val_path) + if ds_pct < 1.0: + trn_tok = trn_tok[:int(len(trn_tok) * ds_pct)] + val_tok = val_tok[:int(len(val_tok) * ds_pct)] # create the vocabulary cnt = Counter(word for sent in trn_tok for word in sent) @@ -123,7 +130,11 @@ def pretrain_lm(dir_path, cuda_id=0, qrnn=True, clean=True, max_vocab=60000, # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.true_wd = False - learn.metrics=[] # accuracy does not work when we have multiple dimensions at the end. + + if bidir: + learn.metrics = [accuracy_fwd, accuracy_bwd] + else: + learn.metrics = [accuracy] # save vocabulary print('Saving vocabulary...') with open(model_dir / f'itos_{name}.pkl', 'wb') as f: From aa6b59a0b3638d6e4a8a83c7ada708d143a94629 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 16 Nov 2018 23:19:15 +0100 Subject: [PATCH 07/22] Fix vocab size so that it remains 60k as in case of pretrain_lm + add longer lm training --- ulmfit/train_clas.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 4bfbb8e..23b3ab6 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -18,7 +18,7 @@ from pathlib import Path def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_dir='models', qrnn=False, - fine_tune=True, max_vocab=30000, bs=20, bptt=70, name='imdb-clas', + fine_tune=True, max_vocab=60000, bs=20, bptt=70, name='imdb-clas', dataset='imdb', ds_pct=1.0): """ :param data_dir: The path to the `data` directory @@ -105,7 +105,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ f'Test size: {len(ids[TST])}.') if ds_pct < 1.0: - print(f"Makeing the dataset smaller {ds_pct}") + print(f"Making the dataset smaller {ds_pct}") for split in [TRN, VAL, TST]: ids[split] = ids[split][:int(len(ids[split])*ds_pct)] @@ -127,11 +127,13 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ pad_token=PAD_TOKEN_ID, pretrained_fnames=pretrained_fname, path=model_dir.parent, model_dir=model_dir.name) - lm_enc_finetuned = f"{lm_name}_{dataset}_enc" + + lm_enc_finetuned = f"{lm_name}_{dataset}_{name}_enc" if fine_tune and not (model_dir / f"lm_enc_finetuned.pth").exists(): print('Fine-tuning the language model...') + learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) learn.unfreeze() - learn.fit(2, slice(1e-4, 1e-2)) + learn.fit_one_cycle(10, 1e-3, moms=(0.8, 0.7)) # save encoder learn.save_encoder(lm_enc_finetuned) From 5cc4bfd444e3c7f1a4ff28d40d65ee9c9920e5e7 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 16 Nov 2018 23:48:22 +0100 Subject: [PATCH 08/22] Add loading weights and itos, so that we can extend training --- ulmfit/pretrain_lm.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 060c4ff..12e48f4 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -92,23 +92,26 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo val_tok = val_tok[:max(20, int(len(val_tok) * ds_pct))] print(f"Limiting data sets to {ds_pct*100}%, trn {len(trn_tok)}, val: {len(val_tok)}") - # create the vocabulary - cnt = Counter(word for sent in trn_tok for word in sent) - itos = [o for o,c in cnt.most_common(n=max_vocab)] - itos.insert(1, PAD) #  set pad id to 1 to conform to fast.ai standard - assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' - stoi = {w: i for i, w in enumerate(itos)} + itos_fn=dir_path / model_dir / f'itos_{name}.pkl' + if not itos_fn.exists(): + # create the vocabulary + cnt = Counter(word for sent in trn_tok for word in sent) + itos = [o for o,c in cnt.most_common(n=max_vocab)] + itos.insert(1, PAD) #  set pad id to 1 to conform to fast.ai standard + assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' + + # save vocabulary + print(f"Saving vocabulary as {dir_path / model_dir}") + results['itos_fname'] = itos_fn + with open(results['itos_fname'], 'wb') as f: + pickle.dump(itos, f) + else: + print("Loading itos:", itos_fn) + itos = np.load(itos_fn) vocab = Vocab(itos) stoi = vocab.stoi - # save vocabulary - print(f"Saving vocabulary as {dir_path / model_dir}") - results['itos_fname'] = dir_path / model_dir / f'itos_{name}.pkl' - with open(results['itos_fname'], 'wb') as f: - pickle.dump(itos, f) - - trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) @@ -152,10 +155,15 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo else: learn.metrics = [accuracy] + try: + learn.load(f'{model_name}_{name}') + print("Weights loaded") + except FileNotFoundError: + print("Starting from random weights") + pass + learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) - - if not subword and max_vocab is None: # only if we use the unpreprocessed version and the full vocabulary # are the perplexity results comparable to previous work From e1418b21144add65f602fcf05ae1745e2e1e694f Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 17 Nov 2018 00:11:56 +0100 Subject: [PATCH 09/22] fix model dir mkdir --- ulmfit/pretrain_lm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 12e48f4..2449bab 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -56,8 +56,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo dir_path = Path(dir_path) assert dir_path.exists() - model_dir = Path(model_dir) - model_dir.mkdir(exist_ok=True) + (dir_path/model_dir).mkdir(exist_ok=True) print('Batch size:', bs) print('Max vocab:', max_vocab) model_name = 'qrnn' if qrnn else 'lstm' @@ -94,6 +93,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo itos_fn=dir_path / model_dir / f'itos_{name}.pkl' if not itos_fn.exists(): + itos_fn.parent.mkdir(exist_ok=True) # create the vocabulary cnt = Counter(word for sent in trn_tok for word in sent) itos = [o for o,c in cnt.most_common(n=max_vocab)] From e97085337e5fd4b32eef481657cb0e5ae0b93776 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 17 Nov 2018 16:47:17 +0100 Subject: [PATCH 10/22] Fix dropout and classification accuracy. 0.91 on imdb --- ulmfit/pretrain_lm.py | 2 +- ulmfit/train_clas.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 2449bab..cc1ecdc 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -140,7 +140,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo dps = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) drop_mult = 0.1 - fastai.text.learner.default_dropout['language'] = dps * drop_mult + fastai.text.learner.default_dropout['language'] = dps lm_learner = bilm_learner if bidir else language_model_learner learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 23b3ab6..6bcb84d 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -126,7 +126,8 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, qrnn=qrnn, pad_token=PAD_TOKEN_ID, pretrained_fnames=pretrained_fname, - path=model_dir.parent, model_dir=model_dir.name) + path=model_dir.parent, model_dir=model_dir.name, + drop_mult=0.3) lm_enc_finetuned = f"{lm_name}_{dataset}_{name}_enc" if fine_tune and not (model_dir / f"lm_enc_finetuned.pth").exists(): @@ -141,20 +142,20 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ print("Starting classifier training") learn = text_classifier_learner(data_clas, bptt=bptt, pad_token=PAD_TOKEN_ID, path=model_dir.parent, model_dir=model_dir.name, - qrnn=qrnn, emb_sz=emb_sz, nh=nh, nl=nl) + qrnn=qrnn, emb_sz=emb_sz, nh=nh, nl=nl, drop_mult=0.5) learn.load_encoder(lm_enc_finetuned) learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7), wd=1e-7) learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) + learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7), wd=1e-7) + learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) learn.unfreeze() - learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7), wd=1e-7) + learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) results['accuracy'] = learn.validate()[1] print(f"Saving models at {learn.path / learn.model_dir}") learn.save(f'{model_name}_{name}') From 2674a713fca14d2abbaf8d76a03321354652d8f0 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 17 Nov 2018 17:13:15 +0100 Subject: [PATCH 11/22] fix resuming training of classifier --- ulmfit/train_clas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 6bcb84d..27f014e 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -130,7 +130,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ drop_mult=0.3) lm_enc_finetuned = f"{lm_name}_{dataset}_{name}_enc" - if fine_tune and not (model_dir / f"lm_enc_finetuned.pth").exists(): + if fine_tune and not (model_dir/f"{lm_enc_finetuned}.pth").exists(): print('Fine-tuning the language model...') learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) learn.unfreeze() From 37b73e262f467f33fd08b050684016c2271ff543 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 17 Nov 2018 17:14:13 +0100 Subject: [PATCH 12/22] Fix bug where de-all was de-100 --- ulmfit/create_wikitext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/create_wikitext.py b/ulmfit/create_wikitext.py index cca6a35..942475a 100644 --- a/ulmfit/create_wikitext.py +++ b/ulmfit/create_wikitext.py @@ -100,7 +100,7 @@ def main(args): write_wikitext(lrg_wiki_train, text_iter, mt, 98000000, mode='a') all_wiki_train = all_wiki / f'{args.lang}.wiki.train.tokens' copyfile(lrg_wiki_train, all_wiki_train) - write_wikitext(lrg_wiki_train, text_iter, mt, None, mode='a') # TODO fix it (change lrg to all) + write_wikitext(all_wiki_train, text_iter, mt, None, mode='a') if __name__ == '__main__': From c821d2e7838d7f746d77d1deb1d3d48a1a159639 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Mon, 19 Nov 2018 09:59:08 +0100 Subject: [PATCH 13/22] first version of bi classifier --- fastai_contrib/learner.py | 84 +++++++++++++++++++++- fastai_contrib/models.py | 44 +++++++++--- results/bilm_qrnn-wt-103-unk-wd1 | 20 ++++++ results/bilm_qrnn-wt-103-wd0 | 20 ++++++ results/lm_lstm-wt-103-wd0 | 22 ++++++ results/lm_qrnn-wt-103-unk-wd0 | 23 ++++++ results/lm_qrnn-wt-103-wd0 | 20 ++++++ tests/test_text_train.py | 48 +++++++++++-- ulmfit/pretrain_lm.py | 16 ++--- ulmfit/train_clas.py | 119 ++++++++++++++++++------------- 10 files changed, 339 insertions(+), 77 deletions(-) create mode 100644 results/bilm_qrnn-wt-103-unk-wd1 create mode 100644 results/bilm_qrnn-wt-103-wd0 create mode 100644 results/lm_lstm-wt-103-wd0 create mode 100644 results/lm_qrnn-wt-103-unk-wd0 create mode 100644 results/lm_qrnn-wt-103-wd0 diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index d787867..3b9bafd 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -1,7 +1,8 @@ +from fastai import GradientClipping, accuracy from fastai.callbacks import * from fastai.basic_data import * from fastai.datasets import untar_data -from fastai_contrib.models import get_bilm +from fastai_contrib.models import get_bilm, get_rnn_classifier, get_birnn_classifier from fastai.text.learner import * @@ -26,7 +27,88 @@ def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:in return learn +def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400, + nh: int = 1150, nl: int = 3, + lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1, + drop_mult: float = 1., qrnn: bool = False, **kwargs) -> 'TextClassifierLearner': + "Create a RNN classifier." + dps = default_dropout['classifier'] * drop_mult + if lin_ftrs is None: lin_ftrs = [50] + if ps is None: ps = [0.1] + ds = data.train_ds + vocab_size, n_class = len(data.vocab.itos), data.c + layers = [emb_sz * 3] + lin_ftrs + [n_class] + ps = [dps[4]] + ps + model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, + layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], + qrnn=qrnn) + learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs) + return learn + + def bilm_split(model:nn.Module) -> List[nn.Module]: "Split a RNN `model` in groups for differential learning rates." return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))] + +def birnn_classifier_split(model:nn.Module) -> List[nn.Module]: + "Split a RNN `model` in groups for differential learning rates." + f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm + groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]] + groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )] + groups.append([model[1]]) + return groups + +# learner extensions +class RNNLearner(Learner): + "Basic class for a Learner in RNN." + def __init__(self, data:DataBunch, model:nn.Module, bptt:int=70, split_func:OptSplitFunc=None, clip:float=None, + adjust:bool=False, alpha:float=2., beta:float=1., **kwargs): + super().__init__(data, model, **kwargs) + self.callbacks.append(RNNTrainer(self, bptt, alpha=alpha, beta=beta, adjust=adjust)) + if clip: self.callback_fns.append(partial(GradientClipping, clip=clip)) + if split_func: self.split(split_func) + self.metrics = [accuracy] + + def model_path(self, name:str): + return self.path/self.model_dir/f'{name}.pth' + + def _get_encoder(self): + return self.model.encoder if hasattr(self.model, 'encoder') else self.model[0] + + def save_encoder(self, name:str): + "Save the encoder to `name` inside the model directory." + torch.save(self._get_encoder().state_dict(), self.model_path(name)) + + def load_encoder(self, name:str): + "Load the encoder `name` from the model directory." + self._get_encoder().load_state_dict(torch.load(self.model_path(name))) + self.freeze() + + def load_pretrained(self, wgts_fname:str, itos_fname:str): + "Load a pretrained model and adapts it to the data vocabulary." + old_itos = pickle.load(open(itos_fname, 'rb')) + old_stoi = {v:k for k,v in enumerate(old_itos)} + wgts = torch.load(wgts_fname, map_location=lambda storage, loc: storage) + wgts = convert_weights(wgts, old_stoi, self.data.train_ds.vocab.itos) + self.model.load_state_dict(wgts) + + def get_preds(self, ds_type:DatasetType=DatasetType.Valid, with_loss:bool=False, n_batch:Optional[int]=None, pbar:Optional[PBar]=None, + ordered:bool=False) -> List[Tensor]: + "Return predictions and targets on the valid, train, or test set, depending on `ds_type`." + self.model.reset() + preds = super().get_preds(ds_type=ds_type, with_loss=with_loss, n_batch=n_batch, pbar=pbar) + if ordered and hasattr(self.dl(ds_type), 'sampler'): + sampler = [i for i in self.dl(ds_type).sampler] + reverse_sampler = np.argsort(sampler) + preds[0] = preds[0][reverse_sampler,:] if preds[0].dim() > 1 else preds[0][reverse_sampler] + preds[1] = preds[1][reverse_sampler,:] if preds[1].dim() > 1 else preds[1][reverse_sampler] + return(preds) + + +def accuracy_fwd(input, targs): + return accuracy(input[...,0], targs[...,0]) + + +def accuracy_bwd(input, targs): + return accuracy(input[...,1], targs[...,1]) \ No newline at end of file diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index df323af..2b420e7 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -9,17 +9,30 @@ class BiLMModel(nn.Module): self.fwd_lm = fwd_lm self.bwd_lm = bwd_lm + def __getitem__(self, idx): + return BiLMModel(self.fwd_lm[idx], self.bwd_lm[idx]) + + def __len__(self): + return len(self.fwd_lm) + + def stack(self, fwd_o, bwd_o): + if is_listy(fwd_o): + return [self.stack(f, b) for f,b in zip(fwd_o,bwd_o)] + else: + return torch.stack([fwd_o, bwd_o], dim=len(fwd_o.shape)) + def forward(self, input): - sl, bs, tracks = input.size() + if len(input) == 3: # sl, bs, tracks + f = input[..., 0] + b = input[..., 1] + elif len(input) == 2: # sl, bs - support during classification mode + f = input + b = torch.flip(input, [0]) - decoded = [] - raw_outputs = [] - outputs = [] + fwd_o = self.fwd_lm(f) + bwd_o = self.bwd_lm(b) - fwd_o = self.fwd_lm(input[..., 0]) - bwd_o = self.bwd_lm(input[..., 1]) - - return torch.stack([fwd_o[0], bwd_o[0]], dim=2), (fwd_o[1]+bwd_o[1]), (fwd_o[2] + bwd_o[2]) + return self.stack(fwd_o, bwd_o) def reset(self): "Reset the hidden states of underlaying lms." @@ -42,4 +55,17 @@ def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, t return BiLMModel( fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), - bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias))) \ No newline at end of file + bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias))) + +def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, + pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False, + hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5)->nn.Module: + "Create a RNN classifier model." + fwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, + qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) + bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, + qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) + + model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), PoolingLinearClassifier(layers, drops)) + model.reset() + return model \ No newline at end of file diff --git a/results/bilm_qrnn-wt-103-unk-wd1 b/results/bilm_qrnn-wt-103-unk-wd1 new file mode 100644 index 0000000..dc5c8ce --- /dev/null +++ b/results/bilm_qrnn-wt-103-unk-wd1 @@ -0,0 +1,20 @@ +(fastaiv1) n-waves@GV100:~/workspace/ulmfit-multilingual$ python -m ulmfit.pretrain_lm data/wiki/wikitext-103-unk --qrnn=True --cuda-id=0 - +-name=bilm-wt-103-unk --num_epochs=10 --bs=64 --bidir=True +Batch size: 64 +Max vocab: 60000 +Using QRNNs... +Loading itos: data/wiki/wikitext-103-unk/models/itos_bilm-wt-103-unk.pkl +Size of vocabulary: 60001 +First 10 words in vocab: the, , ,, ., of, and, to, in, , a +Starting from random weights +epoch train_loss valid_loss accuracy_fwd accuracy_bwd +1 4.184118 4.113817 0.308417 0.318290 +2 4.106002 4.039978 0.312220 0.320315 +3 4.160282 4.086650 0.306327 0.314152 +4 4.136789 4.049762 0.309202 0.317373 +5 4.086520 4.001534 0.313158 0.321543 +6 4.036415 3.960494 0.318333 0.325623 +7 3.988382 3.913568 0.324409 0.330398 +8 3.937409 3.874283 0.328386 0.334786 +9 3.912215 3.856325 0.330593 0.337102 +10 3.876535 3.848783 0.331202 0.337649 \ No newline at end of file diff --git a/results/bilm_qrnn-wt-103-wd0 b/results/bilm_qrnn-wt-103-wd0 new file mode 100644 index 0000000..291173e --- /dev/null +++ b/results/bilm_qrnn-wt-103-wd0 @@ -0,0 +1,20 @@ +(fastaiv1) n-waves@GV100:~/workspace/ulmfit-multilingual$ python -m ulmfit.pretrain_lm +ki/wikitext-103 --qrnn=True --cuda-id=0 --name=bilm-wt-103 --num_epochs=10 --bs=64 --bidir=True +Batch size: 64 +Max vocab: 60000 +Using QRNNs... +Loading itos: data/wiki/wikitext-103/models/itos_bilm-wt-103.pkl +Size of vocabulary: 60001 +First 10 words in vocab: the, , ,, ., of, and, to, in, , a +Starting from random weights +epoch train_loss valid_loss accuracy_fwd accuracy_bwd +1 4.141750 4.114931 0.307700 0.320260 +2 4.078711 4.061186 0.310221 0.320550 +3 4.108722 4.081286 0.308383 0.316431 +4 4.054437 4.040497 0.311229 0.319658 +5 4.014880 3.966010 0.318588 0.327607 +6 3.926770 3.897305 0.326874 0.333171 +7 3.855364 3.816452 0.335632 0.342045 +8 3.761600 3.745798 0.343413 0.350675 + +10 3.614097 3.687730 0.351651 0.358280 \ No newline at end of file diff --git a/results/lm_lstm-wt-103-wd0 b/results/lm_lstm-wt-103-wd0 new file mode 100644 index 0000000..41c178f --- /dev/null +++ b/results/lm_lstm-wt-103-wd0 @@ -0,0 +1,22 @@ +(fastaiv1) pczapla@galatea ~/w/ulmfit-multilingual ❯❯❯ time python -m ulmfit.pretrain_lm data/wiki/wikitext-103 --qrnn=False --cuda-id=1 --name=wt-103 --num_epochs=10 --bs=32 +Batch size: 32 +Max vocab: 60000 +Loading itos: data/wiki/wikitext-103/models/itos_wt-103.pkl +Size of vocabulary: 60001 +First 10 words in vocab: the, , ,, ., of, and, to, in, , a +Starting from random weights +epoch train_loss valid_loss accuracy +1 4.317837 4.245714 0.309034 +2 4.347099 4.251623 0.306932 +3 4.360198 4.302850 0.302014 +4 4.329998 4.264225 0.306828 +5 4.301852 4.209807 0.311763 +6 4.218213 4.131647 0.319867 +7 4.167365 4.047796 0.327259 +8 4.095695 3.980298 0.336255 +9 4.032371 3.919622 0.343671 +10 3.983160 3.906227 0.345863 +Saving models at data/wiki/wikitext-103/models +Saving optimiser state at data/wiki/wikitext-103/models/lstm3_wt-103_state.pth +accuracy: tensor(0.3461) +python -m ulmfit.pretrain_lm data/wiki/wikitext-103 --qrnn=False --cuda-id=1 44147.64s user 16746.69s system 99% cpu 16:58:47.57 total \ No newline at end of file diff --git a/results/lm_qrnn-wt-103-unk-wd0 b/results/lm_qrnn-wt-103-unk-wd0 new file mode 100644 index 0000000..d009387 --- /dev/null +++ b/results/lm_qrnn-wt-103-unk-wd0 @@ -0,0 +1,23 @@ +time python -m ulmfit.pretrain_lm data/wiki/wikitext-103-unk --qrnn=True --cuda-id=0 --name=wt-103 --num_epochs=10 --bs=32 +Batch size: 32 +Max vocab: 60000 +Using QRNNs... +Loading itos: data/wiki/wikitext-103-unk/models/itos_wt-103.pkl +Size of vocabulary: 60001 +First 10 words in vocab: the, , ,, ., of, and, to, in, , a +Starting from random weights +epoch train_loss valid_loss accuracy +1 4.393613 4.282537 0.305135 +2 4.394575 4.299973 0.300028 +3 4.444056 4.336682 0.295496 +4 4.414292 4.332908 0.297408 +5 4.406248 4.269964 0.302962 +6 4.327769 4.209948 0.309025 +7 4.244819 4.140769 0.315675 +8 4.210000 4.075574 0.323544 +9 4.139524 4.034881 0.328550 +10 4.150556 4.021361 0.331256 +Saving models at data/wiki/wikitext-103-unk/models +Saving optimiser state at data/wiki/wikitext-103-unk/models/qrnn3_wt-103_state.pth +accuracy: tensor(0.3312) +python -m ulmfit.pretrain_lm data/wiki/wikitext-103-unk --qrnn=True --bs=3 32119.72s user 12439.18s system 99% cpu 12:25:03.65 total \ No newline at end of file diff --git a/results/lm_qrnn-wt-103-wd0 b/results/lm_qrnn-wt-103-wd0 new file mode 100644 index 0000000..33d243b --- /dev/null +++ b/results/lm_qrnn-wt-103-wd0 @@ -0,0 +1,20 @@ +time python -m ulmfit.pretrain_lm data/wiki/wikitext-103 --qrnn=True --cuda-id=1 --name=wt-103 --num_epochs=10 --bs=32 ✘ 1 +Batch size: 32 +Max vocab: 60000 +Using QRNNs... +Size of vocabulary: 60001 +First 10 words in vocab: the, , ,, ., of, and, to, in, , a +Saving vocabulary as data/wiki/wikitext-103/models +epoch train_loss valid_loss accuracy +1 4.328219 4.268609 0.306414 +2 4.351323 4.282290 0.300560 +3 4.376134 4.345460 0.294145 +4 4.394314 4.319661 0.298142 +5 4.337362 4.265630 0.303437 +6 4.295763 4.199119 0.309354 +7 4.165526 4.121037 0.318062 +8 4.151789 4.060387 0.325899 +9 4.066542 4.014568 0.332732 +10 4.013142 3.997691 0.335299 +Saving models at data/wiki/wikitext-103/models +Saving optimiser state at data/wiki/wikitext-103/models/qrnn3_wt-103_state.pth \ No newline at end of file diff --git a/tests/test_text_train.py b/tests/test_text_train.py index 5116cfa..0103939 100644 --- a/tests/test_text_train.py +++ b/tests/test_text_train.py @@ -4,9 +4,11 @@ from fastai.text import * pytestmark = pytest.mark.integration +print(sys.path) import fastai_contrib.data as contrib_data -from fastai_contrib.learner import bilm_learner +from fastai_contrib.learner import bilm_learner, accuracy_fwd, bilm_text_classifier_learner + def read_file(fname): texts = [] @@ -38,11 +40,44 @@ def learn(): learn.fit_one_cycle(4, 5e-3) return learn +def text_df(n_labels): + data = [] + texts = ["fast ai is a cool project", "hello world"] + for ind, text in enumerate(texts): + sample = {} + for label in range(n_labels): sample[label] = ind%2 + sample["text"] = text + data.append(sample) + df = pd.DataFrame(data) + return df + ###################### NEW CODE def test_val_loss(learn): assert learn.validate()[1] > 0.5 + +def test_bilm_classifier_loads_encoder(): + n_labels=2 + path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp') + os.makedirs(path) + try: + df = text_df(n_labels=1) + lmdf = df#[["text"]] + print(lmdf.head()) + lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer), + lm_type=contrib_data.LanguageModelType.BiLM) + learn = bilm_learner(lmdata, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + learn.save_encoder("enc") + data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"]) + classifier = bilm_text_classifier_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) + print(last_layer(classifier.model), ) + classifier.load_encoder("enc") + classifier.fit(1) + finally: + shutil.rmtree(path) + + def test_bilm_lstm_can_be_trained(): manual_seed() path, df_trn, df_val = prep_human_numbers() @@ -50,9 +85,10 @@ def test_bilm_lstm_can_be_trained(): lm_type = contrib_data.LanguageModelType.BiLM) learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) - learn.metrics = [] - learn.fit_one_cycle(4, 5e-3) - assert learn.validate()[0] < 2 #TODO Change to accuracy once it is fixed + learn.metrics = [accuracy_fwd] + learn.fit_one_cycle(2, 5e-3) + assert learn.validate()[1] > 0.3 + def test_bwdlm_lstm_can_be_trained(): manual_seed() @@ -61,5 +97,5 @@ def test_bwdlm_lstm_can_be_trained(): lm_type = contrib_data.LanguageModelType.BwdLM) learn = language_model_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) - learn.fit_one_cycle(4, 5e-3) - assert learn.validate()[1] > 0.5 + learn.fit_one_cycle(2, 5e-3) + assert learn.validate()[1] > 0.3 diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index cc1ecdc..856fc82 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -6,14 +6,13 @@ to be split. """ import fastai import fire -import numpy as np from fastai import * from fastai.text import * import torch -from fastai_contrib.utils import read_file, read_whitespace_file,\ - DataStump, validate, PAD, UNK, get_sentencepiece -from fastai_contrib.learner import bilm_learner +from fastai_contrib.utils import read_file, read_whitespace_file, \ + validate, PAD, UNK, get_sentencepiece +from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd import pickle from pathlib import Path @@ -25,11 +24,6 @@ import fastai_contrib.data as contrib_data # conda install -c pytorch -c fastai fastai pytorch-nightly [cuda92] # cupy needs to be installed for QRNN -def accuracy_fwd(input, targs): - return accuracy(input[...,0], targs[...,0]) -def accuracy_bwd(input, targs): - return accuracy(input[...,1], targs[...,1]) - def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vocab=60000, bs=70, bptt=70, name='wt-103', num_epochs=10, bidir=False, ds_pct=1.0): @@ -148,7 +142,9 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo bias=True, qrnn=qrnn, clip=0.12) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) - learn.true_wd = False + + #learn.true_wd = False + print("true_wd: ", learn.true_wd) if bidir: learn.metrics = [accuracy_fwd, accuracy_bwd] diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 27f014e..747c421 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -8,6 +8,7 @@ import pickle import torch from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner from fastai import fit_one_cycle +from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists from fastai.text.transform import Vocab @@ -19,7 +20,7 @@ from pathlib import Path def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_dir='models', qrnn=False, fine_tune=True, max_vocab=60000, bs=20, bptt=70, name='imdb-clas', - dataset='imdb', ds_pct=1.0): + dataset='imdb', bidir=False, ds_pct=1.0, train=True): """ :param data_dir: The path to the `data` directory :param lang: the language unicode @@ -67,11 +68,73 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ model_dir/f"{pretrained_fname[0]}.pth", model_dir/f"{pretrained_fname[1]}.pkl") + data_clas, data_lm = get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct) + if qrnn: + emb_sz, nh, nl = 400, 1550, 3 + else: + emb_sz, nh, nl = 400, 1150, 3 + + if bidir: + classifier_learner = bilm_text_classifier_learner + lm_learner = bilm_learner + else: + classifier_learner = text_classifier_learner + lm_learner = language_model_learner + + lm_enc_finetuned = f"{lm_name}_{dataset}_{name}_enc" + if fine_tune and not (model_dir/f"{lm_enc_finetuned}.pth").exists(): + print('Fine-tuning the language model...') + learn = lm_learner( + data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, qrnn=qrnn, + pad_token=PAD_TOKEN_ID, + pretrained_fnames=pretrained_fname, + path=model_dir.parent, model_dir=model_dir.name, + drop_mult=0.3) + learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) + learn.unfreeze() + learn.fit_one_cycle(10, 1e-3, moms=(0.8, 0.7)) + + # save encoder + learn.save_encoder(lm_enc_finetuned) + + + learn = classifier_learner(data_clas, bptt=bptt, pad_token=PAD_TOKEN_ID, + path=model_dir.parent, model_dir=model_dir.name, + qrnn=qrnn, emb_sz=emb_sz, nh=nh, nl=nl, drop_mult=0.5) + + try: + learn.load(f'{model_name}_{name}') + print("Loading classifier") + except FileNotFoundError: + learn.load_encoder(lm_enc_finetuned) + print("loading encoder") + train = True + + if train: + print("Starting classifier training") + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7), wd=1e-7) + + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) + + learn.unfreeze() + learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + + print(f"Saving models at {learn.path / learn.model_dir}") + learn.save(f'{model_name}_{name}') + + results['accuracy'] = learn.validate()[1] + return results + + +def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct): tmp_dir = dataset_dir / 'tmp' tmp_dir.mkdir(exist_ok=True) vocab_file = tmp_dir / f'vocab_{lang}.pkl' - if not (tmp_dir / f'{TRN}_{lang}_ids.npy').exists(): print('Reading the data...') toks, lbls = read_clas_data(dataset_dir, dataset, lang) @@ -100,66 +163,20 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ lbls[split] = np.load(tmp_dir / f'{split}_{lang}_lbl.npy') with open(vocab_file, 'rb') as f: vocab = pickle.load(f) - print(f'Train size: {len(ids[TRN])}. Valid size: {len(ids[VAL])}. ' f'Test size: {len(ids[TST])}.') - if ds_pct < 1.0: print(f"Making the dataset smaller {ds_pct}") for split in [TRN, VAL, TST]: - ids[split] = ids[split][:int(len(ids[split])*ds_pct)] - - + ids[split] = ids[split][:int(len(ids[split]) * ds_pct)] data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL], bs=bs, bptt=bptt) - - # TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls? + #  TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls? data_clas = TextClasDataBunch.from_ids( path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL], train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs) + return data_clas, data_lm - if qrnn: - emb_sz, nh, nl = 400, 1550, 3 - else: - emb_sz, nh, nl = 400, 1150, 3 - learn = language_model_learner( - data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, qrnn=qrnn, - pad_token=PAD_TOKEN_ID, - pretrained_fnames=pretrained_fname, - path=model_dir.parent, model_dir=model_dir.name, - drop_mult=0.3) - - lm_enc_finetuned = f"{lm_name}_{dataset}_{name}_enc" - if fine_tune and not (model_dir/f"{lm_enc_finetuned}.pth").exists(): - print('Fine-tuning the language model...') - learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) - learn.unfreeze() - learn.fit_one_cycle(10, 1e-3, moms=(0.8, 0.7)) - - # save encoder - learn.save_encoder(lm_enc_finetuned) - - print("Starting classifier training") - learn = text_classifier_learner(data_clas, bptt=bptt, pad_token=PAD_TOKEN_ID, - path=model_dir.parent, model_dir=model_dir.name, - qrnn=qrnn, emb_sz=emb_sz, nh=nh, nl=nl, drop_mult=0.5) - - learn.load_encoder(lm_enc_finetuned) - - learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7), wd=1e-7) - - learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) - - learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) - - learn.unfreeze() - learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) - results['accuracy'] = learn.validate()[1] - print(f"Saving models at {learn.path / learn.model_dir}") - learn.save(f'{model_name}_{name}') - return results if __name__ == '__main__': fire.Fire(new_train_clas) From 7f1f8efcc332b5147dd4104acf20b1266c609ba8 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Mon, 19 Nov 2018 12:58:58 +0100 Subject: [PATCH 14/22] Working version of biclassfier --- fastai_contrib/learner.py | 83 +++++++++++++++++---------------------- fastai_contrib/models.py | 48 ++++++++++++++++++++-- ulmfit/pretrain_lm.py | 2 +- ulmfit/train_clas.py | 30 ++++++++------ 4 files changed, 99 insertions(+), 64 deletions(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index 3b9bafd..f3dc5dc 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -27,6 +27,34 @@ def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:in return learn + +def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str]) -> Weights: + "Convert the model weights to go with a new vocabulary." + print(wgts.keys()) + if 'fwd_lm.0.encoder.weight' in wgts: #todo share embedding matrix computation + wgts = convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='fwd_lm.') + return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='bwd_lm.') + else: + return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='') + +def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str], prefix='') -> Weights: + "Convert the model weights to go with a new vocabulary." + dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight'] + bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0) + new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_() + new_b = dec_bias.new_zeros((len(itos_new),)).zero_() + for i,w in enumerate(itos_new): + r = stoi_wgts[w] if w in stoi_wgts else -1 + new_w[i] = enc_wgts[r] if r>=0 else wgts_m + new_b[i] = dec_bias[r] if r>=0 else bias_m + wgts[prefix+'0.encoder.weight'] = new_w + wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone() + wgts[prefix+'1.decoder.weight'] = new_w.clone() + wgts[prefix+'1.decoder.bias'] = new_b + return wgts + + + def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400, nh: int = 1150, nl: int = 3, lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1, @@ -37,7 +65,7 @@ def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = if ps is None: ps = [0.1] ds = data.train_ds vocab_size, n_class = len(data.vocab.itos), data.c - layers = [emb_sz * 3] + lin_ftrs + [n_class] + layers = [emb_sz * 3 * 2] + lin_ftrs + [n_class] ps = [dps[4]] + ps model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], @@ -59,56 +87,15 @@ def birnn_classifier_split(model:nn.Module) -> List[nn.Module]: groups.append([model[1]]) return groups -# learner extensions -class RNNLearner(Learner): - "Basic class for a Learner in RNN." - def __init__(self, data:DataBunch, model:nn.Module, bptt:int=70, split_func:OptSplitFunc=None, clip:float=None, - adjust:bool=False, alpha:float=2., beta:float=1., **kwargs): - super().__init__(data, model, **kwargs) - self.callbacks.append(RNNTrainer(self, bptt, alpha=alpha, beta=beta, adjust=adjust)) - if clip: self.callback_fns.append(partial(GradientClipping, clip=clip)) - if split_func: self.split(split_func) - self.metrics = [accuracy] - - def model_path(self, name:str): - return self.path/self.model_dir/f'{name}.pth' - - def _get_encoder(self): - return self.model.encoder if hasattr(self.model, 'encoder') else self.model[0] - - def save_encoder(self, name:str): - "Save the encoder to `name` inside the model directory." - torch.save(self._get_encoder().state_dict(), self.model_path(name)) - - def load_encoder(self, name:str): - "Load the encoder `name` from the model directory." - self._get_encoder().load_state_dict(torch.load(self.model_path(name))) - self.freeze() - - def load_pretrained(self, wgts_fname:str, itos_fname:str): - "Load a pretrained model and adapts it to the data vocabulary." - old_itos = pickle.load(open(itos_fname, 'rb')) - old_stoi = {v:k for k,v in enumerate(old_itos)} - wgts = torch.load(wgts_fname, map_location=lambda storage, loc: storage) - wgts = convert_weights(wgts, old_stoi, self.data.train_ds.vocab.itos) - self.model.load_state_dict(wgts) - - def get_preds(self, ds_type:DatasetType=DatasetType.Valid, with_loss:bool=False, n_batch:Optional[int]=None, pbar:Optional[PBar]=None, - ordered:bool=False) -> List[Tensor]: - "Return predictions and targets on the valid, train, or test set, depending on `ds_type`." - self.model.reset() - preds = super().get_preds(ds_type=ds_type, with_loss=with_loss, n_batch=n_batch, pbar=pbar) - if ordered and hasattr(self.dl(ds_type), 'sampler'): - sampler = [i for i in self.dl(ds_type).sampler] - reverse_sampler = np.argsort(sampler) - preds[0] = preds[0][reverse_sampler,:] if preds[0].dim() > 1 else preds[0][reverse_sampler] - preds[1] = preds[1][reverse_sampler,:] if preds[1].dim() > 1 else preds[1][reverse_sampler] - return(preds) - def accuracy_fwd(input, targs): return accuracy(input[...,0], targs[...,0]) def accuracy_bwd(input, targs): - return accuracy(input[...,1], targs[...,1]) \ No newline at end of file + return accuracy(input[...,1], targs[...,1]) + + +## Replace code in fastai +import fastai.text.learner +fastai.text.learner.convert_weights = convert_weights \ No newline at end of file diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index 2b420e7..3d04bdb 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -22,13 +22,14 @@ class BiLMModel(nn.Module): return torch.stack([fwd_o, bwd_o], dim=len(fwd_o.shape)) def forward(self, input): - if len(input) == 3: # sl, bs, tracks + if len(input.shape) == 3: # sl, bs, tracks f = input[..., 0] b = input[..., 1] - elif len(input) == 2: # sl, bs - support during classification mode + elif len(input.shape) == 2: # sl, bs - support during classification mode f = input b = torch.flip(input, [0]) - + else: + raise AttributeError(f"Inorrect size of input, {input.shape}") fwd_o = self.fwd_lm(f) bwd_o = self.bwd_lm(b) @@ -39,6 +40,45 @@ class BiLMModel(nn.Module): self.fwd_lm.reset() self.bwd_lm.reset() + +class BiPoolingLinearClassifier(nn.Module): + "Create a linear classifier with pooling." + + def __init__(self, layers:Collection[int], drops:Collection[float]): + super().__init__() + mod_layers = [] + activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None] + for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs): + mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn) + self.layers = nn.Sequential(*mod_layers) + + def pool(self, x:Tensor, bs:int, is_max:bool): + "Pool the tensor along the seq_len dimension." + f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d + return f(x.permute(1,2,0), (1,)).view(bs,-1) + + def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]: + raw_outputs, outputs = input + output = outputs[-1] + if len(output.size()) == 3: + sl,bs,_ = output.size() + avgpool = self.pool(output, bs, False) + mxpool = self.pool(output, bs, True) + x = torch.cat([output[-1], mxpool, avgpool], 1) + x = self.layers(x) + return x, raw_outputs, outputs + elif len(output.size()) == 4: + sl, bs, em_sz, passes = output.size() + + f_avgpool = self.pool(output[...,0], bs, False) + f_mxpool = self.pool(output[...,0], bs, True) + b_avgpool = self.pool(output[..., 1], bs, False) + b_mxpool = self.pool(output[..., 1], bs, True) + x = torch.cat([output[-1][..., 0], f_mxpool, f_avgpool, + output[-1][..., 1], b_mxpool, b_avgpool,], 1) + x = self.layers(x) + return x, raw_outputs, outputs + def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True, qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5)->nn.Module: @@ -66,6 +106,6 @@ def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_s bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), PoolingLinearClassifier(layers, drops)) + model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), BiPoolingLinearClassifier(layers, drops)) model.reset() return model \ No newline at end of file diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 856fc82..c022af8 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -143,7 +143,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) - #learn.true_wd = False + learn.true_wd = False print("true_wd: ", learn.true_wd) if bidir: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 747c421..b4bc636 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -7,8 +7,9 @@ import pickle import torch from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner -from fastai import fit_one_cycle -from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner +from fastai import fit_one_cycle, accuracy +from fastai_contrib.data import LanguageModelType +from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists from fastai.text.transform import Vocab @@ -68,20 +69,22 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ model_dir/f"{pretrained_fname[0]}.pth", model_dir/f"{pretrained_fname[1]}.pkl") - data_clas, data_lm = get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct) - - if qrnn: - emb_sz, nh, nl = 400, 1550, 3 - else: - emb_sz, nh, nl = 400, 1150, 3 - if bidir: + print("BiLM") classifier_learner = bilm_text_classifier_learner lm_learner = bilm_learner else: classifier_learner = text_classifier_learner lm_learner = language_model_learner + lm_type = LanguageModelType.BiLM if bidir else LanguageModelType.FwdLM + data_clas, data_lm = get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_type=lm_type) + + if qrnn: + emb_sz, nh, nl = 400, 1550, 3 + else: + emb_sz, nh, nl = 400, 1150, 3 + lm_enc_finetuned = f"{lm_name}_{dataset}_{name}_enc" if fine_tune and not (model_dir/f"{lm_enc_finetuned}.pth").exists(): print('Fine-tuning the language model...') @@ -91,6 +94,11 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ pretrained_fnames=pretrained_fname, path=model_dir.parent, model_dir=model_dir.name, drop_mult=0.3) + if bidir: + learn.metrics = [accuracy_fwd, accuracy_bwd] + else: + learn.metrics = [accuracy] + learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) learn.unfreeze() learn.fit_one_cycle(10, 1e-3, moms=(0.8, 0.7)) @@ -131,7 +139,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ return results -def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct): +def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_type): tmp_dir = dataset_dir / 'tmp' tmp_dir.mkdir(exist_ok=True) vocab_file = tmp_dir / f'vocab_{lang}.pkl' @@ -170,7 +178,7 @@ def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct): for split in [TRN, VAL, TST]: ids[split] = ids[split][:int(len(ids[split]) * ds_pct)] data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=ids[TRN], - valid_ids=ids[VAL], bs=bs, bptt=bptt) + valid_ids=ids[VAL], bs=bs, bptt=bptt, lm_type=lm_type) #  TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls? data_clas = TextClasDataBunch.from_ids( path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL], From 9f526a98495bd03eef596eb963b7bdc0e4d7222b Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Mon, 19 Nov 2018 15:30:28 +0100 Subject: [PATCH 15/22] Add experiment where wd_true is set to True. --- results/lm_qrnn-wt-103-unk-wd1 | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 results/lm_qrnn-wt-103-unk-wd1 diff --git a/results/lm_qrnn-wt-103-unk-wd1 b/results/lm_qrnn-wt-103-unk-wd1 new file mode 100644 index 0000000..0ee4a69 --- /dev/null +++ b/results/lm_qrnn-wt-103-unk-wd1 @@ -0,0 +1,24 @@ +ime python -m ulmfit.pretrain_lm data/wiki/wikitext-103-unk --qrnn=True --cuda-id=0 --name=wt-103-wd1 --num_epochs=10 --bs=32 +Batch size: 32 +Max vocab: 60000 +Using QRNNs... +Saving vocabulary as data/wiki/wikitext-103-unk/models +Size of vocabulary: 60001 +First 10 words in vocab: the, , ,, ., of, and, to, in, , a +true_wd: True +Starting from random weights +epoch train_loss valid_loss accuracy +1 4.355600 4.271729 0.306827 +2 4.384312 4.288582 0.299482 +3 4.527577 4.366407 0.289836 +4 4.519645 4.387311 0.288076 +5 4.514816 4.371242 0.289862 +6 4.506230 4.337718 0.294208 +7 4.444272 4.295349 0.298786 +8 4.432104 4.257432 0.301785 +9 4.416849 4.242803 0.304147 +10 4.389035 4.239752 0.304871 +Saving models at data/wiki/wikitext-103-unk/models +Saving optimiser state at data/wiki/wikitext-103-unk/models/qrnn3_wt-103-wd1_state.pth +itos_fname: data/wiki/wikitext-103-unk/models/itos_wt-103-wd1.pkl +accuracy: tensor(0.2829) \ No newline at end of file From 2bde30a40262b1a5a43a049d123c8006a75ca04a Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Mon, 19 Nov 2018 15:31:10 +0100 Subject: [PATCH 16/22] Add experiment with biclassifier (concat) --- results/bilcls_qrnn-wt-103-wd0 | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 results/bilcls_qrnn-wt-103-wd0 diff --git a/results/bilcls_qrnn-wt-103-wd0 b/results/bilcls_qrnn-wt-103-wd0 new file mode 100644 index 0000000..4f2a68f --- /dev/null +++ b/results/bilcls_qrnn-wt-103-wd0 @@ -0,0 +1,26 @@ +pretraining +0 - 8 lost, about 0.30 0.32 after 5 epochs +8 3.744213 4.021358 0.308377 0.325672 +9 3.700674 4.021499 0.308637 0.325881 +10 3.674045 4.022058 0.308656 0.325903 +--- crash--- +$ python -m ulmfit.train_clas --data_dir data --model_dir data/wiki/wikitext-103/models --pretrain_name=bilm-wt-103 --qrnn=True --name 'concat-2x' --cuda-id=0 --bs 40 --train=True --bidir=True +Dataset: imdb. Language: en. +Using QRNNs... +BiLM +Loading the pickled data... +Train size: 22500. Valid size: 2500. Test size: 25000. +loading encoder +Starting classifier training +epoch train_loss valid_loss accuracy +1 0.346874 0.276838 0.881200 +epoch train_loss valid_loss accuracy +1 0.321121 0.243946 0.901200 +epoch train_loss valid_loss accuracy +1 0.303174 0.234627 0.908400 +epoch train_loss valid_loss accuracy +1 0.295207 0.227533 0.912800 +2 0.269754 0.221328 0.914400 +Saving models at data/wiki/wikitext-103/models +accuracy: tensor(0.9144) + From 6e3ef21b1ff65b9e6570fbe50a9f037d2b33e963 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 21 Nov 2018 18:44:49 +0100 Subject: [PATCH 17/22] Add Avg BiClassifier --- fastai_contrib/learner.py | 2 +- fastai_contrib/models.py | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index f3dc5dc..a1dfc27 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -65,7 +65,7 @@ def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = if ps is None: ps = [0.1] ds = data.train_ds vocab_size, n_class = len(data.vocab.itos), data.c - layers = [emb_sz * 3 * 2] + lin_ftrs + [n_class] + layers = [emb_sz * 3] + lin_ftrs + [n_class] ps = [dps[4]] + ps model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index 3d04bdb..b99db97 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -79,6 +79,43 @@ class BiPoolingLinearClassifier(nn.Module): x = self.layers(x) return x, raw_outputs, outputs + +class AvgPoolingLinearClassifier(nn.Module): + "Create a linear classifier with pooling." + + def __init__(self, layers:Collection[int], drops:Collection[float]): + super().__init__() + mod_layers = [] + activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None] + for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs): + mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn) + self.layers = nn.Sequential(*mod_layers) + + def pool(self, x:Tensor, bs:int, is_max:bool): + "Pool the tensor along the seq_len dimension." + f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d + return f(x.permute(1,2,0), (1,)).view(bs,-1) + + def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]: + raw_outputs, outputs = input + output = outputs[-1] + if len(output.size()) == 3: + sl,bs,_ = output.size() + avgpool = self.pool(output, bs, False) + mxpool = self.pool(output, bs, True) + x = torch.cat([output[-1], mxpool, avgpool], 1) + x = self.layers(x) + return x, raw_outputs, outputs + elif len(output.size()) == 4: + sl, bs, em_sz, passes = output.size() + + avgpool = (self.pool(output[...,0], bs, False) + self.pool(output[..., 1], bs, False))/2 + mxpool = (self.pool(output[...,0], bs, True) +self.pool(output[..., 1], bs, True))/2 + x = torch.cat([(output[-1][..., 0]+output[-1][..., 1])/2, mxpool, avgpool], 1) + x = self.layers(x) + return x, raw_outputs, outputs + + def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True, qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5)->nn.Module: @@ -106,6 +143,6 @@ def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_s bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), BiPoolingLinearClassifier(layers, drops)) + model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), AvgPoolingLinearClassifier(layers, drops)) model.reset() return model \ No newline at end of file From 979eb196d8bed8aa5a0322183ed3a7691505d8c7 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 21 Nov 2018 18:46:05 +0100 Subject: [PATCH 18/22] Change the classfication training learning rate to the one that was working te best in my exp. on bidirectional clasification --- ulmfit/train_clas.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index b4bc636..0976e1e 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -85,7 +85,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ else: emb_sz, nh, nl = 400, 1150, 3 - lm_enc_finetuned = f"{lm_name}_{dataset}_{name}_enc" + lm_enc_finetuned = f"{lm_name}_{dataset}_{pretrain_name}_enc" if fine_tune and not (model_dir/f"{lm_enc_finetuned}.pth").exists(): print('Fine-tuning the language model...') learn = lm_learner( @@ -120,22 +120,23 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ train = True if train: + learn.true_wd = False print("Starting classifier training") - learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7), wd=1e-7) + learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) + learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) learn.unfreeze() - learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) print(f"Saving models at {learn.path / learn.model_dir}") learn.save(f'{model_name}_{name}') - results['accuracy'] = learn.validate()[1] + results['accuracy'] = learn.metrics[-1][0] return results From 9aa877dcd0a87e49115904b5a31953ea822475b1 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 21 Nov 2018 18:51:42 +0100 Subject: [PATCH 19/22] Share trained LM between different classfiication runs --- ulmfit/train_clas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0976e1e..953d384 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -85,9 +85,9 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ else: emb_sz, nh, nl = 400, 1150, 3 - lm_enc_finetuned = f"{lm_name}_{dataset}_{pretrain_name}_enc" + lm_enc_finetuned = f"{lm_name}_{dataset}_enc" if fine_tune and not (model_dir/f"{lm_enc_finetuned}.pth").exists(): - print('Fine-tuning the language model...') + print('Fine-tuning the language model...', lm_enc_finetuned) learn = lm_learner( data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, qrnn=qrnn, pad_token=PAD_TOKEN_ID, From dbd4884228a09bd805d28cb84a0b49803f9d075e Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 22 Nov 2018 01:14:38 +0100 Subject: [PATCH 20/22] Fixes after mergin with master and updateing to newset fastai --- fastai_contrib/utils.py | 2 +- ulmfit/pretrain_lm.py | 11 ++++------- ulmfit/train_clas.py | 8 ++++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 3017ffc..37a76b3 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -225,7 +225,7 @@ def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], Li reader = csv.reader(f) for row in reader: label, text = row - lbls.append(label) + lbls.append(int(label)) raw_tokens = mt.tokenize(text, return_str=True).split(' ') tokens = [] diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 67d72fb..5b7098a 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -42,7 +42,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo :param bidir: whether the language model is bidirectional """ results = {} - model_dir = 'models' # removed from params, as it is absolute models location in train_clas and here it is relative + if not torch.cuda.is_available(): print('CUDA not available. Setting device=-1.') cuda_id = -1 @@ -50,7 +50,8 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo dir_path = Path(dir_path) assert dir_path.exists() - (dir_path/model_dir).mkdir(exist_ok=True) + model_dir = dir_path / 'models' # removed from params, as it is absolute models location in train_clas and here it is relative + model_dir.mkdir(exist_ok=True) print('Batch size:', bs) print('Max vocab:', max_vocab) model_name = 'qrnn' if qrnn else 'lstm' @@ -93,11 +94,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo itos.insert(1, PAD) #  set pad id to 1 to conform to fast.ai standard assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' - vocab = Vocab(itos) - stoi = vocab.stoi - # save vocabulary - print(f"Saving vocabulary as {itos_fname}") results['itos_fname'] = itos_fname with open(itos_fname, 'wb') as f: @@ -139,7 +136,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo lm_learner = bilm_learner if bidir else language_model_learner learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, - drop_mult=drop_mult, tie_weights=True, model_dir=model_dir, + drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name, bias=True, qrnn=qrnn, clip=0.12) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 953d384..a5e981d 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -136,7 +136,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ print(f"Saving models at {learn.path / learn.model_dir}") learn.save(f'{model_name}_{name}') - results['accuracy'] = learn.metrics[-1][0] + results['accuracy'] = learn.recorder.metrics[-1][0] return results @@ -149,7 +149,7 @@ def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_typ toks, lbls = read_clas_data(dataset_dir, dataset, lang) # create the vocabulary - counter = Counter(word for example in toks[TRN] for word in example) + counter = Counter(word for example in np.concatenate([toks[TRN],toks[TST],toks[VAL]]) for word in example) itos = [word for word, count in counter.most_common(n=max_vocab)] itos.insert(0, PAD) itos.insert(0, UNK) @@ -178,12 +178,12 @@ def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_typ print(f"Making the dataset smaller {ds_pct}") for split in [TRN, VAL, TST]: ids[split] = ids[split][:int(len(ids[split]) * ds_pct)] - data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=ids[TRN], + data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=np.concatenate([ids[TRN],ids[TST]]), valid_ids=ids[VAL], bs=bs, bptt=bptt, lm_type=lm_type) #  TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls? data_clas = TextClasDataBunch.from_ids( path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL], - train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs) + train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs, classes={l:l for l in lbls[VAL]}) return data_clas, data_lm From 8da47324c2737e5727c0d997df1e123f31571e2c Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 22 Nov 2018 15:32:40 +0100 Subject: [PATCH 21/22] Clean ups and fixes --- fastai_contrib/data.py | 11 ++--- fastai_contrib/learner.py | 90 ++++++++++++++++++------------------- fastai_contrib/models.py | 6 ++- fastai_contrib/utils.py | 19 ++++---- tests/test_end_to_end.py | 94 ++++++++++++++++++++------------------- ulmfit/train_clas.py | 17 ++++--- 6 files changed, 125 insertions(+), 112 deletions(-) diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py index 3db59c5..e14706d 100644 --- a/fastai_contrib/data.py +++ b/fastai_contrib/data.py @@ -5,8 +5,7 @@ from fastai.text.transform import * from fastai.basic_data import * from fastai.data_block import * - -###################### UPDATED CODE +#region Modified fastai classes LanguageModelType=Enum('LanguageModelType', 'FwdLM BwdLM BiLM') @@ -24,7 +23,7 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader if getattr(self.dataset, 'item', None) is not None: yield LongTensor(getattr(self.dataset, 'item')).unsqueeze(1),LongTensor([0]) idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset)) - data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx])) + data = self.batchify(np.concatenate([np.array(self.dataset.x.items[i], dtype=np.int) for i in idx])) pos, itr = 0,0 while pos < self.n-1 and itr 'TextClassifierLearner': + "Create a RNN classifier." + dps = default_dropout['classifier'] * drop_mult + if lin_ftrs is None: lin_ftrs = [50] + if ps is None: ps = [0.1] + ds = data.train_ds + vocab_size, n_class = len(data.vocab.itos), data.c + layers = [emb_sz * 3] + lin_ftrs + [n_class] + ps = [dps[4]] + ps + model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, + layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], + qrnn=qrnn) + learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs) + return learn +def bilm_split(model:nn.Module) -> List[nn.Module]: + "Split a RNN `model` in groups for differential learning rates." + + return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))] + +def birnn_classifier_split(model:nn.Module) -> List[nn.Module]: + "Split a RNN `model` in groups for differential learning rates." + f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm + groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]] + groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )] + groups.append([model[1]]) + return groups + +def accuracy_fwd(input, targs): + return accuracy(input[...,0], targs[...,0]) + +def accuracy_bwd(input, targs): + return accuracy(input[...,1], targs[...,1]) + + +#endregion +#region Modified fastai code def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str]) -> Weights: "Convert the model weights to go with a new vocabulary." - print(wgts.keys()) if 'fwd_lm.0.encoder.weight' in wgts: #todo share embedding matrix computation wgts = convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='fwd_lm.') return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='bwd_lm.') @@ -53,49 +92,10 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new: wgts[prefix+'1.decoder.bias'] = new_b return wgts +#endregion +#region Replace code in fastai - -def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400, - nh: int = 1150, nl: int = 3, - lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1, - drop_mult: float = 1., qrnn: bool = False, **kwargs) -> 'TextClassifierLearner': - "Create a RNN classifier." - dps = default_dropout['classifier'] * drop_mult - if lin_ftrs is None: lin_ftrs = [50] - if ps is None: ps = [0.1] - ds = data.train_ds - vocab_size, n_class = len(data.vocab.itos), data.c - layers = [emb_sz * 3] + lin_ftrs + [n_class] - ps = [dps[4]] + ps - model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, - layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], - qrnn=qrnn) - learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs) - return learn - - -def bilm_split(model:nn.Module) -> List[nn.Module]: - "Split a RNN `model` in groups for differential learning rates." - - return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))] - -def birnn_classifier_split(model:nn.Module) -> List[nn.Module]: - "Split a RNN `model` in groups for differential learning rates." - f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm - groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]] - groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )] - groups.append([model[1]]) - return groups - - -def accuracy_fwd(input, targs): - return accuracy(input[...,0], targs[...,0]) - - -def accuracy_bwd(input, targs): - return accuracy(input[...,1], targs[...,1]) - - -## Replace code in fastai import fastai.text.learner -fastai.text.learner.convert_weights = convert_weights \ No newline at end of file +fastai.text.learner.convert_weights = convert_weights + +#endregion \ No newline at end of file diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index b99db97..57ccc47 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -2,6 +2,8 @@ from fastai.torch_core import * from fastai.layers import * from fastai.text.models import * +#region New code + class BiLMModel(nn.Module): def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module): @@ -145,4 +147,6 @@ def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_s model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), AvgPoolingLinearClassifier(layers, drops)) model.reset() - return model \ No newline at end of file + return model + +#endregion \ No newline at end of file diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 37a76b3..2dd21f9 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -55,7 +55,7 @@ class SentencepieceTokenizer(BaseTokenizer): pass -def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=None, +def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, pre_rules:ListRules=None, post_rules:ListRules=None, vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, pad_idx:int=PAD_TOKEN_ID): try: @@ -67,15 +67,15 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N cache_name = 'tmp' os.makedirs(path / cache_name, exist_ok=True) os.makedirs(path / 'models', exist_ok=True) - rules = rules if rules is not None else [] - + pre_rules = pre_rules if pre_rules is not None else [] + post_rules = post_rules if post_rules is not None else [] # load the text frmo the train tokens file text = [line.rstrip('\n') for line in open(trn_path)] text = list(filter(None, text)) if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'): - raw_text = reduce(lambda t, rule: rule(t), rules, '\n'.join(text)) + raw_text = reduce(lambda t, rule: rule(t), pre_rules, '\n'.join(text)) raw_text_path = path / cache_name / 'all_text.txt' with open(raw_text_path, 'w') as f: f.write(raw_text) @@ -86,18 +86,18 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, rules:ListRules=N f"--model_prefix={path / 'models' / 'spm'} " \ f"--vocab_size={vocab_size} --model_type={model_type} " spm.SentencePieceTrainer.Train(sp_params) - + with open(path / 'models' / 'spm.vocab', 'r') as f: vocab = [line.split('\t')[0] for line in f.readlines()] vocab[0] = UNK vocab[pad_idx] = PAD pickle.dump(vocab, open(path / 'models' / f'itos_{name}.pkl', 'wb')) - + # todo add post rules vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb'))) # We cannot use lambdas or local methods here, since `tok_func` needs to be # pickle-able in order to be called in subprocesses when multithread tokenizing - tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), rules=rules) + tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), pre_rules=pre_rules, post_rules=post_rules) clear_cache_directory(path, cache_name) @@ -127,7 +127,7 @@ def ensure_paths_exists(*paths, message="One or more required files cannot be fo if error: raise FileNotFoundError(message) -def get_data_folder(): +def get_data_folder() -> Path: """ return data folder to use for future processing """ @@ -317,7 +317,8 @@ def read_clas_data(dir_path, dataset, lang) -> Tuple[Dict[str, List[List[str]]], # for IMDb, we need to split off a separate validation set # note that we train and fine-tune ULMFiT on the full training set in the paper # to do this, we can just keep the training set the same - trn_len = int(len(toks[TRN]) * 0.9) + val_len = max(int(len(toks[TRN]) * 0.1), 2) # fastai does not work with validation set of size 1 + trn_len = len(toks[TRN]) - val_len toks[TRN], toks[VAL] = toks[TRN][:trn_len], toks[TRN][trn_len:] lbls[TRN], lbls[VAL] = lbls[TRN][:trn_len], lbls[TRN][trn_len:] else: diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index f17bbfb..355ffbf 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -1,52 +1,55 @@ import os import glob import fire -import pytest - import ulmfit.pretrain_lm import ulmfit.train_clas from fastai import * from fastai.text import * -from fastai_contrib.utils import * +from fastai_contrib.utils import * + """ It is a mixture of a pytest unit test and woven together to compose an end to end functional test. """ + def delete_test_models(): - wt2 = data / 'wiki' / 'wikitext-2' - imdb = data / 'imdb' - - # delete test models from the pretraining step - for test_file in glob.iglob(f'{str(wt2)}/models/end-to-end-test*'): - if os.path.isfile(test_file): os.remove(test_file) - - # delete test vocab and model of sentencepiece training - for test_file in [wt2 / 'models' / 'spm.model', - wt2 / 'models' / 'spm.vocab']: - if os.path.isfile(test_file): os.remove(test_file) - - # delete test models from the finetuning/classifier training step - for test_file in glob.iglob(f'{str(imdb)}/models/end-to-end-test*'): - if os.path.isfile(test_file): os.remove(test_file) - - -def check_data_exists(): data = get_data_folder() - wt2 = data / 'wiki' / 'wikitext-2' - imdb = data / 'imdb' - ensure_paths_exists(wt2 / 'en.wiki.train.tokens', - imdb / 'train.csv', - message="We don't run data preparation" - " scripts automatically as it takes ages," - " run prepare_wiki-en.sh & prepare_imdb.sh") - return imdb, wt2 + +def copy_head(src_fn, dst_fn, n=1000): + with src_fn.open("r") as s, dst_fn.open("w") as d: + for i in range(n): + d.write(s.readline()) + + +def get_test_data(): + data = get_data_folder() + wt = data / "wiki" / "wikitext-2" + imdb = data / "imdb" + + test_data = data / "test" + shutil.rmtree(test_data) + + test_wt = test_data / 'wikitext-s' + test_imdb = test_data / 'imdb' + test_wt.mkdir(exist_ok=True, parents=True) + test_imdb.mkdir(exist_ok=True, parents=True) + + sz=10 + # we use the same text to see if models can overfit + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=10*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=6*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=6*sz) + copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) + copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz) + + return test_data, test_wt def test_ulmfit_default_end_to_end(): """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. """ - imdb, wt2 = check_data_exists() + test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-default' cuda_id = 0 results = ulmfit.pretrain_lm.pretrain_lm( @@ -56,23 +59,20 @@ def test_ulmfit_default_end_to_end(): qrnn=True, subword=False, max_vocab=1000, - bs=80, + bs=2, num_epochs=1, - name=lm_name, - ds_pct=0.03 - ) - assert results['accuracy'] > 0.30 + name=lm_name) + assert results['accuracy'] > 0.02 results = ulmfit.train_clas.new_train_clas( - data_dir=get_data_folder(), - lang='en', pretrain_name=lm_name, model_dir=wt2/'models', - qrnn=True, - cuda_id=cuda_id, - fine_tune=True, - max_vocab=1000, - bs=20, bptt=70, name=lm_name+'-imdb-clas', - dataset='imdb', - ds_pct=0.03) + data_dir=test_data, + lang='en', pretrain_name=lm_name, model_dir=wt2 / 'models', + qrnn=True, + cuda_id=cuda_id, + fine_tune=True, + max_vocab=1000, + bs=2, bptt=70, name=lm_name + '-imdb-clas', + dataset='imdb') delete_test_models() @@ -80,7 +80,7 @@ def test_ulmfit_default_end_to_end(): def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ - imdb, wt2 = check_data_exists() + imdb, wt2 = get_test_data() lm_name = 'end-to-end-test-spm' cuda_id = 0 results = ulmfit.pretrain_lm.pretrain_lm( @@ -101,3 +101,7 @@ def test_ulmfit_sentencepiece_end_to_end(): # sentencepiece for finetuning/classification is currently not implemented delete_test_models() + + +if __name__ == "__main__": + fire.Fire() # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index a5e981d..7b617dd 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -51,7 +51,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ 'Error: IMDb is only available in English.' data_dir = Path(data_dir) - assert data_dir.name == 'data',\ + assert data_dir.name in ['data', 'test'],\ f'Error: Name of data directory should be data, not {data_dir.name}.' dataset_dir = data_dir / dataset model_dir = Path(model_dir) @@ -112,8 +112,9 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ qrnn=qrnn, emb_sz=emb_sz, nh=nh, nl=nl, drop_mult=0.5) try: + print(f"Loading classifier {model_name}_{name}") learn.load(f'{model_name}_{name}') - print("Loading classifier") + except FileNotFoundError: learn.load_encoder(lm_enc_finetuned) print("loading encoder") @@ -147,9 +148,8 @@ def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_typ if not (tmp_dir / f'{TRN}_{lang}_ids.npy').exists(): print('Reading the data...') toks, lbls = read_clas_data(dataset_dir, dataset, lang) - # create the vocabulary - counter = Counter(word for example in np.concatenate([toks[TRN],toks[TST],toks[VAL]]) for word in example) + counter = Counter(word for example in toks[TRN]+toks[TST]+toks[VAL] for word in example) itos = [word for word, count in counter.most_common(n=max_vocab)] itos.insert(0, PAD) itos.insert(0, UNK) @@ -176,14 +176,17 @@ def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_typ f'Test size: {len(ids[TST])}.') if ds_pct < 1.0: print(f"Making the dataset smaller {ds_pct}") - for split in [TRN, VAL, TST]: - ids[split] = ids[split][:int(len(ids[split]) * ds_pct)] + for split in [TRN, VAL, TST]: + ids[split] = np.array([np.array(e, dtype=np.int) for e in ids[split]]) + lbls[split] = np.array([np.array(e, dtype=np.int) for e in lbls[split]]) data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=np.concatenate([ids[TRN],ids[TST]]), valid_ids=ids[VAL], bs=bs, bptt=bptt, lm_type=lm_type) #  TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls? data_clas = TextClasDataBunch.from_ids( path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL], - train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs, classes={l:l for l in lbls[VAL]}) + train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs, classes={l:l for l in lbls[TRN]}) + + print(f"Sizes of train_ds {len(data_clas.train_ds)}, valid_ds {len(data_clas.valid_ds)}") return data_clas, data_lm From be117abac4dca4c897f4b566d7e5a165b232d532 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 24 Nov 2018 23:51:23 +0100 Subject: [PATCH 22/22] Make the end to end test run correctly --- fastai_contrib/data.py | 10 +++++++++- tests/test_end_to_end.py | 22 ++++++++++------------ ulmfit/pretrain_lm.py | 4 +++- ulmfit/train_clas.py | 4 ++-- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py index e14706d..274d1f6 100644 --- a/fastai_contrib/data.py +++ b/fastai_contrib/data.py @@ -37,9 +37,17 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader itr += 1 yield res - def __len__(self) -> int: return (self.n-1) // self.bptt + def __len__(self) -> int: return int(math.ceil((self.n-1) / self.bptt)) # so that it is always at least 1 def __getattr__(self,k:str)->Any: return getattr(self.dataset, k) + @property + def batch_size(self): + return self.bs + + @batch_size.setter + def batch_size(self, v): + self.bs = v + def batchify(self, data:np.ndarray) -> LongTensor: "Split the corpus `data` in batches." nb = data.shape[0] // self.bs diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 355ffbf..9b80a6b 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -11,10 +11,8 @@ from fastai_contrib.utils import * It is a mixture of a pytest unit test and woven together to compose an end to end functional test. """ - -def delete_test_models(): - data = get_data_folder() - +import fastai.core +fastai.core.turn_off_parallel_execution=True def copy_head(src_fn, dst_fn, n=1000): with src_fn.open("r") as s, dst_fn.open("w") as d: @@ -35,7 +33,7 @@ def get_test_data(): test_wt.mkdir(exist_ok=True, parents=True) test_imdb.mkdir(exist_ok=True, parents=True) - sz=10 + sz=1 # we use the same text to see if models can overfit copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=10*sz) copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=6*sz) @@ -71,11 +69,12 @@ def test_ulmfit_default_end_to_end(): cuda_id=cuda_id, fine_tune=True, max_vocab=1000, - bs=2, bptt=70, name=lm_name + '-imdb-clas', + num_lm_epochs=0, + bs=4, # minimum size is 4 otherwise it somewhere becomes 1 and fit stops working + bptt=70, + name=lm_name + '-imdb-clas', dataset='imdb') - delete_test_models() - def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. @@ -89,8 +88,8 @@ def test_ulmfit_sentencepiece_end_to_end(): cuda_id=cuda_id, qrnn=True, subword=True, - max_vocab=1000, - bs=80, + max_vocab=100, + bs=2, num_epochs=1, name=lm_name, ) @@ -100,8 +99,7 @@ def test_ulmfit_sentencepiece_end_to_end(): # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset # sentencepiece for finetuning/classification is currently not implemented - delete_test_models() - if __name__ == "__main__": fire.Fire() # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz + diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 5b7098a..aaca9e7 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -74,7 +74,9 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab) - data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp) + lm_type = contrib_data.LanguageModelType.BiLM if bidir else contrib_data.LanguageModelType.FwdLM + + data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type) itos = data_lm.train_ds.vocab.itos stoi = data_lm.train_ds.vocab.stoi else: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 7b617dd..7b88963 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -19,7 +19,7 @@ from pathlib import Path def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_dir='models', - qrnn=False, + qrnn=False, num_lm_epochs=10, fine_tune=True, max_vocab=60000, bs=20, bptt=70, name='imdb-clas', dataset='imdb', bidir=False, ds_pct=1.0, train=True): """ @@ -101,7 +101,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_ learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) learn.unfreeze() - learn.fit_one_cycle(10, 1e-3, moms=(0.8, 0.7)) + if num_lm_epochs > 0: learn.fit_one_cycle(num_lm_epochs, 1e-3, moms=(0.8, 0.7)) # save encoder learn.save_encoder(lm_enc_finetuned)