From 80d4d4da29a8860f9c588db2fd873fbe4af2b17b Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sun, 25 Nov 2018 12:37:45 +0100 Subject: [PATCH 01/41] Extract params to an experiment data class You can run this as follows: `python -m ulmfit.pretrain_lm --dir-path 'data/wiki/wikitext-2' --qrnn=True train_lm --num_epochs=1` --- tests/test_end_to_end.py | 16 +-- ulmfit/pretrain_lm.py | 292 ++++++++++++++++++++++----------------- 2 files changed, 170 insertions(+), 138 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 9b80a6b..a956e40 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -50,17 +50,18 @@ def test_ulmfit_default_end_to_end(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-default' cuda_id = 0 - results = ulmfit.pretrain_lm.pretrain_lm( + exp = ulmfit.pretrain_lm.Experiment( dir_path=wt2, lang='en', - cuda_id=cuda_id, qrnn=True, subword=False, max_vocab=1000, bs=2, - num_epochs=1, name=lm_name) - assert results['accuracy'] > 0.02 + + exp.train_lm(num_epochs=1) + + assert exp.results['accuracy'] > 0.02 results = ulmfit.train_clas.new_train_clas( data_dir=test_data, @@ -82,7 +83,7 @@ def test_ulmfit_sentencepiece_end_to_end(): imdb, wt2 = get_test_data() lm_name = 'end-to-end-test-spm' cuda_id = 0 - results = ulmfit.pretrain_lm.pretrain_lm( + exp = ulmfit.pretrain_lm.Experiment( dir_path=wt2, lang='en', cuda_id=cuda_id, @@ -90,11 +91,10 @@ def test_ulmfit_sentencepiece_end_to_end(): subword=True, max_vocab=100, bs=2, - num_epochs=1, name=lm_name, ) - - assert results['accuracy'] > 0.30 + exp.train_lm(num_epochs=1) + assert exp.results['accuracy'] > 0.30 # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset # sentencepiece for finetuning/classification is currently not implemented diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index aaca9e7..4ac57aa 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -25,159 +25,191 @@ import fastai_contrib.data as contrib_data # cupy needs to be installed for QRNN -def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vocab=60000, - bs=70, bptt=70, name='wt-103', num_epochs=10, bidir=False, ds_pct=1.0): - """ - :param dir_path: The path to the directory of the file. - :param lang: the language unicode - :param cuda_id: The id of the GPU. Uses GPU 0 by default or no GPU when - run on CPU. - :param qrnn: Use a QRNN. Requires installing cupy. - :param subword: Use sub-word tokenization on the cleaned data. - :param max_vocab: The maximum size of the vocabulary. - :param bs: The batch size. - :param bptt: The back-propagation-through-time sequence length. - :param name: The name used for both the model and the vocabulary. - :param model_dir: The path to the directory where the models should be saved - :param bidir: whether the language model is bidirectional - """ - results = {} +# """ +# :param dir_path: The path to the directory of the file. +# :param lang: the language unicode +# :param cuda_id: The id of the GPU. Uses GPU 0 by default or no GPU when +# run on CPU. +# :param qrnn: Use a QRNN. Requires installing cupy. +# :param subword: Use sub-word tokenization on the cleaned data. +# :param max_vocab: The maximum size of the vocabulary. +# :param bs: The batch size. +# :param bptt: The back-propagation-through-time sequence length. +# :param name: The name used for both the model and the vocabulary. +# :param model_dir: The path to the directory where the models should be saved +# :param bidir: whether the language model is bidirectional +# """ - if not torch.cuda.is_available(): - print('CUDA not available. Setting device=-1.') - cuda_id = -1 - torch.cuda.set_device(cuda_id) - dir_path = Path(dir_path) - assert dir_path.exists() - model_dir = dir_path / 'models' # removed from params, as it is absolute models location in train_clas and here it is relative - model_dir.mkdir(exist_ok=True) - print('Batch size:', bs) - print('Max vocab:', max_vocab) - model_name = 'qrnn' if qrnn else 'lstm' - if qrnn: - print('Using QRNNs...') +@dataclass +class Experiment: + dir_path: str + bidir: bool =False + bptt: int = 70 + bs: int = 70 + lang: str = 'en' + max_vocab: int = 60000 + name: str = 'wt-103' + subword: bool = False + ds_pct: float = 1.0 + qrnn: bool = True + cuda_id:int = 0 + def __post_init__(self): + self.results = {} - trn_path = dir_path / f'{lang}.wiki.train.tokens' - val_path = dir_path / f'{lang}.wiki.valid.tokens' - tst_path = dir_path / f'{lang}.wiki.test.tokens' - for path_ in [trn_path, val_path, tst_path]: - assert path_.exists(), f'Error: {path_} does not exist.' + if not torch.cuda.is_available(): + print('CUDA not available. Setting device=-1.') + cuda_id = -1 + torch.cuda.set_device(self.cuda_id) - if subword: - # apply sentencepiece tokenization - trn_path = dir_path / f'{lang}.wiki.train.tokens' - val_path = dir_path / f'{lang}.wiki.valid.tokens' + self.dir_path = Path(self.dir_path) + assert self.dir_path.exists() + self.model_dir = self.dir_path / 'models' # removed from params, as it is absolute models location in train_clas and here it is relative + self.model_dir.mkdir(exist_ok=True) + print('Batch size:', self.bs) + print('Max vocab:', self.max_vocab) - read_file(trn_path, 'train') - read_file(val_path, 'valid') + if self.qrnn: + print('Using QRNNs...') - sp = get_sentencepiece(dir_path, trn_path, name, vocab_size=max_vocab) + @property + def model_name(self): + return 'qrnn' if self.qrnn else 'lstm' - lm_type = contrib_data.LanguageModelType.BiLM if bidir else contrib_data.LanguageModelType.FwdLM + def train_lm(self, num_epochs=10): + data_lm = self.load_data() + exe = Executor(self.create_lm_learner(data_lm), exp=self) + if num_epochs > 0: + exe.learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) + exe.validate() + exe.save() + return exe - data_lm = TextLMDataBunch.from_csv(dir_path, 'train.csv', **sp, bs=bs, bptt=bptt, lm_type=lm_type) - itos = data_lm.train_ds.vocab.itos - stoi = data_lm.train_ds.vocab.stoi - else: - # read the already whitespace separated data without any preprocessing - trn_tok = read_whitespace_file(trn_path) - val_tok = read_whitespace_file(val_path) - if ds_pct < 1.0: - trn_tok = trn_tok[:max(20, int(len(trn_tok) * ds_pct))] - val_tok = val_tok[:max(20, int(len(val_tok) * ds_pct))] - print(f"Limiting data sets to {ds_pct*100}%, trn {len(trn_tok)}, val: {len(val_tok)}") - - itos_fname = model_dir / f'itos_{name}.pkl' - if not itos_fname.exists(): - # create the vocabulary - cnt = Counter(word for sent in trn_tok for word in sent) - itos = [o for o,c in cnt.most_common(n=max_vocab)] - itos.insert(1, PAD) #  set pad id to 1 to conform to fast.ai standard - assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' - - # save vocabulary - print(f"Saving vocabulary as {itos_fname}") - results['itos_fname'] = itos_fname - with open(itos_fname, 'wb') as f: - pickle.dump(itos, f) + def create_lm_learner(self, data_lm): + # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) + # for training on smaller datasets, more dropout is necessary + if self.qrnn: + emb_sz, nh, nl = 400, 1550, 3 + # dps = np.array([0.0, 0.0, 0.0, 0.0, 0.0]) + dps = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) + drop_mult = 0.1 else: - print("Loading itos:", itos_fname) - itos = np.load(itos_fname) - vocab = Vocab(itos) - stoi = vocab.stoi + emb_sz, nh, nl = 400, 1150, 3 + # emb_sz, nh, nl = 400, 1150, 3 + dps = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) + drop_mult = 0.1 + fastai.text.learner.default_dropout['language'] = dps + lm_learner = bilm_learner if self.bidir else language_model_learner + learn = lm_learner(data_lm, bptt=self.bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, + drop_mult=drop_mult, tie_weights=True, model_dir=self.model_dir.name, + bias=True, qrnn=self.qrnn, clip=0.12) + # compared to standard Adam, we set beta_1 to 0.8 + learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) + learn.true_wd = False + print("true_wd: ", learn.true_wd) + if self.bidir: + learn.metrics = [accuracy_fwd, accuracy_bwd] + else: + learn.metrics = [accuracy] + return learn - trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) - val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) - lm_type = contrib_data.LanguageModelType.BiLM if bidir else contrib_data.LanguageModelType.FwdLM + def load_data(self): + trn_path = self.dir_path / f'{self.lang}.wiki.train.tokens' + val_path = self.dir_path / f'{self.lang}.wiki.valid.tokens' + tst_path = self.dir_path / f'{self.lang}.wiki.test.tokens' + for path_ in [trn_path, val_path, tst_path]: + assert path_.exists(), f'Error: {path_} does not exist.' + if self.subword: + # apply sentencepiece tokenization + trn_path = self.dir_path / f'{self.lang}.wiki.train.tokens' + val_path = self.dir_path / f'{self.lang}.wiki.valid.tokens' - # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) - data_lm = TextLMDataBunch.from_ids(path=dir_path, vocab=vocab, train_ids=trn_ids, - valid_ids=val_ids, bs=bs, bptt=bptt, - lm_type=lm_type - ) + read_file(trn_path, 'train') + read_file(val_path, 'valid') - print('Size of vocabulary:', len(itos)) - print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) + sp = get_sentencepiece(self.dir_path, trn_path, self.name, vocab_size=self.max_vocab) - # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) - # for training on smaller datasets, more dropout is necessary - if qrnn: - emb_sz, nh, nl = 400, 1550, 3 - #dps = np.array([0.0, 0.0, 0.0, 0.0, 0.0]) - dps = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) - drop_mult = 0.1 - else: - emb_sz, nh, nl = 400, 1150, 3 - # emb_sz, nh, nl = 400, 1150, 3 - dps = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) - drop_mult = 0.1 + lm_type = contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - fastai.text.learner.default_dropout['language'] = dps + data_lm = TextLMDataBunch.from_csv(self.dir_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=lm_type) + itos = data_lm.train_ds.vocab.itos + stoi = data_lm.train_ds.vocab.stoi + else: + # read the already whitespace separated data without any preprocessing + trn_tok = read_whitespace_file(trn_path) + val_tok = read_whitespace_file(val_path) + if self.ds_pct < 1.0: + trn_tok = trn_tok[:max(20, int(len(trn_tok) * self.ds_pct))] + val_tok = val_tok[:max(20, int(len(val_tok) * self.ds_pct))] + print(f"Limiting data sets to {self.ds_pct * 100}%, trn {len(trn_tok)}, val: {len(val_tok)}") - lm_learner = bilm_learner if bidir else language_model_learner - learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, - drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name, - bias=True, qrnn=qrnn, clip=0.12) - # compared to standard Adam, we set beta_1 to 0.8 - learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) + itos_fname = self.model_dir / f'itos_{self.name}.pkl' + if not itos_fname.exists(): + # create the vocabulary + cnt = Counter(word for sent in trn_tok for word in sent) + itos = [o for o, c in cnt.most_common(n=self.max_vocab)] + itos.insert(1, PAD) #   set pad id to 1 to conform to fast.ai standard + assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' - learn.true_wd = False - print("true_wd: ", learn.true_wd) + # save vocabulary + print(f"Saving vocabulary as {itos_fname}") + self.results['itos_fname'] = itos_fname + with open(itos_fname, 'wb') as f: + pickle.dump(itos, f) + else: + print("Loading itos:", itos_fname) + itos = np.load(itos_fname) + vocab = Vocab(itos) + stoi = vocab.stoi - if bidir: - learn.metrics = [accuracy_fwd, accuracy_bwd] - else: - learn.metrics = [accuracy] + trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) + val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) - try: - learn.load(f'{model_name}_{name}') - print("Weights loaded") - except FileNotFoundError: - print("Starting from random weights") - pass + lm_type = contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) + # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) + data_lm = TextLMDataBunch.from_ids(path=self.dir_path, vocab=vocab, train_ids=trn_ids, + valid_ids=val_ids, bs=self.bs, bptt=self.bptt, + lm_type=lm_type) + itos, stoi, trn_path = data_lm.vocab.itos, data_lm.vocab.stoi, data_lm.path + print('Size of vocabulary:', len(itos)) + print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) + return data_lm - if not subword and max_vocab is None: - # only if we use the unpreprocessed version and the full vocabulary - # are the perplexity results comparable to previous work - print(f"Validating model performance with test tokens from: {trn_path}") - tst_tok = read_whitespace_file(trn_path) - tst_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in tst_tok]) - logloss, perplexity = validate(learn.model, tst_ids, bptt) - print('Test logloss:', logloss.item(), 'perplexity:', perplexity.item()) - print(f"Saving models at {learn.path / learn.model_dir}") - learn.save(f'{model_name}_{name}') +class Executor: + def __init__(self, learn, exp): + self.exp = exp + self.learn = learn + try: + self.learn.load(f'{self.exp.model_name}_{self.exp.name}') + print("Weights loaded") + except FileNotFoundError: + print("Starting from random weights") + pass - opt_state_path = learn.path / learn.model_dir / f'{model_name}3_{name}_state.pth' - print(f"Saving optimiser state at {opt_state_path}") - torch.save(learn.opt.opt.state_dict(), opt_state_path) + def validate(self): + if not self.exp.subword and self.exp.max_vocab is None: + raise NotImplementedError("figure out how to validate and save results") + # only if we use the unpreprocessed version and the full vocabulary + # are the perplexity results comparable to previous work + print(f"Validating model performance with test tokens from: {trn_path}") + tst_tok = read_whitespace_file(trn_path) + tst_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in tst_tok]) + logloss, perplexity = validate(learn.model, tst_ids, self.exp.bptt) + print('Test logloss:', logloss.item(), 'perplexity:', perplexity.item()) + + def save(self): + print(f"Saving models at {self.learn.path / self.learn.model_dir}") + self.learn.save(f'{self.exp.model_name}_{self.exp.name}') + + opt_state_path = self.learn.path / self.learn.model_dir / f'{self.exp.model_name}_{self.exp.name}_state.pth' + print(f"Saving optimiser state at {opt_state_path}") + torch.save(self.learn.opt.opt.state_dict(), opt_state_path) + + self.exp.results['accuracy'] = self.learn.validate()[1] #TODO rewrite - results['accuracy'] = learn.validate()[1] - return results if __name__ == '__main__': - fire.Fire(pretrain_lm) + fire.Fire(Experiment) From 6d6ebef1ca5d30ec4027c24b36bb4b3655850bad Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 10:57:20 +0100 Subject: [PATCH 02/41] Update to newst fastai --- fastai_contrib/data.py | 10 ++++------ fastai_contrib/learner.py | 27 +++++++++++++++------------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py index 274d1f6..d7be826 100644 --- a/fastai_contrib/data.py +++ b/fastai_contrib/data.py @@ -16,8 +16,9 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader max_len:int=25): self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle = dataset,bs,bptt,lm_type,shuffle self.first,self.i,self.iter = True,0,0 - self.n = len(np.concatenate(dataset.x.items)) // self.bs + self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0 self.max_len,self.num_workers = max_len,0 + self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len) def __iter__(self): if getattr(self.dataset, 'item', None) is not None: @@ -41,12 +42,9 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader def __getattr__(self,k:str)->Any: return getattr(self.dataset, k) @property - def batch_size(self): - return self.bs - + def batch_size(self): return self.bs @batch_size.setter - def batch_size(self, v): - self.bs = v + def batch_size(self, v): self.bs = v def batchify(self, data:np.ndarray) -> LongTensor: "Split the corpus `data` in batches." diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index e060b7a..bdcea7c 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -78,18 +78,21 @@ def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[s def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str], prefix='') -> Weights: "Convert the model weights to go with a new vocabulary." - dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight'] - bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0) - new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_() - new_b = dec_bias.new_zeros((len(itos_new),)).zero_() - for i,w in enumerate(itos_new): - r = stoi_wgts[w] if w in stoi_wgts else -1 - new_w[i] = enc_wgts[r] if r>=0 else wgts_m - new_b[i] = dec_bias[r] if r>=0 else bias_m - wgts[prefix+'0.encoder.weight'] = new_w - wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone() - wgts[prefix+'1.decoder.weight'] = new_w.clone() - wgts[prefix+'1.decoder.bias'] = new_b + if 'model' in wgts: + wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix) + else: + dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight'] + bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0) + new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_() + new_b = dec_bias.new_zeros((len(itos_new),)).zero_() + for i,w in enumerate(itos_new): + r = stoi_wgts[w] if w in stoi_wgts else -1 + new_w[i] = enc_wgts[r] if r>=0 else wgts_m + new_b[i] = dec_bias[r] if r>=0 else bias_m + wgts[prefix+'0.encoder.weight'] = new_w + wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone() + wgts[prefix+'1.decoder.weight'] = new_w.clone() + wgts[prefix+'1.decoder.bias'] = new_b return wgts #endregion From b3f5ae1ad517578ace733fcb9153dfc88f50070d Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 10:58:23 +0100 Subject: [PATCH 03/41] Clean the way we save models --- ulmfit/pretrain_lm.py | 226 +++++++++++++++++++++++------------------- 1 file changed, 124 insertions(+), 102 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 4ac57aa..3325ab5 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -4,6 +4,8 @@ expected to have been tokenized with Moses and processed with `postprocess_wikit That is, the data is expected to be white-space separated and numbers are expected to be split. """ +from dataclasses import InitVar + import fastai import fire @@ -11,8 +13,8 @@ from fastai import * from fastai.text import * import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ - validate, PAD, UNK, get_sentencepiece -from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd + validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID +from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner import pickle from pathlib import Path @@ -42,109 +44,153 @@ import fastai_contrib.data as contrib_data @dataclass -class Experiment: - dir_path: str +class LMHyperParams: + dataset_path: str # data_dir + + base_lm_path: str = None bidir: bool =False + qrnn: bool = True + max_vocab: int = 60000 + subword: bool = False + + emb_sz:int = 400 + nh: int = None + nl: int = 3 + + # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) + # for training on smaller datasets, more dropout is necessary + drop_mult = 0.1 + dps = [0.25, 0.1, 0.2, 0.02, 0.15] + clip: float = 0.12 bptt: int = 70 bs: int = 70 - lang: str = 'en' - max_vocab: int = 60000 - name: str = 'wt-103' - subword: bool = False - ds_pct: float = 1.0 - qrnn: bool = True - cuda_id:int = 0 - def __post_init__(self): - self.results = {} + lang: str = 'en' + name: str = '' + cuda_id: InitVar[int] = 0 + + def __post_init__(self, cuda_id): if not torch.cuda.is_available(): print('CUDA not available. Setting device=-1.') cuda_id = -1 - torch.cuda.set_device(self.cuda_id) + torch.cuda.set_device(cuda_id) + self.dataset_path = Path(self.dataset_path) + self.base_lm_path = Path(self.base_lm_path) if self.base_lm_path is not None else None - self.dir_path = Path(self.dir_path) - assert self.dir_path.exists() - self.model_dir = self.dir_path / 'models' # removed from params, as it is absolute models location in train_clas and here it is relative - self.model_dir.mkdir(exist_ok=True) + assert self.dataset_path.exists() + self.cache_dir = self.dataset_path / 'models' / self.tok_name + self.model_dir = self.cache_dir / self.full_name + + self.model_dir.mkdir(exist_ok=True, parents=True) print('Batch size:', self.bs) print('Max vocab:', self.max_vocab) + print('Cache dir:', self.cache_dir) + print('Model dir:', self.model_dir) + self.dps = np.array(self.dps) + if self.nh is None: self.nh = 1550 if self.qrnn else 1150 - if self.qrnn: - print('Using QRNNs...') + @classmethod + def based_on(cls, base_lm_path, dataset_path, **kwargs) -> 'LMHyperParams': + with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) + d['dataset_path'] = dataset_path + d['base_lm_path'] = base_lm_path + d.update(kwargs) + return cls(**d) + + def save_info(self): + from dataclasses import asdict + vals = {k: (str(v) if isinstance(v, Path) else v) for k,v in asdict(self).items()} + vals.pop('name', None) + vals.pop('lang', None) + with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) + print("Saving info", self.model_dir / 'info.json') @property - def model_name(self): - return 'qrnn' if self.qrnn else 'lstm' + def tok_name(self): + pref = 'sp' if self.subword else 'v' + voc_size = self.max_vocab // 1000 + return f"{pref}{voc_size}k" - def train_lm(self, num_epochs=10): - data_lm = self.load_data() - exe = Executor(self.create_lm_learner(data_lm), exp=self) - if num_epochs > 0: - exe.learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) - exe.validate() - exe.save() - return exe + @property + def full_name(self): return f"{self.model_name}_{self.name}.m" + + # todo rework + @property + def model_name(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm') + + @property + def pretrained_fnames(self): return [self.base_lm_path / 'lm_best', self.base_lm_path / '../itos'] if self.base_lm_path else None + + def train_lm(self, num_lm_epochs=10, data_lm=None): + data_lm = self.load_wiki_data() if data_lm is None else data_lm + learn = self.create_lm_learner(data_lm) + + if num_lm_epochs > 0: + if self.pretrained_fnames : + learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) + learn.unfreeze() + if num_lm_epochs > 0: learn.fit_one_cycle(num_lm_epochs, 1e-3, moms=(0.8, 0.7)) + else: + try: + learn.load("lm_best") + print("Weights loaded") + except FileNotFoundError: + print("Starting from random weights") + learn.fit_one_cycle(num_lm_epochs, 5e-3, (0.8, 0.7), wd=1e-7) + opt_state_path = self.model_dir / 'opt_state.pth' + print(f"Saving optimiser state at {opt_state_path}") + torch.save(learn.opt.opt.state_dict(), opt_state_path) + learn.save_encoder("enc_best") + learn.save("lm_best", with_opt=False) + print(learn.path) + + self.save_info() + return learn def create_lm_learner(self, data_lm): - # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) - # for training on smaller datasets, more dropout is necessary - if self.qrnn: - emb_sz, nh, nl = 400, 1550, 3 - # dps = np.array([0.0, 0.0, 0.0, 0.0, 0.0]) - dps = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) - drop_mult = 0.1 - else: - emb_sz, nh, nl = 400, 1150, 3 - # emb_sz, nh, nl = 400, 1150, 3 - dps = np.array([0.25, 0.1, 0.2, 0.02, 0.15]) - drop_mult = 0.1 - fastai.text.learner.default_dropout['language'] = dps + fastai.text.learner.default_dropout['language'] = self.dps lm_learner = bilm_learner if self.bidir else language_model_learner - learn = lm_learner(data_lm, bptt=self.bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1, - drop_mult=drop_mult, tie_weights=True, model_dir=self.model_dir.name, - bias=True, qrnn=self.qrnn, clip=0.12) + + learn = lm_learner(data_lm, bptt=self.bptt, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, + drop_mult=self.drop_mult, tie_weights=True, model_dir= self.model_dir.relative_to(data_lm.path), + bias=True, qrnn=self.qrnn, clip=self.clip, pretrained_fnames=self.pretrained_fnames) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.true_wd = False print("true_wd: ", learn.true_wd) - if self.bidir: - learn.metrics = [accuracy_fwd, accuracy_bwd] - else: - learn.metrics = [accuracy] + learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] return learn + @property + def lm_type(self): + return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - def load_data(self): - trn_path = self.dir_path / f'{self.lang}.wiki.train.tokens' - val_path = self.dir_path / f'{self.lang}.wiki.valid.tokens' - tst_path = self.dir_path / f'{self.lang}.wiki.test.tokens' + def load_wiki_data(self): + trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' + val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' + tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' if self.subword: # apply sentencepiece tokenization - trn_path = self.dir_path / f'{self.lang}.wiki.train.tokens' - val_path = self.dir_path / f'{self.lang}.wiki.valid.tokens' + trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' + val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' read_file(trn_path, 'train') read_file(val_path, 'valid') - sp = get_sentencepiece(self.dir_path, trn_path, self.name, vocab_size=self.max_vocab) + sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab) lm_type = contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - data_lm = TextLMDataBunch.from_csv(self.dir_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=lm_type) + data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=lm_type) itos = data_lm.train_ds.vocab.itos stoi = data_lm.train_ds.vocab.stoi else: # read the already whitespace separated data without any preprocessing trn_tok = read_whitespace_file(trn_path) val_tok = read_whitespace_file(val_path) - if self.ds_pct < 1.0: - trn_tok = trn_tok[:max(20, int(len(trn_tok) * self.ds_pct))] - val_tok = val_tok[:max(20, int(len(val_tok) * self.ds_pct))] - print(f"Limiting data sets to {self.ds_pct * 100}%, trn {len(trn_tok)}, val: {len(val_tok)}") - - itos_fname = self.model_dir / f'itos_{self.name}.pkl' + itos_fname = self.cache_dir / f'itos.pkl' if not itos_fname.exists(): # create the vocabulary cnt = Counter(word for sent in trn_tok for word in sent) @@ -154,7 +200,6 @@ class Experiment: # save vocabulary print(f"Saving vocabulary as {itos_fname}") - self.results['itos_fname'] = itos_fname with open(itos_fname, 'wb') as f: pickle.dump(itos, f) else: @@ -166,50 +211,27 @@ class Experiment: trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) - lm_type = contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM + # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) - data_lm = TextLMDataBunch.from_ids(path=self.dir_path, vocab=vocab, train_ids=trn_ids, + data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids, valid_ids=val_ids, bs=self.bs, bptt=self.bptt, - lm_type=lm_type) + lm_type=self.lm_type) itos, stoi, trn_path = data_lm.vocab.itos, data_lm.vocab.stoi, data_lm.path print('Size of vocabulary:', len(itos)) print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) return data_lm - -class Executor: - def __init__(self, learn, exp): - self.exp = exp - self.learn = learn - try: - self.learn.load(f'{self.exp.model_name}_{self.exp.name}') - print("Weights loaded") - except FileNotFoundError: - print("Starting from random weights") - pass - - def validate(self): - if not self.exp.subword and self.exp.max_vocab is None: - raise NotImplementedError("figure out how to validate and save results") - # only if we use the unpreprocessed version and the full vocabulary - # are the perplexity results comparable to previous work - print(f"Validating model performance with test tokens from: {trn_path}") - tst_tok = read_whitespace_file(trn_path) - tst_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in tst_tok]) - logloss, perplexity = validate(learn.model, tst_ids, self.exp.bptt) - print('Test logloss:', logloss.item(), 'perplexity:', perplexity.item()) - - def save(self): - print(f"Saving models at {self.learn.path / self.learn.model_dir}") - self.learn.save(f'{self.exp.model_name}_{self.exp.name}') - - opt_state_path = self.learn.path / self.learn.model_dir / f'{self.exp.model_name}_{self.exp.name}_state.pth' - print(f"Saving optimiser state at {opt_state_path}") - torch.save(self.learn.opt.opt.state_dict(), opt_state_path) - - self.exp.results['accuracy'] = self.learn.validate()[1] #TODO rewrite - +def validate_lm(self): + if not self.exp.subword and self.exp.max_vocab is None: + raise NotImplementedError("figure out how to validate and save results") + # only if we use the unpreprocessed version and the full vocabulary + # are the perplexity results comparable to previous work + print(f"Validating model performance with test tokens from: {trn_path}") + tst_tok = read_whitespace_file(trn_path) + tst_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in tst_tok]) + logloss, perplexity = validate(learn.model, tst_ids, self.exp.bptt) + print('Test logloss:', logloss.item(), 'perplexity:', perplexity.item()) if __name__ == '__main__': - fire.Fire(Experiment) + fire.Fire(LMHyperParams) From 4b29376b44639ff99d211c7136173390928b0bb0 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 10:58:46 +0100 Subject: [PATCH 04/41] Rewrite classifier to use changed pretrain_lm --- tests/test_end_to_end.py | 33 ++---- ulmfit/train_clas.py | 231 +++++++++++++-------------------------- 2 files changed, 89 insertions(+), 175 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index a956e40..1ac4e2b 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -12,7 +12,7 @@ It is a mixture of a pytest unit test and woven together to compose an end to en """ import fastai.core -fastai.core.turn_off_parallel_execution=True +fastai.core.defaults.cpus = 1 def copy_head(src_fn, dst_fn, n=1000): with src_fn.open("r") as s, dst_fn.open("w") as d: @@ -50,8 +50,8 @@ def test_ulmfit_default_end_to_end(): test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-default' cuda_id = 0 - exp = ulmfit.pretrain_lm.Experiment( - dir_path=wt2, + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, lang='en', qrnn=True, subword=False, @@ -59,23 +59,12 @@ def test_ulmfit_default_end_to_end(): bs=2, name=lm_name) - exp.train_lm(num_epochs=1) + exp.train_lm(num_lm_epochs=1) - assert exp.results['accuracy'] > 0.02 - - results = ulmfit.train_clas.new_train_clas( - data_dir=test_data, - lang='en', pretrain_name=lm_name, model_dir=wt2 / 'models', - qrnn=True, - cuda_id=cuda_id, - fine_tune=True, - max_vocab=1000, - num_lm_epochs=0, - bs=4, # minimum size is 4 otherwise it somewhere becomes 1 and fit stops working - bptt=70, - name=lm_name + '-imdb-clas', - dataset='imdb') + #assert exp.results['accuracy'] > 0.02 + exp2 = ulmfit.train_clas.CLSHyperParams.based_on(exp.model_dir, test_data/'imdb') + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4,) def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. @@ -83,8 +72,8 @@ def test_ulmfit_sentencepiece_end_to_end(): imdb, wt2 = get_test_data() lm_name = 'end-to-end-test-spm' cuda_id = 0 - exp = ulmfit.pretrain_lm.Experiment( - dir_path=wt2, + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, lang='en', cuda_id=cuda_id, qrnn=True, @@ -93,8 +82,8 @@ def test_ulmfit_sentencepiece_end_to_end(): bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) - assert exp.results['accuracy'] > 0.30 + exp.train_lm(num_lm_epochs=1) + #assert exp.results['accuracy'] > 0.30 # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset # sentencepiece for finetuning/classification is currently not implemented diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 7b88963..9447d68 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -2,6 +2,7 @@ Train a classifier on top of a language model trained with `pretrain_lm.py`. Optionally fine-tune LM before. """ +import fastai import numpy as np import pickle @@ -17,178 +18,102 @@ import fire from collections import Counter from pathlib import Path - -def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_dir='models', - qrnn=False, num_lm_epochs=10, - fine_tune=True, max_vocab=60000, bs=20, bptt=70, name='imdb-clas', - dataset='imdb', bidir=False, ds_pct=1.0, train=True): - """ - :param data_dir: The path to the `data` directory - :param lang: the language unicode - :param cuda_id: The id of the GPU. Uses GPU 0 by default or no GPU when - run on CPU. - :param pretrain_name: name of the pretrained model - :param model_dir: The path to the directory where the pretrained model is saved - :param qrrn: Use a QRNN. Requires installing cupy. - :param fine_tune: Fine-tune the pretrained language model - :param max_vocab: The maximum size of the vocabulary. - :param bs: The batch size. - :param bptt: The back-propagation-through-time sequence length. - :param name: The name used for both the model and the vocabulary. - :param dataset: The dataset used for evaluation. Currently only IMDb and - XNLI are implemented. Assumes dataset is located in `data` - folder and that name of folder is the same as dataset name. - """ - results={} - if not torch.cuda.is_available(): - print('CUDA not available. Setting device=-1.') - cuda_id = -1 - torch.cuda.set_device(cuda_id) - - print(f'Dataset: {dataset}. Language: {lang}.') - assert dataset in DATASETS, f'Error: {dataset} processing is not implemented.' - assert (dataset == 'imdb' and lang == 'en') or not dataset == 'imdb',\ - 'Error: IMDb is only available in English.' - - data_dir = Path(data_dir) - assert data_dir.name in ['data', 'test'],\ - f'Error: Name of data directory should be data, not {data_dir.name}.' - dataset_dir = data_dir / dataset - model_dir = Path(model_dir) +from ulmfit.pretrain_lm import LMHyperParams - if qrnn: - print('Using QRNNs...') - model_name = 'qrnn' if qrnn else 'lstm' - lm_name = f'{model_name}_{pretrain_name}' - pretrained_fname = (lm_name, f'itos_{pretrain_name}') +class CLSHyperParams(LMHyperParams): + # dir_path -> data/imdb/ - ensure_paths_exists(data_dir, - dataset_dir, - model_dir, - model_dir/f"{pretrained_fname[0]}.pth", - model_dir/f"{pretrained_fname[1]}.pkl") + def __post_init__(self, *args, **kwargs): + super().__post_init__(*args, **kwargs) + self.dataset_dir=self.dataset_path - if bidir: - print("BiLM") - classifier_learner = bilm_text_classifier_learner - lm_learner = bilm_learner - else: - classifier_learner = text_classifier_learner - lm_learner = language_model_learner + @property + def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() - lm_type = LanguageModelType.BiLM if bidir else LanguageModelType.FwdLM - data_clas, data_lm = get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_type=lm_type) + def train_cls(self, num_lm_epochs, unfreeze=True, bs=70): + data_clas, data_lm = self.load_cls_data(bs) - if qrnn: - emb_sz, nh, nl = 400, 1550, 3 - else: - emb_sz, nh, nl = 400, 1150, 3 + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm) + learn = self.create_cls_learner(data_clas) - lm_enc_finetuned = f"{lm_name}_{dataset}_enc" - if fine_tune and not (model_dir/f"{lm_enc_finetuned}.pth").exists(): - print('Fine-tuning the language model...', lm_enc_finetuned) - learn = lm_learner( - data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, qrnn=qrnn, - pad_token=PAD_TOKEN_ID, - pretrained_fnames=pretrained_fname, - path=model_dir.parent, model_dir=model_dir.name, - drop_mult=0.3) - if bidir: - learn.metrics = [accuracy_fwd, accuracy_bwd] - else: - learn.metrics = [accuracy] + try: + learn.load('cls_last') + print("Loading last classfier") + except FileNotFoundError: + learn.load_encoder("enc_best") - learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) - learn.unfreeze() - if num_lm_epochs > 0: learn.fit_one_cycle(num_lm_epochs, 1e-3, moms=(0.8, 0.7)) - - # save encoder - learn.save_encoder(lm_enc_finetuned) - - - learn = classifier_learner(data_clas, bptt=bptt, pad_token=PAD_TOKEN_ID, - path=model_dir.parent, model_dir=model_dir.name, - qrnn=qrnn, emb_sz=emb_sz, nh=nh, nl=nl, drop_mult=0.5) - - try: - print(f"Loading classifier {model_name}_{name}") - learn.load(f'{model_name}_{name}') - - except FileNotFoundError: - learn.load_encoder(lm_enc_finetuned) - print("loading encoder") - train = True - - if train: learn.true_wd = False print("Starting classifier training") learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) + if unfreeze: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) - learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) - learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) - - learn.unfreeze() - learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) + learn.unfreeze() + learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) print(f"Saving models at {learn.path / learn.model_dir}") - learn.save(f'{model_name}_{name}') + learn.save('cls_last', with_opt=False) + return learn - results['accuracy'] = learn.recorder.metrics[-1][0] - return results + def create_cls_learner(self, data_clas): + fastai.text.learner.default_dropout['language'] = self.dps + classifier_learner = bilm_text_classifier_learner if self.bidir else text_classifier_learner + learn = classifier_learner(data_clas, bptt=self.bptt, pad_token=PAD_TOKEN_ID, + path=self.model_dir.parent, model_dir=self.model_dir.name, + qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, drop_mult=self.drop_mult) + learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] + return learn -def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_type): - tmp_dir = dataset_dir / 'tmp' - tmp_dir.mkdir(exist_ok=True) - vocab_file = tmp_dir / f'vocab_{lang}.pkl' - if not (tmp_dir / f'{TRN}_{lang}_ids.npy').exists(): - print('Reading the data...') - toks, lbls = read_clas_data(dataset_dir, dataset, lang) - # create the vocabulary - counter = Counter(word for example in toks[TRN]+toks[TST]+toks[VAL] for word in example) - itos = [word for word, count in counter.most_common(n=max_vocab)] - itos.insert(0, PAD) - itos.insert(0, UNK) - vocab = Vocab(itos) - stoi = vocab.stoi - with open(vocab_file, 'wb') as f: - pickle.dump(vocab, f) - - ids = {} + def load_cls_data(self, bs): + tmp_dir = self.cache_dir + tmp_dir.mkdir(exist_ok=True) + vocab_file = tmp_dir / f'vocab_{self.lang}.pkl' + if not (tmp_dir / f'{TRN}_{self.lang}_ids.npy').exists(): + print('Reading the data...') + toks, lbls = read_clas_data(self.dataset_dir, self.dataset_dir.name, self.lang) + # create the vocabulary + counter = Counter(word for example in toks[TRN] + toks[TST] + toks[VAL] for word in example) + itos = [word for word, count in counter.most_common(n=self.max_vocab)] + itos.insert(0, PAD) + itos.insert(0, UNK) + vocab = Vocab(itos) + stoi = vocab.stoi + with open(vocab_file, 'wb') as f: + pickle.dump(vocab, f) + ids = {} + for split in [TRN, VAL, TST]: + ids[split] = np.array([([stoi.get(w, stoi[UNK]) for w in s]) + for s in toks[split]]) + np.save(tmp_dir / f'{split}_{self.lang}_ids.npy', ids[split]) + np.save(tmp_dir / f'{split}_{self.lang}_lbl.npy', lbls[split]) + else: + print('Loading the pickled data...') + ids, lbls = {}, {} + for split in [TRN, VAL, TST]: + ids[split] = np.load(tmp_dir / f'{split}_{self.lang}_ids.npy') + lbls[split] = np.load(tmp_dir / f'{split}_{self.lang}_lbl.npy') + with open(vocab_file, 'rb') as f: + vocab = pickle.load(f) + print(f'Train size: {len(ids[TRN])}. Valid size: {len(ids[VAL])}. ' + f'Test size: {len(ids[TST])}.') for split in [TRN, VAL, TST]: - ids[split] = np.array([([stoi.get(w, stoi[UNK]) for w in s]) - for s in toks[split]]) - np.save(tmp_dir / f'{split}_{lang}_ids.npy', ids[split]) - np.save(tmp_dir / f'{split}_{lang}_lbl.npy', lbls[split]) - else: - print('Loading the pickled data...') - ids, lbls = {}, {} - for split in [TRN, VAL, TST]: - ids[split] = np.load(tmp_dir / f'{split}_{lang}_ids.npy') - lbls[split] = np.load(tmp_dir / f'{split}_{lang}_lbl.npy') - with open(vocab_file, 'rb') as f: - vocab = pickle.load(f) - print(f'Train size: {len(ids[TRN])}. Valid size: {len(ids[VAL])}. ' - f'Test size: {len(ids[TST])}.') - if ds_pct < 1.0: - print(f"Making the dataset smaller {ds_pct}") - for split in [TRN, VAL, TST]: - ids[split] = np.array([np.array(e, dtype=np.int) for e in ids[split]]) - lbls[split] = np.array([np.array(e, dtype=np.int) for e in lbls[split]]) - data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=np.concatenate([ids[TRN],ids[TST]]), - valid_ids=ids[VAL], bs=bs, bptt=bptt, lm_type=lm_type) - #  TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls? - data_clas = TextClasDataBunch.from_ids( - path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL], - train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs, classes={l:l for l in lbls[TRN]}) - - print(f"Sizes of train_ds {len(data_clas.train_ds)}, valid_ds {len(data_clas.valid_ds)}") - return data_clas, data_lm + ids[split] = np.array([np.array(e, dtype=np.int) for e in ids[split]]) + lbls[split] = np.array([np.array(e, dtype=np.int) for e in lbls[split]]) + data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=np.concatenate([ids[TRN], ids[TST]]), + valid_ids=ids[VAL], bs=bs, bptt=self.bptt, lm_type=self.lm_type) + #  TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls? + data_clas = TextClasDataBunch.from_ids( + path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL], + train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs, classes={l: l for l in lbls[TRN]}) + print(f"Sizes of train_ds {len(data_clas.train_ds)}, valid_ds {len(data_clas.valid_ds)}") + return data_clas, data_lm if __name__ == '__main__': - fire.Fire(new_train_clas) + fire.Fire(CLSHyperParams) From ab9faa2ad9d1460cbe343dbb69134ac1ec06488d Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 13:42:10 +0100 Subject: [PATCH 05/41] Clean up Fire interface. --- tests/test_end_to_end.py | 8 ++--- ulmfit/__init__.py | 1 + ulmfit/__main__.py | 26 ++++++++++++++++ ulmfit/pretrain_lm.py | 65 +++++++++++++++++++--------------------- 4 files changed, 62 insertions(+), 38 deletions(-) create mode 100644 ulmfit/__main__.py diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 1ac4e2b..7e4c56c 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -59,11 +59,11 @@ def test_ulmfit_default_end_to_end(): bs=2, name=lm_name) - exp.train_lm(num_lm_epochs=1) + exp.train_lm(num_epochs=1) #assert exp.results['accuracy'] > 0.02 - exp2 = ulmfit.train_clas.CLSHyperParams.based_on(exp.model_dir, test_data/'imdb') + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4,) def test_ulmfit_sentencepiece_end_to_end(): @@ -82,7 +82,7 @@ def test_ulmfit_sentencepiece_end_to_end(): bs=2, name=lm_name, ) - exp.train_lm(num_lm_epochs=1) + exp.train_lm(num_epochs=1) #assert exp.results['accuracy'] > 0.30 # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset @@ -90,5 +90,5 @@ def test_ulmfit_sentencepiece_end_to_end(): if __name__ == "__main__": - fire.Fire() # allows using all functions via CLI e.g. python utils.py prepare_imdb aclImdb.tgz + fire.Fire() # allows using all functions via CLI diff --git a/ulmfit/__init__.py b/ulmfit/__init__.py index e69de29..8b13789 100644 --- a/ulmfit/__init__.py +++ b/ulmfit/__init__.py @@ -0,0 +1 @@ + diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py new file mode 100644 index 0000000..5c1684a --- /dev/null +++ b/ulmfit/__main__.py @@ -0,0 +1,26 @@ +from functools import wraps + +import fire +from .pretrain_lm import LMHyperParams +from .train_clas import CLSHyperParams + +class FireView: + def __init__(self, **kwargs): + for k,v in kwargs.items(): + setattr(self, k, v) + +class ULMFiT: + @wraps(LMHyperParams) + def lm(self, dataset_path, **changes): + changes['dataset_path'] = dataset_path + params = LMHyperParams(**changes) + return FireView(train=params.train_lm) + + lm2 = LMHyperParams + @wraps(CLSHyperParams) + def cls(self, dataset_path, baseon_path, **changes): + params = CLSHyperParams.from_lm(dataset_path, baseon_path, **changes) + return FireView(train=params.train_cls) + +if __name__ == '__main__': + fire.Fire(ULMFiT()) \ No newline at end of file diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 3325ab5..eada9be 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -66,7 +66,7 @@ class LMHyperParams: bs: int = 70 lang: str = 'en' - name: str = '' + name: str = None cuda_id: InitVar[int] = 0 def __post_init__(self, cuda_id): @@ -78,8 +78,8 @@ class LMHyperParams: self.base_lm_path = Path(self.base_lm_path) if self.base_lm_path is not None else None assert self.dataset_path.exists() - self.cache_dir = self.dataset_path / 'models' / self.tok_name - self.model_dir = self.cache_dir / self.full_name + self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix + self.model_dir = self.cache_dir / self.model_name self.model_dir.mkdir(exist_ok=True, parents=True) print('Batch size:', self.bs) @@ -88,14 +88,23 @@ class LMHyperParams: print('Model dir:', self.model_dir) self.dps = np.array(self.dps) if self.nh is None: self.nh = 1550 if self.qrnn else 1150 + if self.name is None: self.name = self.lang - @classmethod - def based_on(cls, base_lm_path, dataset_path, **kwargs) -> 'LMHyperParams': - with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) - d['dataset_path'] = dataset_path - d['base_lm_path'] = base_lm_path - d.update(kwargs) - return cls(**d) + @property + def tokenizer_prefix(self): return f"{'sp' if self.subword else 'v'}{self.max_vocab // 1000}k" + + @property + def model_prefix(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm') + + @property + def model_name(self): return f"{self.model_prefix}_{self.name}.m" + + @property + def pretrained_fnames(self): return [self.base_lm_path / 'lm_best', self.base_lm_path / '../itos'] if self.base_lm_path else None + + @property + def lm_type(self): + return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM def save_info(self): from dataclasses import asdict @@ -105,38 +114,22 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - @property - def tok_name(self): - pref = 'sp' if self.subword else 'v' - voc_size = self.max_vocab // 1000 - return f"{pref}{voc_size}k" - - @property - def full_name(self): return f"{self.model_name}_{self.name}.m" - - # todo rework - @property - def model_name(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm') - - @property - def pretrained_fnames(self): return [self.base_lm_path / 'lm_best', self.base_lm_path / '../itos'] if self.base_lm_path else None - - def train_lm(self, num_lm_epochs=10, data_lm=None): + def train_lm(self, num_epochs=10, data_lm=None): data_lm = self.load_wiki_data() if data_lm is None else data_lm learn = self.create_lm_learner(data_lm) - if num_lm_epochs > 0: + if num_epochs > 0: if self.pretrained_fnames : learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) learn.unfreeze() - if num_lm_epochs > 0: learn.fit_one_cycle(num_lm_epochs, 1e-3, moms=(0.8, 0.7)) + if num_epochs > 0: learn.fit_one_cycle(num_epochs, 1e-3, moms=(0.8, 0.7)) else: try: learn.load("lm_best") print("Weights loaded") except FileNotFoundError: print("Starting from random weights") - learn.fit_one_cycle(num_lm_epochs, 5e-3, (0.8, 0.7), wd=1e-7) + learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) opt_state_path = self.model_dir / 'opt_state.pth' print(f"Saving optimiser state at {opt_state_path}") torch.save(learn.opt.opt.state_dict(), opt_state_path) @@ -161,10 +154,6 @@ class LMHyperParams: learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] return learn - @property - def lm_type(self): - return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - def load_wiki_data(self): trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' @@ -222,6 +211,14 @@ class LMHyperParams: print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) return data_lm + @classmethod + def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams': + with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) + d['dataset_path'] = dataset_path + d['base_lm_path'] = base_lm_path + d.update(kwargs) + return cls(**d) + def validate_lm(self): if not self.exp.subword and self.exp.max_vocab is None: raise NotImplementedError("figure out how to validate and save results") From c17dcce75e3f0b57fe3b3fd470be4a0bfb7e6af0 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 15:24:30 +0100 Subject: [PATCH 06/41] spelling --- ulmfit/train_clas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 9447d68..c327f3e 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -39,7 +39,7 @@ class CLSHyperParams(LMHyperParams): try: learn.load('cls_last') - print("Loading last classfier") + print("Loading last classifier") except FileNotFoundError: learn.load_encoder("enc_best") From 887211137a92f9465edbf855f81ecb0a415e1e4b Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 16:42:11 +0100 Subject: [PATCH 07/41] Add fastai tokenizer to pretrain_lm --- fastai_contrib/utils.py | 7 +++++-- tests/test_end_to_end.py | 21 +++++++++++++++++++-- ulmfit/pretrain_lm.py | 40 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 60 insertions(+), 8 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 2dd21f9..92b89d1 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -332,14 +332,17 @@ def replace_number(token): return token -def read_file(file_path, outname): +def read_file(file_path, outname=None): """Reads a text file and writes it to a .csv.""" with open(file_path, encoding='utf8') as f: text = f.readlines() df = pd.DataFrame( {'text': np.array(text), 'labels': np.zeros(len(text))}, columns=['labels', 'text']) - df.to_csv(file_path.parent / f'{outname}.csv', header=False, index=False) + if outname is not None: + df.to_csv(file_path.parent / f'{outname}.csv', header=False, index=False) + return df + def read_whitespace_file(filepath): diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 7e4c56c..9e3e412 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -54,7 +54,6 @@ def test_ulmfit_default_end_to_end(): dataset_path=wt2, lang='en', qrnn=True, - subword=False, max_vocab=1000, bs=2, name=lm_name) @@ -66,6 +65,24 @@ def test_ulmfit_default_end_to_end(): exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4,) +def test_ulmfit_fastai_end_to_end(): + """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. + """ + imdb, wt2 = get_test_data() + lm_name = 'end-to-end-test-fastai' + cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, + lang='en', + cuda_id=cuda_id, + qrnn=True, + tokenizer=ulmfit.pretrain_lm.Tokenizers.FASTAI, + max_vocab=100, + bs=2, + name=lm_name, + ) + exp.train_lm(num_epochs=1) + def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ @@ -77,7 +94,7 @@ def test_ulmfit_sentencepiece_end_to_end(): lang='en', cuda_id=cuda_id, qrnn=True, - subword=True, + tokenizer=ulmfit.pretrain_lm.Tokenizers.SUBWORD, max_vocab=100, bs=2, name=lm_name, diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index eada9be..da14833 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -41,7 +41,16 @@ import fastai_contrib.data as contrib_data # :param model_dir: The path to the directory where the models should be saved # :param bidir: whether the language model is bidirectional # """ +class Tokenizers(Enum): + SUBWORD='sb' + MOSES='v' + FASTAI='f' +# tokenizers ={ +# Tok.MOSES: MosesTok, +# Tok.SUBWORD: SentencepieceTok, +# Tok.FASTAI: FastaiTok +# } @dataclass class LMHyperParams: @@ -51,7 +60,7 @@ class LMHyperParams: bidir: bool =False qrnn: bool = True max_vocab: int = 60000 - subword: bool = False + tokenizer: Tokenizers = Tokenizers.MOSES emb_sz:int = 400 nh: int = None @@ -89,9 +98,10 @@ class LMHyperParams: self.dps = np.array(self.dps) if self.nh is None: self.nh = 1550 if self.qrnn else 1150 if self.name is None: self.name = self.lang + self.tokenizer = Tokenizers[self.tokenizer] if type(self.tokenizer) is str else self.tokenizer @property - def tokenizer_prefix(self): return f"{'sp' if self.subword else 'v'}{self.max_vocab // 1000}k" + def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k" @property def model_prefix(self): return ('bi' if self.bidir else '') + ('qrnn' if self.qrnn else 'lstm') @@ -111,6 +121,7 @@ class LMHyperParams: vals = {k: (str(v) if isinstance(v, Path) else v) for k,v in asdict(self).items()} vals.pop('name', None) vals.pop('lang', None) + vals['tokenizer'] = self.tokenizer.value with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') @@ -160,7 +171,7 @@ class LMHyperParams: tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' - if self.subword: + if self.tokenizer is Tokenizers.SUBWORD: # apply sentencepiece tokenization trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' @@ -175,7 +186,7 @@ class LMHyperParams: data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=lm_type) itos = data_lm.train_ds.vocab.itos stoi = data_lm.train_ds.vocab.stoi - else: + elif self.tokenizer is Tokenizers.MOSES: # read the already whitespace separated data without any preprocessing trn_tok = read_whitespace_file(trn_path) val_tok = read_whitespace_file(val_path) @@ -206,6 +217,17 @@ class LMHyperParams: data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids, valid_ids=val_ids, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type) + elif self.tokenizer is Tokenizers.FASTAI: + try: + data_lm = TextLMDataBunch.load(self.cache_dir, '.') + print("Tokenized data loaded") + except FileNotFoundError: + print("Running tokenization") + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), + test_df=read_file(tst_path), classes=None) + data_lm.save('.') + else: + raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") itos, stoi, trn_path = data_lm.vocab.itos, data_lm.vocab.stoi, data_lm.path print('Size of vocabulary:', len(itos)) print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) @@ -216,6 +238,16 @@ class LMHyperParams: with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) d['dataset_path'] = dataset_path d['base_lm_path'] = base_lm_path + + subword = d.pop('subword', False) + tokenizer = d.pop('tokenizer', None) + if tokenizer is not None: + d['tokenizer'] = Tokenizers(tokenizer) + elif subword: + d['tokenizer'] = Tokenizers.SUBWORD + else: + d['tokenizer'] = Tokenizers.MOSES + d.update(kwargs) return cls(**d) From e14c967cc87b25bf71633a18a4d76163892e97a6 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 16:46:25 +0100 Subject: [PATCH 08/41] Fix string parsing in tokenzier --- ulmfit/pretrain_lm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index da14833..95a8068 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -98,7 +98,7 @@ class LMHyperParams: self.dps = np.array(self.dps) if self.nh is None: self.nh = 1550 if self.qrnn else 1150 if self.name is None: self.name = self.lang - self.tokenizer = Tokenizers[self.tokenizer] if type(self.tokenizer) is str else self.tokenizer + self.tokenizer = Tokenizers[self.tokenizer] if isinstance(self.tokenizer, str) else self.tokenizer @property def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k" From 0c4aed6d0534c90a13a83f81596d7f4855b50ff4 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 16:51:21 +0100 Subject: [PATCH 09/41] Really fix conversion from str to Tokenzier --- tests/test_end_to_end.py | 2 +- ulmfit/pretrain_lm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 9e3e412..5d62a70 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -76,7 +76,7 @@ def test_ulmfit_fastai_end_to_end(): lang='en', cuda_id=cuda_id, qrnn=True, - tokenizer=ulmfit.pretrain_lm.Tokenizers.FASTAI, + tokenizer='f', max_vocab=100, bs=2, name=lm_name, diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 95a8068..64fde72 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -85,6 +85,7 @@ class LMHyperParams: torch.cuda.set_device(cuda_id) self.dataset_path = Path(self.dataset_path) self.base_lm_path = Path(self.base_lm_path) if self.base_lm_path is not None else None + self.tokenizer = Tokenizers(self.tokenizer) if isinstance(self.tokenizer, str) else self.tokenizer assert self.dataset_path.exists() self.cache_dir = self.dataset_path / 'models' / self.tokenizer_prefix @@ -98,7 +99,6 @@ class LMHyperParams: self.dps = np.array(self.dps) if self.nh is None: self.nh = 1550 if self.qrnn else 1150 if self.name is None: self.name = self.lang - self.tokenizer = Tokenizers[self.tokenizer] if isinstance(self.tokenizer, str) else self.tokenizer @property def tokenizer_prefix(self): return f"{self.tokenizer.value}{self.max_vocab // 1000}k" From b9eb7388f6e4413933bc4ab9551dada38f56f2e9 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 1 Dec 2018 23:58:14 +0100 Subject: [PATCH 10/41] Fix bidir for fastai tokenizer --- tests/test_end_to_end.py | 19 +++++++++++++++++++ ulmfit/pretrain_lm.py | 8 +++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 5d62a70..c3be22f 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -83,6 +83,25 @@ def test_ulmfit_fastai_end_to_end(): ) exp.train_lm(num_epochs=1) +def test_ulmfit_fastai_bidir_end_to_end(): + """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. + """ + imdb, wt2 = get_test_data() + lm_name = 'end-to-end-test-fastai' + cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, + lang='en', + cuda_id=cuda_id, + qrnn=True, + bidir=True, + tokenizer='f', + max_vocab=100, + bs=2, + name=lm_name, + ) + exp.train_lm(num_epochs=1) + def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 64fde72..dd8142d 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -181,9 +181,7 @@ class LMHyperParams: sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab) - lm_type = contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM - - data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=lm_type) + data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type) itos = data_lm.train_ds.vocab.itos stoi = data_lm.train_ds.vocab.stoi elif self.tokenizer is Tokenizers.MOSES: @@ -219,12 +217,12 @@ class LMHyperParams: lm_type=self.lm_type) elif self.tokenizer is Tokenizers.FASTAI: try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.') + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), - test_df=read_file(tst_path), classes=None) + test_df=read_file(tst_path), classes=None, lm_type=self.lm_type) data_lm.save('.') else: raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") From 83427aadc6244e8ecbc73f7437f3acd780780048 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sun, 2 Dec 2018 21:43:57 +0100 Subject: [PATCH 11/41] Add moses with fastai preprocessing --- results/sp30k_biqrnn_bs70 | 13 +++++++++++++ tests/test_end_to_end.py | 19 +++++++++++++++++++ ulmfit/pretrain_lm.py | 17 +++++++++++++---- 3 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 results/sp30k_biqrnn_bs70 diff --git a/results/sp30k_biqrnn_bs70 b/results/sp30k_biqrnn_bs70 new file mode 100644 index 0000000..5c658fd --- /dev/null +++ b/results/sp30k_biqrnn_bs70 @@ -0,0 +1,13 @@ +Starting from random weights +epoch train_loss valid_loss accuracy_fwd accuracy_bwd +1 3.669261 3.641705 0.399714 0.380689 +2 3.574335 3.547273 0.404729 0.385104 +3 3.573150 3.549644 0.403350 0.384167 +4 3.518714 3.499090 0.408166 0.389015 +5 3.477355 3.441828 0.413880 0.394777 +6 3.408005 3.366269 0.422041 0.402934 +7 3.314280 3.284519 0.431068 0.411727 +8 3.244735 3.205757 0.440180 0.421078 +9 3.170936 3.152495 0.446947 0.428045 +10 3.131996 3.138446 0.448782 0.430013 +Saving optimiser state at data/wiki/wikitext-103/models/sp30k/biqrnn_bs70.m \ No newline at end of file diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index c3be22f..787c5a3 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -102,6 +102,25 @@ def test_ulmfit_fastai_bidir_end_to_end(): ) exp.train_lm(num_epochs=1) +def test_ulmfit_moses_fa_bidir_end_to_end(): + """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. + """ + imdb, wt2 = get_test_data() + lm_name = 'end-to-end-test-fastai' + cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2, + lang='en', + cuda_id=cuda_id, + qrnn=True, + bidir=True, + tokenizer='vf', + max_vocab=100, + bs=2, + name=lm_name, + ) + exp.train_lm(num_epochs=1) + def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index dd8142d..6169edf 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -44,6 +44,7 @@ import fastai_contrib.data as contrib_data class Tokenizers(Enum): SUBWORD='sb' MOSES='v' + MOSES_FA='vf' FASTAI='f' # tokenizers ={ @@ -182,8 +183,6 @@ class LMHyperParams: sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab) data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type) - itos = data_lm.train_ds.vocab.itos - stoi = data_lm.train_ds.vocab.stoi elif self.tokenizer is Tokenizers.MOSES: # read the already whitespace separated data without any preprocessing trn_tok = read_whitespace_file(trn_path) @@ -209,12 +208,22 @@ class LMHyperParams: trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) - - # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids, valid_ids=val_ids, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type) + elif self.tokenizer is Tokenizers.MOSES_FA: + try: + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) + print("Tokenized data loaded") + except FileNotFoundError: + print("Running tokenization") + + pretokenized = Tokenizer(tok_func=BaseTokenizer, lang='en', pre_rules=None, post_rules=None) + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), + valid_df=read_file(val_path), tokenizer=pretokenized, + test_df=read_file(tst_path), classes=None, lm_type=self.lm_type) + data_lm.save('.') elif self.tokenizer is Tokenizers.FASTAI: try: data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) From 82c955ce6a489ad6efe22a12aa0aaccd825848da Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 00:51:21 +0100 Subject: [PATCH 12/41] Add different tokenization algorithms to train_clas --- tests/test_end_to_end.py | 21 ++++++---- ulmfit/pretrain_lm.py | 3 +- ulmfit/train_clas.py | 88 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 99 insertions(+), 13 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 787c5a3..635c9a6 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -68,7 +68,7 @@ def test_ulmfit_default_end_to_end(): def test_ulmfit_fastai_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ - imdb, wt2 = get_test_data() + test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' cuda_id = 0 exp = ulmfit.pretrain_lm.LMHyperParams( @@ -82,11 +82,13 @@ def test_ulmfit_fastai_end_to_end(): name=lm_name, ) exp.train_lm(num_epochs=1) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) def test_ulmfit_fastai_bidir_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ - imdb, wt2 = get_test_data() + test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' cuda_id = 0 exp = ulmfit.pretrain_lm.LMHyperParams( @@ -101,11 +103,13 @@ def test_ulmfit_fastai_bidir_end_to_end(): name=lm_name, ) exp.train_lm(num_epochs=1) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) def test_ulmfit_moses_fa_bidir_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ - imdb, wt2 = get_test_data() + test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' cuda_id = 0 exp = ulmfit.pretrain_lm.LMHyperParams( @@ -120,11 +124,13 @@ def test_ulmfit_moses_fa_bidir_end_to_end(): name=lm_name, ) exp.train_lm(num_epochs=1) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ - imdb, wt2 = get_test_data() + test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-spm' cuda_id = 0 exp = ulmfit.pretrain_lm.LMHyperParams( @@ -138,10 +144,9 @@ def test_ulmfit_sentencepiece_end_to_end(): name=lm_name, ) exp.train_lm(num_epochs=1) - #assert exp.results['accuracy'] > 0.30 - - # NOTE: ds_pct is not available for sentencepiece -- tests are on the complete dataset - # sentencepiece for finetuning/classification is currently not implemented + # not supported yet + # exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + # exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) if __name__ == "__main__": diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 6169edf..e68ed95 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -219,6 +219,7 @@ class LMHyperParams: except FileNotFoundError: print("Running tokenization") + # wikitext is pretokenized with Moses pretokenized = Tokenizer(tok_func=BaseTokenizer, lang='en', pre_rules=None, post_rules=None) data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), tokenizer=pretokenized, @@ -237,7 +238,7 @@ class LMHyperParams: raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") itos, stoi, trn_path = data_lm.vocab.itos, data_lm.vocab.stoi, data_lm.path print('Size of vocabulary:', len(itos)) - print('First 10 words in vocab:', ', '.join([itos[i] for i in range(10)])) + print('First 20 words in vocab:', data_lm.vocab.itos[:20]) return data_lm @classmethod diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index c327f3e..0782995 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -2,27 +2,46 @@ Train a classifier on top of a language model trained with `pretrain_lm.py`. Optionally fine-tune LM before. """ +from sacremoses import MosesTokenizer + import fastai import numpy as np import pickle +from fastai import * +from fastai.text import * + import torch from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner from fastai import fit_one_cycle, accuracy from fastai_contrib.data import LanguageModelType from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd -from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists +from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ + get_sentencepiece from fastai.text.transform import Vocab + import fire from collections import Counter from pathlib import Path -from ulmfit.pretrain_lm import LMHyperParams +from ulmfit.pretrain_lm import LMHyperParams, Tokenizers +class MosesTokenizerFunc(BaseTokenizer): + "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." + def __init__(self, lang:str): + self.tok = MosesTokenizer(lang) + + def tokenizer(self, t:str) -> List[str]: + return self.tok.tokenize(t, return_str=False, escape=False) + + def add_special_cases(self, toks:Collection[str]): + for w in toks: + assert len(self.tokenizer(w))==1, f"Tokenizer is unable to keep {w} as one token!" class CLSHyperParams(LMHyperParams): # dir_path -> data/imdb/ + use_test_for_validation=False def __post_init__(self, *args, **kwargs): super().__post_init__(*args, **kwargs) @@ -66,11 +85,72 @@ class CLSHyperParams(LMHyperParams): learn = classifier_learner(data_clas, bptt=self.bptt, pad_token=PAD_TOKEN_ID, path=self.model_dir.parent, model_dir=self.model_dir.name, qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, drop_mult=self.drop_mult) - - learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] + learn.true_wd = False + print("true_wd: ", learn.true_wd) return learn def load_cls_data(self, bs): + if self.dataset_dir.name == 'imdb': + return self.load_cls_data_imdb(bs) + else: + assert self.tokenizer is Tokenizers.MOSES, "XNLI does not support other tokenizers than Moses" + return self.load_cls_data_old_for_xnli(bs) + + def load_cls_data_imdb(self, bs): + trn_df = pd.read_csv(self.dataset_path / 'train.csv', header=None) + tst_df = pd.read_csv(self.dataset_path / 'test.csv', header=None) + + if self.use_test_for_validation: + val_len = max(int(len(tst_df) * 0.1), 2) + tst_len = len(tst_df) - val_len + val_df = trn_df[tst_len:] + else: + val_len = max(int(len(trn_df) * 0.1), 2) + trn_len = len(trn_df) - val_len + trn_df, val_df = trn_df[trn_len:], trn_df[trn_len:] + + if self.tokenizer is Tokenizers.SUBWORD: + #TODO Fix me to make sure it trains correct dictionary + args = get_sentencepiece(self.dataset_path, self.dataset_path / 'train.csv', self.name, vocab_size=self.max_vocab) + elif self.tokenizer is Tokenizers.MOSES: + args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[])) + elif self.tokenizer is Tokenizers.MOSES_FA: + args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en')) # use default pre/post rules + elif self.tokenizer is Tokenizers.FASTAI: + args = dict() + else: + raise ValueError( + f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") + + try: + data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type) + print("Tokenized data loaded") + except FileNotFoundError: + print("Running tokenization") + + # wikitext is pretokenized with Moses + + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=trn_df, + valid_df=val_df, test_df=tst_df, + lm_type=self.lm_type, **args) + data_lm.save('lm') + + args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation + try: + data_cls = TextClasDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) + print("Tokenized data loaded") + except FileNotFoundError: + print("Running tokenization") + data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, + valid_df=val_df, test_df=tst_df, + **args) + data_cls.save('.') + print('Size of vocabulary:', len(data_lm.vocab.itos)) + print('First 20 words in vocab:', data_lm.vocab.itos[:20]) + return data_cls, data_lm + + + def load_cls_data_old_for_xnli(self, bs): tmp_dir = self.cache_dir tmp_dir.mkdir(exist_ok=True) vocab_file = tmp_dir / f'vocab_{self.lang}.pkl' From 35a5dcb75a6a88bdbb1702cabbe94aa96e859c18 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 00:59:36 +0100 Subject: [PATCH 13/41] Fix issue when running cls training from command line --- tests/test_end_to_end.py | 2 +- ulmfit/__main__.py | 4 ++-- ulmfit/pretrain_lm.py | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 635c9a6..7d551d7 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -103,7 +103,7 @@ def test_ulmfit_fastai_bidir_end_to_end(): name=lm_name, ) exp.train_lm(num_epochs=1) - exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir)) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) def test_ulmfit_moses_fa_bidir_end_to_end(): diff --git a/ulmfit/__main__.py b/ulmfit/__main__.py index 5c1684a..23aec10 100644 --- a/ulmfit/__main__.py +++ b/ulmfit/__main__.py @@ -18,8 +18,8 @@ class ULMFiT: lm2 = LMHyperParams @wraps(CLSHyperParams) - def cls(self, dataset_path, baseon_path, **changes): - params = CLSHyperParams.from_lm(dataset_path, baseon_path, **changes) + def cls(self, dataset_path, base_lm_path, **changes): + params = CLSHyperParams.from_lm(dataset_path, base_lm_path, **changes) return FireView(train=params.train_cls) if __name__ == '__main__': diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index e68ed95..0bab350 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -243,6 +243,8 @@ class LMHyperParams: @classmethod def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams': + base_lm_path = Path(base_lm_path) + dataset_path = Path(dataset_path) with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) d['dataset_path'] = dataset_path d['base_lm_path'] = base_lm_path From a499bf9a20a76bf9aed573be10c351a465841aa4 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 01:24:44 +0100 Subject: [PATCH 14/41] Make cls train work with relative paths --- tests/test_end_to_end.py | 25 +++++++++++++++++++++++++ ulmfit/pretrain_lm.py | 4 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 7d551d7..85d8cd7 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -44,6 +44,31 @@ def get_test_data(): return test_data, test_wt +def test_ulmfit_works_with_relative_paths(): + """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. + """ + os.chdir(get_data_folder()/"..") + + + test_data, wt2 = get_test_data() + lm_name = 'end-to-end-test-default' + cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( + dataset_path=wt2.relative_to(Path.cwd()), + lang='en', + qrnn=True, + max_vocab=1000, + bs=2, + name=lm_name) + + exp.train_lm(num_epochs=1) + + #assert exp.results['accuracy'] > 0.02 + + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=1, unfreeze=False, bs=4,) + + def test_ulmfit_default_end_to_end(): """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. """ diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 0bab350..05af939 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -243,8 +243,8 @@ class LMHyperParams: @classmethod def from_lm(cls, dataset_path, base_lm_path, **kwargs) -> 'LMHyperParams': - base_lm_path = Path(base_lm_path) - dataset_path = Path(dataset_path) + base_lm_path = Path(base_lm_path).resolve() + dataset_path = Path(dataset_path).resolve() with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) d['dataset_path'] = dataset_path d['base_lm_path'] = base_lm_path From 039624870a17336f38bf3eddccc3af8317335ca1 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 01:28:03 +0100 Subject: [PATCH 15/41] Fix loading tokenized data set in train cls --- tests/test_end_to_end.py | 5 +++++ ulmfit/train_clas.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 85d8cd7..a5de301 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -68,6 +68,11 @@ def test_ulmfit_works_with_relative_paths(): exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=1, unfreeze=False, bs=4,) + # should work for the second time as well + + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) + def test_ulmfit_default_end_to_end(): """ Test ulmfit with (default) Moses tokenizer on small wikipedia dataset. diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0782995..4bada19 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -137,7 +137,7 @@ class CLSHyperParams(LMHyperParams): args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation try: - data_cls = TextClasDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) + data_cls = TextClasDataBunch.load(self.cache_dir, '.') print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") From 9826f6881cbc5c8d7509d094325c62318ab95b15 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 01:35:59 +0100 Subject: [PATCH 16/41] Fix the way trn & val set is created in imdb --- ulmfit/train_clas.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 4bada19..6ee1b59 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -103,11 +103,11 @@ class CLSHyperParams(LMHyperParams): if self.use_test_for_validation: val_len = max(int(len(tst_df) * 0.1), 2) tst_len = len(tst_df) - val_len - val_df = trn_df[tst_len:] + val_df = trn_df[:tst_len] else: val_len = max(int(len(trn_df) * 0.1), 2) trn_len = len(trn_df) - val_len - trn_df, val_df = trn_df[trn_len:], trn_df[trn_len:] + trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] if self.tokenizer is Tokenizers.SUBWORD: #TODO Fix me to make sure it trains correct dictionary @@ -124,7 +124,7 @@ class CLSHyperParams(LMHyperParams): try: data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type) - print("Tokenized data loaded") + print(f"Tokenized data loaded, trn.trn {len(data_trn.train_ds)}, trn.val {len(data_trn.valid_ds)}") except FileNotFoundError: print("Running tokenization") @@ -138,7 +138,7 @@ class CLSHyperParams(LMHyperParams): args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation try: data_cls = TextClasDataBunch.load(self.cache_dir, '.') - print("Tokenized data loaded") + print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") except FileNotFoundError: print("Running tokenization") data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, From f25deb3049bd6d452839e5b22fa31c1ee87cde44 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 01:43:01 +0100 Subject: [PATCH 17/41] Use trn + tst for LM training --- ulmfit/train_clas.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 6ee1b59..652eba3 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -124,13 +124,13 @@ class CLSHyperParams(LMHyperParams): try: data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type) - print(f"Tokenized data loaded, trn.trn {len(data_trn.train_ds)}, trn.val {len(data_trn.valid_ds)}") + print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") except FileNotFoundError: - print("Running tokenization") + print("Running tokenization, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") # wikitext is pretokenized with Moses - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=trn_df, + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=pd.concat([trn_df,tst_df]), valid_df=val_df, test_df=tst_df, lm_type=self.lm_type, **args) data_lm.save('lm') @@ -140,7 +140,7 @@ class CLSHyperParams(LMHyperParams): data_cls = TextClasDataBunch.load(self.cache_dir, '.') print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") except FileNotFoundError: - print("Running tokenization") + print("Running tokenization, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, valid_df=val_df, test_df=tst_df, **args) From 908c3d7e8aeb4d6ef9a1575a4d48b0830ef79152 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 16:41:07 +0100 Subject: [PATCH 18/41] bug fix --- ulmfit/train_clas.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 652eba3..91d75d1 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -126,24 +126,26 @@ class CLSHyperParams(LMHyperParams): data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type) print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") except FileNotFoundError: - print("Running tokenization, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") + print(f"Running tokenization...") # wikitext is pretokenized with Moses data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=pd.concat([trn_df,tst_df]), valid_df=val_df, test_df=tst_df, - lm_type=self.lm_type, **args) + lm_type=self.lm_type, max_vocab=self.max_vocab, **args) data_lm.save('lm') + print(f" cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation try: data_cls = TextClasDataBunch.load(self.cache_dir, '.') print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") except FileNotFoundError: - print("Running tokenization, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") + print(f"Running tokenization...") data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, - valid_df=val_df, test_df=tst_df, + valid_df=val_df, test_df=tst_df, max_vocab=self.max_vocab, **args) + print(f" cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") data_cls.save('.') print('Size of vocabulary:', len(data_lm.vocab.itos)) print('First 20 words in vocab:', data_lm.vocab.itos[:20]) From 5524006d81375f942fba721a97d873232c122eff Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 4 Dec 2018 16:41:22 +0100 Subject: [PATCH 19/41] Respect max_vocab --- ulmfit/pretrain_lm.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 05af939..d40ad8a 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -62,6 +62,7 @@ class LMHyperParams: qrnn: bool = True max_vocab: int = 60000 tokenizer: Tokenizers = Tokenizers.MOSES + pretrained_model: str = None emb_sz:int = 400 nh: int = None @@ -132,7 +133,7 @@ class LMHyperParams: if num_epochs > 0: if self.pretrained_fnames : - learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) + learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) # TODO Fix the learning rates learn.unfreeze() if num_epochs > 0: learn.fit_one_cycle(num_epochs, 1e-3, moms=(0.8, 0.7)) else: @@ -158,7 +159,8 @@ class LMHyperParams: learn = lm_learner(data_lm, bptt=self.bptt, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, drop_mult=self.drop_mult, tie_weights=True, model_dir= self.model_dir.relative_to(data_lm.path), - bias=True, qrnn=self.qrnn, clip=self.clip, pretrained_fnames=self.pretrained_fnames) + bias=True, qrnn=self.qrnn, clip=self.clip, pretrained_fnames=self.pretrained_fnames, + pretrained_model=self.pretrained_model) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.true_wd = False @@ -223,7 +225,8 @@ class LMHyperParams: pretokenized = Tokenizer(tok_func=BaseTokenizer, lang='en', pre_rules=None, post_rules=None) data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), tokenizer=pretokenized, - test_df=read_file(tst_path), classes=None, lm_type=self.lm_type) + test_df=read_file(tst_path), classes=None, lm_type=self.lm_type, + max_vocab=self.max_vocab) data_lm.save('.') elif self.tokenizer is Tokenizers.FASTAI: try: @@ -232,7 +235,7 @@ class LMHyperParams: except FileNotFoundError: print("Running tokenization") data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), - test_df=read_file(tst_path), classes=None, lm_type=self.lm_type) + test_df=read_file(tst_path), classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab,) data_lm.save('.') else: raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") From 4454af167f09dec8988d8af07f345e2406691451 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 5 Dec 2018 16:26:16 +0100 Subject: [PATCH 20/41] Use more text during pretraining of imdb --- prepare_imdb.sh | 4 ++-- tests/test_end_to_end.py | 1 + ulmfit/train_clas.py | 15 +++++++++------ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/prepare_imdb.sh b/prepare_imdb.sh index 6d2488c..89815b7 100644 --- a/prepare_imdb.sh +++ b/prepare_imdb.sh @@ -6,6 +6,6 @@ mkdir -p "${DATA_DIR}" echo "Saving data in $DATA_DIR" wget -c "http://files.fast.ai/data/aclImdb.tgz" -P "${DATA_DIR}" -echo "Imdb is raw text so we are tokenizing it with Moses" -python -m fastai_contrib.utils prepare_imdb "${DATA_DIR}/aclImdb.tgz" --prepare_lm==False +echo "Imdb is raw text no preparation is done" +python -m fastai_contrib.utils prepare_imdb "${DATA_DIR}/aclImdb.tgz" diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index a5de301..014b6f1 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -40,6 +40,7 @@ def get_test_data(): copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=6*sz) copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz) + copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz) return test_data, test_wt diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 91d75d1..c9b5826 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -99,6 +99,12 @@ class CLSHyperParams(LMHyperParams): def load_cls_data_imdb(self, bs): trn_df = pd.read_csv(self.dataset_path / 'train.csv', header=None) tst_df = pd.read_csv(self.dataset_path / 'test.csv', header=None) + unsp_df = pd.read_csv(self.dataset_path / 'unsup.csv', header=None) + + lm_trn_df = pd.concat([unsp_df, trn_df, tst_df]) + val_len = max(int(len(lm_trn_df) * 0.1), 2) + lm_trn_df = lm_trn_df[val_len:] + lm_val_df = lm_trn_df[:val_len] if self.use_test_for_validation: val_len = max(int(len(tst_df) * 0.1), 2) @@ -127,12 +133,9 @@ class CLSHyperParams(LMHyperParams): print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") except FileNotFoundError: print(f"Running tokenization...") - - # wikitext is pretokenized with Moses - - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=pd.concat([trn_df,tst_df]), - valid_df=val_df, test_df=tst_df, - lm_type=self.lm_type, max_vocab=self.max_vocab, **args) + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=lm_trn_df, valid_df=lm_val_df, + max_vocab=self.max_vocab, bs=bs, lm_type=self.lm_type, **args) + print(f"Saving tokenized: cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") data_lm.save('lm') print(f" cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") From 2acbf15555cb9c371f67a98cf9e8241bc4479fbb Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 5 Dec 2018 16:27:14 +0100 Subject: [PATCH 21/41] Tweak hyper training params of cls (drop_mul, bs, true_wd=True) I've set the same hyperparams as in lesson3 --- ulmfit/pretrain_lm.py | 62 ++++++++++++++++++------------ ulmfit/train_clas.py | 88 ++++++++++++++++++++++++------------------- 2 files changed, 86 insertions(+), 64 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index d40ad8a..c323ac5 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -41,6 +41,10 @@ import fastai_contrib.data as contrib_data # :param model_dir: The path to the directory where the models should be saved # :param bidir: whether the language model is bidirectional # """ +LM_BEST = "lm_best" +ENC_BEST = "enc_best" + + class Tokenizers(Enum): SUBWORD='sb' MOSES='v' @@ -127,44 +131,52 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=10, data_lm=None): + def train_lm(self, num_epochs=20, data_lm=None, true_wd=False, drop_mult=0.1): data_lm = self.load_wiki_data() if data_lm is None else data_lm - learn = self.create_lm_learner(data_lm) + learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) + learn.true_wd = true_wd + try: + learn.load("lm_best_with_opt") + print("Continuing training") + except FileNotFoundError: + pass if num_epochs > 0: - if self.pretrained_fnames : - learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) # TODO Fix the learning rates - learn.unfreeze() - if num_epochs > 0: learn.fit_one_cycle(num_epochs, 1e-3, moms=(0.8, 0.7)) + if self.pretrained_fnames or self.pretrained_model: + print("Training lm from: ", self.pretrained_fnames or self.pretrained_model) + if learn.true_wd: + learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) + learn.unfreeze() + learn.fit_one_cycle(num_epochs, 1e-3, moms=(0.8, 0.7)) + else: + learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7), wd=1e-7) # TODO Fix the learning rates + learn.unfreeze() + learn.fit_one_cycle(num_epochs, 1e-3, moms=(0.8, 0.7), wd=1e-7) else: - try: - learn.load("lm_best") - print("Weights loaded") - except FileNotFoundError: - print("Starting from random weights") - learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) - opt_state_path = self.model_dir / 'opt_state.pth' - print(f"Saving optimiser state at {opt_state_path}") - torch.save(learn.opt.opt.state_dict(), opt_state_path) - learn.save_encoder("enc_best") - learn.save("lm_best", with_opt=False) + print("Training lm from random weights") + if not learn.true_wd: learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) + else: learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7)) # TODO find proper values + learn.save("lm_best_with_opt", with_opt=False) + learn.save_encoder(ENC_BEST) + learn.save(LM_BEST, with_opt=False) print(learn.path) self.save_info() return learn - def create_lm_learner(self, data_lm): - fastai.text.learner.default_dropout['language'] = self.dps + def create_lm_learner(self, data_lm, dps=None, **kwargs): + fastai.text.learner.default_dropout['language'] = dps or self.dps lm_learner = bilm_learner if self.bidir else language_model_learner - learn = lm_learner(data_lm, bptt=self.bptt, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, - drop_mult=self.drop_mult, tie_weights=True, model_dir= self.model_dir.relative_to(data_lm.path), - bias=True, qrnn=self.qrnn, clip=self.clip, pretrained_fnames=self.pretrained_fnames, - pretrained_model=self.pretrained_model) + trn_args = dict(drop_mult=self.drop_mult, tie_weights=True, clip=self.clip, bptt=self.bptt, + pretrained_fnames=self.pretrained_fnames, + pretrained_model=self.pretrained_model) + trn_args.update(kwargs) + print ("Training args: ", trn_args, "dps: ", dps) + learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, + bias=True, qrnn=self.qrnn, model_dir=self.model_dir.relative_to(data_lm.path), **trn_args) # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) - learn.true_wd = False - print("true_wd: ", learn.true_wd) learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] return learn diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index c9b5826..439915b 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -25,7 +25,8 @@ import fire from collections import Counter from pathlib import Path -from ulmfit.pretrain_lm import LMHyperParams, Tokenizers +from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST + class MosesTokenizerFunc(BaseTokenizer): "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." @@ -50,53 +51,61 @@ class CLSHyperParams(LMHyperParams): @property def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() - def train_cls(self, num_lm_epochs, unfreeze=True, bs=70): + def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5): data_clas, data_lm = self.load_cls_data(bs) - if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm) - learn = self.create_cls_learner(data_clas) - + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm) + learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls) try: learn.load('cls_last') print("Loading last classifier") except FileNotFoundError: - learn.load_encoder("enc_best") - - learn.true_wd = False - print("Starting classifier training") - learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) - if unfreeze: - learn.freeze_to(-2) - learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) - - learn.freeze_to(-3) - learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) - - learn.unfreeze() - learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) - + learn.load_encoder(ENC_BEST) + if true_wd: + learn.true_wd = True + print("Starting classifier training") + learn.freeze_to(-1) + learn.fit_one_cycle(1, 2e-2, moms=(0.8, 0.7)) + if unfreeze: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) + learn.freeze_to(-3) + learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + learn.unfreeze() + learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + else: + learn.true_wd = False + print("Starting classifier training") + learn.fit_one_cycle(1, 5e-2, moms=(0.8, 0.7), wd=1e-7) + if unfreeze: + learn.freeze_to(-2) + learn.fit_one_cycle(1, slice(5e-2 / (2.6 ** 4), 5e-2), moms=(0.8, 0.7), wd=1e-7) + learn.freeze_to(-3) + learn.fit_one_cycle(1, slice(5e-4 / (2.6 ** 4), 5e-4), moms=(0.8, 0.7), wd=1e-7) + learn.unfreeze() + learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) return learn - def create_cls_learner(self, data_clas): - fastai.text.learner.default_dropout['language'] = self.dps + def create_cls_learner(self, data_clas, dps=None, **kwargs): + fastai.text.learner.default_dropout['language'] = dps or self.dps + trn_args=dict(drop_mult=self.drop_mult, bptt=self.bptt, clip=self.clip,) + trn_args.update(kwargs) classifier_learner = bilm_text_classifier_learner if self.bidir else text_classifier_learner - learn = classifier_learner(data_clas, bptt=self.bptt, pad_token=PAD_TOKEN_ID, + learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID, path=self.model_dir.parent, model_dir=self.model_dir.name, - qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, drop_mult=self.drop_mult) - learn.true_wd = False - print("true_wd: ", learn.true_wd) + qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args) return learn - def load_cls_data(self, bs): + def load_cls_data(self, bs, **kwargs): if self.dataset_dir.name == 'imdb': - return self.load_cls_data_imdb(bs) + return self.load_cls_data_imdb(bs, **kwargs) else: assert self.tokenizer is Tokenizers.MOSES, "XNLI does not support other tokenizers than Moses" - return self.load_cls_data_old_for_xnli(bs) + return self.load_cls_data_old_for_xnli(bs, **kwargs) - def load_cls_data_imdb(self, bs): + def load_cls_data_imdb(self, bs, force=False, use_test_for_validation=False): trn_df = pd.read_csv(self.dataset_path / 'train.csv', header=None) tst_df = pd.read_csv(self.dataset_path / 'test.csv', header=None) unsp_df = pd.read_csv(self.dataset_path / 'unsup.csv', header=None) @@ -106,7 +115,7 @@ class CLSHyperParams(LMHyperParams): lm_trn_df = lm_trn_df[val_len:] lm_val_df = lm_trn_df[:val_len] - if self.use_test_for_validation: + if use_test_for_validation: val_len = max(int(len(tst_df) * 0.1), 2) tst_len = len(tst_df) - val_len val_df = trn_df[:tst_len] @@ -115,6 +124,7 @@ class CLSHyperParams(LMHyperParams): trn_len = len(trn_df) - val_len trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] + if self.tokenizer is Tokenizers.SUBWORD: #TODO Fix me to make sure it trains correct dictionary args = get_sentencepiece(self.dataset_path, self.dataset_path / 'train.csv', self.name, vocab_size=self.max_vocab) @@ -129,7 +139,8 @@ class CLSHyperParams(LMHyperParams): f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") try: - data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type) + if force: raise FileNotFoundError("Forcing reloading of caches") + data_lm = TextLMDataBunch.load(self.cache_dir, 'lm', lm_type=self.lm_type, bs=bs) print(f"Tokenized data loaded, lm.trn {len(data_lm.train_ds)}, lm.val {len(data_lm.valid_ds)}") except FileNotFoundError: print(f"Running tokenization...") @@ -137,18 +148,17 @@ class CLSHyperParams(LMHyperParams): max_vocab=self.max_vocab, bs=bs, lm_type=self.lm_type, **args) print(f"Saving tokenized: cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") data_lm.save('lm') - print(f" cls.trn {len(data_lm.train_ds)}, cls.val {len(data_lm.valid_ds)}") - args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation try: - data_cls = TextClasDataBunch.load(self.cache_dir, '.') + if force: raise FileNotFoundError("Forcing reloading of caches") + data_cls = TextClasDataBunch.load(self.cache_dir, '.', bs=bs) print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") except FileNotFoundError: + args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation print(f"Running tokenization...") - data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, - valid_df=val_df, test_df=tst_df, max_vocab=self.max_vocab, - **args) - print(f" cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") + data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, valid_df=val_df, + test_df=tst_df, max_vocab=self.max_vocab, bs=bs, **args) + print(f"Saving tokenized: cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") data_cls.save('.') print('Size of vocabulary:', len(data_lm.vocab.itos)) print('First 20 words in vocab:', data_lm.vocab.itos[:20]) From 6a899015edef94ce72aaf5e1761aa6f5bd1233f0 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 5 Dec 2018 16:27:41 +0100 Subject: [PATCH 22/41] Generate imdb unsp.csv --- fastai_contrib/utils.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 92b89d1..927426a 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -69,13 +69,12 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, pre_rules:ListRul os.makedirs(path / 'models', exist_ok=True) pre_rules = pre_rules if pre_rules is not None else [] post_rules = post_rules if post_rules is not None else [] - - # load the text frmo the train tokens file - text = [line.rstrip('\n') for line in open(trn_path)] - text = list(filter(None, text)) if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'): - raw_text = reduce(lambda t, rule: rule(t), pre_rules, '\n'.join(text)) + # load the text from the train tokens file + text = [line.rstrip('\n') for line in open(trn_path)] + text = list(filter(None, text)) + raw_text = reduce(lambda t, rule: rule(t), pre_rules, '\n'.join(text)) # FIXME: possibly does not work with pre_rules raw_text_path = path / cache_name / 'all_text.txt' with open(raw_text_path, 'w') as f: f.write(raw_text) @@ -187,21 +186,9 @@ def prepare_imdb(file_path: str, prepare_lm = False): print(f"Writing them to {CLAS_PATH}") df_trn[df_trn['labels'] != 2].to_csv(CLAS_PATH / 'train.csv', header=False, index=False) df_val.to_csv(CLAS_PATH / 'test.csv', header=False, index=False) - + df_trn[df_trn['labels'] == 2].to_csv(CLAS_PATH / 'unsup.csv', header=False, index=False) (CLAS_PATH / 'classes.txt').open('w', encoding='utf-8').writelines(f'{o}\n' for o in CLASSES) - if prepare_lm: - print("Preparing LM data") - trn_texts, val_texts = model_selection.train_test_split( - np.concatenate([trn_texts, val_texts]), test_size=0.1) - print(f"trn_texts has {len(trn_texts)} samples, while val_texts has {len(val_texts)} rows") - print(f"Writing them to {LM_PATH}") - df_trn = pd.DataFrame({'text': trn_texts, 'labels': [0] * len(trn_texts)}, columns=col_names) - df_val = pd.DataFrame({'text': val_texts, 'labels': [0] * len(val_texts)}, columns=col_names) - - df_trn.to_csv(LM_PATH / 'train.csv', header=False, index=False) - df_val.to_csv(LM_PATH / 'test.csv', header=False, index=False) - def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], List[str]]: """ @@ -354,9 +341,6 @@ def read_whitespace_file(filepath): tokens.append(line.split() + [EOS]) return np.array(tokens) - - - class DataStump: """Placeholder class as LanguageModelLoader requires object with ids attribute.""" def __init__(self, ids): From 4ae2a158a270a40467d18fbde2c9c1491e4bbf2a Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 6 Dec 2018 23:29:43 +0100 Subject: [PATCH 23/41] Respect batch size in training lm model. --- ulmfit/pretrain_lm.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index c323ac5..b2483cf 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -228,7 +228,7 @@ class LMHyperParams: lm_type=self.lm_type) elif self.tokenizer is Tokenizers.MOSES_FA: try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") @@ -238,16 +238,18 @@ class LMHyperParams: data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), tokenizer=pretokenized, test_df=read_file(tst_path), classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab) + max_vocab=self.max_vocab, bs=self.bs) data_lm.save('.') elif self.tokenizer is Tokenizers.FASTAI: try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type) + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), - test_df=read_file(tst_path), classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab,) + test_df=read_file(tst_path), classes=None, lm_type=self.lm_type, + max_vocab=self.max_vocab,bs=self.bs, + ) data_lm.save('.') else: raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") From 50ab34eea959eb5d08fc2375a7e15f9d02f65d3a Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 7 Dec 2018 16:10:20 +0100 Subject: [PATCH 24/41] Fix classification scripts and give a way to test accuracy on test set --- ulmfit/train_clas.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 439915b..5581498 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -70,7 +70,7 @@ class CLSHyperParams(LMHyperParams): learn.freeze_to(-2) learn.fit_one_cycle(1, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7)) learn.freeze_to(-3) - learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) + learn.fit_one_cycle(1, slice(5e-3 / (2.6 ** 4), 5e-3), moms=(0.8, 0.7)) learn.unfreeze() learn.fit_one_cycle(2, slice(1e-3 / (2.6 ** 4), 1e-3), moms=(0.8, 0.7)) else: @@ -117,13 +117,13 @@ class CLSHyperParams(LMHyperParams): if use_test_for_validation: val_len = max(int(len(tst_df) * 0.1), 2) - tst_len = len(tst_df) - val_len - val_df = trn_df[:tst_len] + val_df = tst_df + cls_cache = 'notst' else: val_len = max(int(len(trn_df) * 0.1), 2) trn_len = len(trn_df) - val_len trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] - + cls_cache = '.' if self.tokenizer is Tokenizers.SUBWORD: #TODO Fix me to make sure it trains correct dictionary @@ -151,7 +151,7 @@ class CLSHyperParams(LMHyperParams): try: if force: raise FileNotFoundError("Forcing reloading of caches") - data_cls = TextClasDataBunch.load(self.cache_dir, '.', bs=bs) + data_cls = TextClasDataBunch.load(self.cache_dir, cls_cache, bs=bs) print(f"Tokenized data loaded, cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") except FileNotFoundError: args['vocab'] = data_lm.vocab # make sure we use the same vocab for classifcation @@ -159,7 +159,7 @@ class CLSHyperParams(LMHyperParams): data_cls = TextClasDataBunch.from_df(path=self.cache_dir, train_df=trn_df, valid_df=val_df, test_df=tst_df, max_vocab=self.max_vocab, bs=bs, **args) print(f"Saving tokenized: cls.trn {len(data_cls.train_ds)}, cls.val {len(data_cls.valid_ds)}") - data_cls.save('.') + data_cls.save(cls_cache) print('Size of vocabulary:', len(data_lm.vocab.itos)) print('First 20 words in vocab:', data_lm.vocab.itos[:20]) return data_cls, data_lm From 0b7dc7ec6bde9363a0f9bab230aca08f3292b163 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Fri, 7 Dec 2018 16:10:32 +0100 Subject: [PATCH 25/41] Add two experiments showing how to train imdb classification to 94% accuracy --- experiments/cls_test_wt103_1_f.ipynb | 736 ++++++++++++++++++++++++++ experiments/cls_test_wt103_1_vf.ipynb | 467 ++++++++++++++++ 2 files changed, 1203 insertions(+) create mode 100644 experiments/cls_test_wt103_1_f.ipynb create mode 100644 experiments/cls_test_wt103_1_vf.ipynb diff --git a/experiments/cls_test_wt103_1_f.ipynb b/experiments/cls_test_wt103_1_f.ipynb new file mode 100644 index 0000000..26a6e2e --- /dev/null +++ b/experiments/cls_test_wt103_1_f.ipynb @@ -0,0 +1,736 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test of classifcation Fastai tokenization" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline\n", + "%cd .." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from fastai import *\n", + "from fastai.text import *" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from ulmfit.train_clas import *" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch size: 70\n", + "Max vocab: 60000\n", + "Cache dir: data/imdb/models/f60k\n", + "Model dir: data/imdb/models/f60k/lstm_None.m\n" + ] + } + ], + "source": [ + "exp = CLSHyperParams('data/imdb', qrnn=False,tokenizer='f', lang='en', cuda_id=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "exp.pretrained_model = URLs.WT103_1\n", + "exp.drop_mult=0.3" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving tokenized: cls.trn 25000, cls.val 25000\n", + "Size of vocabulary: 60002\n", + "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', '.', ',', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", '-', '\\n\\n']\n" + ] + } + ], + "source": [ + "data_clas, data_lm = exp.load_cls_data(bs=40,force=False, use_test_for_validation=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "40" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_lm.train_dl.batch_size" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "true_wd: False\n" + ] + } + ], + "source": [ + "learn = exp.create_lm_learner(data_lm)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "learn.true_wd=True\n", + "learn.opt=None" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n" + ] + } + ], + "source": [ + "learn.lr_find()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8VfX9+PHXOxMICSEkrLDCBpEZtoKCX+rE4rYu0JZatVKttfVnv7Zfq3XX8W2/RcVN60KpeyCKOFhho0DAsJJAEhISsuf798c9wQAJCXBP7sj7+XjcR8494573Jze57/s5n3FEVTHGGGO8LcTXARhjjAlOlmCMMca4whKMMcYYV1iCMcYY4wpLMMYYY1xhCcYYY4wrLMEYY4xxhSUYY4wxrrAEY4wxxhVhvg7AW+Lj47VXr16+DsMYYwLK6tWr96tqghuvHTQJplevXqSkpPg6DGOMCSgissut17ZLZMYYY1xhCcYYY4wrLMEYY4xxhSUYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXWIIxxpgA9tbqdF5dudvXYdTLEowxxgSwBavTeXtNuq/DqJclGGOMCWB5xRXERUX4Oox6WYIxxpgAlltcQVxUpK/DqJclGGOMCVA1NcqBkgo6WA3GGGOMNxWUVlJdo3aJzBhjjHflFlcA0KGtJRhjjDFelOckmBZZgxGRnSKyUUTWichRN2sRj6dEZLuIbBCRkUdsjxGRDBH5u5txGmNMIMorLgf8N8E0xw3HzlTV/Q1sOwfo5zzGAv90ftb6C/Clu+EZY0xgOnSJzHqR1etC4GX1WA7EikgXABEZBXQCPvVlgMYY46/yijwJpn1UuI8jqZ/bCUaBT0VktYjMrmd7IrCnzvN0IFFEQoDHgN8d68VFZLaIpIhISk5OjteCNsaYQJBbXEF0ZBiRYaG+DqVebieYiao6Es+lsJtFZNIR26WeYxS4CfhQVffUs/3HHVWfUdVkVU1OSEjwTsTGGBMg8ooriPPTHmTgchuMqmY6P7NFZCEwBlhaZ5d0oHud592ATGA8cLqI3AS0BSJEpEhV/+BmvMYYE0j8eZoYcLEGIyJRIhJduwxMAzYdsdu7wLVOb7JxQIGq7lXVq1S1h6r2Au7A005jycUYY+rILfbfUfzgbg2mE7BQRGrP829V/VhEbgRQ1bnAh8C5wHagBJjlYjzGGBNU8orLOTUxxtdhNMi1BKOqacCwetbPrbOswM2NvM6LwIteDs8YYwKaqjqXyPyzizL4vpuyMcaYE1BYXkVltfr1JTJLMMYYE4Bqx8C0yEZ+Y4wx7qkdxe/P3ZQtwRhjTADKOzRNjCUYY4wxXuTvE12CJRhjjAlI/j7RJViCMcaYgJRXVEHr8FBaR/jnPGRgCcYYYwJSXnGF397JspYlGGOMCUD+Pk0MWIIxxpiA5O8TXYIlGGOMCUj+Pk0MWIIxxpiAlFtcbm0wxhhjvKukooqyyhq7RGaMMca7cgNgHjKwBGOMMQEnEKaJAUswxhgTcGoTTIuuwYjIThHZKCLrRCSlnu0iIk+JyHYR2SAiI531w0VkmYh856y/3M04jTEmkATCNDHg7i2Ta52pqvsb2HYO0M95jAX+6fwsAa5V1W0i0hVYLSKfqGp+M8RrjDF+7dBEl37ei6w5EsyxXAi87Nw6ebmIxIpIF1VNrd1BVTNFJBtIACzBGGNavNziCiLCQojy43nIwP02GAU+FZHVIjK7nu2JwJ46z9OddYeIyBggAvjBtSiNMSaA5BZ5pokREV+Hckxu12AmOjWQjsAiEdmiqkvrbK/vt6OHNop0AV4BrlPVmiN3dJLWbIAePXp4N3JjjPFTgTBNDLhcg1HVTOdnNrAQGHPELulA9zrPuwGZACISA3wA/FFVlzfw+s+oarKqJickJJxQjPklFTz52Ta+yyw4oeONMaa55bb0BCMiUSISXbsMTAM2HbHbu8C1Tm+ycUCBqu4VkQg8CellVX3TrRid2Pjfz7fx3vq9bp7GGGO8Jq+43O/HwIC7NZhOwNcish5YCXygqh+LyI0icqOzz4dAGrAdeBa4yVl/GTAJmOl0cV4nIsPdCLJd63BG94pj8eYsN17eGGO8Lq/I/ye6BBfbYFQ1DRhWz/q5dZYVuLmefeYD892K7UhTB3Xkvg82szu3hB4d2jTXaY0x5riVVVZTXFHt9xNdgo3kB+CsQZ0AWLzFajHGGP8WKKP4wRIMAL3io+iTEMXizdm+DsUYY47JEkwAOmtQJ1bsyKWwrNLXoRhjTINyA2SiS7AEc8jUQZ2orFaWpjY0q40xxvjeoWliLMEEjpE9YoltE269yYwxfq32XjD+PtElWII5JCw0hDMHdOSLrdlU12jjBxhjjA/kFVcQFiLEtPb1VJKNswRTx9RBHTlQUsma3Qd8HYoxxtQrr7iC9gEwDxlYgjnMpP4JhIUIn9llMmOMn8otrgiIBn6wBHOYmFbhjO0dZ92VjTF+K1AmugRLMEeZOrAT27OL2JVb7OtQjDHmKHnFFXRo6/8N/GAJ5iiHRvVbLcYY44dyCstJsAQTmHp0aENSfBTf/pDr61CMMeYwJRVVFJVXkRBtCSZgjekVx6qdedRYd2VjjB/ZX+gZA2MJJoCNSYqjoLSS1OxCX4dijDGH5BSVAZZgAtrY3nEArEjL83Ekxhjzo5xCzzQx1gYTwLq1b0NibGtW7rAEY4zxH4cSjNVgQER2ishG546UKfVsFxF5SkS2i8gGERlZZ9t1IrLNeVznZpz1GZMUx4odeXjuiWaMMb6XU1hOiATGRJfQPDWYM1V1uKom17PtHKCf85gN/BNAROKAPwFjgTHAn0SkfTPEesiYpDj2F5WzY7+NhzHG+IeconI6tI0kNMT/p4kB318iuxB4WT2WA7Ei0gX4CbBIVfNU9QCwCDi7OQMbm+S0w9hlMmOMnwikMTDgfoJR4FMRWS0is+vZngjsqfM83VnX0PpmkxQfRXzbSGuHMcb4jZzC8oBpfwFwe77niaqaKSIdgUUiskVVl9bZXl89T4+x/jBO0poN0KNHD2/EW/e1GZsUZwnGGOM3cgrL6dcp2tdhNJmrNRhVzXR+ZgML8bSn1JUOdK/zvBuQeYz1R77+M6qarKrJCQkJ3gwd8LTDZOSXkn6gxOuvbYwxx0NVySkKrBqMawlGRKJEJLp2GZgGbDpit3eBa53eZOOAAlXdC3wCTBOR9k7j/jRnXbOy8TDGGH9RUFpJZbUGVBuMm5fIOgELnZvihAH/VtWPReRGAFWdC3wInAtsB0qAWc62PBH5C7DKea17VbXZP+X7d4ymXetwVu7I4+JR3Zr79MYYc0igjYEBFxOMqqYBw+pZP7fOsgI3N3D888DzbsXXFCEhwuhecazcaTUYY4xvBWKC8XU3Zb83NimOHfuLyT5Y5utQjDEtWE6RJZigc6gdxnqTGWN8yGowQWhwlxiiIkJZlmb3hzHG+E5OYTmRYSFER7o9usR7LME0Iiw0hEn9E/j0uyyq7f4wxhgfqR1k6XScCgiWYJrggmFd2V9UznKrxRhjfCTQxsCAJZgmmTKwI1ERoby3/qixnsYY0yxyCsuJD6AxMGAJpklahYcy7ZTOfLRpHxVVNb4OxxjTAgXaPGRgCabJLhjWhYLSSr7enuPrUIwxLUxldQ15JRUBNYofLME02Wl9E2jXOpx319llMmNM88orrkA1sLoogyWYJosIC+GcIZ1Z9H0WpRXVvg7HGNOCBOIYGLAEc1ymD+tKcUU1X2zN9nUoxpgWxBJMCzC2dwfi20baZTJjTLM6lGCsDSZ4hYYI5w/twudbsyksq/R1OMaYFiIQ5yEDSzDH7YJhXaioqmHR91m+DsUY00LkFJYT3SqMVuGhvg7luFiCOU4je7QnMbY1c7/8gYz8Ul+HY4xpAQJxDAxYgjluIsJ9Px1CZn4Z5z31FUuswd8Y47KcwvKAa38BSzAn5MyBHXnv16fROaYVs15cxd8WpdpEmMYY1wTiPGTQDAlGREJFZK2IvF/Ptp4islhENojIEhHpVmfbwyLynYhsFpGnxM+mEE2Kj2LhTRO5ZGQ3nlq8jd+8vs7XIRljgpRdImvYHGBzA9seBV5W1aHAvcADACIyAZgIDAWGAKOBye6HenxaR4TyyKXDmDWxF+9vyCS/pMLXIRljgkxJRRVF5VWWYI7k1EjOA+Y1sMtgYLGz/AVwobOsQCsgAogEwgG/7bZ1/tCuqMLX2/f7OhRjTJDZX+j54mptMEd7ArgTaGgK4vXAxc7yDCBaRDqo6jI8CWev8/hEVY+qBYnIbBFJEZGUnBzfTUI5rFs7YlqFsTTVJsI0xnhXTlEZEHhjYMDFBCMi5wPZqrr6GLvdAUwWkbV4LoFlAFUi0hcYBHQDEoEpIjLpyINV9RlVTVbV5ISEBO8XoonCQkOY2Deer7btR9Ua+40x3hOo08SAuzWYicB0EdkJvIYnScyvu4OqZqrqRao6ArjbWVeApzazXFWLVLUI+AgY52KsJ21S/wT2FpSxPbvI16EYY4KIJZh6qOpdqtpNVXsBVwCfq+rVdfcRkXgRqY3hLuB5Z3k3nppNmIiE46ndNNRRwC9M6u+pQX1pl8mMMV6UU1hOiECHqCBNMCLSR0QineUzRORWEYk9kROKyL0iMt15egawVURSgU7A/c76BcAPwEY87TTrVfW9Ezlfc0mMbU2fhCiWbrOGfmOM9+QUlRMXFUloiF+N1GiSsCbu9xaQ7LSNPAe8C/wbOLcpB6vqEmCJs3xPnfUL8CSTI/evBn7ZxNj8xun9Enh15W7KKqsDbs4gY4x/CtQxMND0S2Q1qlqFp23kCVW9DejiXliBaXL/BMqrali1M8/XoRhjgkRLSDCVInIlcB1QOyI/3J2QAtfY3nFEhIZYd2VjjNcE6jxk0PQEMwsYD9yvqjtEJAmY38gxLU6biDCSe7XnK2uHMcZ4QU2NklNUTseYIE4wqvq9qt6qqq+KSHsgWlUfdDm2gDSpfwJb9hWSdbDM16G0SNU1amORTNDILiynslrpGtva16GckCY18ovIEmC6s/86IEdEvlTV212MLSBN6pfAgx9tYWlqDpcmd2/SMUXlVfz53e84WOq5S6YCoSLcOrUfg7vGuBhtcFmelsutr66lb8e2PHbZMLq0C8x/SmNqZeSXANAtQBNMUy+RtVPVg8BFwAuqOgo4y72wAtfAztHEt408rstkzy5NY8HqdHbnlbA7r4Q9eSV8tS2HP727ybVv4zmF5SxPyw2Kb/uqytwvf+CqeStoFR7Kuj35nPPkV3zy3T5fh2bMSUk/4LmpYWL7wEwwTe2mHCYiXYDLcEbcm/qFhAiT+sXzxdZsqqprCAs9dg4/UFzBc1/v4OxTOjP3mlGH1r+yfBf//Z9NLEnN4cwBHb0SW1llNYs3Z/PWmnS+TM2hukaZM7Uft/1X/0aPPVhWye2vr6d7XGsuGdWNU7q280pMJ6ugtJI73lzPou+zOO/ULjx0yVByCsu59dW1/PKV1fxsbA/++7zBtI7w327jReVVRIaFEN7I34ppeWrvmpsYoDWYpiaYe4FPgG9UdZWI9Aa2uRdWYPvJkM68vTaDr7bvbzQ5zF36A8UVVdw+7fAP+cuTu/Ps0jQe+Xgrk/slEHLEICtV5Vi3yNmxv5gPN+4lp7Cc/UXl5BSWs3nvQQ6WVdEpJpLZk3qTVVDGk4u3ER4q3DKlX4OvVVOj3P76Or7YmkOoCC98s5OBnaO5ZFQ3rhjTg7aRTf0z8q7Vu/K47fX1ZOaX8t/nD+b6ib0QEdpGhvHWrybw2KKtPP1lGst+yOWhi4cyJinOJ3Ee6d31mXz2fdah2mpucQVd27XiuZmjGdTFLomaH2XmlxLbJpwoH/2PnawmRa2qbwJv1nmexo+zIJsjnDmgI7Ftwnl7TcYxE0z2wTJe+nYnPx2eSP9O0YdtiwgL4bb/6sdtr6/ng417uWBY10PbcgrLmfnCSjrFtOKpK0cc9QH/feZBfjZvOfkllURHhhEfHUlC20imndKZ6cO6MrFvPKEhcugunI9+mkpYaAg3Tu5Tb5x//2I7n23O5s8XDOanIxJ5b30mC1anc98Hm1n2Qy7PzRx9or+qE1JWWc3jn6XyzNI0EmNb89rscST3Ojx5RISFcNc5g5jcL4Hfv72By55exrXje3Ln2QN9lhBVlacWb+fxz1Lp0q4VfRLaMu2UziTGtmL+8t1cOncZ/3fVyEPTDhmTcaCUrgHcltjURv5uwP/imcBSga+BOaqa7mJsASsiLIQLhnbljZQ9FJZVEt2q/iFD//hiO5XVnstU9Zk+LJG5S9L426JUzhnSmbDQEPYXlXPVvOXsyi1hy75Cfvbscp6fOZp4p5/85r0HuWreclqHh/KfOybSKz6qwThDQ4RHLh1GVY3y4EdbCAsRfn5678P2+WJLNo9/lsqMEYlcN8FTQ7hmfC+uGd+LuV/+wIMfbWHx5iymDup0gr+t47Mpo4Db31hHalYRV47pzt3nDT5mwpjQN55PfjOJRz7Zyovf7mTx5mweuXQoE/rEey2mvOIK2rcJP2aNsqZGuff973nx251cPLIbD1186mGXTy8e1Y1ZL6xi1our+OuMIVw+uofX4jOBKyO/lJ4dGv4f9ndNvej7Ap7pYbrimT7/PWedacBFIxMpr6rho431NzSnHyjh3yt3c1lytwaTQGiIcMdPBrBjfzELVqeTW1TO1fNWsDuvhBdmjuaZa0aRmlXIJf/8lj15JWzZd/BQQ/drs8cdM7nUPcffLhvGead24b4PNnPOk1/x3Nc72F9Uzq7cYua8tpaBnWP464xTj/oAvX5iEn0Sorj3/e8pq6w+/l/Scdqy7yAX/d+35JdU8sKs0Txw0dAm1UbaRITxpwtO4c1fjicyPISZz69ieVquV2L65Lt9JN+3iD+8tZGamvo7TFRW1/DbN9fz4rc7uX5iEo9cMvSotrku7Vrz5o3jmdg3nt+/tZGHP97S4OuZlkFVyThQGrDtLwDSlF5EIrJOVYc3ts6XkpOTNSUlxddhHKKqTH3sSxKiI3n9l+OP2v77BRtYuDaDJb8745h93FWVi/75LfsKymjXOpwd+4t5fuZoJvb1fANfvesA17+4ioiwEKprlIjQkCYnl7oqq2t4bdUeFqTsYX16AWEhQrvW4VTVKO/dcho9OrSp97ivt+3n6udW8Nv/6s+vG6iJeYOqcvVzK/gu8yCf3T75UI3teB0oruDSp5eRVVDGGzeOP6k2j43pBVz69LdEtwonp7CcS0Z146GLhx42KWHWwTJ+t2ADS1NzuGNaf24+s+8xazqV1TXc8853vLpyN2cOSOCJy0fQro1NmtESFZRUMuzeT/njeYOOurLgTSKyWlWT3XjtptZg9ovI1SIS6jyuBrzzFTBIiQgXjUxkxY489uSVHLZt675CFqxJ56pxPRodQCUi/O4nA9hbUEba/mLmXZd8KLkAjOrZngU3jicsRAgPFV49geQCEB4awjXjevLOLafx6W2TuOG0JDq0jeDvPxvRYHIBOK1fPOee2pl/LNlO+oGSBvc7WZ9tzuab7bncdlb/E04uAO2jInjp+jG0iQxl5gsrTzjmzPxSbnhpFR2iIvnw1tP5zVn9WLA6nTsXbDg02PO1lbs5629fsiItl7/OOJVbpvQ7ZnIBz/vw1xlD+MtPh/D19v1M/8fXbNl38IRiNIEt3RkD0xJqMD2Av+OZLkaBb4FbVXW3u+E1nb/VYAD25JVw+sNfHPbtvqCkkgv/8TVF5VV8NGdSkyexe/7rHQxJbNdgT6ii8ipqVIlpoL3HTRn5pZz12JecMSCBf149qvEDjlN5VTU/eXwp4aEhfDTn9Ea7fjfFln0HuXTuMhKiI3nrxgm0j4po8rFF5VVcOncZ6XklvHXThEMdNJ78bBuPf5bK+UO7kFdcwbc/5DI2KY4HLx5K0gkk/dW78vjV/DUUllXx8CVDD+voYYLfp9/tY/Yrq3nn5okM635Cd0dpEjdrME3tRbYbz0j+ukH9BnjCjaCCRfe4NoxNimPh2gxumdKX6hrlllfXkJFfyqu/GHdcM6Ref1rSMbf7qmcUeL5h3TKlL498spWnFm+jbWQYxeVVFFdU07NDGy4amUhk2ImPQ3n5213szC3hpevHeCW5AAzsHMO8a5O55vmVXDVvBX+7fBgDOzd+uWxfQRl3vb2B1KxCXpg5+rDef3PO6ocI/G1RKtGRYfx1xqlcMbr7UV3Mm2pUzzje//Vp3PSvNfz61bWEhwpnD7FJzFuKQ2NgAnSQJTSxBlPvgSK7VdVvurr4Yw0G4I1Ve7jzrQ0svGkC72/Yy3Nf7+Dhi4dy2eimTSMTKMqrqjnnya9Iyyk+tC48VKisVhJjWzNnaj8uGpl43Akit6icMx5ZQnKv9rwwa4y3w+bzLVn89o31FJZVccPpScyZ2o82ET8m69KKatbuPsCXqTl8mZrDln2FANw/YwhXje1Z72t+tS2H/p2i6RTTyisxllVWc+Wzy9m89yBv/nICp3bzj0Guxl33vf8981fsYvO9Zzd6afVkuFmDOZkEs0dVG/2UFJFQIAXIUNXzj9jWE89tkhOAPODq2q7PzmW5eUB3PJflzlXVnQ2dx18TTGFZJcn3fUaPuDZsyy5i5oRe/Hn6Kb4OyxXF5VXkl1YSFRFKm4gwwkOFr7bt57FFqazfk0+vDm34/dkDOefUpn8Lv3vhRl5btYdPfjOJvh3buhL3geIKHvhoM2+kpJMY25obTktiZ24xa3fns3nvQapqlPBQYXSvOCb3T2DKwI70O2LckttyCsv56T++oaqmhnduPo3O7byTvIz/uulfq9myr5DPf3uGq+fx1wTTpBqMiNwOJAMx9SSYN4H3VfUlEZkCzFLVa5xtS/DcHmCRiLTFc9OzBltk/TXBAPz61bW8tz6TiX078NIs713mCRSqyuLN2Tz66Va27CtkztR+/Oasxhu8v0zNYdYLK7l2fPMk5ZU78rh74Ua2ZRcRFRHKsO6xjOgRy8ge7Rnbu4NPL0OCZ4zTJf/8lt4JbXnjl+P9evobc/Iu/PvXxLQO55Ubxrp6Hp+1wYhIIZ7aw1GbgEYvDDoDNM8D7gfqm3l5MHCbs/wF8B/nuMFAmKouAlDVosbO5c9uPrMPYSHCPecPbnHJBTw94c4a3InJAxK46+2NPLl4G/sKyrh/xpAGfx/vrMvgt2+sp3+naG47q/G50rxhTFIcH845nYwDpXSPa+N390Af1CWGp64cwc9fTuE3r6/lscuG+zzpGfdk5JcG/NRBx/y0U9VoVY2p5xGtqk35y34CuBOoaWD7en6ccmYGEC0iHYD+QL6IvC0ia0XkEedSW0Aa2DmGxy8fflw9lYJReGgIj1wylFun9OX1lD384uUUSiqqjtrvua93MOe1dYzq2Z43bhzfrONAwkND6BUf5XfJpdbUQZ3443mD+eS7LCY8sJiHPt5i9x4KQmWV1ewvqgjoLsrQ9Mkuj5uInA9kq+pqETmjgd3uAP4uIjOBpUAGUOXEdTowAtgNvA7MBJ474hyzgdkAPXr4TX8Dcwwiwu3TBtCpXSv++z+bOO+prxnTK47eCVH0TmjLqp15PLM0jXOGdObxy4fTKjxgv1e45obTkhjZI5Z5X+3g6S9/YN5XaUw7pTOhIuwrKGPvwVKKyqr48/RTuHB4oq/DNScgGHqQgYsJBs+8ZdNF5FygFRAjIvNV9eraHVQ1E889ZnDaWS5W1QIRSQfWOpNqIiL/AcZxRIJR1WeAZ8DTBuNiWYyXXTW2J13atWLukjQWb8ni9ZSKQ9uuHteD/5k+xG9rEf5gRI/2/OOq9uzOLeH5b3bw7vpMoluF0TmmFaN6tGdbdhF/eGsjQxLb0SfBnc4Rxj2ZAT5Nf60TbuQ/rpN4ajB31NPIHw/kqWqNiNwPVKvqPc7lsDXAWaqaIyIvACmq+o+GzuHPjfymcQUllaTtL6KiqoYxSXGudstsCbIOlnH2E0vp0q41C2+ecFLjkEzze23lbv7w9ka+/v2ZdGvf8Ewa3uAPU8V4jYjcKyK1gzbPALaKSCrQCU9nAFS1Gs/ls8UishFPp4JnmztW03zatQlnhNNby5LLyesU04pHLx3G93sP8uBHW3wdjjlOGfmlhAheG0vlK83SBUVVlwBLnOV76qxfACxo4JhFwNBmCM+YoDR1UCdmTujFC9/s5LS+8c12SwVz8jIOlNI5plXA3+U0sKM3xhzTXecOZHCXGO54cz37Cqy3WaBIzy8N+AZ+sARjTFCLDAvlf382grLKGv707iZfh2OaKDM/sO8DU8sSjDFBrk9CW26Z0pdPvsviq205vg7HNKK6RtlXUGY1GGNMYPj56Un07NCGP7/7HRVVDY17Nv4g62AZVTXa6L2iAoElGGNagMiwUO45fzA/5BTz0rc7fR2OOYaMIBkDA5ZgjGkxpg7qxJkDEnhy8TayC63B31/VDrLsZpfIjDGB5J4LTqGiqoaHPtrq61BMA9IPeBKMXSIzxgSUpPgobjg9ibfWpLN61wFfh2PqkZFfSlxUxGE3vgtUlmCMaWFuObMvXdq14qZ/rWbn/uLGDzDNKuNAKV1jA3sEfy1LMMa0MFGRYbw4awyV1cqVzy5nd26D9/EzPpARJGNgwBKMMS3SgM7RzL9hLKWV1Vz57HLSD1iS8Qeq6gyydHeCy+ZiCcaYFmpw1xjm3zCWwrJKrnx2+aHeS8Z3cosrKKmoDooeZGAJxpgWbUhiO165YSz5xZVc/dwK8ksqGj/IuCY1qxCAfp2C4x4+lmCMaeGGdY/luZmjSc8rZfbLqymrrPZ1SC3WtqwiAPp3ivZxJN5hCcYYw5ikOB67bBgrd+bxuwUbqKk5/EaEB8sqSc0qPGq98a7UrEJiWoXRMTrS16F4ReB3tDbGeMUFw7qSfqCUhz7eQmJsa/5wzkB+yCnipW93smB1OiUV1XSIimBi33hO7xfPgM7RbN1XyMaMAjakF1BQWskjlwwluVecr4sSsFKzCunfKTpobrrneoJxbn+cAmTUc8vknsDzQAKQB1ytqul1tscAm4GFqnqL27Ea09LdOLk36QdKmPvlDyxtI6sKAAAWOklEQVRPy2XdnnwiQkO4YFhXxiS1Z0VaHku37efd9ZmHjmkbGcaQxBhyi2uY9eIqXv3FOIYktvNhKQKTqpKaVcR5Q7v4OhSvaY4azBw8SSKmnm2PAi+r6ksiMgV4ALimzva/AF+6H6IxBkBE+J/pp5BTWM66PfncdlZ/fja2BwnOJZvLR/dAVdmyr5C0nGIGdokmqUMUISFCRn4pl/7zW657fiVv3DiePgnB0VDdXHIKyykoraR/x+D5vbnaBiMi3YDzgHkN7DIYWOwsfwFcWOfYUUAn4FM3YzTGHC4sNISnrxnFiv83lTln9TuUXGqJCIO6xHDe0C70SWhLSIjnck5ibGvm/3wsInD1vBU2tuY4pQZZAz+438j/BHAn0NANKNYDFzvLM4BoEekgIiHAY8DvXI7PGFMPETmhdoDeCW15+fqxFJdXcfW8FRwotm7PTbX1UBdlSzCNEpHzgWxVXX2M3e4AJovIWmAykAFUATcBH6rqnkbOMVtEUkQkJSfH7tRnjD8Y3DWGF2aNZmduCS8t2+nrcALGtqxC4qIiiG8b4etQvMbNGsxEYLqI7AReA6aIyPy6O6hqpqpepKojgLuddQXAeOAW59hHgWtF5MEjT6Cqz6hqsqomJyQkuFgUY8zxGNUzjsn9E3h15W4qq+0Omk2RmlVIv45tg6YHGbiYYFT1LlXtpqq9gCuAz1X16rr7iEi8czkM4C48PcpQ1atUtYdz7B14OgL8wa1YjTHed824nmQdLGfR91m+DsXvqSrbsoqCqv0FfDDQUkTuFZHpztMzgK0ikoqnQf/+5o7HGOOOMwd2JDG2Na8s2+XrUPze3oIyCsur6N85uBJMswy0VNUlwBJn+Z466xcACxo59kXgRdeCM8a4IjREuGpcDx7+eCvbsgqDqvHa22rnIAumLspgU8UYY1x0eXJ3IkJDmL/cajHHcijBBFkStgRjjHFNh7aRnDe0C2+tyaCovMrX4fit1Kwi4ttG0j4qeHqQgSUYY4zLrhnfk6LyKv6zNsPXofitbVmFDOgcXJfHwBKMMcZlI7rHckrXGF5ZtgtVJSO/lPc3ZPLAR5vZlFHg6/B8rqbGMwdZv47BdXkMbDZlY4zLRIRrx/fk929tJPm+z8itM7r/3XWZfDTndGLbBNeloeORkV9KaWV10LW/gCUYY0wzmD4skUXfZxPTKozhPWIZ0b09FdU1XPHMMu5csIGnrxkVVAMMj0dtA38wXiKzBGOMcV3riFDmXZd81Po7fzKQ+z/czPzlu7hmfK/mD8wP1M5B1jcIL5FZG4wxxmduOC2Jyf0T+MsHm9m896Cvw/GJbVlFdI5pRbvW4b4OxesswRhjfCYkRHjssmG0ax3Or19dS0lFy+vKnJpVSL9OwXd5DCzBGGN8LL5tJI9fNpwfcor43YINVLWgyTGra5Tt2UUMCMIGfrAEY4zxA6f1i+cPZw/kgw17ue2N9S0myezKLaa8qiYoe5CBNfIbY/zELyf3oUbhoY+3APD4ZcMICw3u78ApOw8AMLxHrI8jcYclGGOM3/jVGX0QgQc/ahlJZnlaLnFREfQLskkua1mCMcb4lRsn9wE8SaZtZBgPXHSqjyNyz4odeYxNigvaMUDB+9XAGBOwbpzch+snJvHaqt18nxmc3Zf35JWQkV/K2KQ4X4fiGkswxhi/NGdqP6Ijw3j0062+DsUVy9NyARjXp4OPI3GPJRhjjF9q1yacG8/ow+dbslm5I8/X4Xjdih15xLYJp38QjuCv5XqCEZFQEVkrIu/Xs62niCwWkQ0iskREujnrh4vIMhH5ztl2udtxGmP8z6wJSXSMjuThj7egqr4Ox6tW7MhlTK84QkKCs/0FmqcGMwfY3MC2R4GXVXUocC/wgLO+BLhWVU8BzgaeEJHg7MdnjGlQ64hQbp3aj5RdB/h8S7avw/GajPxS9uSVMq538F4eA5cTjFMjOQ+Y18Aug4HFzvIXwIUAqpqqqtuc5UwgG0hwM1ZjjH+6fHR3enZow8Mfb6W6JjhqMSuc9pexvYO3gR/cr8E8AdwJNDQsdz1wsbM8A4gWkcNSuoiMASKAH448WERmi0iKiKTk5OR4L2pjjN8IDw3ht9MGsDWrkHfXB8ddMVek5RHTKoyBnWN8HYqrXEswInI+kK2qq4+x2x3AZBFZC0wGMoBDs92JSBfgFWCWqh6VpFT1GVVNVtXkhASr4BgTrM4/tQundI3h0U9Sg2JCzBU7chmT1IHQIG5/AXdrMBOB6SKyE3gNmCIi8+vuoKqZqnqRqo4A7nbWFQCISAzwAfBHVV3uYpzGGD8XEiLcc/5gMvJL+fvn230dzknZV1DGztwSxgX55TFwMcGo6l2q2k1VewFXAJ+r6tV19xGReBGpjeEu4HlnfQSwEE8HgDfditEYEzjG9u7ARSMTefarNLZnF/o6nBO2YofT/pIU3A384INxMCJyr4hMd56eAWwVkVSgE3C/s/4yYBIwU0TWOY/hzR2rMca//L9zB9EmIow//mdTwHZbXp6WR3RkGIO7Bnf7CzTTXGSqugRY4izfU2f9AmBBPfvPB+Yfud4Y07LFt43kzrMHcPfCTSxcm8FFI7v5OqTjtiItl9FJcUHf/gI2kt8YE2CuHN2D4d1juf+DzRSUVPo6nOOSfbCMtP3FQT3/WF2WYIwxASUkRLjvp0M4UFLBn9/7jgPFFb4OqckWO4NFJ/SJ93EkzcOm6zfGBJwhie34xem9eXppGu+syyC5ZxxTBnXknCGd6dkhytfhNei1lbsZ0CmaIYnB3/4CVoMxxgSoP5wzkHdunsgtZ/alqLyKBz/awrTHl7Irt9jXodVrU0YB69MLuGJM96C9/8uRLMEYYwKSiDCseyy3TxvAh3NOZ/FvJ1OjynNf7/B1aPV6bdVuIsNCmDEi0dehNBtLMMaYoNAnoS0/HZ7IGyl7/K5dpqSiinfWZnLeqV2IbRPh63CajSUYY0zQ+MWk3pRV1jB/+S5fh3KY9zfspbC8iivG9PB1KM3KEowxJmj07xTNGQMSeGnZTsoqq30dziGvrtxNn4QoRvdq7+tQmpUlGGNMUJk9qTf7iypYuNY/Zl7esu8ga3fnc+WYHi2mcb+WJRhjTFAZ37sDQxJjeParNGr84P4xr63cQ0RoSEDOOnCyLMEYY4KKiPCL03uTllPs87tgllVW8/aadM4e0pm4qJbTuF/LEowxJuice2oXEmNb88xXaT6N46NNezlYVsUVY7r7NA5fsQRjjAk64aEhzJrYi5U78li7+4DP4nh7TQbd41ozrgVMzV8fSzDGmKB0xZgexLQK4+kvfVOLyTpYxjfb9zNjeCIhLWDm5PpYgjHGBKW2kWFcM74nn3y/j7ScomY//zvrMqhRmNECG/drWYIxxgStmROSCA8N4VkftMW8vSaD4d1jSYr338k33eZ6ghGRUBFZKyLv17Otp4gsFpENIrJERLrV2XadiGxzHte5HacxJvgkREdy6ahuvLU6g+yDZc123u8zD7JlXyEXjWw5847VpzlqMHOAzQ1sexR4WVWHAvcCDwCISBzwJ2AsMAb4k4i0rCGwxhivmD2pN1U1Nbzw7c5mO+fCtemEhQjnD+3abOf0R64mGKdGch4wr4FdBgOLneUvgAud5Z8Ai1Q1T1UPAIuAs92M1RgTnHp2iOKcU7swf/kuCsvcvwNmdY3yzrpMzhjQsUWOfanL7RrME8CdQE0D29cDFzvLM4BoEekAJAJ76uyX7qwzxpjjduOkPhSWVfHvFbtdP9c32/eTXVje4i+PgYsJRkTOB7JVdfUxdrsDmCwia4HJQAZQBdTXp++oOR9EZLaIpIhISk5OjjfCNsYEoVO7tWNi3w489/UOSivcnQRz4doMoluFMWVgR1fPEwjcrMFMBKaLyE7gNWCKiMyvu4OqZqrqRao6ArjbWVeAp8ZSd+hrNyDzyBOo6jOqmqyqyQkJCS4VwxgTDG46oy/ZheWM/etn3PX2Rlak5Xp9rrLi8io+3rSP84d2oVV4qFdfOxC5lmBU9S5V7aaqvYArgM9V9eq6+4hIvIjUxnAX8Lyz/AkwTUTaO43705x1xhhzQib2jeffPx/L1EGd+M/aDC5/ZjmnP/wFq3d5b6T/u+szKa2sZsaIljv2pa5mHwcjIveKyHTn6RnAVhFJBToB9wOoah7wF2CV87jXWWeMMSdsQt94Hr98OCl/PIsnLh8OwJ0L1lNR1VAzcdOt3nWA/3nvO4Z3jyW5p3V6BRBV309n7Q3JycmakpLi6zCMMQFk8eYsbngphbvPHcQvJvU+4dfZnl3IJXOXEds6nAW/mkB820gvRukuEVmtqsluvLaN5DfGtFhTB3ViysCOPLl42wkPxNxXUMZ1z68iLCSEl68fG1DJxW2WYIwxLdo95w+moqqGBz/actzHFpRWct3zKykoreTFWaPp0aGNCxEGLkswxpgWrVd8FL+YlMTbazNYtbNpTb2qyrvrMznvqa9I21/E3KtHMSSxncuRBp4wXwdgjDG+dvOZfXl7TQb3vPMd828Yw4odeXy1LYdvtufSJiKU0/rGc1q/eMYmdeC7zALu+2Az6/bkM7BzNI/eMJZxvVvm/V4aY438xhgDfLBhLzf/e82h59GRYYzr04GSiipW7ThARXUN4aFCZbXSMTqSO34ygItHdiM0wO/14mYjv9VgjDEGOPfUztw6tR8CTOofz7BusYSFeloRSiuqWbkzj2+276d9mwium9CTNhH28dkYq8EYY0wLZt2UjTHGBBxLMMYYY1xhCcYYY4wrLMEYY4xxhSUYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXBM1ASxHJAXYdsbodUHCc6xpbjgf2n2CY9Z37ePZpSnmaqyyNxdrYPsdbliOf1y7XXWfvTdNibWwfe298+xlwrP3cKEuUqrpzz3lVDdoH8MzxrmtsGUjxZjzHs09TytNcZTnZ8hxvWY5Rhrrr7L2x98av35umlMWb743bf2eNPYL9Etl7J7CuKcvejOd49mlKeZqrLE19nYb2Od6yHPn8vQb2OVH23hx7vb03zfcZcKz9/KksjQqaS2TNRURS1KV5e5pbMJUFgqs8wVQWCK7yWFmaLthrMG54xtcBeFEwlQWCqzzBVBYIrvJYWZrIajDGGGNcYTUYY4wxrmjRCUZEnheRbBHZdALHjhKRjSKyXUSeEhGps+3XIrJVRL4TkYe9G3WD8Xi9LCLyZxHJEJF1zuNc70feYEyuvDfO9jtEREUk3nsRHzMeN96bv4jIBud9+VREuno/8nrjcaMsj4jIFqc8C0Uk1vuRNxiTG+W51PnfrxER19tqTqYMDbzedSKyzXlcV2f9Mf+v6uVmFzV/fwCTgJHAphM4diUwHhDgI+AcZ/2ZwGdApPO8YwCX5c/AHcHy3jjbugOf4BkzFR+oZQFi6uxzKzA3gMsyDQhzlh8CHgrkvzNgEDAAWAIk+2sZnPh6HbEuDkhzfrZ3ltsfq7zHerToGoyqLgXy6q4TkT4i8rGIrBaRr0Rk4JHHiUgXPP/gy9Tzm38Z+Kmz+VfAg6pa7pwj291SeLhUFp9xsTyPA3cCzdb46EZZVPVgnV2jaKbyuFSWT1W1ytl1OdDN3VL8yKXybFbVrc0Rv3O+EypDA34CLFLVPFU9ACwCzj7Rz4kWnWAa8Azwa1UdBdwB/F89+yQC6XWepzvrAPoDp4vIChH5UkRGuxrtsZ1sWQBucS5dPC8i7d0LtUlOqjwiMh3IUNX1bgfaBCf93ojI/SKyB7gKuMfFWBvjjb+zWtfj+XbsS94sj680pQz1SQT21HleW64TKm9YE0/aIohIW2AC8Gady4uR9e1az7rab5BheKqW44DRwBsi0tvJ+s3GS2X5J/AX5/lfgMfwfAA0u5Mtj4i0Ae7GcznGp7z03qCqdwN3i8hdwC3An7wcaqO8VRbnte4GqoB/eTPG4+HN8vjKscogIrOAOc66vsCHIlIB7FDVGTRcrhMqryWYw4UA+ao6vO5KEQkFVjtP38XzwVu3Gt8NyHSW04G3nYSyUkRq8Mz3k+Nm4PU46bKoalad454F3ncz4EacbHn6AEnAeuefrhuwRkTGqOo+l2M/kjf+zur6N/ABPkgweKksTmPy+cDU5v4ydgRvvze+UG8ZAFT1BeAFABFZAsxU1Z11dkkHzqjzvBuetpp0TqS8bjdA+fsD6EWdxjHgW+BSZ1mAYQ0ctwpPLaW2wetcZ/2NwL3Ocn881U0J0LJ0qbPPbcBrgfzeHLHPTpqpkd+l96ZfnX1+DSwI4LKcDXwPJDTn35fbf2c0UyP/iZaBhhv5d+C5CtPeWY5rSnnrjcsXb6i/PIBXgb1AJZ4MfQOeb7kfA+udP/p7Gjg2GdgE/AD8nR8HrUYA851ta4ApAVyWV4CNwAY839q6NEdZ3CrPEfvspPl6kbnx3rzlrN+AZ16pxAAuy3Y8X8TWOY9m6RHnYnlmOK9VDmQBn/hjGagnwTjrr3fek+3ArMbKe6yHjeQ3xhjjCutFZowxxhWWYIwxxrjCEowxxhhXWIIxxhjjCkswxhhjXGEJxgQ1ESlq5vPNE5HBXnqtavHMlrxJRN5rbJZhEYkVkZu8cW5jvMG6KZugJiJFqtrWi68Xpj9OzOiqurGLyEtAqqref4z9ewHvq+qQ5ojPmMZYDca0OCKSICJvicgq5zHRWT9GRL4VkbXOzwHO+pki8qaIvAd8KiJniMgSEVkgnvuY/Kv23hjO+mRnuciZkHK9iCwXkU7O+j7O81Uicm8Ta1nL+HHSzrYislhE1ojn/hwXOvs8CPRxaj2POPv+zjnPBhH5Hy/+Go1plCUY0xI9CTyuqqOBi4F5zvotwCRVHYFnduK/1jlmPHCdqk5xno8AfgMMBnoDE+s5TxSwXFWHAUuBX9Q5/5PO+Rudz8mZB2sqntkUAMqAGao6Es/9hx5zEtwfgB9Udbiq/k5EpgH9gDHAcGCUiExq7HzGeItNdmlaorOAwXVmmo0RkWigHfCSiPTDM1NseJ1jFqlq3XturFTVdAARWYdnLqivjzhPBT9OELoa+C9neTw/3kvj38CjDcTZus5rr8Zzbw7wzAX1VydZ1OCp2XSq5/hpzmOt87wtnoSztIHzGeNVlmBMSxQCjFfV0rorReR/gS9UdYbTnrGkzubiI16jvM5yNfX/L1Xqj42cDe1zLKWqOlxE2uFJVDcDT+G5/0sCMEpVK0VkJ9CqnuMFeEBVnz7O8xrjFXaJzLREn+K5fwoAIlI7rXk7IMNZnuni+ZfjuTQHcEVjO6tqAZ7bIt8hIuF44sx2ksuZQE9n10Igus6hnwDXO/cHQUQSRaSjl8pgTKMswZhg10ZE0us8bsfzYZ3sNHx/j+cWCwAPAw+IyDdAqIsx/Qa4XURWAl2AgsYOUNW1eGbGvQLPDbmSRSQFT21mi7NPLvCN0635EVX9FM8luGUishFYwOEJyBhXWTdlY5qZc3fNUlVVEbkCuFJVL2zsOGMCjbXBGNP8RgF/d3p+5eOj21Ab4zarwRhjjHGFtcEYY4xxhSUYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXWIIxxhjjiv8Pjaxti3ClkuwAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "learn.recorder.plot(skip_end=15)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 23:05\n", + "epoch train_loss valid_loss accuracy\n", + "1 4.216088 4.007746 0.300710 (23:05)\n", + "\n" + ] + } + ], + "source": [ + "learn.fit_one_cycle(1, 1e-02, moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save('fit_head')" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load('fit_head');" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 8:42:10\n", + "epoch train_loss valid_loss accuracy\n", + "1 3.969496 3.845486 0.315829 (26:05)\n", + "2 3.918823 3.763834 0.326209 (26:03)\n", + "3 3.839340 3.699592 0.335176 (26:08)\n", + "4 3.806484 3.647501 0.342129 (26:06)\n", + "5 3.774351 3.600486 0.347226 (26:12)\n", + "6 3.743252 3.558260 0.352164 (26:09)\n", + "7 3.699686 3.523352 0.356172 (26:09)\n", + "8 3.713620 3.493051 0.360080 (26:11)\n", + "9 3.655959 3.463499 0.363502 (26:11)\n", + "10 3.639268 3.436016 0.366489 (26:08)\n", + "11 3.612611 3.406506 0.370597 (26:11)\n", + "12 3.583289 3.374530 0.374582 (26:07)\n", + "13 3.538116 3.351603 0.378152 (26:08)\n", + "14 3.525463 3.321655 0.382012 (26:06)\n", + "15 3.490068 3.292532 0.385994 (26:08)\n", + "16 3.455298 3.272834 0.388984 (26:11)\n", + "17 3.408465 3.253918 0.391583 (26:05)\n", + "18 3.420946 3.241025 0.393331 (25:56)\n", + "19 3.382529 3.235213 0.394229 (25:57)\n", + "20 3.367180 3.233525 0.394359 (25:49)\n", + "\n" + ] + } + ], + "source": [ + "learn.unfreeze()\n", + "learn.fit_one_cycle(20, 1e-3, moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(\"afteroom\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "learn.load(\"afteroom\");" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[3.2334335, tensor(0.3944)]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learn.validate()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# odl: [3.6938994, tensor(0.3338)]" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save_encoder(\"enc_best\")\n", + "learn.save(\"lm_best\", with_opt=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Classification" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validation = 100% of test" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "exp.drop_mult=0.5\n", + "learn=exp.create_cls_learner(data_clas)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "learn.true_wd=True" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load_encoder('enc_best')\n", + "learn.freeze()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 04:03\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.286372 0.176776 0.933840 (04:03)\n", + "\n" + ] + } + ], + "source": [ + "learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 04:21\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.235155 0.160959 0.940560 (04:21)\n", + "\n" + ] + } + ], + "source": [ + "learn.freeze_to(-2)\n", + "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(\"2\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 05:38\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.213980 0.148502 0.946840 (05:38)\n", + "\n" + ] + } + ], + "source": [ + "learn.freeze_to(-3)\n", + "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(\"3\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 13:45\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.198522 0.153674 0.947800 (06:52)\n", + "2 0.169954 0.157253 0.947320 (06:52)\n", + "\n" + ] + } + ], + "source": [ + "learn.unfreeze()\n", + "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(\"all\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Validation = 10% of train" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tokenized data loaded, lm.trn 90000, lm.val 10000\n", + "Tokenized data loaded, cls.trn 22500, cls.val 2500\n", + "Size of vocabulary: 60002\n", + "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', '.', ',', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", '-', '\\n\\n']\n" + ] + } + ], + "source": [ + "data_clas, data_lm = exp.load_cls_data(bs=40,force=False, use_test_for_validation=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "exp.drop_mult=0.5\n", + "learn=exp.create_cls_learner(data_clas)\n", + "learn.true_wd=True" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load_encoder('enc_best')\n", + "learn.freeze()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 02:43\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.288307 1.597378 0.575200 (02:43)\n", + "\n" + ] + } + ], + "source": [ + "learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 03:08\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.246644 0.435174 0.779200 (03:08)\n", + "\n" + ] + } + ], + "source": [ + "learn.freeze_to(-2)\n", + "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 04:19\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.202743 0.212411 0.923200 (04:19)\n", + "\n" + ] + } + ], + "source": [ + "learn.freeze_to(-3)\n", + "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 10:51\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.189055 0.162822 0.944000 (05:25)\n", + "2 0.168473 0.165473 0.941600 (05:25)\n", + "\n" + ] + } + ], + "source": [ + "learn.unfreeze()\n", + "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(\"valid\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Running models on testset\n", + "There is a small bug that does not let us use test_ds so we load the test set in to valid_ds and run validate() on that data set" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tokenized data loaded, lm.trn 90000, lm.val 10000\n", + "Tokenized data loaded, cls.trn 25000, cls.val 25000\n", + "Size of vocabulary: 60002\n", + "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', '.', ',', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", '-', '\\n\\n']\n" + ] + } + ], + "source": [ + "data_clas, data_lm = exp.load_cls_data(bs=40,force=False, use_test_for_validation=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "learn=exp.create_cls_learner(data_clas)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load('valid');" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.15947564, tensor(0.9483)]" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learn.validate()" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load(\"all\");" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[0.15725298, tensor(0.9473)]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learn.validate()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:fastaiv1]", + "language": "python", + "name": "conda-env-fastaiv1-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/experiments/cls_test_wt103_1_vf.ipynb b/experiments/cls_test_wt103_1_vf.ipynb new file mode 100644 index 0000000..c405b6b --- /dev/null +++ b/experiments/cls_test_wt103_1_vf.ipynb @@ -0,0 +1,467 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test of classifcation Moses + Fastai pre/post processing" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%reload_ext autoreload\n", + "%autoreload 2\n", + "%matplotlib inline\n", + "%cd .." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from fastai import *\n", + "from fastai.text import *" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from ulmfit.train_clas import *" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Batch size: 70\n", + "Max vocab: 60000\n", + "Cache dir: data/imdb/models/vf60k\n", + "Model dir: data/imdb/models/vf60k/lstm_None.m\n" + ] + } + ], + "source": [ + "exp = CLSHyperParams('data/imdb', qrnn=False,tokenizer='vf', lang='en', cuda_id=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "exp.pretrained_model = URLs.WT103_1\n", + "exp.drop_mult=0.3" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "exp.load_cls_data??" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Saving tokenized: cls.trn 25000, cls.val 25000\n", + "Size of vocabulary: 60002\n", + "First 20 words in vocab: ['xxunk', 'xxpad', 'xxmaj', 'the', ',', '.', 'and', 'a', 'of', 'to', 'is', 'it', 'in', 'i', 'this', 'that', '\"', \"'s\", 'was', 'as']\n" + ] + } + ], + "source": [ + "data_clas, data_lm = exp.load_cls_data(bs=40,force=True, use_test_for_validation=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Training args: {'drop_mult': 0.3, 'tie_weights': True, 'clip': 0.12, 'bptt': 70, 'pretrained_fnames': None, 'pretrained_model': 'https://s3.amazonaws.com/fast-ai-modelzoo/wt103-1'} dps: None\n" + ] + } + ], + "source": [ + "learn = exp.create_lm_learner(data_lm)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n" + ] + } + ], + "source": [ + "learn.true_wd=True\n", + "learn.opt=None\n", + "learn.lr_find()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd8VtX9wPHPN4MEyIIkIBAgTBmyI4pMtVpFBAVnpXXWDrXa1lrpsIq1zipVa1t+1FFttYijigOUIeCChAjIDEsSRgYJIYPs7++P5wYDJCRA7rPyfb9ez4v73Hvuc7+HB/LNOefec0RVMcYYY5pbiK8DMMYYE5wswRhjjHGFJRhjjDGusARjjDHGFZZgjDHGuMISjDHGGFdYgjHGGOMKSzDGGGNcYQnGGGOMK8J8HUBzSUhI0OTkZF+HYYwxASUtLS1PVRPd+OygSTDJycmkpqb6OgxjjAkoIvKNW59tXWTGGGNcYQnGGGOMKyzBGGOMcYUlGGOMMa6wBGOMMcYVlmCMMca4whKMMcYYV1iCMcaYAPZGWhavrtzl6zDqZQnGGGMC2Ly0LN5cneXrMOplCcYYYwJYQWkFcW1a+TqMelmCMcaYAJZfUkF7SzDGGGOak6pyoLSSuLbhvg6lXpZgjDEmQJVUVFNRXWMtGGOMMc2roKQCgHZtLcEYY4xpRgWlToKxFowxxpjmVFBaCUB7G4MxxhjTnGq7yOw2ZWOMMc2qtovMBvmNMcY0q4KSCkQgprV1kRljjGlG+aUVxLUOJzREfB1KvVxPMCISKiLpIjK/nmPdRWSRiKwVkaUiknTU8RgR2S0iz7odpzHGBJqC0kq/vYMMvNOCuRPY2MCxJ4B/qepgYCbw8FHHHwQ+cTE2Y4wJWAUlFX77DAy4nGCcFsklwJwGigwAFjnbS4Apdc4dAXQEFroZozHGBCpPC8Y/x1/A/RbMLOAeoKaB42uAac725UC0iMSLSAjwZ+BXLsdnjDEBq6CkomV2kYnIJCBHVdOOU+xuYLyIpAPjgd1AFfBT4H1VzWzkGreKSKqIpObm5jZX6MYY4/dUlYLSCtr7cRdZmIufPRqYLCITgUggRkReUdXptQVUdQ8wFUBEooBpqlooIqOAsSLyUyAKaCUixap6b90LqOpsYDZASkqKulgXY4zxK4cqqymvqvHbhyzBxQSjqjOAGQAiMgG4u25ycfYnAPmqWuOUfd4597o6ZW4AUo5OLsYY05L5+zQx4IPnYERkpohMdt5OADaLyBY8A/oPeTseY4wJRP4+TQy420V2mKouBZY62/fV2T8PmNfIuS8CL7oWnDHGBKB8J8H48xiMPclvjDEB6Nup+q2LzBhjTDM6vNiYH3eRWYIxxpgAVDvIH+unE12CJRhjjAlIBaUVxLYOJyzUf3+M+29kxhhjGlRQWunXA/xgCcYYYwJSQUkFcX48wA+WYIwxJiAVlFb47UqWtSzBGGNMAPK0YCzBGGOMaWb5pRV+PU0MWIIxxpiAc6iimrJK/57oEizBGGNMwKl9it/uIgsCt/9nNU8vyvB1GMYYAwTGNDHgpckuA9mGPQeZv3YvrUKzmTYiiS5xrX0dkjGmhSso8TzF78/TxIC1YBr131W7aBXm+Wt6xloxxhg/cLgFY11kgausspq30ndz8Rmn8b2zuvF6WhY78kp8HZYxpoX7tovMEozfq6iqqXf/++v2crCsiqvP7Mpt5/amVWgIsz7e4uXojDHmSLVdZPYkv5/LzC/lu7OW8dGG7GOOvbYyk+T4NozqGU9idAQ3jE7mnTV72LyvyAeRGmOMR0FpBdGRYYT78USXYAmGxOgIoiLC+MXcr9i1v/Tw/q05xazcmc/VZ3ZDRAD40bieRLUK48mPNvsqXGOMIb+kwu9vUQYvJBgRCRWRdBGZX8+x7iKySETWishSEUly9g8Vkc9FZL1z7Gq34osMD+W564YTIsJP/p1GWWU1AHNTMwkLEa4YkXS4bFybVtwyticL1mezNuuAWyEZY8xxFZT6/zQx4J0WzJ3AxgaOPQH8S1UHAzOBh539pcAPVHUgcBEwS0Ti3Aqwa/s2PHX1ENbvOcgD766noqqGN9Ky+E7/jiRGRxxR9qYxybRrE84zi7e6FY4xxhyXZ6JL/x5/AZcTjNMiuQSY00CRAcAiZ3sJMAVAVbeoaoazvQfIARLdjPW8fh257dxevLoyk7v+m87+kgquHtn1mHLRkeFMGdqF5Rm5lFdVuxmSMcbUq6Ck0u/vIAP3WzCzgHuA+m/TgjXANGf7ciBaROLrFhCRkUArYNvRJ4vIrSKSKiKpubm5pxzsz7/Tl1E943l/3T66xLVmXJ/6c9rYPgmUVdaQtrPglK9pjDEnqqC0wu+fgQEXE4yITAJyVDXtOMXuBsaLSDowHtgNVNX5jE7Ay8CNqnpMklLV2aqaoqopiYmn3sAJCw3h6WuHcXrHaH4yoRehIVJvubN7xhMeKizLyDvlaxpjzIkoq6ymtKLa76eJAXenihkNTBaRiUAkECMir6jq9NoCTvfXVAARiQKmqWqh8z4GeA/4nap+4WKcR0iMjuDDu8YevnOsPm0jwhjWrR3LM3K59+J+3grNGGM4UOpME9OSWzCqOkNVk1Q1GbgGWFw3uQCISIKI1MYwA3je2d8KeAvPDQCvuxVjQ46XXGqN65PA+j0H2V9c7oWIjDHG4/BMyjYGcywRmSkik523E4DNIrIF6Ag85Oy/ChgH3CAiXzmvod6O9XjGOuMzK7ZaN5kxxnsKSjwJJhBuU/bKbMqquhRY6mzfV2f/PGBePeVfAV7xRmwn64wuscS2Dmd5Rh5ThnbxdTjGmBaiwOkiswctg1hoiDCmdwLLM3JRVV+HY4xpIfIDZC0YsARzSsb2SSD7YDlbc4p9HYoxpoUIpC4ySzCnYEyfBAC7XdkY4zUFpRVERYQdXqfKn/l/hH4sqV0beia0ZXnGkQ95frl9P498sImsgtIGzjTGmJNTUFJBu7b+3z0GtmTyKRvbJ4G5qVmUV1UTERbK++v2ctdrX1FRXcOc5duZOrwLP53Qm+SEthworWDhhmzeX7eXjXsPMu/H59C1fRtfV8EYE0AKSgNjmhiwBHPKxvZJ5KXPvyHtmwK25ZZw3/++ZkS3dvzx8jN4bWUmr67cxby0LAYlxbF+dyFVNUqXuNZkHyxnwfp93DK2p6+rYIwJIAWlFZZgWoqze8UTFiL87u2v2Z5bwnf6d+CZa4fTulUo908eyE/P7cX/LdvOF9vzuWVsTyYOOo1BXWL5zpOf8MmWXEswxpgTsr+4gl6JUb4Oo0kswZyiqIgwhndrx8qd+Vw5IomHpw4irM4qcx2iI/ntJQOOOW983w78+8tvKKusJjI81JshG2MClKqSW1x+zDIi/soG+ZvBvRP78afLB/HYFYOPSC7HM65vAuVVNXyxfb/L0RljgkVReRUVVTUkRlmCaTGGd2vH987q1qQ5zGqd3TOeiLAQlm2xW5yNMU2TV+SZ+zAhOjDGYCzB+EhkeCgje7Tnky05vg7FGBMgcmsTjLVgTGPG901kW26JPS9jjGmSvGLPU/w2BmMaNb6vZ0Zm6yYzxjRFblEZYC0Y0wS9O0TRKTaSZVtOfblnY0zwyyuuIEQImOdgLMH4kIgwvm8in27No7L6mBWhjTHmCHnF5cRHRTS4nLu/sQTjY+P6JlJUXsVXmQd8HYoxxs/lFpUHTPcYWILxudG9EwgNEesmM8Y0Ki+AHrIELyQYEQkVkXQRmV/Pse4iskhE1orIUhFJqnPsehHJcF7Xux2nr8S2Dmdo1zg+sQRjjGmEpwUTGOMv4J0WzJ3AxgaOPQH8S1UHAzOBhwFEpD3wB+AsYCTwBxFp54VYfWJ830TW7S5kf3G5r0MxxvgpVSWvuCJgnuIHlxOM0yK5BJjTQJEBwCJnewkwxdn+LvCRquaragHwEXCRm7H60vi+iajC0s3WijHG1O9gWRUV1TXWRVbHLOAeoKFbpNYA05zty4FoEYkHugCZdcplOfuC0uCkWLrEtebdtXt8HcoJqayuQVV9HYYxLUKgPcUPLiYYEZkE5Khq2nGK3Q2MF5F0YDywG6gC6rsH75ifZCJyq4ikikhqbm7g/vYvIkwa0okVGXnkO+tt+7MDpRU8uXAzw2d+xI9fSaOiym6xNsZteU4XurVgPEYDk0VkJ/AacJ6IvFK3gKruUdWpqjoM+K2zrxBPi6VrnaJJwDG/3qvqbFVNUdWUxMREl6rhHZOHdKaqRvng671euV5haSX/XbWLwkOVTT6nNrGMfXQJTy/eSv/OMSxYn81PXkmjvKraxWiNMbUJxlowgKrOUNUkVU0GrgEWq+r0umVEJEFEamOYATzvbC8ALhSRds7g/oXOvqA1oFMMPRPb8s5X3ukme2D+en79xjrGPrqYv3ycwcGy4yealTvymfDEUp5evJWxfRP48K6xzP3RKB687AwWbcrhxy+nUVZpScYYt3zbRWZ3kTVIRGaKyGTn7QRgs4hsAToCDwGoaj7wILDKec109gUtEWHykM6s3JnPvsIyV6+1NusAb67ezdThXTi7ZzxPfbyFMY8s5ulFGRTVk2g+/Hof0//5Je3btuKDO8fy3HUj6HdaDADfP7s7D08dxJLNudxqScYY1+QVlxMaIgEzTQx4KcGo6lJVneRs36eq7zjb81S1j6r2VdVbVLW8zjnPq2pv5/WCN+L0tUuHdEYV5rs42K+q/HH+RuLbtuKByQOZ/YMU5t8xhrN7xvPkR1sY+9gS/rpkK8XlVQC88sU3/PTfaQzsHMO8H59D/04xx3zmtSO78di0wSzPyOXXb6x1LXZjWrLconLi27YiJECmiQFbMtmv9EqMYmDnGN5du5dbxvZ05Roffr2PlTvzeejyM4iODAfgjC6xzP5BCl/vLmTWx1t4fMFm5izfzjm9E3hv7V7O79eBZ783nNatGl7a+aozu5J14BBPL8rgurO6M7JHe1fiN6alyiuuCKjxF7CpYvzOpUM6sybzAN/sL2n2zy6vqubhDzZxesdork7peszxM7rEMuf6M/nfbaMZ0jWO99bu5eqUrvzj+yOOm1xq/WR8LzrFRjJz/npqauz2ZWOaU6BNEwOWYPzOpMGdAJi/tvnvJnvps53syi/ld5P6Exba8Fc/pGscL944kpW/OZ9Hpg06btm6WrcK5d6L+/H17oPMW53VXGG3OFXVNRQEwO3qxrsCbaJLsATjd5LatWFE93andDdZZXUNzy7O4Pv//JLfvLWO2cu28d7avTyzaCvnnp7I2D5Nu6W7Q0wkIifW3zt5SGeGdYvj8QWbD4/j1CqrrD58q6Vp2K/mreXshxfxzprAevDWuEdV2V9cYS0Yc+omD+nM5uwiNu8rOuFzt+UWc8XfPuOJhVvIPljGB+v28qf3N3Hbf1ZzqLKa317S34WIvyUi/OHSgeQWlfPckq0A1NQob67OYvzjSxj9yGL+uWKHdaE1YNXOfN5K303biDB+9mo6jy/YZH9XhoOHPNPEBNItymCD/H5p4qBOPPTeRp5YuJnZ3x/RpFaEqvLKF9/w0PsbiQwP5a/fG84lTndbYWkl3+SXEBYSQu8O0W6Hz9CucUwd1oU5K3YwoHMM/7d8B2syDzA4KZb+nWJ4cP4GFq7fxxNXDqFr+zauxxMoqmuU+99ZT6fYSD68axwPv7+Rvy7ZxuZ9RTx19dDDN2WYlie32PPogrVgzClLjI7gnotO56MN2fz7y12Nli+vquYnr6zm9/9bz8ge8Sy4a9zh5AIQ2yacwUlxDOh87C3Gbrnnon6EinD7f9LZe+AQf75yCG//dDQv3HAmj10xmA17DvLdWcuYuyqz8Q/zYxnZRfzs1XTSvjn1x7Tmpmayfs9B7r24H7Gtw3l46iAemDyQJZtzufSZFTy/Yod1MbZQuUWeMblAmkkZrAXjt24a3YNlGXk8OH8DI3u0p2/H+lsehyqqufXlVJZn5PHbif25ZWyPEx43ccNpsZE8ceUQdu4v4YZzkmkb8e0/tatSujK6dwK/en0N97yxlgGdYzijS6wPoz1xNTXKi5/t5JEPN1FRVcOijdm8dNNIUpKPvT07M7+UA6WVxEe1on3bVkSGH3tHXuGhSp5YsJmU7u2YPKQz4OluvP6cZPp0jOLh9zcxc/4GHnp/IxP6JnL58C6M7ZNIbGtr1bQEubXTxARYC0aCZTbclJQUTU1N9XUYzSq3qJyL/7KMhKgI3r5t9DE/mIrLq7jpxVWs2pnPo1MHc9WZx9567M8OllUy5pHFjOwRz5zrU3wdTpPtLTzEr15fy4qteZzfrwM/v6Avd7yaTs7BsiOSzKGKamYt2sKc5TuorjOOEh0RxtBucVw/Kpnz+nUgJER4cP4Gnv90B+/ePqbBZLslu4g3Vmfxdvpusg+WEyKeO/7G9k5gTJ9EhnWLI7yJd/yZwPL8ih3MnL+B1b+/gPZtm3ccRkTSVNWV/4CWYPzc0s053PDCKq4f1Z0HppxxeH9haSXXv7CSdbsLefKqIUwZGpirGTy9KIMnP9rCu7ePYVCS/7diMvNLmfTMCiqra/j9pAFcc2ZXRITsg2VcM/sLcg6W8a+bR1JWWcNv3lrHN/tLuSolifP7dyS/pIL9xeXkFpWzcEM2ewvL6B7fhsuHdeHZxVu5YkQSj0wb3GgM1TVK2jcFLM/IZXlGHmuzDlCjnsQ1uncC409PZHzfRDrHtfbC34jxhsc+3MTsZdvZ8seLm/1JfkswTRCsCQZg5rue325H946nqKyKgtIK8ooqqK5RnvneML478DRfh3jSisoqGfPoElK6t+OfN5zp63Aa9fAHG5mzfAcL7hpH7w5RRxyrTTK7DxyioqqG5Pg2/GnqIM7plXDM51RW1/Dh1/t44dMdrN51gOiIMJb8asJJPedQWFrJZ9vyWJaRy9LNuex15rI7LSaSAZ1jGOi8zu3XgYiwxh+YNf7nV6+vYVlGLl/+5jvN/tluJhgbgwkAv774dHKLy9m1v4R2bVrRKzGKuDbhTBzUiTPr6fMPJNGR4fxwbA+eWLiFNZkHGNI1ztchNaiyuoY30nZzXr8OxyQXgI4xkbx269nc+Vo6w7u142fn96l3vAUgPDSES4d05tIhnVmXVUhoiJz0Q3SxbcK5eFAnLh7UCVUlI6eYZVtyWb/nIOv3FPLJllyqa5SLBp7G36YP94sxOnNiAvEpfrAEExAiwkJ55tphvg7DNdefk8ycFTuY9fEWXrhxpK/DadDSzbnkFZfXO81OLU+SGXVCn9ucXYMiQt+O0UfcFFJWWc3fP9nGrI8zeHP1bqaNSGq26xnvyC0OvKf4wW5TNn7A04rpyZLNuaTvKvB1OA2am5pJYnQEE04PrMXtIsNDueO8PoxMbs8f3llPZn6pr0MyJyivKPAmugRLMMZPXH9OMu3btmLWxxm+DqVeOUVlLN6Uw9ThXZo8N5s/CQ0R/nzVEFSVu19fY7MDBJCaGmV/SWB2kQXe/xQTlKIiwvjRuJ58siWX11P97+HLt1bvprpGueo43WP+rmv7Nvxh8kC+3JHPP1fs8HU4pokKD1VSWa3B24IRkV4iEuFsTxCRn4mI/47GmoB005gejOmdwIw31/Hp1jxfh3OYqjI3NZOU7u3olXjs4H4guXJEEhcO6MjjCzazad9BX4djmqB29oZgbsG8AVSLSG/gn0AP4D+uRWVapPDQEJ6bPpxeiVH8+OU0tmSf+GSfbli9q4BtuSUB3XqpJSI8PHUQMa3DuGfe2iMeADX+6fBT/AE20SU0PcHUqGoVcDkwS1V/DnRq5BwARCRURNJFZH49x7qJyBLn+FoRmejsDxeRl0RknYhsFJEZTa2QCWwxkeE8f+OZRLYK5cYXVpFTVObrkJi7Kos2rUKPmN8tkMVHRfD7SQNYm1XIf778xtfhmEbkFjktmGDtIgMqReRa4HqgNlE0dRKkO4GNDRz7HTBXVYcB1wDPOfuvBCJUdRAwAviRiCQ38XomwHWJa80LN5xJQWkFN7+Yyq79vrvrqaS8ivlr9zBpcKcj5lMLdJOHdGZ073ge+3CzXyRx07C8YmeiyyDuIrsRGAU8pKo7RKQH8EpjJ4lIEnAJMKeBIgrUTvEbC+yps7+tiIQBrYEKwDqMW5AzusTyzLXD2LyviAlPLOG2f6/mq8wDXo/j1ZW7KKmoDorusbpEhAennEF5VQ0PvdfQ73/GH+QWlRMeKgE5sWmTEoyqblDVn6nqqyLSDohW1UeacOos4B6gpoHj9wPTRSQLeB+4w9k/DygB9gK7gCdU9dTnQzcB5fz+HVn+63O5dVwvlmXkctlfP+Wqf3zOngOHmnS+qnLzi6uY+JflzF2VSVll9Qldf19hGU99tIUJpycyonu7k6mCX+uZGMWPJ/Tif1/t8aubKsyR8orLiW8bEZAzMDT1LrKlIhIjIu2BNcALIvJkI+dMAnJUNe04xa4FXlTVJGAi8LKIhAAjgWqgM54bCn4pIj3rucatIpIqIqm5ublNqYoJMB1jIrn34n58PuN8fj9pAF/vLuSP721o0rlLNuewaFMOecXl3PPGWkY/spg/L9xMzsGmdQn98b0NVNYoD0weGJD/uZvipxN60T2+Db9/+2vKq04sARvvCNRpYqDpXWSxqnoQmAq8oKojgMZmXRsNTBaRncBrwHkicnS32s3AXABV/RyIBBKA7wEfqmqlquYAnwLHTMamqrNVNUVVUxITA+vpanNioiLCuHlMD24d15P31+1jdSNP/NfUKI8v2EL3+Das+PV5/OeWsxjWrR3PLtnK6EcX84v/fsXXuwsbPH95Ri7z1+7ltgm96R7ftrmr4zciw0OZOeUMtueV8NySbb4Ox9Qjt6g8IO8gg6YnmDAR6QRcxbeD/MelqjNUNUlVk/EM4C9W1elHFdsFnA8gIv3xJJhcZ/954tEWOBvY1MRYTRD74dieJERF8PD7GzneTODz1+1l496D/OKCvrQKC+Gc3gnMuT6FJb+cwHVndefD9fuY9MwKrv7H53y0IfuIJ9vLq6q573/rSY5vw4/GH9NwDjrj+yZy2dDOPLM4g8+sq8zvtIQWzExgAbBNVVc53VUnNaeHiMwUkcnO218CPxSRNcCrwA3q+anxVyAK+BpYhafVtPZkrmeCS9uIMH5+QR9W7Szgow3Z9ZaprK7hyYWb6XdaNJcO7nzEseSEttw/eSCfzzif30zsR1bBIX74r1Qu+ssy3k7fTVV1Df+3bDs78kqYOeWMBmdDDjZ/vHwQPROjuOPVdPYWNm2My7ivsrqG3KJyOsZE+jqUk2LrwZiAU1Vdw4WzliHAgrvGHTM32KsrdzHjzXXM+UEK3xnQsdHPmr92L88t3cqW7GK6tW9D9sEyzu/fgeeuG+FiLfzP1pxipjy7gr6nRfPfW0fRKsxmkvK1b/aXMP7xpTx2xWDX7mR0cz2Ypg7yJ4nIWyKSIyLZIvKGcwuyMV4XFhrCvRf1Y1tuCXNTs444VlZZzV8+zmB4tzjO79+hSZ912bAufHjnOGZ/fwTt2oTTulUov580wK3w/VbvDlE8ceUQ0ncdaPKNFMZdu5yZr7u1b+PjSE5OU58cewHP1DBXOu+nO/sucCMoYxpzwYCOnJncjqc+3sI5veIJEaFalbfTd7PvYBlPXT30hO78CgkRLhx4GhcM6Eh1jQbkjMnN4eJBnbh1XE9mL9vO0K5xTB1uv0f6Uma+p7sy2BNMoqq+UOf9iyJylxsBGdMUIsK9F/dn2t8+Y8ITS484NrZPAqN6xZ/054aFBuctyU11z3dPZ03mAe7733rG9EmgQ3Rg9v8Hg135pYSHSsCOwTQ1weSJyHQ8A/HgeX5lvzshGdM0I7q344UbzyTnYBmhISGEhXiSw7mnN941ZhoWFhrCI9MGc+FTn/D4h5t5/Mohvg6pxcrMLyWpXRtCQwLzl56mJpibgGeBp/BM4/IZnuljjPEpSybu6JHQlpvG9OAfn2znurO7M7Srrc7hC7vyS+kaoN1j0PSpYnap6mRVTVTVDqp6GZ6HLo0xQeqO8/qQGB3B/e+stxUwfWRXfind2rf2dRgn7VRGMn/RbFEYY/xOVEQY917Uj68yD/Bm+m5fh9PiFB6qpPBQZcAO8MOpJZjA7BQ0xjTZ5cO6MKxbHI98sImiskpfh9OiZDq3KHdt1zITjLWZjQlyISHC/ZcOJK+4nGcWb/V1OC3K4QQTwC2Y4w7yi0gR9ScSwbNOizEmyA3pGsc1Z3Zl9rLthIUIv7zw9IC9qymQHH7IMj5IE4yqRnsrEGOM/3pgykBCQoTnlm5j3e5Cnr5mGO3aBuYMv4FiV34pcW3CiYkMvIXGarXMx5WNMSckIiyUP10+iEenDeLL7flc+uyK4y53YE5dZsGhgB7gB0swxpgTcPWZ3Xj9x6OorlGu+Ptnja7LY05eZn5pQA/wgyUYY8wJGtI1jnduH8NpMZHc/OIqtucW+zqkoFNdo2QVBPZDlmAJxhhzEhKjI3jpppGEiHD9CyvJKWraMtSmafYdLKOyWq2LzBjTMnWPb8vzN5xJXlEFN7+YSkl5la9DChq79gf2NP21LMEYY07akK5x/PW6YWzYe5Cf/ns1VdU1vg4pKGQWWIIxxhjO69eRB6ecwSdbcnn+0x2+DicoZOaXEiLQKS4wp+mv5XqCEZFQEUkXkfn1HOsmIkuc42tFZGKdY4NF5HMRWS8i60QksP+mjQli147sygUDOvLkR1v4Zn+Jr8MJeLvyS+kc15rwAF/4zhvR3wlsbODY74C5qjoMuAZ4DkBEwoBXgB+r6kBgAmATIRnjp0SEB6ecQXhICDPeXIeqzSR1KjyzKAd29xi4nGBEJAm4BJjTQBEFYpztWGCPs30hsFZV1wCo6n5VrXYzVmPMqTktNpJfX9yPz7bt5/W0LF+HE9AyLcE0ySzgHqChkb/7gekikgW8D9zh7O8LqIgsEJHVInJPfSeLyK0ikioiqbm5uc0cujHmRH1vZDdGJrfnofc22q3LJ6m0ooq84oqAfwYGXEwwIjIJyFHVtOMUuxZ4UVWTgInAyyISgmeOtDHAdc6fl4vI+UefrKqzVTVFVVMSExObvxLGmBPvWJCZAAAXYElEQVQSEiI8PG0QhyqqeeDdDb4OJyBl5h8CAnsW5VputmBGA5NFZCfwGnCeiLxyVJmbgbkAqvo5EAkkAFnAJ6qap6qleFo3w12M1RjTTHolRvGz83vz3tq9vLZyl6/DCTiHZ1G2BNMwVZ2hqkmqmoxnAH+xqk4/qtgu4HwAEemPJ8HkAguAwSLSxhnwHw/Yr0PGBIgfje/FuL6J/Oatdby3dq+vwwkolmBOgYjMFJHJzttfAj8UkTXAq8AN6lEAPAmsAr4CVqvqe96O1RhzcsJDQ/jH9BGM6N6Ou/6bzpJNOb4OKWBk5pcSFRFGuzaBO01/LQmW2wlTUlI0NTXV12EYY+o4WFbJ9/7vCzKyi3npppGc3TPe1yH5vZtfXMWewjI+uHOsV64nImmqmuLGZwf2UzzGGL8WExnOv246i67t23DLS6ls2nfQ1yH5vV35pXRtFxwLBluCMca4qn3bVrxy81mEhwqPfrDJ1+H4NVUNmocswRKMMcYLTouN5JaxPVmyOZc1mQd8HY7f2newjPKqGrrHW4Ixxpgm+8Go7sS2DufpRRm+DsVvZWR7Fm/r3SHax5E0D0swxhiviI4M55YxPVi0KYd1WYW+DscvZeR4EkyfjlE+jqR5WIIxxnjN9aOTiYkM4+nF1oqpz9acItq1CSe+bStfh9IsLMEYY7wmJjKcm8f05KMN2azfY62Yo2VkF9OnYzQi4utQmoUlGGOMV90wOpnoyDAbizmKqpKRU0yfDsHRPQaWYIwxXhbbOpybRvdgwfpsNu6152Jq5RaXU3io0hKMMcacipvG9CA6MownP9ri61D8xtbs2gH+4LiDDCzBGGN8ILZ1OLeO9YzFrN5V4Otw/MLhO8isBWOMMafmpjE9SIhqxWMfbrIlloEt2UXERIaRGB3h61CajSUYY4xPtI0I4/Zze/PF9nyWZ+T5Ohyfy8gJrjvIwBKMMcaHrj2rG13iWvP4gs3U1LTsVszWnGL6BskDlrUswRhjfCYiLJSfX9CXdbsL+eDrfb4Ox2f2F5eTX1IRNFPE1LIEY4zxqcuHdaFPhyj+vHAzVdU1vg7HJ4JxgB8swRhjfCw0RLj7u6ezPa+EeWlZvg7HJ4JtDrJaricYEQkVkXQRmV/PsW4issQ5vlZEJtZzvFhE7nY7TmOM71w4oCNDu8bxl0UZlFVW+zocr9uaXURURBinxUT6OpRm5Y0WzJ3AxgaO/Q6Yq6rDgGuA5446/hTwgYuxGWP8gIhw94Wns7ewjFdX7vJ1OF6XkVNM7w5RQXUHGbicYEQkCbgEmNNAEQVinO1YYE+dcy8DtgPr3YzRGOMfRveO5+ye7fnrkq2UVlT5Ohyv2pIdXHOQ1XK7BTMLuAdoaOTufmC6iGQB7wN3AIhIW+DXwAMux2eM8RO1rZi84gpe+uwbX4fjNQUlFeQVlwfd+Au4mGBEZBKQo6ppxyl2LfCiqiYBE4GXRSQET2J5SlWLG7nGrSKSKiKpubm5zRa7McY3UpLbM+H0RP7+yTYOllX6Ohyv2JobfHOQ1XKzBTMamCwiO4HXgPNE5JWjytwMzAVQ1c+BSCABOAt4zDn3LuA3InL70RdQ1dmqmqKqKYmJia5VxBjjPb+84HQKD1Xyz+U7fB2KV9Quk2xdZCdAVWeoapKqJuMZwF+sqtOPKrYLOB9ARPrjSTC5qjpWVZOdc2cBf1LVZ92K1RjjPwYlxXLRwNP454odFJRU+Doc12XkFNGmVSidY1v7OpRm5/XnYERkpohMdt7+EvihiKwBXgVuUJv1zpgW7xcX9qWkoopHPtgU9A9fbnXuIAsJCa47yADCvHERVV0KLHW276uzfwOerrTjnXu/i6EZY/xQ347R3DS6B/9csYOMnCKeunoo3ePb+josV2RkF3NO73hfh+EKe5LfGOOXfj9pAH+5Zihbc4q5+C/LeXXlrqCb1r/wUCX7DpbRJ8jmIKvllRaMMcacjClDuzCyR3vufn0NM95cx39XZTK6dzwjurdjWNd2tGvbytchnpIvtu8H4IwuMY2UDEyWYIwxfq1TbGtevuksXvnyG15PzeLvn2yn2pna/+ye7XnxxpFEhof6OMqT80ZaFglREYzqGZxdZJZgjDF+LyRE+MGoZH4wKpnSiirWZhXy6dY8nlm8lWcXb+Xu757u6xBPWH5JBUs253D9qGTCQoNztMISjDEmoLRpFcbZPeM5u2c8ew6U8fdPtnHJ4E707xRY3UzvrtlDZbUybUSSr0NxTXCmTWNMi/C7S/oT2zqce99cd7jbLFC8sTqLAZ1iAi4xnghLMMaYgNWubSvuu3QAazIP8K/Pd/o6nCbLyC5ibVZhULdewBKMMSbATR7SmfF9E3l8wWZ2Hzjk63CaZN7qLEJDhClDO/s6FFdZgjHGBDQR4Y+XnYEq/P7tr/3+WZnqGuXt9N1M6JtIQlSEr8NxlSUYY0zA69q+DXd9pw+LN+Wwbnehr8M5rhVb88g+WB703WNgCcYYEySuGdmNVmEhvJGW5etQjuuNtCxiW4dzfv8Ovg7FdZZgjDFBIbZ1OBcM6Mg7a/ZQUeWfE2QWlVWyYP0+Lh3SiYiwwHw49ERYgjHGBI0rhidRUFrJks05vg6lXh+s20d5VQ1Thwd/9xhYgjHGBJGxfRJIiIrw226yt9J30yOhLcO6xvk6FK+wBGOMCRphoSFcNrQzSzbnkO9ni5XtOXCIL3bs57KhXRAJvrVf6mMJxhgTVKaNSKKyWnl3zR5fh3KE/321B1W4bFhwP/tSlyUYY0xQ6d8phgGdYnhjtf90k6kqb6VnMbxbXNAunFYfSzDGmKAzdXgX1mYVkpFd5OtQANiw9yBbsou5vIUM7tdyPcGISKiIpIvI/HqOdRORJc7xtSIy0dl/gYikicg658/z3I7TGBM8pgztQmiIMM9PWjFvp+8mLESYNKiTr0PxKm+0YO4ENjZw7HfAXFUdBlwDPOfszwMuVdVBwPXAy65HaYwJGonREUzom8jb6bt9PstydY3yv6/2MOH0DgG/AueJcjXBiEgScAkwp4EiCtTOVR0L7AFQ1XRVrR2hWw9EikhwT9pjjGlWU4cnkX2wnBVb83wax2fb8sgpKmfq8C4+jcMX3G7BzALuARp6rPZ+YLqIZAHvA3fUU2YakK6q5UcfEJFbRSRVRFJzc3ObKWRjTDD4zoAOxLUJ5/XUTJ/G8Vb6bqIjwjivX/BPDXM01xKMiEwCclQ17TjFrgVeVNUkYCLwsogcjklEBgKPAj+q72RVna2qKaqakpiY2IzRG2MCXURYKJcN7cLC9dkcKPXNMzGlFVUs+HofEwd1IjI8+KeGOZqbLZjRwGQR2Qm8BpwnIq8cVeZmYC6Aqn4ORAIJcLh77S3gB6q6zcU4jTFB6sqUJCqqa3jHR8/EfLQhm5KKai5vgd1j4GKCUdUZqpqkqsl4BvAXq+r0o4rtAs4HEJH+eBJMrojEAe8BM1T1U7diNMYEt4GdYxnQKYa5Puomm5eWRZe41oxMbu+T6/ua15+DEZGZIjLZeftL4IcisgZ4FbhBPasF3Q70Bn4vIl85r5bXgWmMOWVXpSTx9e6DbNhz0KvX3XPgECu25jFtRBIhIS1japijeSXBqOpSVZ3kbN+nqu842xtUdbSqDlHVoaq60Nn/R1Vt6+yrffnn9KjGGL82ZWgXWoWG8Hqad1sxb6XvRhWmtdDuMbAn+Y0xQa5d21ZcMKAjb6fv9to6MarKvLQsRvZo36KmhjmaJRhjTNC7MsWzTsyijdleud7qXQXsyCvhyhawLPLxWIIxxgS9sX0SOS0mkte9tE7MvLQs2rQKZWILmxrmaJZgjDFBLzREmDaiC0s357CvsMzVax2qqGb+mr1cfEYn2kaEuXotf2cJxhjTIlyd0o2wkBDufXMtNS7OT7Zwwz6Kyqu4ooV3j4ElGGNMC9Etvg33XTqApZtz+euSra5d5/XULJLateasHi3z2Ze6LMEYY1qM687qxmVDO/Pkx1tYkdH8k2DuPnCIT7flcUULfvalLkswxpgWQ0T409RB9OkQxZ2vpTfreIyq8vTHGc6zL9Y9BpZgjDEtTJtWYfxt+gjKKqu57T+rm+3ZmKc+2sJ/UzP50fiedG3fplk+M9BZgjHGtDi9EqN49IrBpH1TwMSnl/POmj2ntDDZnOXbeXrxVq5O6cq9F/VrxkgDmyUYY0yLNGlwZ/4+fQQhAj97NZ2LZi3j3ZNINK+nZvLH9zYycdBp/GnqIERs7KWWJRhjTIt10Rmn8eGd43j2e8MAuOPVdKbP+ZLcomPWN6zXwvX7+PUbaxnbJ4Gnrh5KqA3sH8ESjDGmRQsJESYN7syCu8bxyNRBrN5VwKRnlpO6M/+4532VeYCfvZbO4KQ4/j59BBFhLW9BscZYgjHGGDyJ5pqR3Xjrp6OJDA/lmtlfMGf5djwriBwpM7+UW15aRWJ0BHOuT2nxT+w3xBKMMcbUMaBzDO/cPoZz+3Xgj+9t5Pv/XMlXmQcOHy88VMmNL66ioqqGF24YSUJUhA+j9W+WYIwx5iixrcOZ/f0R3H/pANbvKeSyv37KzS+uIn1XAT95JY1v9pfwj++n0LtDlK9D9WtSX/MvEKWkpGhqaqqvwzDGBJni8ipe+mwns5dtp/BQJQB/vnII04JkrjERSVPVFDc+2/WOQxEJBVKB3bWrWtY51g14CYgDQoF7VfV959gM4GagGviZqi5wO1ZjjDlaVEQYt53bm++P6s6/PttJ+7YRQZNc3OaNkak7gY1ATD3HfgfMVdW/icgA4H0g2dm+BhgIdAY+FpG+qlrthXiNMeYYMZHh3H5eH1+HEVBcHYMRkSTgEmBOA0WUbxNPLLDH2Z4CvKaq5aq6A9gKjHQzVmOMMc3L7RbMLOAeILqB4/cDC0XkDqAt8B1nfxfgizrlspx9xhhjAoRrLRgRmQTkqGracYpdC7yoqknAROBlEQkB6nsc9pi7EUTkVhFJFZHU3NzcZonbGGNM83Czi2w0MFlEdgKvAeeJyCtHlbkZmAugqp8DkUACnhZL1zrlkvi2++wwVZ2tqimqmpKYmNj8NTDGGHPSXEswqjpDVZNUNRnPgP1iVZ1+VLFdwPkAItIfT4LJBd4BrhGRCBHpAfQBVroVqzHGmObn9fkNRGQmkKqq7wC/BP5PRH6OpwvsBvU8mLNeROYCG4Aq4Da7g8wYYwKLPWhpjDEtmJsPWtpUMcYYY1wRNC0YEckFvjlqdyxQeIL7GttOAPJOMsz6rn0iZZpSH2/VpbFYGytzonU5+n3tdt199t00LdbGyth349ufAccr50Zd2qqqO3dJqWrQvoDZJ7qvsW0840fNFs+JlGlKfbxVl1Otz4nW5Th1qLvPvhv7bvz6u2lKXZrzu3H731ljr2DvInv3JPY1Zbs54zmRMk2pj7fq0tTPaajMidbl6PfvNlDmZNl3c/z99t1472fA8cr5U10aFTRdZN4iIqnq0oCYtwVTXSC46hNMdYHgqo/VpemCvQXjhtm+DqAZBVNdILjqE0x1geCqj9WliawFY4wxxhXWgjHGGOOKFp1gROR5EckRka9P4twRIrJORLaKyNMiInWO3SEim0VkvYg81rxRNxhPs9dFRO4Xkd0i8pXzmtj8kTcYkyvfjXP8bhFREUlovoiPG48b382DIrLW+V4Wikjn5o+83njcqMvjIrLJqc9bIhLX/JE3GJMb9bnS+b9fIyKuj9WcSh0a+LzrRSTDeV1fZ/9x/1/Vy81b1Pz9BYwDhgNfn8S5K4FReGZ+/gC42Nl/LvAxEOG87xDAdbkfuDtYvhvnWFdgAZ5nphICtS5ATJ0yPwP+HsB1uRAIc7YfBR4N5H9nQH/gdGApkOKvdXDiSz5qX3tgu/NnO2e73fHqe7xXi27BqOoyIL/uPhHpJSIfikiaiCwXkX5HnycinfD8B/9cPX/z/wIucw7/BHhEVcuda+S4WwsPl+riMy7W5yk8axR5bfDRjbqo6sE6Rdvipfq4VJeFqlrlFP0Cz+zpXuFSfTaq6mZvxO9c76Tq0IDvAh+par6qFgAfARed7M+JFp1gGjAbuENVRwB3A8/VU6YLniUFatVdEK0vMFZEvhSRT0TkTFejPb5TrQvA7U7XxfMi0s69UJvklOojIpOB3aq6xu1Am+CUvxsReUhEMoHrgPtcjLUxzfHvrNZNeH479qXmrI+vNKUO9ekCZNZ5X1uvk6qv12dT9mciEgWcA7xep3sxor6i9eyr/Q0yDE/T8mzgTGCuiPR0sr7XNFNd/gY86Lx/EPgznh8AXneq9RGRNsBv8XTH+FQzfTeo6m+B34rIDOB24A/NHGqjmqsuzmf9Fs/s6f9uzhhPRHPWx1eOVwcRuRG409nXG3hfRCqAHap6OQ3X66TqawnmSCHAAVUdWneniIQCtStzvoPnB2/dZnzdBdGygDedhLJSRGrwzPfj7SU3T7kuqppd57z/A+a7GXAjTrU+vYAewBrnP10SsFpERqrqPpdjP1pz/Dur6z/Ae/ggwdBMdXEGkycB53v7l7GjNPd34wv11gFAVV8AXgAQkaV4lkjZWadIFjChzvskPGM1WZxMfd0egPL3F5BMncEx4DPgSmdbgCENnLcKTyuldsBrorP/x8BMZ7svnuamBGhdOtUp83PgtUD+bo4qsxMvDfK79N30qVPmDmBeANflIjxrPyV689+X2//O8NIg/8nWgYYH+Xfg6YVp52y3b0p9643LF1+ov7yAV4G9QCWeDH0znt9yPwTWOP/o72vg3BTga2Ab8CzfPrTaCnjFObYaOC+A6/IysA5Yi+e3tk7eqItb9TmqzE68dxeZG9/NG87+tXjmleoSwHXZiucXsa+cl1fuiHOxPpc7n1UOZAML/LEO1JNgnP03Od/JVuDGxup7vJc9yW+MMcYVdheZMcYYV1iCMcYY4wpLMMYYY1xhCcYYY4wrLMEYY4xxhSUYE9REpNjL15sjIgOa6bOqxTNb8tci8m5jswyLSJyI/LQ5rm1Mc7DblE1QE5FiVY1qxs8L028nZnRV3dhF5CVgi6o+dJzyycB8VT3DG/EZ0xhrwZgWR0QSReQNEVnlvEY7+0eKyGciku78ebqz/wYReV1E3gUWisgEEVkqIvPEs47Jv2vXxnD2pzjbxc6ElGtE5AsR6ejs7+W8XyUiM5vYyvqcbyftjBKRRSKyWjzrc0xxyjwC9HJaPY87ZX/lXGetiDzQjH+NxjTKEoxpif4CPKWqZwLTgDnO/k3AOFUdhmd24j/VOWcUcL2qnue8HwbcBQwAegKj67lOW+ALVR0CLAN+WOf6f3Gu3+h8Ts48WOfjmU0BoAy4XFWH41l/6M9OgrsX2KaqQ1X1VyJyIdAHGAkMBUaIyLjGrmdMc7HJLk1L9B1gQJ2ZZmNEJBqIBV4SkT54ZooNr3POR6pad82NlaqaBSAiX+GZC2rFUdep4NsJQtOAC5ztUXy7lsZ/gCcaiLN1nc9Ow7M2B3jmgvqTkyxq8LRsOtZz/oXOK915H4Un4Sxr4HrGNCtLMKYlCgFGqeqhujtF5Blgiape7oxnLK1zuOSozyivs11N/f+XKvXbQc6GyhzPIVUdKiKxeBLVbcDTeNZ/SQRGqGqliOwEIus5X4CHVfUfJ3hdY5qFdZGZlmghnvVTABCR2mnNY4HdzvYNLl7/CzxdcwDXNFZYVQvxLIt8t4iE44kzx0ku5wLdnaJFQHSdUxcANznrgyAiXUSkQzPVwZhGWYIxwa6NiGTVef0Czw/rFGfgewOeJRYAHgMeFpFPgVAXY7oL+IWIrAQ6AYWNnaCq6Xhmxr0Gz4JcKSKSiqc1s8kpsx/41Lmt+XFVXYinC+5zEVkHzOPIBGSMq+w2ZWO8zFld85CqqohcA1yrqlMaO8+YQGNjMMZ43wjgWefOrwP4aBlqY9xmLRhjjDGusDEYY4wxrrAEY4wxxhWWYIwxxrjCEowxxhhXWIIxxhjjCkswxhhjXPH/jDiXn8khkQMAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "learn.recorder.plot(skip_end=15)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 23:37\n", + "epoch train_loss valid_loss accuracy\n", + "1 4.225404 4.029806 0.302210 (23:37)\n", + "\n" + ] + } + ], + "source": [ + "learn.fit_one_cycle(1, 1e-02, moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save('fit_head')" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load('fit_head');" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 8:58:03\n", + "epoch train_loss valid_loss accuracy\n", + "1 3.970405 3.867857 0.316788 (26:53)\n", + "2 3.920167 3.786381 0.327380 (26:54)\n", + "3 3.851934 3.726164 0.336203 (26:53)\n", + "4 3.814150 3.671940 0.343449 (26:54)\n", + "5 3.798842 3.626528 0.348548 (26:53)\n", + "6 3.753443 3.585267 0.353221 (26:57)\n", + "7 3.723497 3.546739 0.357538 (26:55)\n", + "8 3.676399 3.515404 0.361083 (26:53)\n", + "9 3.668247 3.487297 0.364337 (26:52)\n", + "10 3.633528 3.454587 0.368336 (26:51)\n", + "11 3.611737 3.425386 0.372090 (26:49)\n", + "12 3.586109 3.396924 0.375549 (26:54)\n", + "13 3.568666 3.371741 0.379174 (26:53)\n", + "14 3.536366 3.337895 0.383549 (26:59)\n", + "15 3.507154 3.313276 0.387211 (26:56)\n", + "16 3.465269 3.290863 0.390346 (26:56)\n", + "17 3.479237 3.274435 0.392694 (26:54)\n", + "18 3.407071 3.261680 0.394350 (26:55)\n", + "19 3.425704 3.255773 0.395224 (26:52)\n", + "20 3.390900 3.253836 0.395450 (26:52)\n", + "\n" + ] + } + ], + "source": [ + "learn.unfreeze()\n", + "learn.fit_one_cycle(20, 1e-3, moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save(\"afteroom\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load(\"afteroom\");" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[3.2538004, tensor(0.3954)]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "learn.validate()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save_encoder(\"enc_best\")\n", + "learn.save(\"lm_best\", with_opt=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xd4VNXWwOHfIhVI6KGjoXcIISII0kUBRVRUUGyoXFGvBb1X7Mpnwd6uvYAVFBtKVZogIBB671E6obdA2v7+OCdhkswkM5OZzCRZ7/PMw+TMKSuTkDX77L3XFmMMSimlVGGVCXQASimlSgZNKEoppXxCE4pSSimf0ISilFLKJzShKKWU8glNKEoppXxCE4pSSimf0ISilFLKJzShKKWU8onQQAfgqWrVqpnY2NhAh6GUUsXKsmXLDhpjYvx5jWKXUGJjY0lMTAx0GEopVayIyN/+vobe8lJKKeUTmlCUUkr5hCYUpZRSPlHs+lCUUiVHWloau3bt4syZM4EOpcSIjIykbt26hIWFFfm1NaEopQJm165dREdHExsbi4gEOpxizxjDoUOH2LVrF/Xr1y/y6+stL6VUwJw5c4aqVatqMvEREaFq1aoBa/FpQlFKBZQmE98K5PtZahLK0qTDvP7bJlLTMwMdilJKlUilJqEs//sIb8/eSnqmJhSlFBw6dIi4uDji4uKoWbMmderUyf46NTXVrXPcdtttbNq0yc+RFh/aKa+UKpWqVq3KypUrAXjmmWeIiori4YcfzrGPMQZjDGXKOP/sPXbsWL/HWZyUmhaKUkq5Y+vWrbRq1Yq77rqL+Ph49u7dy/Dhw0lISKBly5aMHj06e98uXbqwcuVK0tPTqVSpEqNGjaJt27Z06tSJAwcOBPC7CIxS10IxJtARKKWcefbXdazfc9yn52xRuwJPX9HS4+PWr1/P2LFj+eCDDwAYM2YMVapUIT09nR49ejBo0CBatGiR45hjx47RrVs3xowZw8iRI/nss88YNWqUT76P4qLUtFB0IIlSyl0NGzbkggsuyP56/PjxxMfHEx8fz4YNG1i/fn2eY8qWLUvfvn0BaN++PUlJSUUVbtAodS0UpVRw8qYl4S/ly5fPfr5lyxbeeustlixZQqVKlRg6dKjTeR7h4eHZz0NCQkhPTy+SWINJqWmhKKWUN44fP050dDQVKlRg7969zJgxI9AhBa1S10LRLhSllCfi4+Np0aIFrVq1okGDBnTu3DnQIQUtMcWslzohIcF4s8DWx/O28/zUDax99lKiIkpdHlUqKG3YsIHmzZsHOowSx9n7KiLLjDEJ/ryu3vJSSinlE5pQlFJK+USpSyjF7RafUkoVF6Umoeg8FKWU8q9Sk1CUUkr5lyYUpZRSPlHqEor2oCilsnTv3j3PRMU333yTu+++2+UxUVFRAOzZs4dBgwa5PG9B0xvefPNNTp8+nf11v379OHr0qLuhB6VSl1CUUirLkCFDmDBhQo5tEyZMYMiQIQUeW7t2bb7//nuvr507oUydOpVKlSp5fb5goAlFKVVqDRo0iMmTJ3P27FkAkpKS2LNnD3FxcfTq1Yv4+Hhat27NpEmT8hyblJREq1atAEhJSWHw4MG0adOG66+/npSUlOz9RowYkV36/umnnwbg7bffZs+ePfTo0YMePXoAEBsby8GDBwF4/fXXadWqFa1ateLNN9/Mvl7z5s258847admyJX369MlxnWCgU8aVUsFh2ijYt8a356zZGvqOcfly1apV6dChA9OnT+fKK69kwoQJXH/99ZQtW5affvqJChUqcPDgQTp27MiAAQNcrtf+/vvvU65cOVavXs3q1auJj4/Pfu3555+nSpUqZGRk0KtXL1avXs19993H66+/zpw5c6hWrVqOcy1btoyxY8eyePFijDFceOGFdOvWjcqVK7NlyxbGjx/Pxx9/zHXXXccPP/zA0KFDffNe+YC2UJRSpZrjba+s213GGB577DHatGlD79692b17N/v373d5jnnz5mX/YW/Tpg1t2rTJfu27774jPj6edu3asW7dOqel7x39+eefXHXVVZQvX56oqCiuvvpq5s+fD0D9+vWJi4sDgrNEvt9bKCISAiQCu40xl+d6bSRwB5AOJAPDjDF/+zMendeoVJDKpyXhTwMHDmTkyJEsX76clJQU4uPjGTduHMnJySxbtoywsDBiY2Odlqx35Kz1smPHDl599VWWLl1K5cqVufXWWws8T36TryMiIrKfh4SEBN0tr6JoodwPbHDx2gogwRjTBvgeeNlfQbhqqiqlSreoqCi6d+/OsGHDsjvjjx07RvXq1QkLC2POnDn8/Xf+n3O7du3K119/DcDatWtZvXo1YJW+L1++PBUrVmT//v1MmzYt+5jo6GhOnDjh9Fw///wzp0+f5tSpU/z0009cfPHFvvp2/cqvCUVE6gL9gU+cvW6MmWOMyRrm8BdQ15/xKKWUM0OGDGHVqlUMHjwYgBtvvJHExEQSEhL4+uuvadasWb7HjxgxgpMnT9KmTRtefvllOnToAEDbtm1p164dLVu2ZNiwYTlK3w8fPpy+fftmd8pniY+P59Zbb6VDhw5ceOGF3HHHHbRr187H37F/+LV8vYh8D7wIRAMP577llWvf/wH7jDHP5XdOb8vXf/rnDv5v8npWPd2HimXDPD5eKeV7Wr7eP0pc+XoRuRw4YIxZ5sa+Q4EE4BUXrw8XkUQRSUxOTi5cYNqHopRSfuHPW16dgQEikgRMAHqKyFe5dxKR3sDjwABjzFlnJzLGfGSMSTDGJMTExHgVjPagKKWUf/ktoRhjHjXG1DXGxAKDgdnGmBwDpkWkHfAhVjI54K9YlFLBS5eU8K1Avp9FPg9FREaLyAD7y1eAKGCiiKwUkV+KOh6lVOBERkZy6NAhTSo+Yozh0KFDREZGBuT6RTJT3hgzF5hrP3/KYXvvorh+jli0E0WpoFG3bl127dpFoftGVbbIyEjq1g3MgNlSU3pFp6EoFXzCwsKoX79+oMNQPqKlV5RSSvlEqUko+49bA8hOnEkPcCRKKVUylZqE8sEf2wD4ZdWeAEeilFIlU6lJKFkyM7VTXiml/KH0JRTNJ0op5RelLqG8MXMza3cfC3QYSilV4pS6hAJw3YeLAh2CUkqVOKUyoZxOzQh0CEopVeKUyoSilFLK9zShKKWU8glNKEoppXxCE4pSSimfKLUJ5ctFSYEOQSmlSpRSm1CenLSOn1fsDnQYSilVYpSahFKvStk82x74dmUAIlFKqZKp1CSUX+/t4nS7MYZF2w6x79iZIo5IKaVKllKTUCqVC3e6/ff1+xny8V/0fWteEUeklFIlS6lJKK4M/3IZAEdOpwU4EqWUKt5KfUJxlJqeGegQlFKq2CpVCeWP/3TP9/UmT0xj64ETRROMUkqVMKUqoZxftXyB+zw0cTUZumiKUkp5rFQlFICGMfknlVU7j/Lt0p1FFI1SSpUcpS6h/HJvF6IiQvPd57Gf1jB97T5W7jxaRFEppVTxV+oSSvmIUBKf6F3gfnd9tYyB7y4ogoiUUqpkKHUJBSAyLCTQISilVIlTKhMKwFOXt3Brvx+X7+LAcZ1Fr5RSBRFjiteIpoSEBJOYmOiTc8WOmuLR/pueu4yIUG3dKKWKHxFZZoxJ8Oc1/N5CEZEQEVkhIpOdvBYhIt+KyFYRWSwisf6OpzAOnkwNdAhKKRW0iuKW1/3ABhev3Q4cMcY0At4AXiqCeLLd3Ol8j/Y/ckoTilJKueLXhCIidYH+wCcudrkS+Nx+/j3QS0TEnzE5erx/86K6lFJKlXj+bqG8CfwXcFUkqw6wE8AYkw4cA6rm3klEhotIoogkJicn+yy4iNAQZj/UjY4Nqri1/4ivl7H3WIrPrq+UUiWJ3xKKiFwOHDDGLMtvNyfb8owSMMZ8ZIxJMMYkxMTE+CxGgAYxUbwzJN6tfXceTqHvW/N9en2llCop/NlC6QwMEJEkYALQU0S+yrXPLqAegIiEAhWBw36MyamsmfNt61UqcN+jp9Po99Z81u05xvwtvmstKaVUcVckw4ZFpDvwsDHm8lzb7wFaG2PuEpHBwNXGmOvyO5cvhw07OnIqlejIUBo9Ps2j45LG9Pd5LEop5WslYthwbiIyWkQG2F9+ClQVka3ASGBUUceTpXL5cEJDynicII6l5FyYK/nEWR7/aY2uraKUKnXyr5LoI8aYucBc+/lTDtvPANcWRQz+8uC3KwktI9xyUSydGlTlgudnAtCxQVWuaFs7wNEppVTRKZKEUtzc0aU+n/y5w619Z288AMBv6/czpEO97O0G+PvQKdIyMmlUPdofYSqlVFAptbW88vPE5S1YP/pSBrWvy4c3tXf7uPFLcq6j0u2VufR+fZ6vw1NKqaCkCcWFcuGhvHptWy5tWZPZD3Xz+Pi5dstFKaVKC00obggL8fxt+nHFbj9EopRSwUsTihtioiMKdfwPy3b5KBKllApemlDcUNgFuR6auIqMzOK1TIBSSnlKE0oRufeb5YEOQSml/EoTioeqe3n7a9rafU63p2Vk6iRIpVSJoAnFTTdceB6NqkexcFRPn5638ePTaPLENNbtOebT8yqlVFHThOKmF65qzcyR3QgNKeNWEUlnXv9tE3M3WcOJ9x8/w8B3F2S/1v/tP30Sp1JKBYrOlPfCpHs6A56vSf/27K0AXNSwKgu3HXK6z/EzaYSHlCn0QACllCpq2kIJAFfJ5NTZdNo88xvNnpxOSmoG25NPFnFkSinlPU0ohfDqtW2znzeuHlXo87V8ekb28+ZPTafna3/w0vSNZOqQY6VUMaAJpRAGtK1Nv9Y1mTmyG1fF18neflvnWJ9d4/252xj80V8+O59SSvmL9qEUQnhoGd670Soe2aBaQwa0rU3dyuUASE3P5OvF//jkOkuSinwRS6WU8pi2UHykTBnJTiYAz1/Vmk4NqhZ5HAeOn2HdnmNkZhpemLqBvcdSijwGpVTppC0UP6pbuaxfz5+eYU2IDLWLVxpj6PDCLAC+v6sTH83bzupdR5kwvJNf41BKKdAWivtSjnp8SKVyYX4IBBZuPUhmpqHts7/R8cXZACQdPMXoyeuz98nqx0/P0A59pVTR0ITijr8XwlttYPNvHh32UJ+mPDewlU9DmbPpADd8sphXf9vEqdQMDp48C0D3V+cydkFS9n5ZrZdTqRk+vb5SSrmiCcUdMc2gcix8eyNsmu72YZFhIQzteL5PQ0k+YSWQ9+Zuy942bNzSPPvd8MliADbsPe7T6yullCuaUNxRrgrcPAlqtIRvh8JGz2bI+1JoGcmzbbauDqmUCgKaUNxVtjLc9DPUagPf3Qzrf3H70Gvi6+bZ9v1dnnWUr9tzjHV7jrF2t7Y4lFLBSROKJ8pWgpt+gtrxMPFWWPeTW4c9f1Urnrq8RY5tCbFVsp93bxpT4Dn6v/0n/d/+k88W7PAoZKWUKiqaUDwVWRFu+hHqXgDf3w5rvi/4kLAQhnWpz5VxtZ2+fl6Vck63Z2lRq4JXoSqlVFFyK6GISEMRibCfdxeR+0TEuxruJUFENAz9Ac7rCD/eCau/c+uwZwe0BKBBTHkA7uvZCICbO8UCMPa2C+jVrDpT7uvCX4/2yj5uvXasK6WKATGm4HkKIrISSABigRnAL0BTY0w/v0bnREJCgklMTCzqyzqXegq+uR7+XgBXvgdxQwo85ODJs5QLD6FceMFzSj0tj+9K0pj+Be6z8/Bp6lYui4iQkprBiTNpVK8Q6ZPrK6UCT0SWGWMS/HkNd2fKZxpj0kXkKuBNY8w7IrLCn4EVC+Hl4YbvYPxg+HkEmAxoNzTfQ6pFub+EcNKY/j5LKs70fWt+jmHFj/Vrxvtzt3HkdFr2tu0v9KOMk5FlSimVm7t9KGkiMgS4BZhsb/PPNPDiJrwc3PAtNOwBk+6FZZ8HOqJ8fb4wiW+XWkUrc89ReWHqxhzJBGD/iTNFFptSqnhzN6HcBnQCnjfG7BCR+sBX+R0gIpEiskREVonIOhF51sk+54nIHBFZISKrRaTIb6H5RFhZGDweGvWCX++DxM8CHZFTG/Ye5+lf1vHID2sY+d1Kt475cfluP0ellCop3Eooxpj1xpj7jDHjRaQyEG2MGVPAYWeBnsaYtkAccJmIdMy1zxPAd8aYdsBg4D0P4w8eYZEw+BtofClMfhCWfBzoiPLo+9b87OfuJoo0u4SLJ06eTff4GKVU8efuKK+5IlJBRKoAq4CxIvJ6fscYS9YatmH2I/cIAANkjYmtCOxxO/JgFBoB138JTfvB1Ifhrw8CHREAp1PT2bjPu5FiniaU+VuSafX0DBZuO+jV9ZRSxZe7t7wqGmOOA1cDY40x7YHeBR0kIiH2CLEDwO/GmMW5dnkGGCoiu4CpwL9dnGe4iCSKSGJycrKbIQdIaARc+zk0uxymPwKL3i2yS5cPD3G6/fZxiVz25nynrxXE02rFi7YdAuCDP7Z7dT2lVPHlbkIJFZFawHWc65QvkDEmwxgTB9QFOohI7tK7Q4Bxxpi6QD/gSxHJE5Mx5iNjTIIxJiEmpuBZ5QEXGg7XjoMWV8KMx2DB2z6/RI9cs+uTxvRn3n97MOW+LvRpUSPHa4u2H/L6OqkFtFCSDp7iwIkzfPDHNmJHTckuWjlvczJ/btFWilKlibsJZTTW/JNtxpilItIA2OLuRYwxR4G5wGW5Xrod+M7eZxEQCVRz97xBLSQMrvkMWl4Nvz8JMx6H9FSfnX7sbR2yn3/3L6suWNWoCFrWrshHNye4nJXvqaiIvCPLD548S2q6lWi6vzqXDs/PYsy0jXn2S/xbly5WqjRxax6KMWYiMNHh6+3ANfkdIyIxQJox5qiIlMW6RfZSrt3+AXoB40SkOVZCCfJ7Wh4ICYWrP7aqFS/6n7WuyqBPoUoDr0639fm+GCBEcs4LqVI+7wjutwa3Y9LKwndJZSUORwnPzXTrWEHnryhVmrjbKV9XRH4SkQMisl9EfhCRvCV0c6oFzBGR1cBSrD6UySIyWkQG2Ps8BNwpIquA8cCtxp2p+8VJSCj0fw2u+xIOb4MPurpdqsXRumcvJTSkDGEhZbInGi59vDfPDWxFo+rRvo4625k07xfoWpLk/a02pVTx4+5M+bHAN8C19tdD7W2XuDrAGLMaaOdk+1MOz9cDnd0NtlhrMQBqt7Nqf/14J2ybA/1egYgotw4v7+TWU0x0hM8X8MrtbK4Wiif5fsFWTShKlSbu9qHEGGPGGmPS7cc4oBj0jgeZSvXglsnQ7RFYPQE+7Ap73JtgGCi5E8qD37ofb00Pa4GlZ2Ry3/gVusqkUsWUuwnloIgMtYcBh4jIUEA/fnojJBR6PAa3/AppKfBJb1j0Hrj45P/pLQlOF+gqKmfTM8jMNNz99TJ+XbWHnz3ol7myneuBAWfTM3hq0lqOnrYGKrw1cwv3f7uSX1btyTEBUylVfLibUIZhDRneB+wFBmGVY1Heiu0CIxZA40tgxqPwzXVwKu8w217Na/DadW2LJKS3BscBMLzruUEDZ9MyefbXdUxds49/j/esHmibOq5XOJi0cg9fLPqbl6ZvAuCNmZuZsnpv9utJB095dC2lVOC5W3rlH2PMAGNMjDGmujFmINYkR1UY5apY5Vr6vQrb/4D3L4Ltc4s8jMQnerPsid5cGVeHpDH9eaxf8+zXlv9zhM8X/e3VeXcfPZ39fM7GA6Tbc1pmbdjP+j3Wba30jExemLohz7HT1u4r8PwpqRl8Mn87GZmGy9+Zzx2fB8myBkqVUoVZsXGkz6IozUSgw51w5yxrNcgvBsLMZyEjreBjfWDxY72oFhVBVRdl9XNXH/ZE1pzIP7cc5LZxS3l7ljV16fbPExm3MAmAict28dG8vLPqX5qed15Lbq/+tonnpmxgypq9rN19nJkb9nsdq1Kq8AqTUHSSgS/VbA3D50L8TfDn6zC2LxxJ8sulhnSoB1iz7Wu46Djv1qTwYy5ioq0klXzSKoE/f6tvZ84fPHkWgIxMzwtYKqV8rzAJpWTNFwkG4eVhwDsw6DNI3gQfXAxrf/D5ZZrVtOpxXpdQz3UooYX51bBk2gMNth2w+kNW/HPUpwuGnU2zEklE6LkaZpmZ+mupVKDk+1dDRE6IyHEnjxOAb2p7qLxaXQN3zYeYpvD9MPhpBKQc8fg0dSqVdbp9477jJI3pT9/WtVweu/PwaZevuStrzsoBPy3StWn/CQBW7Tyave2vHZ4PPvxiUZJbt9iUUvnLN6EYY6KNMRWcPKKNMe5OilTeqBwLt02Dix+G1d/Cux1h41SPThERdu7H+9PdF2U/79mshrPdcziW4n7fSes6FZ1uzzRW/8l3ibvcPpcndtgjwT506INZ8c9RjnnY7/PUpHW8bxe1VEp5r/D3NZT/hIRBryfhztlQvhpMGALf3+50eLEzoQ5rwbc7rzI/jOhE5XJhdIitUuCxvZpXdzvMIR3OY9I9nWlc3Zr1/8b11jBnY2Dop7lXLCic39fvZ/Sv612+/sqMTVz57p9uncsYwzuz3K5xqpQqgLYyioPacXDnHPjzDZj3Cmy3y7a0vNoaJeZCtagINu8/yb09GgHQ/vwqrHiqj1uXjAx1vraKM1WjwmlbrxK//rsL09fu46KGVQHfd5anZWRy5xfW0ODPFuxwuV/SIfdu1z3+81q+WfyPT2JTSmkLpfgIDYfuj8C//oBK51l9K98OhROu52uUsZNNh/oFt0hyq1QubwVjZ766/cLs9Vciw0IY2K5OdvHKVA8X58rPiTNpNH58WqHOcTo1ndhRU5i6Zi8z1+/XZKKUj2lCKW5qtITbZ0LvZ2HL7/DuhbByvNPSLXdcXB+AVi76OPJzlRvlXq5LqEuXxtWQXK2krEQ2d9MBj6/ryrBxSwt9jp2HUwB44/fN3PFF3kmQ3y3dyZFTvluzRqnSRm95FUchodDlAWjWHybdAz/fZQ0vvuJNqHguEXRvWp2kMf29ukQtF/NTxt56AZ0aVmXR9kN0bex8rkpW1818H6zYuPdYCrM3HmBpkuej3HIz9kj3TBd10/77w2o6rqjChOGdCjzXlv0nqF+tPKEh+plMqSyaUIqzao2tkWBLPoZZz1ojwfr8H7S/Nd++FXeUKZP3eMfk1KOp6057Xy6s1enF2T47V5Ztya7rhCUdLLj/ZXvySS55Yx7/6taAyuXCGTNtI2NvvYDuTWPytNaUKk3041VxVyYEOt4FIxZanfeTH4AvBsBh153W3rjhwvPc3le8+K1aOKonAE/0b17Ant6bsGRngfvsO17wnJndR61bZ2t3H+OVGVZxy9vGLfXJCplKFWeaUEqKKvWtkviXvwm7V1iFJv/6AHw00urpK1q4vW8ZLz6lVygbxrYX+nF7l/rZ256atNbj8zhz8mw6M9bty64fVhjGGMYusM6TmZmz/pCu46JKO00oJYkIJNwG9/wF53eG6Y/AZ31g55JCnbZvq5o5ypsUxMndMpd+GNGJq+PrUD48hJAykuOW0RdeVjnOrdXTM/jXl8t8cq6B7y5g9kZrsMGi7YdIdyj18qGTIpdKlSaaUEqiinXhxokw8AM48jd8egl8exMc8mw2eK9m1RnWuT7vD23v0XH5tVCevqIFSWP6c0eX+nx8cwLtz6/C69fFBW3fw+hf12fXHzt+Jo1Vu4759PwLtx7UtV9UiaGd8iWVCMQNgeZXwKL/wYK3YdNUSLjdWoK4fNUCT/HprRd4fWlXbuts3dJ64nL3b6G54+VBbViy4zDfL7PKvPR8bS4P92nKnI2FG7rsOIGyzTO/FepcuRljuOETq5LAwlE9qe2i9ppSxYW2UEq6iCjoPgruWw7thsLSj+HtOJj/mrUEsR/4cpSXO5rUiOK6hHq8em1b+rWuCcD25FPc/fVyJi7zvI6YcTKs+LnJrsu9FHQsWLP8z6Rl5Nh28mx69vOLxvh+NJtSRU0TSmkRXROueAvutvtXZo2Gd9rDym8gM6Pg4z3grIXi7sx7Tz3WrxmfObSkQsoU/lfaWU745E/3Rs3NczH3ZsD/FtDsyekATF69h9hRU2jt4xaPUoGmCaW0iWkKN0yAW6dAVA34eQR82A22zvLZJUKd9Mr/8XAPrydZ5vbCVa0BqFo+nOFdG1K3crl8r+2pwhSMSU13PqrOcQTYvd+scLrPIXvBMKWKK00opVVsF7hjFlzzKZw9Dl9dDV9eBfvWFPrUzjrYK/qwhXLDhefx+4Nd+X1ktzyveTNkObdMYxj1w2q+WJRU6HPl5uqWGMD+4+4nlMOnUhn0/kL2HfPPWjNKeUMTSmlWpgy0HgT3LoVLX4Ddy61VIn8aAcd2++wyT/qoAz4qIpSZI7sC0LhGNFXKh+fZx5sWSvx5lXJ8bQxMWLqTpyat8/hcjpdPSc0gdtSUHEUoH/h2pctjP/dgnsxzk9eT+PcROr44i9vGFm5YuFK+oglFQWgEdLoH7l8JF90La7+Hd+Jh5jNw+rBXpwx3qHHlOFmxMKbdfzGNqkfnu09GPi2ALIsf68WMB7pmfz321g45XjeFuOm1JOnc+3XPN8sBeOync62+/GbTf5uY/0z+7cknSc+wbqn9uOJcwp+zKdmrWJXyNU0o6pyylaHPc3BvIjQfYK2/8mYbmP2cx4llzn+6+zw8d+5mfe/GqK4aFSJpWjOa6Q9czIO9m1CxXBi9m9egWxOr2KUbOcmlrGrFd3yemD0B0hd2Hj5Nz9f+oNHj09iefNJn51XKl/yWUEQkUkSWiMgqEVknIs+62O86EVlv7/ONv+JRHqh8PlzzsVUfrFEva1EvDxNL2TD3Z9bnZ0Db2nxycwKj+jajjo/naTSrWYH7ezcG4JNbEriwgbVuzJIdniXPns3OFcq8vE1tAGZu2O+jKOGGj//i4pfnnLvea3/k2Scz03drzyjlLX+2UM4CPY0xbYE44DIR6ei4g4g0Bh4FOhtjWgIP+DEe5akaLeG6z/Mmlln/V2BiKRduJZRr2xe8rkp+3hocR+8WNbirW0OfzKYffEE9l6+F2Oe/+TPP+iT+d0O7QsVUkIXbDhW4z/EzaV6de/fRFA7rGjDKR/yWUIwlq20eZj9yf4y6E3jXGHPEPsZ39wiU72S5UZseAAAgAElEQVQnlkVWYpn/aoGJJTIshDXP9GHMNW0KdWlfl2TJb4CAt2ublAsP5Y3r2wLu9eF44pdV7lUwDvMy9s5jZtPh+ZleHatUbn7tQxGREBFZCRwAfjfGLM61SxOgiYgsEJG/ROQyf8ajCqlGi3OJpXFva7Z9PoklOjKMEC9GXQ3v2gCAUX2bFTrkLElj+pM0pj/lI1xXG3JnhNjaZy/l8X55S+w3irEGC2RkGJ/MJ9m47zixo6Zw33jnc1ZyW7XzqEfnd7xFlq63y5SP+DWhGGMyjDFxQF2gg4i0yrVLKNAY6A4MAT4RkUq59kFEhotIoogkJifriJaAq9ECrh1n3QrLTiytrdn3Xo4Kc/RYv+YkjenPXd0aFj5WDxTUd3J5m1pERYRyZ9cGLLDXb8mSlTinrtlL++c8+8R/ftVyebZd9uZ8j85xm4slkj+et50/c83en752Lw0em5pd9FIpXymSUV7GmKPAXCB3C2QXMMkYk2aM2QFswkowuY//yBiTYIxJiIlxvuysCoAcieUSmP+6TxNLUftru+u+ig6xVXhnyLm+ktyDDrISiuNwXnd9fluHgncqwFkXM/Sfn7qBoZ8uJjPTsP/4GY6dTuOur5YX+npKOeO3asMiEgOkGWOOikhZoDfwUq7dfsZqmYwTkWpYt8B0UYniJiuxdF0P8162EsviD621WTreAxVqFVkoZcNCSEnzrjbZIRed07Me6kb9quVz9OeEh+b8LObNrb0s5SJ8MyIuP+//sS17dcnCMsYE7XIDKrD82UKpBcwRkdXAUqw+lMkiMlpEBtj7zAAOich6YA7wH2NMwUNaVHDKSix3L4Iml8Kid+GtNjDpXji4pUhCcEwmV8bVZtsL/dj2Qj+vzxcdGUrDmCjK5EoYYSHW11F2n4y3NcT+3bNRjsXLtiWfdGsujad8lUwmrdxN/UensvPwaZ+cT5UsfmuhGGNWA3nGUxpjnnJ4boCR9kOVFNWbw6DPoOeT1losK76yHs0vh84PQl3PFuzyxle3X0jnRlUL/Ul6usOMekcRoSE83KcJl7SwyuW700IZ0qEe4+117Wc/1I335m7j/l6Nc3SK93IyxySY3D/BKh2z5cAJ6lXJ2/ejSjedKa/8p0p96P8aPLAWLn4IdsyDT3rCuMth68zCTUl3oZ1dl6tu5bIeJ5O6lXNOnLytc2y+kynv7dmYpjXzLwXjyLEETYOYKF69ti2hIWW8HvKb26GTZ5mz6QD/mbiKPm/8kV2mxR98sUyAKnl0xUblf1Ex0OtJ6PIALBsHi96Dr66Bmq2h8wPQYiCE+OZX8YOh7Zm6Zi+x1cp7fOyTl7fIsfb8Y06GB7uSX24Ugd8f7Eaj6lH8q2sDKpTNWXnZ2+4XEXjksmaMmbYRgJs+XcJ6hzL5SYd8d1vq0MmzvDN7a/bX787eSp1KkQXWVlOli37MUEUnIhou+jfcvwqufBfSzsAPt1uFKJd87JMVJGtUiMxeZthT1aIisp+fX7WcRy2HkBDnWaFD/SrseLE/japHAfBov+bc06NRjn28vS3XpVG1HK2erQdy1vjq/brnt89+WbWHcQt2MHtjztIxd3+9nHEO1ZCXJB2m9+vzmLJ6r8fXUCWXtlBU0QsNt5YjbnuDtc79n2/A1Idh7hjoeBdccIdVqLKIVXRoOYy8pIlHx4Y5SSgT7+rEBbFVCh2XKx/dlJAj6aX64BaX40TKJ/o3p1uTGC55Y57L/e/5ZjmXtuzrdpWBI6dSOZ2W4fO6bCo4aAtFBU6ZMlZH/R0zrRUka7ezClC+0QqmPGStz+KHfhZXsloRYI3u8kT16Mg82/yZTADKhvt2uPGRXMOmn5uygT82FzyR2NVM+w17j7PnaM5W50VjZtN5zGyOnk5l2d9HvA9WBSVtoajAE7FWkIztYq0YufAda1TY0k+geguIuxHaXG/1xRSR9uf5Nxl4qnp0BAdO+HeJ4M4vzc6z7bkpGwo8LtNF0u/7ljXbP2vp59T0zOxh3Z3HzOZUagY7Xuync1pKEG2hqOBSszVc/RE8tAn6vw5hZeG3x+H1ZjD+Btg4BTK8q6zrjg9vak+LWhU8bqH4w38ubZr9/PcHuzFzZDdmPNA1zwqTvnI61bsJoRlu1AIb/et6mjwxLfvrU/a1irABqopA4P/XKOVM2Upwwe3W48AGWPk1rPoWNk2B8jFWiyXuRmsypQ9d2rIml7asWahzzH6oGw1iogresQD39GjEoPZ1KRceQnRkGBXLWX08E++6yGWrwF3/6taAD/84V5Tijs8TvT5XQQklv5phmcZQBm2hlBTaQlHBr3pzayXJkethyASodyEs/gDe7wQf9bBujaUEz/14XySTLDUqRBIdmXOYcUgZKdTclRHdG/JAryZMGN6RtvWs1k5hFgTL3YdyOjWdT+a7V0HJ3UrHf2xO5kxaBolJh3lnVtFUXVCe0xaKKj5CwqBpX+txMhnWfAcrvrY68Kc/ZnXwx90IDbpDGf/Xx8otq68gWO14sR/GkF1GpmODqlzUsKrHpe9zy91CeXjiKqau2efWse4klA17j3PLZ0sYfEE9Jiy1Kg38u1eeGrIqCGgLRRVPUTHQ6R4YsQCGz4X4m2HrLPjqanijJUwbBTuXFtub9G8P8f0qkCKSpyZZiA86xI+cTuXk2XRiR01hW/JJt5MJwFk3CnlmrSiZlUxU8NIWiireRKzhxrXbWbfFNk2FNd9D4qew+H2oeB60HAitroZacdb+QSgqIpSTZ9MBaBhTngFta/vs3CO6N6SZixIxh04VfuTYkh2Hs9dc8bQW2VuztjD6ytzLJMHnC5M4fCqValHhPDlpXaFjVEVDWyiq5AiLtBLHkG/gP1th4PsQ0xT+eg8+6g5vt7PWatm3NuhaLgse6Un/1laZ/8KuoBgdGcqch7vTIMYqP/PIZc24Mq6O03037TtRqGsBbNx3wuuhv2t3H8uz7ejpVJ7+ZR1vzdriMpk8PHGV29fYcfAUv6/3vo9IuU9MkP3HKkhCQoJJTPR+RIoqhU4fhg2/wrofrQKVJhOqNYGWV1sJKKZpwecoAmfSMmj25HTeGhznMgE4k3sU1YonL6Fy+XAOnjzLpn0n6NyomtvHuuO1a9vykAd/0AuSNKY/q3YepXGNKMqGhVD/0aluHffDiItof37BFRWyvsfSPudFRJYZYxL8eQ1toaiSr1wVaH8L3DwJHtpsVUAuXx3+eAne7QDvd4Z5r8LhwK7tFhkWQtKY/h4lk9wuaVGDyuXDAas2WX7JxFvnVy1H0pj+OVawLIzjZ9K48t0F/PubFS5XnnRm4VbrNtvBk2c5etr54miOlv9TuMEHqmCaUFTpEhVj1Qq7bQqM3ACXjYGwcjD7/6xbYh/1gMTP4Mzxgs8VhN6/Md4v5136eG/evSEeEWhi98cUdi5MlqveXQDArI0HPLqVNX7JPwAkPDeTuNG/F7j/dR8uYvSv61m47SDHz/hvcmxppp3yqvSqUAs6jrAeR3fCup9g1QSY/CDMeMK6Hdb+VqjTPmg783Nzt0ijJ269KJaY6Aj6t6lF/zbnhkb7KqFsSz6V/XyyB9WL9xw749F1MjINny3YwWcLdgDBP8y7ONKEohRApXrQ+T6rvP7uZbBsLKz9AVZ8CTVaQfwt0OY6awZ/KXJfr8YuKy/vOuz5cgO1Kkay18NE4I3MTMPptAyed6MWGVi33Y6dTtNVKAtJE4pSjkSgboL1uPRFWPs9LPscpv0Hfn8SWl5lJZfzOgZdq8WbhbreHtIuR8l6Rx3qV8m3jP/Xi//x+HqLHu3F0qTDJB08xX++X+3x8e5q8Jh7HftZ+r01n11HUpy2WpIOnqJ2pbKEh2oPQUE0oSjlSmQFSBhmPfashOWfw+qJsGo8VGtqdfS3HWJ1+geBKfdd7PExA9rWzpNQWtWpwIhujejfpla+x1YpH86+4+63Nr68vQNglfVvVbuizxPKnV8kEhMdQZoHHftZdh2xWlurdx0lPdNQr3I5pq/dS/KJs7xtr1Q566FuNHSjrE5GpmHguwu4v1djereo4XEsxZkmFKXcUTvOevR5Dtb+aCWXGY/BzGeg+QArucReHJBWy1+P9uJYSppH69u7Mv7OjnRqWNWtfR2XG87P7w92pXGNnLGFeLvucS4pDhWSfTHXZMD/Frh87ZP5O3jx6tYFnuPkmXTW7D7GHV8klrp+Gk0oSnkivDzE32Q99q+D5V9YLZa130OVBlYV5NbXQtWGRRZSzYqR1KyYd4EvT3n6x+9fXRvw4TznQ627N41h7qZkWtSqkCeZAIT6KKE0f2q6T87jjvFL/mHL/hN8c2fHfG9/icNLN326mAtiq7D/+Bn6t6nFRQ2rsXb3MWpWjMyx5HRJoQlFKW/VaAl9X4Lez8D6X6wO/LljYO6LUDve6sRveTVEl8zbHlERrv98fHhTe4Z+sphRfZs7fT13TbFA8GZSZ+LfR/h4/naOpaRx4ky60xaLcbjjNn/LQebbZWm+XvwPSWP6c/k7f1KjQgRT7ruYCUv+4Z4ejQqccJl84izpmZnUqhjcSydrL5NShRVWFtpeD7dOhgfXWbfFMtNh+ihrYbAvBlpVkc/kLTNSnN3aOTbPtuiIUD66qT0RoSFMvMu9mewF+fqOCwt9Dl96ZcYmPpq3PXsezJXvLuDH5bsAqwO/y8t5V77Mbf/xszw8cRWv/raZ5f8cISU1g7ELdpDpouzOBc/PpNOLBZ830LSFopQvVaxjDT2+6N+QvAnWTLQek+625rc0vcy6Jda4D4QW71se0ZFhhJSR7PL13ZrE8PmwDj4596R7Omev1QLw/FWtePyntQBcEFuZpUnBsf5N37fms2HvcUbuPMrV8XXp/urcfPd/22Etl1N2MdDtyacY9cMathw4Sc0KkfRtnf9giGCmCUUpf4lpCj2fgB6Pw65EK7Gs+xHWT4KIitBigHVb7PzOAVm/Jct3/+rEpn3eVQbISihfDOtA1yYxPolnWOf6OZIJQKcG1kCBzo2q8ub17bjg+Zk+uVZhbXAYmPDunK0F7v/675uzn2clRcfRbjPW7aNv61ocOH6GhdsOYTCs3V18qjZocUililJGOuyYaw0/3jgZUk9CdC1odQ20HhTUJfad+XJREk/9so6tz/fzeORW7j6MsmEhpKRlsH70pZQLz/tZ1xiT3dfgTf9HcfHfy5ry8vRNTl8rzKixoigOqS0UpYpSSCg06m09Uk/D5mnW+i2LP4RF/7NGirW6xnpUd96hHUxu6hTLTZ1ifXKuK9rW4rvEXS4TU2mpFOwqmYA1TLpseOBaswXxW6e8iESKyBIRWSUi60Tk2Xz2HSQiRkT8mj2VCirh5azEMWQ8PLwZrngbKtaD+a/Bex3hvYuCogpyUXn+qtYse6I3EaH++4NZo0Lx7rdatSu4Kyb7s4VyFuhpjDkpImHAnyIyzRjzl+NOIhIN3Acs9mMsSgW3rBL77W+BE/utfpa1P1hVkGf/nzUMudU1VsHKCr5bzTGYhIWUoaqf5mbMfqgbx1LSqFO5LB2en+WXaxSFckHcOgE/JhRjdc6ctL8Msx/OOmz+D3gZeNhfsShVrETXgAuHW4+j/1hVkNf+AL89Dr89AedfZCWWFgOhvO/XOylJStqiWr6qMOAvfp2HIiIhIrISOAD8boxZnOv1dkA9Y8xkf8ahVLFV6TzofD/8ax7cuwx6PAanDsKUh+DVJvDlVdYcl5TgGEbriaw/ju/fGM8YN0qauGPds5cy/789ABjRvaHTZPLxzcX3znqFyLBAh5AvvyYUY0yGMSYOqAt0EJFWWa+JSBngDeChgs4jIsNFJFFEEpOTk/0XsFLBrFoj6PZfuGcxjFhoJZpD26w5Lq80gnGXw6L34PCOQEfqlmeuaAFAn5Y1GdzhvEKf76nLW1A+IpR6Vcqx9fm+/PdS50s7X1JAwcZJ93QudCz+snJncPehFNmwYRF5GjhljHnV/roisI1zt8VqAoeBAcYYl+OCddiwUg6Mgd3LrSHIm6ZBsr3+R0xzaNoXmvazFggrU7KKYjzzyzrGLUzil3s7Zxd09OT2lrNhx6ue7kPFsmFOX+/aJIZ5m4Pjw6y3Q4eL9ZryIhIjIpXs52WB3sDGrNeNMceMMdWMMbHGmFjgLwpIJkqpXESgbnvo/TTc8xfct8Jax6V8NVjwFnzaG15rCpPuhY1TraHKJcAzA1qy4slLaFO3ElXLhwOeDSte+dQlebaVz9XhfXW7OtnPvxjWgfWjL+XJy1t4HGvDmPIeH1Nc+XOUVy3gcxEJwUpc3xljJovIaCDRGPOLH6+tVOlUpQF0utt6pByBLTNh01Rr1NiKLyE0Ehr0sFovTS4r1oUrK9uJZOr9F/P3Ic8SZaVy4VwdX4dWtSsyevJ6IGeH94bRlxEeWoZbLorlWIq1/ny58NA8SccdM0d2497xK5jiwfLGxZXOlFeqNEhPhb8XWLfFNk2DY/Zqi3USrBIwcTfqiDE3zNqwn9s/z/n35+s7LuTGT1zPekga05+VO48y8F3r1pxjLbL1oy+lxVMzPIqhVN7yUkoFkdBwaNgD+r0MD6yGuxZAjyfAZMDvT8HrzeGHO+DvRVa/jHKqZ7PqjL31Ara90I8mNazVGzs3qpbjj/ywzvX57NYElj7emzXP9AGgtr1ezX8va8rEuy4CoN15lSgXHkq/1jWzj60QGZq9X3GkLRSlSrsDGyHxM2uhsLPHoXoLa9njNtdbyyArp46lpHHg+JnsBcSyOvIn/7sLrepUzLP/iTNpREWEIiIcPZ1KZFgIkWEhOY69Mq42k1buYebIrgx8dyEnz6bTrGY0G/edACA8pAybn+/rVbxF0ULRhKKUsqSesuqKJX4Ke1dBWHmrGvIFt0NN38wTKcl2H03hp+W73FowK7e40b9xR5f63N6lAQu3HaRX8xqcSctg8/4TtKlbiVNn0+n12h/MfKhbvgub5UcTihOaUJTys6yhyImfWjP0089A3Q5WYmkxEMIKv9ywKnqaUJzQhKJUETp92LoVlvgZHNoKZStbHfgJw6Bqw0BHpzygCcUJTShKBYAxsGOe1WrZMNnqzG/QAxJus0rxh5eeuRbFla6HopQKDiLQoJv1OL7XmtOybBx8d7M1t6V+V2hyKTS+FCrVC3S0KkC0haKU8k5GOvz9J2yeYc1tOWLXEKvRypo02eQyqBMf0OWN1Tl6y8sJTShKBSFj4OAW2DzdSjD/LLJui5WrBo37QNPLrFtkOgw5YPSWl1KqeBCBmCbWo/N9Vmf+ttlWgtk0FVZ9A2XCILbzudZLlfqBjlr5mLZQlFL+lZEOOxfbrZfpcHCztb1aU2jUy2q5xHbWjn0/01teTmhCUaqYO7QNtvxmJZe/F0HGWQgJh3oXWuVhGvSAWnElruR+oGlCcUITilIlSFqK1d+ybTZsmwv711jby1axRpQ17GklGB05Vmjah6KUKtnCylpJo2FP6+uTB2D7XDvBzIF1P1nbqza2Wi8Ne0JsF4iIDljIyjVtoSilgpMxkLzxXHJJ+hPSU6BMKNS9AM7raN0aq9UWKsdaAwOUS9pCUUqVXiJQvbn16HQPpJ+1Ove3zbZaMQvfgcx0a9/ISlZiqR1nJZnacVC5viaZIqYJRSlVPIRGWDPy63e1vk4/C/vXwd6VsGelVSF50XuQaa2wSGRFK8lktWJqt7OSjHb2+40mFKVU8RQaYc3ErxN/blt6KhxY75BkVsLiDyAj1Xo9ooKVXGq0gmqNoFoT6xFVQ1szPqAJRSlVcoSGW7e7asdBe3tbeiokb7BaMFlJZvkXkHbq3HHh0VCt8blH1cZWoqnSQMv1e0ATilKqZAsNt299tYX4m61txsDxPdYky0NbrX8PboakBbD623PHShmodN65BFOtkXXbrFwVq5R/2SrWhExt3QCaUJRSpZEIVKxjPRr2yPla6ik7yWyxH5vh0JZzo8xyKxNmJZfsJOPiUa6K1a8TGmlN5AwJs/91eF4mrFj38WhCUUopR+Hlz7VoHGVmwvFdcPQfSDmS83H68LnnR/+xbq+lHIG0055fv0yo64TT7RFoPcg336cfaEJRSil3lLFvf1U6z/1j0s7kTDxnjloDBDLS7H89fF62sv++Px/QhKKUUv4SFglhtaBCrUBHUiSK7806pZRSQUUTilJKKZ/QhKKUUsonNKEopZTyCb8lFBGJFJElIrJKRNaJyLNO9hkpIutFZLWIzBKR8/0Vj1JKKf/yZwvlLNDTGNMWiAMuE5GOufZZASQYY9oA3wMv+zEepZRSfuS3hGIsJ+0vw+yHybXPHGNM1syfv4C6/opHKaWUf/m1D0VEQkRkJXAA+N0Yszif3W8HpvkzHqWUUv7j14mNxpgMIE5EKgE/iUgrY8za3PuJyFAgAejm7DwiMhwYbn95UkQ2eRlSNeCgl8cWBY2vcDS+wtH4CifY42vq7wsU2RLAIvI0cMoY82qu7b2Bd4BuxpgDfo4h0d9LYBaGxlc4Gl/haHyFo/H5d5RXjN0yQUTKAr2Bjbn2aQd8CAzwdzJRSinlX/685VUL+FxEQrAS13fGmMkiMhpINMb8ArwCRAETxVpP4B9jzAA/xqSUUspP/JZQjDGrgXZOtj/l8Ly3v67vwkdFfD1PaXyFo/EVjsZXOKU+viLrQ1FKKVWyaekVpZRSvmGMKRUP4DJgE7AVGOXH69QD5gAbgHXA/fb2Z4DdwEr70c/hmEftuDYBlxYUM1AfWAxsAb4Fwj2MMQlYY8eRaG+rAvxun/N3oLK9XYC37RhWA/EO57nF3n8LcIvD9vb2+bfax4oHsTV1eI9WAseBBwL5/gGfYc2lWuuwze/vl6truBnfK1iDYFYDPwGV7O2xQIrD+/iBt3Hk9726EZ/ff55AhP31Vvv1WA/i+9YhtiRgZQDfP1d/U4LmdzD7PN780SxuDyAE2AY0AMKBVUALP12rVtYPEIgGNgMt7P9ADzvZv4UdT4T9H2ObHa/LmIHvgMH28w+AER7GmARUy7Xt5az/pMAo4CX7eT+sCacCdAQWO/yibbf/rWw/z/qFXgJ0so+ZBvQtxM9tH3B+IN8/oCsQT84/OH5/v1xdw834+gCh9vOXHOKLddwv13k8isPV9+pmfH7/eQJ3Y//BBwYD37obX67XXwOeCuD75+pvStD8DmbH6s1/9OL2sN+oGQ5fPwo8WkTXngRcks9/oByxADPseJ3GbP/AD3Luj0WO/dyMKYm8CWUTUMvhF3iT/fxDYEju/YAhwIcO2z+0t9UCNjpsz7Gfh3H2ARbYzwP6/pHrD0lRvF+uruFOfLleuwr4Or/9vInD1ffq5vvn959n1rH281B7P6et5XzeFwF2Ao0D+f7lulbW35Sg+h00xpSaPpQ6WL8UWXbZ2/xKRGKxRrpllZy5166s/JmIZC0O7So2V9urAkeNMem5tnvCAL+JyDK7CgFADWPMXgD73+pexlfHfp57uzcGA+Mdvg6W9w+K5v1ydQ1PDSNnWaP6IrJCRP4QkYsd4vY0jsL+v/L3zzP7GPv1Y/b+nrgY2G+M2eKwLWDvX66/KUH3O1haEoo42Wb8ekGRKOAH4AFjzHHgfaAhVuXlvVjN6Pxi83S7JzobY+KBvsA9ItI1n30DER8iEg4MACbam4Lp/ctPUMUjIo8D6cDX9qa9wHnGmHbASOAbEangZRyFib0ofp6+eG+HkPNDTcDePyd/Uzw9r99/B0tLQtmF1bGVpS6wx18XE5EwrB/818aYHwGMMfuNMRnGmEzgY6BDAbG52n4QqCQiobm2u80Ys8f+9wBWh20HYL+I1LLjr4XVSelNfLvIWTXa2/e6L7DcGLPfjjVo3j9bUbxfrq7hFhG5BbgcuNHY9yyMMWeNMYfs58uw+iWaeBmH1/+viujnmX2M/XpF4LA78TkcczVWB31W3AF5/5z9TfHivH7/HSwtCWUp0FhE6tuffAcDv/jjQmJN+f8U2GCMed1hey2H3a4Csopk/gIMFpEIEakPNMbqIHMas/2HYQ4wyD7+Fqx7qu7GV15EorOeY/VTrLXjuMXJOX8BbhZLR+CY3fSdAfQRkcr27Yo+WPeu9wInRKSj/V7c7El8DnJ8MgyW989BUbxfrq5RIBG5DHgEq6zRaYftMXb1CkSkAdb7td3LOFx9r+7EVxQ/T8e4BwGzsxKrm3pj9S1k3w4KxPvn6m+KF+f1/+9gQR1AJeWBNfJhM9Ynisf9eJ0uWM3F1TgMiQS+xBqWt9r+IdVyOOZxO65NOIyIchUz1kiXJVhD/CYCER7E1wBrhMwqrCGIj9vbqwKzsIYHzgKq2NsFeNeOYQ3WgmhZ5xpmx7AVuM1hewLWH4htwP/wYNiwfXw54BBQ0WFbwN4/rMS2F0jD+jR3e1G8X66u4WZ8W7Hul+cY3gpcY//cVwHLgSu8jSO/79WN+Pz+8wQi7a+32q83cDc+e/s44K5c+wbi/XP1NyVofgezHjpTXimllE+UllteSiml/EwTilJKKZ/QhKKUUsonNKEopZTyCU0oSimlfEITigo6IpIhIitFZJWILBeRiwrYv5KI3O3GeeeKSNCu+R0IIjJORAYVvKdSBdOEooJRijEmzhjTFqsA4IsF7F8Jq7JsUHKYxa1UiaYJRQW7CsARsGoZicgsu9WyRkSutPcZAzS0WzWv2Pv+195nlYiMcTjftSKyREQ2i13YT0RCROQVEVkqVrHCf9nba4nIPPu8a+VcIcBsIpIkIi/Z51wiIo3s7eNE5HURmQO8JCJVRORn+/x/iUgbh+9prB3rahG5xt7eR0QW2d/rRLHqOCEiY0Rkvb3vq/a2a+34VonIvAK+JxGR/9nnmIL3BSeVykM/OalgVFZEVmLNdK4F9LS3nwGuMsYcF5FqwF8i8gvWOg2tjDFxACLSFxgIXGiMOS0iVRzOHWqM6SAi/YCnscpr3I5Vns9wFJ8AAAL7SURBVOICEYkAFojIb1h1nGYYY563y22UcxHvcfucNwNvYtXPAqvGU29jTIaIvAOsMMYMFJGewBdYhRGftK/d2o69sv29PWEfe0pEHgFGisj/sMqUNDPGGBGpZF/nKayFqHY7bHP1PbXDWsSsNVADWI+1wJRShaYJRQWjFIfk0An4QkRaYZWUeEGs6siZWCW2azg5vjcw1tg1rIwxjgUBswrrLcNa2wKsmkZtHPoSKmLVaFoKfCZWYb6fjTErXcQ73uHfNxy2TzTGZNjPu2CV7cAYM1tEqopIRTvWwVkHGGOOiMjlWAsoLbBKKxEOLMJavfIM8InduphsH7YAGCci3zl8f66+p67AeDuuPSIy28X3pJTHNKGooGaMWWR/Yo/Bql8UA7Q3xqSJSBJWKyY3wXX57bP2vxmc+/0X4N/GmBl5TmQlr/7AlyLyijHmC2dhunh+KldMzo5zFqsAvxtjhjiJpwPQCysJ3Qv0NMbcJSIX2nGuFJE4V9+T3TLTekvKL7QPRQU1EWmGtfzrIaxP2QfsZNIDa2lggBNYS6Nm+Q0YJiLl7HM43vJyZgYwwm6JICJNxKrKfL59vY+xqr3Guzj+eod/F7nYZx5wo33+7sBBY61p8RtWYsj6fisDfwGdHfpjytkxRWEVzJwKPIB1ywwRaWiMWWyMeQqrnHs9V9+THcdgu4+lFtCjgPdGKbdpC0UFo6w+FLA+ad9i90N8DfwqIolYFVc3AhhjDonIAhFZC0wzxvzH/pSeKCKpwFTgsXyu9wnW7a/lYt1jSsbqg+kO/EdE0oCTWGW9nYkQkcVYH9DytCpszwBjRWQ1cJpzJcGfA961Y88AnjXG/CgitwLj7f4PsPpUTgCTRCTSfl8etF97RUQa29tmYVXCXe3ie/oJq09qDVbl3j/yeV+U8ohWG1aqEOzbbgnGmIOBjkWpQNNbXkoppXxCWyhKKaV8QlsoSimlfEITilJKKZ/QhKKUUsonNKEopZTyCU0oSimlfEITilJKKZ/4f1jUpczdPVjPAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "learn.recorder.plot_losses()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Classification" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "exp.drop_mult=0.5\n", + "learn=exp.create_cls_learner(data_clas)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "learn.true_wd=True" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load_encoder('enc_best')\n", + "learn.freeze()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 03:53\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.264060 0.178016 0.934240 (03:53)\n", + "\n" + ] + } + ], + "source": [ + "learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 04:21\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.261168 0.164097 0.940720 (04:21)\n", + "\n" + ] + } + ], + "source": [ + "learn.freeze_to(-2)\n", + "learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 05:39\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.222833 0.151485 0.944720 (05:39)\n", + "\n" + ] + } + ], + "source": [ + "learn.freeze_to(-3)\n", + "learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total time: 13:48\n", + "epoch train_loss valid_loss accuracy\n", + "1 0.210751 0.151522 0.948720 (06:54)\n", + "2 0.181168 0.161364 0.948640 (06:53)\n", + "\n" + ] + } + ], + "source": [ + "learn.unfreeze()\n", + "learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "learn.save('best_cls')" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "learn.load(\"best_cls\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:fastaiv1]", + "language": "python", + "name": "conda-env-fastaiv1-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 978bbcc92347983360646087423c7509105715a4 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 8 Dec 2018 23:10:31 +0100 Subject: [PATCH 26/41] Fix issue with finetuning language model (it wasn't freezed) --- ulmfit/pretrain_lm.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index b2483cf..a4b04a1 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -75,7 +75,7 @@ class LMHyperParams: # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) # for training on smaller datasets, more dropout is necessary drop_mult = 0.1 - dps = [0.25, 0.1, 0.2, 0.02, 0.15] + dps = (0.25, 0.1, 0.2, 0.02, 0.15) clip: float = 0.12 bptt: int = 70 bs: int = 70 @@ -131,7 +131,7 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, true_wd=False, drop_mult=0.1): + def train_lm(self, num_epochs=20, data_lm=None, true_wd=False, drop_mult=0.1, lr=5e-3): data_lm = self.load_wiki_data() if data_lm is None else data_lm learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) @@ -145,17 +145,20 @@ class LMHyperParams: if self.pretrained_fnames or self.pretrained_model: print("Training lm from: ", self.pretrained_fnames or self.pretrained_model) if learn.true_wd: + learn.freeze_to(-1) learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7)) learn.unfreeze() learn.fit_one_cycle(num_epochs, 1e-3, moms=(0.8, 0.7)) else: + learn.freeze_to(-1) learn.fit_one_cycle(1, 1e-2, moms=(0.8, 0.7), wd=1e-7) # TODO Fix the learning rates learn.unfreeze() learn.fit_one_cycle(num_epochs, 1e-3, moms=(0.8, 0.7), wd=1e-7) else: print("Training lm from random weights") - if not learn.true_wd: learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7), wd=1e-7) - else: learn.fit_one_cycle(num_epochs, 5e-3, (0.8, 0.7)) # TODO find proper values + learn.unfreeze() + if not learn.true_wd: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7), wd=1e-7) + else: learn.fit_one_cycle(num_epochs, lr, (0.8, 0.7)) # TODO find proper values learn.save("lm_best_with_opt", with_opt=False) learn.save_encoder(ENC_BEST) learn.save(LM_BEST, with_opt=False) From 591393b96c21a6a68d6da166bc391237dee9102e Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sat, 8 Dec 2018 23:48:02 +0100 Subject: [PATCH 27/41] Change AvgPooling to BiPooling as a default --- fastai_contrib/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index 57ccc47..0c328a7 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -145,7 +145,7 @@ def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_s bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), AvgPoolingLinearClassifier(layers, drops)) + model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), BiPoolingLinearClassifier(layers, drops)) model.reset() return model From 23386185636820a9d6593b0b7f939f8ade5afde9 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sun, 9 Dec 2018 00:39:27 +0100 Subject: [PATCH 28/41] Make biclasifier head a hyperparameter. --- fastai_contrib/learner.py | 10 +++++++--- fastai_contrib/models.py | 8 ++++++-- ulmfit/train_clas.py | 15 +++++++++++---- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index bdcea7c..ea2588f 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -30,18 +30,22 @@ def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:in def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400, nh: int = 1150, nl: int = 3, lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1, - drop_mult: float = 1., qrnn: bool = False, **kwargs) -> 'TextClassifierLearner': + drop_mult: float = 1., qrnn: bool = False, bicls_head:str='BiPoolingLinearClassifier', **kwargs) -> 'TextClassifierLearner': "Create a RNN classifier." dps = default_dropout['classifier'] * drop_mult if lin_ftrs is None: lin_ftrs = [50] if ps is None: ps = [0.1] ds = data.train_ds vocab_size, n_class = len(data.vocab.itos), data.c - layers = [emb_sz * 3] + lin_ftrs + [n_class] + if bicls_head == 'BiPoolingLinearClassifier': + count = 3*2 + else: + count = 3 + layers = [emb_sz * count] + lin_ftrs + [n_class] ps = [dps[4]] + ps model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token, layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3], - qrnn=qrnn) + qrnn=qrnn, bicls_head=bicls_head) learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs) return learn diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index 0c328a7..b5de84c 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -138,14 +138,18 @@ def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, t def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False, - hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5)->nn.Module: + hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, bicls_head:str='BiPoolingLinearClassifier')->nn.Module: "Create a RNN classifier model." fwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir, qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p) - model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), BiPoolingLinearClassifier(layers, drops)) + head = BiPoolingLinearClassifier + if bicls_head == 'BiPoolingLinearClassifier': head = BiPoolingLinearClassifier + elif bicls_head == 'AvgPoolingLinearClassifier': head = AvgPoolingLinearClassifier + + model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), head(layers, drops)) model.reset() return model diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 5581498..139f8a4 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -44,6 +44,8 @@ class CLSHyperParams(LMHyperParams): # dir_path -> data/imdb/ use_test_for_validation=False + bicls_head:str = 'BiPoolingLinearClassifier' + def __post_init__(self, *args, **kwargs): super().__post_init__(*args, **kwargs) self.dataset_dir=self.dataset_path @@ -92,7 +94,10 @@ class CLSHyperParams(LMHyperParams): fastai.text.learner.default_dropout['language'] = dps or self.dps trn_args=dict(drop_mult=self.drop_mult, bptt=self.bptt, clip=self.clip,) trn_args.update(kwargs) - classifier_learner = bilm_text_classifier_learner if self.bidir else text_classifier_learner + classifier_learner = text_classifier_learner + if self.bidir: + classifier_learner = bilm_text_classifier_learner + trn_args['bicls_head'] = self.bicls_head learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID, path=self.model_dir.parent, model_dir=self.model_dir.name, qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args) @@ -116,7 +121,6 @@ class CLSHyperParams(LMHyperParams): lm_val_df = lm_trn_df[:val_len] if use_test_for_validation: - val_len = max(int(len(tst_df) * 0.1), 2) val_df = tst_df cls_cache = 'notst' else: @@ -126,8 +130,11 @@ class CLSHyperParams(LMHyperParams): cls_cache = '.' if self.tokenizer is Tokenizers.SUBWORD: - #TODO Fix me to make sure it trains correct dictionary - args = get_sentencepiece(self.dataset_path, self.dataset_path / 'train.csv', self.name, vocab_size=self.max_vocab) + args = get_sentencepiece(self.dataset_path, self.dataset_path / 'train.csv', + self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[]) + if self.tokenizer is Tokenizers.SUBWORD: + args = get_sentencepiece(self.dataset_path, self.dataset_path / 'train.csv', + self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[]) elif self.tokenizer is Tokenizers.MOSES: args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[])) elif self.tokenizer is Tokenizers.MOSES_FA: From 34b1d18600ac58ef3a4994e652902aa1f9b21d1e Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sun, 9 Dec 2018 22:42:17 +0100 Subject: [PATCH 29/41] load wikipedia as articles for v & fv tok. --- ulmfit/pretrain_lm.py | 35 ++++++++++++++++++++++++++--------- ulmfit/train_clas.py | 2 +- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index a4b04a1..278ec43 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -57,6 +57,22 @@ class Tokenizers(Enum): # Tok.FASTAI: FastaiTok # } +def istitle(line): + return len(re.findall(r'^ = [^=]* = $', line)) != 0 + +def read_wiki_articles(filename): + articles = [] + with open(filename, encoding='utf8') as f: + lines = f.readlines() + current_article = '' + for i,line in enumerate(lines): + current_article += line + if i < len(lines)-2 and lines[i+1] == ' \n' and istitle(lines[i+2]): + articles.append(current_article) + current_article = '' + articles.append(current_article) + return pd.DataFrame({'texts':np.array(articles)}) + @dataclass class LMHyperParams: dataset_path: str # data_dir @@ -175,7 +191,7 @@ class LMHyperParams: pretrained_fnames=self.pretrained_fnames, pretrained_model=self.pretrained_model) trn_args.update(kwargs) - print ("Training args: ", trn_args, "dps: ", dps) + print ("Training args: ", trn_args, "dps: ", dps or self.dps) learn = lm_learner(data_lm, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, pad_token=PAD_TOKEN_ID, bias=True, qrnn=self.qrnn, model_dir=self.model_dir.relative_to(data_lm.path), **trn_args) # compared to standard Adam, we set beta_1 to 0.8 @@ -230,6 +246,7 @@ class LMHyperParams: valid_ids=val_ids, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type) elif self.tokenizer is Tokenizers.MOSES_FA: + try: data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs) print("Tokenized data loaded") @@ -238,10 +255,10 @@ class LMHyperParams: # wikitext is pretokenized with Moses pretokenized = Tokenizer(tok_func=BaseTokenizer, lang='en', pre_rules=None, post_rules=None) - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), - valid_df=read_file(val_path), tokenizer=pretokenized, - test_df=read_file(tst_path), classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab, bs=self.bs) + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), + valid_df=read_wiki_articles(val_path), tokenizer=pretokenized, + classes=None, lm_type=self.lm_type, + max_vocab=self.max_vocab, bs=self.bs, text_cols='texts') data_lm.save('.') elif self.tokenizer is Tokenizers.FASTAI: try: @@ -249,10 +266,10 @@ class LMHyperParams: print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_file(trn_path), valid_df=read_file(val_path), - test_df=read_file(tst_path), classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab,bs=self.bs, - ) + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), + valid_df=read_wiki_articles(val_path), + classes=None, lm_type=self.lm_type, + max_vocab=self.max_vocab,bs=self.bs, text_cols='texts') data_lm.save('.') else: raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 139f8a4..b1c430b 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -104,7 +104,7 @@ class CLSHyperParams(LMHyperParams): return learn def load_cls_data(self, bs, **kwargs): - if self.dataset_dir.name == 'imdb': + if 'imdb' in self.dataset_dir.name: return self.load_cls_data_imdb(bs, **kwargs) else: assert self.tokenizer is Tokenizers.MOSES, "XNLI does not support other tokenizers than Moses" From 1b9d04d7ef230ddb97a343c729e1222bd270b76a Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Sun, 9 Dec 2018 22:45:21 +0100 Subject: [PATCH 30/41] Expose use_test_for_validation as param to train --- ulmfit/train_clas.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index b1c430b..93c0561 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -53,9 +53,11 @@ class CLSHyperParams(LMHyperParams): @property def need_fine_tune_lm(self): return not (self.model_dir/f"enc_best.pth").exists() - def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5): - data_clas, data_lm = self.load_cls_data(bs) + def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5, + use_test_for_validation=False): + data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=use_test_for_validation) + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm) learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls) try: From fac3ce343b5acdb6b16c2601be8a84b12bd405eb Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 12 Dec 2018 00:29:23 +0100 Subject: [PATCH 31/41] Fix BiLM implementation after upgrade of fastai --- fastai_contrib/learner.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index ea2588f..9699c41 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -1,3 +1,5 @@ +from torch.nn import CrossEntropyLoss + from fastai import GradientClipping, accuracy from fastai.callbacks import * from fastai.basic_data import * @@ -25,6 +27,7 @@ def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:in fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])] learn.load_pretrained(*fnames) learn.freeze() + learn.loss_func = CrossEntropyLoss() # I'm not sure why fast ai is using CrossEntropyFlat but it breaks bilm return learn def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400, From f9394b9af132b75af000c97b29e65f6385685037 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 12 Dec 2018 00:31:16 +0100 Subject: [PATCH 32/41] Add callbacks to save history and best weights remove bs & drop_mult --- tests/test_end_to_end.py | 45 ++++++++++++++++++++-------------------- ulmfit/pretrain_lm.py | 41 ++++++++++++++++++------------------ ulmfit/train_clas.py | 15 ++++++++++++-- 3 files changed, 56 insertions(+), 45 deletions(-) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 014b6f1..6f26c24 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -13,13 +13,12 @@ It is a mixture of a pytest unit test and woven together to compose an end to en import fastai.core fastai.core.defaults.cpus = 1 - +cuda_id=0 def copy_head(src_fn, dst_fn, n=1000): with src_fn.open("r") as s, dst_fn.open("w") as d: for i in range(n): d.write(s.readline()) - def get_test_data(): data = get_data_folder() wt = data / "wiki" / "wikitext-2" @@ -35,9 +34,9 @@ def get_test_data(): sz=1 # we use the same text to see if models can overfit - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=10*sz) - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=6*sz) - copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=6*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.train.tokens', n=1000*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.valid.tokens', n=600*sz) + copy_head(wt / 'en.wiki.train.tokens', test_wt / 'en.wiki.test.tokens', n=600*sz) copy_head(imdb / 'train.csv', test_imdb / 'train.csv', n=10*sz) copy_head(imdb / 'train.csv', test_imdb / 'test.csv', n=6*sz) copy_head(imdb / 'train.csv', test_imdb / 'unsup.csv', n=1*sz) @@ -59,10 +58,10 @@ def test_ulmfit_works_with_relative_paths(): lang='en', qrnn=True, max_vocab=1000, - bs=2, - name=lm_name) + name=lm_name, + cuda_id=cuda_id) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) #assert exp.results['accuracy'] > 0.02 @@ -86,10 +85,10 @@ def test_ulmfit_default_end_to_end(): lang='en', qrnn=True, max_vocab=1000, - bs=2, - name=lm_name) + name=lm_name, + cuda_id=cuda_id) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) #assert exp.results['accuracy'] > 0.02 @@ -101,7 +100,7 @@ def test_ulmfit_fastai_end_to_end(): """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -109,10 +108,9 @@ def test_ulmfit_fastai_end_to_end(): qrnn=True, tokenizer='f', max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) @@ -121,7 +119,7 @@ def test_ulmfit_fastai_bidir_end_to_end(): """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -130,10 +128,9 @@ def test_ulmfit_fastai_bidir_end_to_end(): bidir=True, tokenizer='f', max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir)) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) @@ -142,7 +139,7 @@ def test_ulmfit_moses_fa_bidir_end_to_end(): """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-fastai' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -151,19 +148,22 @@ def test_ulmfit_moses_fa_bidir_end_to_end(): bidir=True, tokenizer='vf', max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) +# def test_classification_model_work_with_different_dropmul(): +# learn = self.create_cls_learner(data_clas, drop_mult=0.1) +# learn = self.create_cls_learner(data_clas, drop_mult=0.0) + def test_ulmfit_sentencepiece_end_to_end(): """ Test ulmfit with sentencepiece tokenizer on small wikipedia dataset. """ test_data, wt2 = get_test_data() lm_name = 'end-to-end-test-spm' - cuda_id = 0 + exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', @@ -171,10 +171,9 @@ def test_ulmfit_sentencepiece_end_to_end(): qrnn=True, tokenizer=ulmfit.pretrain_lm.Tokenizers.SUBWORD, max_vocab=100, - bs=2, name=lm_name, ) - exp.train_lm(num_epochs=1) + exp.train_lm(num_epochs=1, bs=2) # not supported yet # exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) # exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 278ec43..3b499b8 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -10,6 +10,7 @@ import fastai import fire from fastai import * +from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ @@ -90,11 +91,9 @@ class LMHyperParams: # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103) # for training on smaller datasets, more dropout is necessary - drop_mult = 0.1 - dps = (0.25, 0.1, 0.2, 0.02, 0.15) + dps = (0.25, 0.1, 0.2, 0.02, 0.15) # consider removing dps & clip from the default hyperparams and put them to train clip: float = 0.12 bptt: int = 70 - bs: int = 70 lang: str = 'en' name: str = None @@ -114,7 +113,6 @@ class LMHyperParams: self.model_dir = self.cache_dir / self.model_name self.model_dir.mkdir(exist_ok=True, parents=True) - print('Batch size:', self.bs) print('Max vocab:', self.max_vocab) print('Cache dir:', self.cache_dir) print('Model dir:', self.model_dir) @@ -147,16 +145,16 @@ class LMHyperParams: with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp) print("Saving info", self.model_dir / 'info.json') - def train_lm(self, num_epochs=20, data_lm=None, true_wd=False, drop_mult=0.1, lr=5e-3): - data_lm = self.load_wiki_data() if data_lm is None else data_lm + def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3): + data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) learn.true_wd = true_wd - try: - learn.load("lm_best_with_opt") - print("Continuing training") - except FileNotFoundError: - pass + # try: + # learn.load("lm_best_with_opt") + # print("Continuing training") + # except FileNotFoundError: + # pass if num_epochs > 0: if self.pretrained_fnames or self.pretrained_model: print("Training lm from: ", self.pretrained_fnames or self.pretrained_model) @@ -187,7 +185,7 @@ class LMHyperParams: fastai.text.learner.default_dropout['language'] = dps or self.dps lm_learner = bilm_learner if self.bidir else language_model_learner - trn_args = dict(drop_mult=self.drop_mult, tie_weights=True, clip=self.clip, bptt=self.bptt, + trn_args = dict(tie_weights=True, clip=self.clip, bptt=self.bptt, pretrained_fnames=self.pretrained_fnames, pretrained_model=self.pretrained_model) trn_args.update(kwargs) @@ -197,9 +195,11 @@ class LMHyperParams: # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), + partial(SaveModelCallback, every='epoch', name='lm')] return learn - def load_wiki_data(self): + def load_wiki_data(self, bs=70): trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' @@ -215,7 +215,7 @@ class LMHyperParams: sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab) - data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=self.bs, bptt=self.bptt, lm_type=self.lm_type) + data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=bs, bptt=self.bptt, lm_type=self.lm_type) elif self.tokenizer is Tokenizers.MOSES: # read the already whitespace separated data without any preprocessing trn_tok = read_whitespace_file(trn_path) @@ -243,12 +243,12 @@ class LMHyperParams: # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids, - valid_ids=val_ids, bs=self.bs, bptt=self.bptt, + valid_ids=val_ids, bs=bs, bptt=self.bptt, lm_type=self.lm_type) elif self.tokenizer is Tokenizers.MOSES_FA: try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs) + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") @@ -258,18 +258,18 @@ class LMHyperParams: data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), valid_df=read_wiki_articles(val_path), tokenizer=pretokenized, classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab, bs=self.bs, text_cols='texts') + max_vocab=self.max_vocab, bs=bs, text_cols='texts') data_lm.save('.') elif self.tokenizer is Tokenizers.FASTAI: try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=self.bs) + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) print("Tokenized data loaded") except FileNotFoundError: print("Running tokenization") data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), valid_df=read_wiki_articles(val_path), classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab,bs=self.bs, text_cols='texts') + max_vocab=self.max_vocab, bs=bs, text_cols='texts') data_lm.save('.') else: raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") @@ -285,7 +285,8 @@ class LMHyperParams: with open(base_lm_path/'info.json', 'r') as f: d = json.load(f) d['dataset_path'] = dataset_path d['base_lm_path'] = base_lm_path - + d.pop('bs', None) + d.pop('drop_mult', None) subword = d.pop('subword', False) tokenizer = d.pop('tokenizer', None) if tokenizer is not None: diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 93c0561..627cb09 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -9,6 +9,7 @@ import numpy as np import pickle from fastai import * +from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * import torch @@ -57,7 +58,7 @@ class CLSHyperParams(LMHyperParams): def train_cls(self, num_lm_epochs, unfreeze=True, bs=40, true_wd=True, drop_mul_lm=0.3, drop_mul_cls=0.5, use_test_for_validation=False): data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=use_test_for_validation) - + if self.need_fine_tune_lm: self.train_lm(num_lm_epochs, data_lm=data_lm, true_wd=true_wd, drop_mult=drop_mul_lm) learn = self.create_cls_learner(data_clas, drop_mult=drop_mul_cls) try: @@ -90,11 +91,19 @@ class CLSHyperParams(LMHyperParams): learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) + self.validate_cls('cls_last') + self.validate_cls('cls_best') return learn + def validate_cls(self, save_name='cls_last', bs=40): + data_clas, data_lm = self.load_cls_data(bs, use_test_for_validation=True) + learn = self.create_cls_learner(data_clas, drop_mult=0.1) + learn.load(save_name) + print(f"Loss and accuracy using ({save_name}):", learn.validate()) + def create_cls_learner(self, data_clas, dps=None, **kwargs): fastai.text.learner.default_dropout['language'] = dps or self.dps - trn_args=dict(drop_mult=self.drop_mult, bptt=self.bptt, clip=self.clip,) + trn_args=dict(bptt=self.bptt, clip=self.clip,) trn_args.update(kwargs) classifier_learner = text_classifier_learner if self.bidir: @@ -103,6 +112,8 @@ class CLSHyperParams(LMHyperParams): learn = classifier_learner(data_clas, pad_token=PAD_TOKEN_ID, path=self.model_dir.parent, model_dir=self.model_dir.name, qrnn=self.qrnn, emb_sz=self.emb_sz, nh=self.nh, nl=self.nl, **trn_args) + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), + partial(SaveModelCallback, every='improvement', name='cls_best')] return learn def load_cls_data(self, bs, **kwargs): From dca502a398bc3edec1c4a3639dda96f10af409e3 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Wed, 12 Dec 2018 00:31:48 +0100 Subject: [PATCH 33/41] Add docs how to train a classfier --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 6b65884..a04fca8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,18 @@ # ulmfit-multilingual Temporary repository used for collaboration on application of for multiple languages. +# How to train classifier + +``` +$ python -m ulmfit lm --dataset-path data/wiki/wikitext-103 --bidir=False --qrnn=False --tokenizer=vf --name 'bs40' --bs=40 --cuda-id=0 - train 20 --drop-mult=0.9 +... +Model dir: data/wiki/wikitext-103/models/vf60k/lstm_bs40.m +... +$ python -m ulmfit cls --dataset-path data/imdb --base-lm-path data/wiki/wikitext-103/models/vf60k/lstm_bs40.m - train 20 +``` + + + ## data directory strucutre Directory structure after changes to the way we process wiki dumps. From ff30fb5642fb54a6d8e519f7eb682e0e72274dcb Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 27 Dec 2018 14:57:15 +0100 Subject: [PATCH 34/41] Fastai upgrade --- fastai_contrib/data.py | 26 +++++++++++++------------- fastai_contrib/models.py | 2 +- ulmfit/train_clas.py | 10 +++------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py index d7be826..34a205e 100644 --- a/fastai_contrib/data.py +++ b/fastai_contrib/data.py @@ -13,24 +13,24 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader "Create a dataloader with bptt slightly changing." def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70, lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False, - max_len:int=25): - self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle = dataset,bs,bptt,lm_type,shuffle + max_len:int=25, p_bptt:int=0.95): + self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt self.first,self.i,self.iter = True,0,0 self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0 self.max_len,self.num_workers = max_len,0 - self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len) + self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt) def __iter__(self): if getattr(self.dataset, 'item', None) is not None: - yield LongTensor(getattr(self.dataset, 'item')).unsqueeze(1),LongTensor([0]) + yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0]) idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset)) - data = self.batchify(np.concatenate([np.array(self.dataset.x.items[i], dtype=np.int) for i in idx])) + data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx])) pos, itr = 0,0 while pos < self.n-1 and itr LongTensor: "Split the corpus `data` in batches." nb = data.shape[0] // self.bs - data = np.array(data[:nb*self.bs]).reshape(self.bs, -1).T - if self.lm_type == LanguageModelType.BwdLM: data=data[::-1].copy() - elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[::-1].copy()], axis=2) + data = np.array(data[:nb*self.bs]).reshape(self.bs, -1) + if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy() + elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2) return LongTensor(data) def get_batch(self, data:LongTensor, i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]: "Create a batch at `i` of a given `seq_len`." - seq_len = min(seq_len, len(data) - 1 - i) - x = data[i:i+seq_len] - y = data[i+1:i+1+seq_len].contiguous() # x & y has 2 elements on the last dimension - y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1) + seq_len = min(seq_len, data.shape[1] - 1 - i) + x = data[:,i:i+seq_len] + y = data[:,i+1:i+1+seq_len] + #y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1) return x,y #endregion diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index b5de84c..03f406b 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -29,7 +29,7 @@ class BiLMModel(nn.Module): b = input[..., 1] elif len(input.shape) == 2: # sl, bs - support during classification mode f = input - b = torch.flip(input, [0]) + b = torch.flip(input, [1]) # todo test if we are duplicating the backward pass correctly else: raise AttributeError(f"Inorrect size of input, {input.shape}") fwd_o = self.fwd_lm(f) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 627cb09..264be7c 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -5,16 +5,12 @@ Optionally fine-tune LM before. from sacremoses import MosesTokenizer import fastai -import numpy as np -import pickle +import torch from fastai import * from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * -import torch -from fastai.text import TextLMDataBunch, TextClasDataBunch, language_model_learner, text_classifier_learner -from fastai import fit_one_cycle, accuracy from fastai_contrib.data import LanguageModelType from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ @@ -91,8 +87,8 @@ class CLSHyperParams(LMHyperParams): learn.fit_one_cycle(2, slice(1e-2 / (2.6 ** 4), 1e-2), moms=(0.8, 0.7), wd=1e-7) print(f"Saving models at {learn.path / learn.model_dir}") learn.save('cls_last', with_opt=False) - self.validate_cls('cls_last') - self.validate_cls('cls_best') + self.validate_cls('cls_last', bs=bs) + self.validate_cls('cls_best', bs=bs) return learn def validate_cls(self, save_name='cls_last', bs=40): From 82d6a30b11c5294ef093b7ee2af1c0ff450d45cc Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 27 Dec 2018 15:14:41 +0100 Subject: [PATCH 35/41] Fix SentencePiece implementation, to get 94.5% on imdb - train on whole articles when tokenizer is sentencepice - add moses tokenizer to get the same tokens on imdb as on wt103 - apply pre / post processing rules to moses tokenzier so that sentencepiece works on correctly preprocessed text (lowercased with html removed) - add alpha implementation on xnli (no tests) - remove vocab adaptation when swiching from wiki to imdb. As otherwise we get 50% of missing words and 93% accuracy on imdb --- fastai_contrib/utils.py | 199 +++++++++++++++++++--------------------- ulmfit/pretrain_lm.py | 57 +++++------- ulmfit/train_clas.py | 17 +++- 3 files changed, 128 insertions(+), 145 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 927426a..977b6bb 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -1,18 +1,9 @@ """ Utility methods for data processing. """ -import pandas as pd -import numpy as np import fire -from fastai import F, to_device -import torch -from tqdm import tqdm -import re -import csv - -from functools import reduce -from fastai.text.transform import Tokenizer, BaseTokenizer, Vocab -from fastai.torch_core import * +from fastai import * +from fastai.text import * import shutil import pathlib @@ -22,6 +13,7 @@ from sacremoses import MosesTokenizer from typing import Dict, Tuple, List EOS = '' +BOS = '' UNK = '' PAD = '' SEP = '' # special separator token for NLI @@ -39,67 +31,102 @@ CLASSES = ['neg', 'pos', 'unsup'] number_match_re = re.compile(r'^([0-9]+[,.]?)+$') number_split_re = re.compile(r'([,.])') -class SentencepieceTokenizer(BaseTokenizer): - def __init__(self, model_dir:PathOrStr): +class MosesTokenizerFunc(BaseTokenizer): + "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." + def __init__(self, lang:str): + self.tok = MosesTokenizer(lang) + + def tokenizer(self, t:str) -> List[str]: + return self.tok.tokenize(t, return_str=False, escape=False) + + def add_special_cases(self, toks:Collection[str]): + for w in toks: + assert len(self.tokenizer(w))==1, f"Tokenizer is unable to keep {w} as one token!" + +class SentencePieceTokenizer(Tokenizer): + "Put together rules and a tokenizer function to tokenize text with multiprocessing." + def __init__(self, spm_model, lang:str='en', pre_rules:ListRules=None, + post_rules:ListRules=None, special_cases:Collection[str]=None, n_cpus:int=None, use_moses=False): + super().__init__(self.tok_fun_with_sp, lang, pre_rules, post_rules, special_cases, n_cpus) + self.spm_model = spm_model + self.use_moses = use_moses + + def tok_fun_with_sp(self, lang): try: - import sentencepiece as spm + import sentencepiece as spm except ImportError: raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') - self.tok = spm.SentencePieceProcessor() - self.tok.Load(str(pathlib.Path(model_dir) / 'spm.model')) - - def tokenizer(self, t:str) -> List[str]: - return self.tok.EncodeAsPieces(t) - - def add_special_cases(self, toks:Collection[str]): - pass + tok = MosesTokenizerFunc(lang) if self.use_moses else BaseTokenizer(lang) + tok.sp = spm.SentencePieceProcessor() + tok.sp.Load(str(self.spm_model)) + return tok + def process_text(self, t:str, tok:BaseTokenizer) -> List[str]: + "Process one text `t` with tokenizer `tok`." + toks = super().process_text(t, tok) + toks = tok.sp.EncodeAsPieces(" ".join(toks)) + return toks -def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, pre_rules:ListRules=None, post_rules:ListRules=None, +def get_sentencepiece(cache_dir:PathOrStr, load_text, name:str, pre_rules:ListRules=None, post_rules:ListRules=None, vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, - pad_idx:int=PAD_TOKEN_ID): + pad_idx:int=PAD_TOKEN_ID, use_moses=False, lang='en'): try: import sentencepiece as spm except ImportError: raise Exception('sentencepiece module is missing: run `pip install sentencepiece`') - - path = pathlib.Path(path) - cache_name = 'tmp' - os.makedirs(path / cache_name, exist_ok=True) - os.makedirs(path / 'models', exist_ok=True) - pre_rules = pre_rules if pre_rules is not None else [] - post_rules = post_rules if post_rules is not None else [] - if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'): + cache_dir = pathlib.Path(cache_dir) + pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules + post_rules = post_rules if post_rules is not None else defaults.text_post_rules + + special_cases = defaults.text_spec_tok + + if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'): # load the text from the train tokens file - text = [line.rstrip('\n') for line in open(trn_path)] - text = list(filter(None, text)) - raw_text = reduce(lambda t, rule: rule(t), pre_rules, '\n'.join(text)) # FIXME: possibly does not work with pre_rules - raw_text_path = path / cache_name / 'all_text.txt' - with open(raw_text_path, 'w') as f: - f.write(raw_text) - - sp_params = f"--input={raw_text_path} --pad_id={pad_idx} --unk_id=0 " \ - f"--character_coverage=1.0 --bos_id=-1 --eos_id=-1 " \ - f"--input_sentence_size={int(input_sentence_size)} " \ - f"--model_prefix={path / 'models' / 'spm'} " \ - f"--vocab_size={vocab_size} --model_type={model_type} " - spm.SentencePieceTrainer.Train(sp_params) + text = load_text() + text = filter(lambda x: len(x.rstrip(" ")), text) + text = (reduce(lambda t, rule: rule(t), pre_rules, line) for line in text) + if use_moses: + mt = MosesTokenizer(lang) + splitter = lambda t: mt.tokenize(t, return_str=False, escape=False) + else: + splitter = lambda t: t.split() + def cleanup_n_postprocess(t): + t = splitter(t) + for r in post_rules: + t = r(t) + return ' '.join(t) + text = map(cleanup_n_postprocess, text) + raw_text_path = cache_dir / 'all_text.txt' + with open(raw_text_path, 'w') as f: f.write("\n".join(text)) - with open(path / 'models' / 'spm.vocab', 'r') as f: + sp_params = [ + f"--input={raw_text_path}", + f"--character_coverage=1.0", + f"--unk_id={len(defaults.text_spec_tok)}", + f"--pad_id=-1", + f"--bos_id=-1", + f"--eos_id=-1", + f"--max_sentence_length=20480", + f"--input_sentence_size={int(input_sentence_size)}", + f"--user_defined_symbols={','.join(special_cases)}", + f"--model_prefix={cache_dir/'spm'}", + f"--vocab_size={vocab_size} --model_type={model_type}"] + spm.SentencePieceTrainer.Train(" ".join(sp_params)) + + with open(cache_dir / 'spm.vocab', 'r') as f: vocab = [line.split('\t')[0] for line in f.readlines()] - vocab[0] = UNK - vocab[pad_idx] = PAD - - pickle.dump(vocab, open(path / 'models' / f'itos_{name}.pkl', 'wb')) + + pickle.dump(vocab, open(cache_dir/ f'itos.pkl', 'wb')) # todo add post rules - vocab = Vocab(pickle.load(open(path / 'models' / f'itos_{name}.pkl', 'rb'))) + vocab = Vocab(pickle.load(open(cache_dir / f'itos.pkl', 'rb'))) # We cannot use lambdas or local methods here, since `tok_func` needs to be # pickle-able in order to be called in subprocesses when multithread tokenizing - tokenizer = Tokenizer(tok_func=SentencepieceTokenizer, lang=str(path / 'models'), pre_rules=pre_rules, post_rules=post_rules) - - clear_cache_directory(path, cache_name) - + tokenizer = SentencePieceTokenizer(cache_dir/'spm.model', + use_moses=use_moses, + lang=lang, + pre_rules=pre_rules, + post_rules=post_rules) return {'tokenizer': tokenizer, 'vocab': vocab} @@ -107,7 +134,6 @@ def clear_cache_directory(path:PathOrStr, cache_name:str='tmp'): path = pathlib.Path(path) shutil.rmtree(path / cache_name) - def get_texts(path): texts, labels = [],[] for idx, label in enumerate(CLASSES): @@ -189,48 +215,6 @@ def prepare_imdb(file_path: str, prepare_lm = False): df_trn[df_trn['labels'] == 2].to_csv(CLAS_PATH / 'unsup.csv', header=False, index=False) (CLAS_PATH / 'classes.txt').open('w', encoding='utf-8').writelines(f'{o}\n' for o in CLASSES) - -def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], List[str]]: - """ - Reads IMDb data. - :param dir_path: the path to the imdb folder - :param lang: the language (not used here as IMDb is only available in English) - :param split: the split of the data that should be read (train, test, val) - :param spm_path: path to sentencepiece model - :return: a tuple consisting of a list of lists of tokens and a list of labels - """ - file_path = dir_path / 'train.csv' if split == TRN else dir_path / 'test.csv' - toks, lbls = [], [] - - mt = MosesTokenizer('en') - if spm_path is not None: - sp = SentencepieceTokenizer(spm_path) - - print(f'Reading {file_path}...') - - with open(file_path, encoding='utf-8') as f: - reader = csv.reader(f) - for row in reader: - label, text = row - lbls.append(int(label)) - raw_tokens = mt.tokenize(text, return_str=True).split(' ') - - tokens = [] - - # fix up occurences of numbers in text - for token in raw_tokens: - if number_match_re.match(token): - tokens += number_split_re.sub(r' @\1@ ', token).split() - else: - tokens.append(token) - - if spm_path is not None: - tokens = sp.tokenizer(' '.join(tokens)) - - toks.append(tokens + [EOS]) - return toks, lbls - - def read_xnli(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], List[str]]: """ Reads XNLI data. @@ -249,7 +233,14 @@ def read_xnli(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], Li file_path = dir_path / file_path if spm_path is not None: - sp = SentencepieceTokenizer(spm_path) + tokenizer = SentencePieceTokenizer(spm_path, + use_moses=False, + lang=lang) + tok = tokenizer.tok_fun_with_sp(lang) + tokenize = lambda x: tokenizer.process_text(x, tok) + print("WARNING: Sentence Piece is not tested on XNLI yet") + else: + tokenize = lambda x: x.split(' ') toks, lbls = [], [] print(f'Reading {file_path}...') @@ -268,13 +259,9 @@ def read_xnli(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], Li premise, hypo, label = row[-3], row[-2], row[1] # TODO add BOS - if spm_path is not None: - premise_toks = sp.tokenizer(premise) + [EOS] - hypo_toks = sp.tokenizer(hypo) + [EOS] - else: - premise_toks = premise.split(' ') + [EOS] - hypo_toks = hypo.split(' ') + [EOS] - + premise_toks = tokenize(premise) + [EOS] + hypo_toks = tokenize(hypo) + [EOS] + toks.append(premise_toks + [SEP] + hypo_toks) lbls.append(label) return toks, lbls diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 3b499b8..e1c4ea3 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -23,41 +23,16 @@ from pathlib import Path from collections import Counter import fastai_contrib.data as contrib_data -# to install, do: -# conda install -c pytorch -c fastai fastai pytorch-nightly [cuda92] -# cupy needs to be installed for QRNN - - -# """ -# :param dir_path: The path to the directory of the file. -# :param lang: the language unicode -# :param cuda_id: The id of the GPU. Uses GPU 0 by default or no GPU when -# run on CPU. -# :param qrnn: Use a QRNN. Requires installing cupy. -# :param subword: Use sub-word tokenization on the cleaned data. -# :param max_vocab: The maximum size of the vocabulary. -# :param bs: The batch size. -# :param bptt: The back-propagation-through-time sequence length. -# :param name: The name used for both the model and the vocabulary. -# :param model_dir: The path to the directory where the models should be saved -# :param bidir: whether the language model is bidirectional -# """ LM_BEST = "lm_best" ENC_BEST = "enc_best" class Tokenizers(Enum): - SUBWORD='sb' + SUBWORD='sp' MOSES='v' MOSES_FA='vf' FASTAI='f' -# tokenizers ={ -# Tok.MOSES: MosesTok, -# Tok.SUBWORD: SentencepieceTok, -# Tok.FASTAI: FastaiTok -# } - def istitle(line): return len(re.findall(r'^ = [^=]* = $', line)) != 0 @@ -195,10 +170,15 @@ class LMHyperParams: # compared to standard Adam, we set beta_1 to 0.8 learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99)) learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy] - learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/cls-history"), + learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"), partial(SaveModelCallback, every='epoch', name='lm')] return learn + def load_train_text(self): + trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' + with open(trn_path) as f: + return [line.rstrip('\n') for line in f] + def load_wiki_data(self, bs=70): trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' @@ -206,16 +186,25 @@ class LMHyperParams: for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' if self.tokenizer is Tokenizers.SUBWORD: - # apply sentencepiece tokenization - trn_path = self.dataset_path / f'{self.lang}.wiki.train.tokens' - val_path = self.dataset_path / f'{self.lang}.wiki.valid.tokens' + sp = get_sentencepiece(self.cache_dir, + self.load_train_text, + self.name, + vocab_size=self.max_vocab, + use_moses=False, + lang=self.lang) - read_file(trn_path, 'train') - read_file(val_path, 'valid') + try: + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) + print("Tokenized data loaded") + except FileNotFoundError: + print("Running tokenization") + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), + valid_df=read_wiki_articles(val_path), + classes=None, lm_type=self.lm_type, **sp, + max_vocab=self.max_vocab, bs=bs, text_cols='texts') + data_lm.save('.') - sp = get_sentencepiece(self.dataset_path, trn_path, self.name, vocab_size=self.max_vocab) - data_lm = TextLMDataBunch.from_csv(self.dataset_path, 'train.csv', **sp, bs=bs, bptt=self.bptt, lm_type=self.lm_type) elif self.tokenizer is Tokenizers.MOSES: # read the already whitespace separated data without any preprocessing trn_tok = read_whitespace_file(trn_path) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 264be7c..22cf91f 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -139,11 +139,18 @@ class CLSHyperParams(LMHyperParams): cls_cache = '.' if self.tokenizer is Tokenizers.SUBWORD: - args = get_sentencepiece(self.dataset_path, self.dataset_path / 'train.csv', - self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[]) - if self.tokenizer is Tokenizers.SUBWORD: - args = get_sentencepiece(self.dataset_path, self.dataset_path / 'train.csv', - self.name, vocab_size=self.max_vocab, pre_rules=[], post_rules=[]) + shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir) + shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir) + shutil.copy(self.base_lm_path / '..' / 'spm.vocab', self.cache_dir) + + args = get_sentencepiece(self.cache_dir, + lambda: trn_df[1], + self.name, + vocab_size=self.max_vocab, + lang='en', + use_moses=True) + + # TODO remove migration of tokens for SentencePiece as more than 50% of tokens are different in imdb elif self.tokenizer is Tokenizers.MOSES: args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[])) elif self.tokenizer is Tokenizers.MOSES_FA: From f1b49a0d342aedccb8276b395c8a2cc5f2ebc466 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Thu, 27 Dec 2018 15:15:09 +0100 Subject: [PATCH 36/41] fix imports in learner (fastai adpatation) --- fastai_contrib/learner.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py index 9699c41..67e79cb 100644 --- a/fastai_contrib/learner.py +++ b/fastai_contrib/learner.py @@ -1,13 +1,11 @@ from torch.nn import CrossEntropyLoss -from fastai import GradientClipping, accuracy -from fastai.callbacks import * -from fastai.basic_data import * -from fastai.datasets import untar_data -from fastai_contrib.models import get_bilm, get_rnn_classifier, get_birnn_classifier -from fastai.text.learner import * +from fastai import * +from fastai.text import * #region New code +from fastai_contrib.models import * + def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:int=3, pad_token:int=1, drop_mult:float=1., tie_weights:bool=True, bias:bool=True, qrnn:bool=False, pretrained_model=None, @@ -92,10 +90,14 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new: bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0) new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_() new_b = dec_bias.new_zeros((len(itos_new),)).zero_() + unk_tokens=[] for i,w in enumerate(itos_new): r = stoi_wgts[w] if w in stoi_wgts else -1 + if r < 0: + unk_tokens.append(w) new_w[i] = enc_wgts[r] if r>=0 else wgts_m new_b[i] = dec_bias[r] if r>=0 else bias_m + print(f"Unknown tokens {len(unk_tokens)}, first 100: {unk_tokens[:100]}") wgts[prefix+'0.encoder.weight'] = new_w wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone() wgts[prefix+'1.decoder.weight'] = new_w.clone() From 514a9e6b864f7a86e779dbe3da152469260cb423 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Mon, 31 Dec 2018 12:13:53 +0100 Subject: [PATCH 37/41] Fix BiLM training after update to newest fastai --- fastai_contrib/data.py | 2 +- fastai_contrib/models.py | 62 +++++++++++++++++++++++----------------- fastai_contrib/utils.py | 3 +- tests/test_end_to_end.py | 18 ++++++------ 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/fastai_contrib/data.py b/fastai_contrib/data.py index 34a205e..5f6e732 100644 --- a/fastai_contrib/data.py +++ b/fastai_contrib/data.py @@ -59,7 +59,7 @@ class LanguageModelLoader(): # copy of the original LanguageModelLoader seq_len = min(seq_len, data.shape[1] - 1 - i) x = data[:,i:i+seq_len] y = data[:,i+1:i+1+seq_len] - #y = y.view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.view(-1) + y = y.contiguous().view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.contiguous().view(-1) return x,y #endregion diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py index 03f406b..92ac7e4 100644 --- a/fastai_contrib/models.py +++ b/fastai_contrib/models.py @@ -6,10 +6,11 @@ from fastai.text.models import * class BiLMModel(nn.Module): - def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module): + def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module, squash_bs_sl=False): super().__init__() self.fwd_lm = fwd_lm self.bwd_lm = bwd_lm + self.squash_bs_sl = squash_bs_sl def __getitem__(self, idx): return BiLMModel(self.fwd_lm[idx], self.bwd_lm[idx]) @@ -35,49 +36,57 @@ class BiLMModel(nn.Module): fwd_o = self.fwd_lm(f) bwd_o = self.bwd_lm(b) - return self.stack(fwd_o, bwd_o) + outs = self.stack(fwd_o, bwd_o) + if self.squash_bs_sl: + o = outs[0] + o = o.view(o.shape[0]*o.shape[1],o.shape[2],o.shape[3]) + outs[0] = o + return outs def reset(self): "Reset the hidden states of underlaying lms." self.fwd_lm.reset() self.bwd_lm.reset() +class MultiBatchBiLMModel(BiLMModel): + "Create a RNNCore module that can process a full sentence." -class BiPoolingLinearClassifier(nn.Module): + def __init__(self, bptt:int, max_seq:int, *args, **kwargs): + self.max_seq,self.bptt = max_seq,bptt + super().__init__(*args, **kwargs) + + def concat(self, arrs:Collection[Tensor])->Tensor: + "Concatenate the `arrs` along the batch dimension." + return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])] + + def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]: + bs,sl = input.size() + self.reset() + raw_outputs, outputs = [],[] + for i in range(0, sl, self.bptt): + r, o = super().forward(input[:,i: min(i+self.bptt, sl)]) + if i>(sl-self.max_seq): + raw_outputs.append(r) + outputs.append(o) + return self.concat(raw_outputs), self.concat(outputs) + +class BiPoolingLinearClassifier(PoolingLinearClassifier): "Create a linear classifier with pooling." - def __init__(self, layers:Collection[int], drops:Collection[float]): - super().__init__() - mod_layers = [] - activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None] - for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs): - mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn) - self.layers = nn.Sequential(*mod_layers) - - def pool(self, x:Tensor, bs:int, is_max:bool): - "Pool the tensor along the seq_len dimension." - f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d - return f(x.permute(1,2,0), (1,)).view(bs,-1) - def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]: raw_outputs, outputs = input output = outputs[-1] if len(output.size()) == 3: - sl,bs,_ = output.size() - avgpool = self.pool(output, bs, False) - mxpool = self.pool(output, bs, True) - x = torch.cat([output[-1], mxpool, avgpool], 1) - x = self.layers(x) - return x, raw_outputs, outputs + return super().forward(input) elif len(output.size()) == 4: - sl, bs, em_sz, passes = output.size() + bs, sl, em_sz, passes = output.size() f_avgpool = self.pool(output[...,0], bs, False) f_mxpool = self.pool(output[...,0], bs, True) b_avgpool = self.pool(output[..., 1], bs, False) b_mxpool = self.pool(output[..., 1], bs, True) - x = torch.cat([output[-1][..., 0], f_mxpool, f_avgpool, - output[-1][..., 1], b_mxpool, b_avgpool,], 1) + x = torch.cat([output[:,-1,..., 0], f_mxpool, f_avgpool, + output[:,-1,..., 1], b_mxpool, b_avgpool,], 1) x = self.layers(x) return x, raw_outputs, outputs @@ -134,7 +143,8 @@ def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, t return BiLMModel( fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), - bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias))) + bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)), + squash_bs_sl=True) def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False, diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 977b6bb..7be5edc 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -31,6 +31,8 @@ CLASSES = ['neg', 'pos', 'unsup'] number_match_re = re.compile(r'^([0-9]+[,.]?)+$') number_split_re = re.compile(r'([,.])') +# FIXME: coping of tokens from one sentencepiece model to another does not work for 50% of tokens +# FIXME: tokens in sentencepiece are uppercase eventhough post-transformation will convert them to lowercase class MosesTokenizerFunc(BaseTokenizer): "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." def __init__(self, lang:str): @@ -278,7 +280,6 @@ def read_clas_data(dir_path, dataset, lang) -> Tuple[Dict[str, List[List[str]]], 2. a dictionary mapping splits to a list of labels """ processors = { - 'imdb': read_imdb, 'xnli': read_xnli } processor = processors[dataset] diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 6f26c24..3d4d112 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -56,7 +56,7 @@ def test_ulmfit_works_with_relative_paths(): exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2.relative_to(Path.cwd()), lang='en', - qrnn=True, + qrnn=False, max_vocab=1000, name=lm_name, cuda_id=cuda_id) @@ -83,7 +83,7 @@ def test_ulmfit_default_end_to_end(): exp = ulmfit.pretrain_lm.LMHyperParams( dataset_path=wt2, lang='en', - qrnn=True, + qrnn=False, max_vocab=1000, name=lm_name, cuda_id=cuda_id) @@ -105,7 +105,7 @@ def test_ulmfit_fastai_end_to_end(): dataset_path=wt2, lang='en', cuda_id=cuda_id, - qrnn=True, + qrnn=False, tokenizer='f', max_vocab=100, name=lm_name, @@ -124,7 +124,7 @@ def test_ulmfit_fastai_bidir_end_to_end(): dataset_path=wt2, lang='en', cuda_id=cuda_id, - qrnn=True, + qrnn=False, bidir=True, tokenizer='f', max_vocab=100, @@ -144,7 +144,7 @@ def test_ulmfit_moses_fa_bidir_end_to_end(): dataset_path=wt2, lang='en', cuda_id=cuda_id, - qrnn=True, + qrnn=False, bidir=True, tokenizer='vf', max_vocab=100, @@ -168,15 +168,15 @@ def test_ulmfit_sentencepiece_end_to_end(): dataset_path=wt2, lang='en', cuda_id=cuda_id, - qrnn=True, + qrnn=False, tokenizer=ulmfit.pretrain_lm.Tokenizers.SUBWORD, - max_vocab=100, + max_vocab=200, name=lm_name, ) exp.train_lm(num_epochs=1, bs=2) # not supported yet - # exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) - # exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) + exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir) + exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, ) if __name__ == "__main__": From d269d53d7d198b0c32eb8ea9ae2f358ea160711c Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 1 Jan 2019 14:40:59 +0100 Subject: [PATCH 38/41] Use fastai tokens instead of f'xx{token_name}' are kept as one token by Moses tokenizer, which is sometimes required if you want to have moses in tokenizers pipeline, and we use that for imdb. --- fastai_contrib/utils.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 7be5edc..1cde194 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -12,11 +12,15 @@ from sklearn import model_selection from sacremoses import MosesTokenizer from typing import Dict, Tuple, List -EOS = '' -BOS = '' -UNK = '' -PAD = '' -SEP = '' # special separator token for NLI +EOS = 'xxeos' # fastai does not use eos, but we do +SEP = 'xxsep' # special separator token for NLI + +def replace_std_toks(x:str) -> str: + "Replace standard token names with fastai supported tokens" + # We change tokens to f'xx{token_name}' as it is not split by Moses tokenizer, + # while f'<{token_name}>' is being split to: '<' f'{token_name}' '>' + return x.replace('', UNK).replace('', BOS).replace('', EOS) + PAD_TOKEN_ID = 1 IMDB, XNLI, TRN, VAL, TST, EN = 'imdb', 'xnli', 'train', 'val', 'test', 'en' DATASETS = ['imdb', 'xnli'] @@ -31,11 +35,10 @@ CLASSES = ['neg', 'pos', 'unsup'] number_match_re = re.compile(r'^([0-9]+[,.]?)+$') number_split_re = re.compile(r'([,.])') -# FIXME: coping of tokens from one sentencepiece model to another does not work for 50% of tokens -# FIXME: tokens in sentencepiece are uppercase eventhough post-transformation will convert them to lowercase class MosesTokenizerFunc(BaseTokenizer): "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." def __init__(self, lang:str): + super().__init__(lang=lang) self.tok = MosesTokenizer(lang) def tokenizer(self, t:str) -> List[str]: @@ -69,9 +72,9 @@ class SentencePieceTokenizer(Tokenizer): toks = tok.sp.EncodeAsPieces(" ".join(toks)) return toks -def get_sentencepiece(cache_dir:PathOrStr, load_text, name:str, pre_rules:ListRules=None, post_rules:ListRules=None, +def get_sentencepiece(cache_dir:PathOrStr, load_text,pre_rules:ListRules=None, post_rules:ListRules=None, vocab_size:int=30000, model_type:str='unigram', input_sentence_size:int=1E7, - pad_idx:int=PAD_TOKEN_ID, use_moses=False, lang='en'): + use_moses=False, lang='en'): try: import sentencepiece as spm except ImportError: From 4a858f357243187901103c2c670605be47477c5f Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 1 Jan 2019 14:43:12 +0100 Subject: [PATCH 39/41] Simplfy and unify input data parsing --- ulmfit/pretrain_lm.py | 118 +++++++++++++----------------------------- ulmfit/train_clas.py | 43 +++------------ 2 files changed, 43 insertions(+), 118 deletions(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index e1c4ea3..27b3fb3 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -14,7 +14,8 @@ from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * import torch from fastai_contrib.utils import read_file, read_whitespace_file, \ - validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID + validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, MosesTokenizerFunc, \ + replace_std_toks from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner import pickle @@ -111,6 +112,30 @@ class LMHyperParams: def lm_type(self): return contrib_data.LanguageModelType.BiLM if self.bidir else contrib_data.LanguageModelType.FwdLM + def tokenzier_to_fastai_args(self, trn_data_loading_func, add_moses): + tok_func = MosesTokenizerFunc if add_moses else BaseTokenizer + if self.tokenizer is Tokenizers.SUBWORD: + if self.base_lm_path: # ensure we are using the same sentence piece model + shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir) + shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir) + shutil.copy(self.base_lm_path / '..' / 'spm.vocab', self.cache_dir) + args = get_sentencepiece(self.cache_dir, + trn_data_loading_func, + vocab_size=self.max_vocab, + use_moses=add_moses, + lang=self.lang) + + elif self.tokenizer is Tokenizers.MOSES: + args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang, pre_rules=[replace_std_toks], post_rules=[])) + elif self.tokenizer is Tokenizers.MOSES_FA: + args = dict(tokenizer=Tokenizer(tok_func=tok_func, lang=self.lang)) # use default pre/post rules + elif self.tokenizer is Tokenizers.FASTAI: + args = dict() + else: + raise ValueError( + f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") + return args + def save_info(self): from dataclasses import asdict vals = {k: (str(v) if isinstance(v, Path) else v) for k,v in asdict(self).items()} @@ -125,11 +150,6 @@ class LMHyperParams: learn = self.create_lm_learner(data_lm, drop_mult=drop_mult) learn.true_wd = true_wd - # try: - # learn.load("lm_best_with_opt") - # print("Continuing training") - # except FileNotFoundError: - # pass if num_epochs > 0: if self.pretrained_fnames or self.pretrained_model: print("Training lm from: ", self.pretrained_fnames or self.pretrained_model) @@ -185,83 +205,19 @@ class LMHyperParams: tst_path = self.dataset_path / f'{self.lang}.wiki.test.tokens' for path_ in [trn_path, val_path, tst_path]: assert path_.exists(), f'Error: {path_} does not exist.' - if self.tokenizer is Tokenizers.SUBWORD: - sp = get_sentencepiece(self.cache_dir, - self.load_train_text, - self.name, - vocab_size=self.max_vocab, - use_moses=False, - lang=self.lang) - try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) - print("Tokenized data loaded") - except FileNotFoundError: - print("Running tokenization") - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), - valid_df=read_wiki_articles(val_path), - classes=None, lm_type=self.lm_type, **sp, - max_vocab=self.max_vocab, bs=bs, text_cols='texts') - data_lm.save('.') + args = self.tokenzier_to_fastai_args(trn_data_loading_func=self.load_train_text, add_moses=False) + try: + data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) + print("Tokenized data loaded") + except FileNotFoundError: + print("Running tokenization") + data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), + valid_df=read_wiki_articles(val_path), + classes=None, lm_type=self.lm_type, max_vocab=self.max_vocab, + bs=bs, text_cols='texts', **args) + data_lm.save('.') - - elif self.tokenizer is Tokenizers.MOSES: - # read the already whitespace separated data without any preprocessing - trn_tok = read_whitespace_file(trn_path) - val_tok = read_whitespace_file(val_path) - itos_fname = self.cache_dir / f'itos.pkl' - if not itos_fname.exists(): - # create the vocabulary - cnt = Counter(word for sent in trn_tok for word in sent) - itos = [o for o, c in cnt.most_common(n=self.max_vocab)] - itos.insert(1, PAD) #   set pad id to 1 to conform to fast.ai standard - assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.' - - # save vocabulary - print(f"Saving vocabulary as {itos_fname}") - with open(itos_fname, 'wb') as f: - pickle.dump(itos, f) - else: - print("Loading itos:", itos_fname) - itos = np.load(itos_fname) - vocab = Vocab(itos) - stoi = vocab.stoi - - trn_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in trn_tok]) - val_ids = np.array([([stoi.get(w, stoi[UNK]) for w in s]) for s in val_tok]) - - # data_lm = TextLMDataBunch.from_ids(dir_path, trn_ids, [], val_ids, [], len(itos)) - data_lm = TextLMDataBunch.from_ids(path=self.dataset_path, vocab=vocab, train_ids=trn_ids, - valid_ids=val_ids, bs=bs, bptt=self.bptt, - lm_type=self.lm_type) - elif self.tokenizer is Tokenizers.MOSES_FA: - - try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) - print("Tokenized data loaded") - except FileNotFoundError: - print("Running tokenization") - - # wikitext is pretokenized with Moses - pretokenized = Tokenizer(tok_func=BaseTokenizer, lang='en', pre_rules=None, post_rules=None) - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), - valid_df=read_wiki_articles(val_path), tokenizer=pretokenized, - classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab, bs=bs, text_cols='texts') - data_lm.save('.') - elif self.tokenizer is Tokenizers.FASTAI: - try: - data_lm = TextLMDataBunch.load(self.cache_dir, '.', lm_type=self.lm_type, bs=bs) - print("Tokenized data loaded") - except FileNotFoundError: - print("Running tokenization") - data_lm = TextLMDataBunch.from_df(path=self.cache_dir, train_df=read_wiki_articles(trn_path), - valid_df=read_wiki_articles(val_path), - classes=None, lm_type=self.lm_type, - max_vocab=self.max_vocab, bs=bs, text_cols='texts') - data_lm.save('.') - else: - raise ValueError(f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") itos, stoi, trn_path = data_lm.vocab.itos, data_lm.vocab.stoi, data_lm.path print('Size of vocabulary:', len(itos)) print('First 20 words in vocab:', data_lm.vocab.itos[:20]) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 22cf91f..0f9221f 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -10,33 +10,20 @@ import torch from fastai import * from fastai.callbacks import CSVLogger, SaveModelCallback from fastai.text import * +from fastai_contrib import utils from fastai_contrib.data import LanguageModelType from fastai_contrib.learner import bilm_text_classifier_learner, bilm_learner, accuracy_fwd, accuracy_bwd from fastai_contrib.utils import PAD, UNK, read_clas_data, PAD_TOKEN_ID, DATASETS, TRN, VAL, TST, ensure_paths_exists, \ - get_sentencepiece + get_sentencepiece, MosesTokenizerFunc from fastai.text.transform import Vocab - import fire from collections import Counter from pathlib import Path from ulmfit.pretrain_lm import LMHyperParams, Tokenizers, ENC_BEST - -class MosesTokenizerFunc(BaseTokenizer): - "Wrapper around a MosesTokenizer to make it a `BaseTokenizer`." - def __init__(self, lang:str): - self.tok = MosesTokenizer(lang) - - def tokenizer(self, t:str) -> List[str]: - return self.tok.tokenize(t, return_str=False, escape=False) - - def add_special_cases(self, toks:Collection[str]): - for w in toks: - assert len(self.tokenizer(w))==1, f"Tokenizer is unable to keep {w} as one token!" - class CLSHyperParams(LMHyperParams): # dir_path -> data/imdb/ use_test_for_validation=False @@ -138,28 +125,7 @@ class CLSHyperParams(LMHyperParams): trn_df, val_df = trn_df[:trn_len], trn_df[trn_len:] cls_cache = '.' - if self.tokenizer is Tokenizers.SUBWORD: - shutil.copy(self.base_lm_path / '..' / 'itos.pkl', self.cache_dir) - shutil.copy(self.base_lm_path / '..' / 'spm.model', self.cache_dir) - shutil.copy(self.base_lm_path / '..' / 'spm.vocab', self.cache_dir) - - args = get_sentencepiece(self.cache_dir, - lambda: trn_df[1], - self.name, - vocab_size=self.max_vocab, - lang='en', - use_moses=True) - - # TODO remove migration of tokens for SentencePiece as more than 50% of tokens are different in imdb - elif self.tokenizer is Tokenizers.MOSES: - args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en', pre_rules=[], post_rules=[])) - elif self.tokenizer is Tokenizers.MOSES_FA: - args = dict(tokenizer=Tokenizer(tok_func=MosesTokenizerFunc, lang='en')) # use default pre/post rules - elif self.tokenizer is Tokenizers.FASTAI: - args = dict() - else: - raise ValueError( - f"self.tokenizer has wrong value {self.tokenizer}, Allowed values are taken from {Tokenizers}") + args = self.tokenzier_to_fastai_args(trn_data_loading_func=lambda: trn_df[1], add_moses=True) try: if force: raise FileNotFoundError("Forcing reloading of caches") @@ -235,3 +201,6 @@ class CLSHyperParams(LMHyperParams): if __name__ == '__main__': fire.Fire(CLSHyperParams) + +## + From 0945c699c786cba4c75ad5127bf317d15d260349 Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 1 Jan 2019 15:07:02 +0100 Subject: [PATCH 40/41] Fixes #25 by adding article title in markdown format to wiki text --- ulmfit/create_wikitext.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/create_wikitext.py b/ulmfit/create_wikitext.py index 942475a..30073d4 100644 --- a/ulmfit/create_wikitext.py +++ b/ulmfit/create_wikitext.py @@ -24,7 +24,7 @@ def get_texts(root): if text.strip() == title: # print('No content continuing...') continue - yield text + yield (f"={title}=\n"+text) def write_wikitext(file_path, text_iter, mt, num_tokens, mode='w'): From f784d7bcd24d9befe4f82c23b77139b0b30a5f5b Mon Sep 17 00:00:00 2001 From: Piotr Czapla Date: Tue, 1 Jan 2019 15:13:14 +0100 Subject: [PATCH 41/41] Make the article detection code work with our wikitext --- ulmfit/pretrain_lm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py index 27b3fb3..9ee559c 100644 --- a/ulmfit/pretrain_lm.py +++ b/ulmfit/pretrain_lm.py @@ -35,7 +35,7 @@ class Tokenizers(Enum): FASTAI='f' def istitle(line): - return len(re.findall(r'^ = [^=]* = $', line)) != 0 + return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0 def read_wiki_articles(filename): articles = [] @@ -48,6 +48,7 @@ def read_wiki_articles(filename): articles.append(current_article) current_article = '' articles.append(current_article) + print(f"Wiki text was split to {len(articles)} articles") return pd.DataFrame({'texts':np.array(articles)}) @dataclass