From dbb929e3ca8997f0d566a652074b607f105f67d0 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 18 Feb 2019 16:54:00 -0300 Subject: [PATCH 1/7] Adding more text cols to use all CLS data --- ulmfit/train_clas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 86ef276..aa72a1c 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -167,6 +167,7 @@ class CLSHyperParams(LMHyperParams): cls_name = f'{cls_name}noise{noise}' args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) + args['text_cols'] = list(csv.columns.values)[1:] data_lm = self.lm_databunch('lm', train_df=lm_trn_df, valid_df=lm_val_df, bs=bs, force=force, **args) args['vocab'] = data_lm.vocab data_cls = self.cls_databunch(cls_name, train_df=trn_df, valid_df=val_df, bs=bs, force=force, **args) From 458c06f779079ea235cd47730951300c6842c6ce Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 18 Feb 2019 17:05:11 -0300 Subject: [PATCH 2/7] Fix df name --- ulmfit/train_clas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index aa72a1c..0fb1993 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -167,7 +167,7 @@ class CLSHyperParams(LMHyperParams): cls_name = f'{cls_name}noise{noise}' args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) - args['text_cols'] = list(csv.columns.values)[1:] + args['text_cols'] = list(trn_df.columns.values)[1:] data_lm = self.lm_databunch('lm', train_df=lm_trn_df, valid_df=lm_val_df, bs=bs, force=force, **args) args['vocab'] = data_lm.vocab data_cls = self.cls_databunch(cls_name, train_df=trn_df, valid_df=val_df, bs=bs, force=force, **args) From 13ae29a95d2179b20726807b6086d753b98d5279 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Mon, 18 Feb 2019 22:20:24 -0300 Subject: [PATCH 3/7] Set mark_fields in True --- ulmfit/train_clas.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 0fb1993..7c3f980 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -168,6 +168,7 @@ class CLSHyperParams(LMHyperParams): args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) args['text_cols'] = list(trn_df.columns.values)[1:] + args['mark_fields'] = True data_lm = self.lm_databunch('lm', train_df=lm_trn_df, valid_df=lm_val_df, bs=bs, force=force, **args) args['vocab'] = data_lm.vocab data_cls = self.cls_databunch(cls_name, train_df=trn_df, valid_df=val_df, bs=bs, force=force, **args) From b39fb5390b3f995689383953eb802d230b3024e5 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Wed, 20 Feb 2019 15:21:58 -0300 Subject: [PATCH 4/7] Adding QRNN benchmark --- results/time_benchmark/qrnn_benchmark.py | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 results/time_benchmark/qrnn_benchmark.py diff --git a/results/time_benchmark/qrnn_benchmark.py b/results/time_benchmark/qrnn_benchmark.py new file mode 100644 index 0000000..154ec9a --- /dev/null +++ b/results/time_benchmark/qrnn_benchmark.py @@ -0,0 +1,52 @@ +import glob +import shutil +import time +from fastai.text import * + +orig_path = untar_data(URLs.IMDB) +path = Path('data') / 'imdb_small' +path.mkdir(parents=True, exist_ok=True) + +for mode in ['train', 'test']: + for label in ['pos', 'neg']: + tgt_path = path / mode / label + tgt_path.mkdir(parents=True, exist_ok=True) + # Keep just 10% of the files + pattern = str(orig_path / mode / label / '3*.txt') + for file in glob.glob(pattern): + shutil.copy(file, tgt_path) + +data_lm = TextLMDataBunch.from_folder(path) +data_clas = TextClasDataBunch.from_folder(path, bs=32, vocab=data_lm.train_ds.vocab) + +print('Vocab size', len(data_lm.train_ds.vocab.itos)) + + +def count_parameters(model, requires_grad): + return sum(p.numel() for p in model.parameters() if p.requires_grad == requires_grad) + + +def test(qrnn, func, config, data, arch=AWD_LSTM): + total = len(list(data.train_dl)) + config = config.copy() + config['qrnn'] = qrnn + + learn = func(data, AWD_LSTM, config=config, pretrained=False) + learn.unfreeze() + params = count_parameters(learn.model, True) + total = len(list(data.train_dl)) + start_time = time.clock() + learn.fit(1) + diff = time.clock() - start_time + + print('Batch size', data.one_batch()[0].shape) + print(f'Params = {params // 1000000} MM') + print(f'Training time is {1000 * diff // total} ms per batch') + + +for qrnn in [True, False]: + print('QRNN' if qrnn else 'LSTM') + print('LM') + test(qrnn, language_model_learner, config=awd_lstm_lm_config, data=data_lm) + print('CLAS') + test(qrnn, text_classifier_learner, config=awd_lstm_clas_config, data=data_clas) \ No newline at end of file From c698c97772f1e9759df46bfc125953e968024f34 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Wed, 20 Feb 2019 17:02:02 -0300 Subject: [PATCH 5/7] Adding logs --- results/time_benchmark/logs.md | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 results/time_benchmark/logs.md diff --git a/results/time_benchmark/logs.md b/results/time_benchmark/logs.md new file mode 100644 index 0000000..d5ec78f --- /dev/null +++ b/results/time_benchmark/logs.md @@ -0,0 +1,50 @@ +# Results + +## Set-up. + - Num Tokens 15K + - GPU V100 + - LM BPTT = 70 + - LM BS = 64 + - CLAS BS = 32 + +| Model | LSTM | QRNN | +|----------------|-----------|-----------| +| LM ms/batch | 143ms | 71ms | +| CLAS ms/batch | 467ms | 156ms | + + +``` +> python results/time_benchmark/qrnn_benchmark.py + +Vocab size 14513 +QRNN +LM +epoch train_loss valid_loss accuracy +1 6.326089 +Total time: 00:11 +Batch size torch.Size([64, 70]) +Params = 22 MM +Training time is 71.0 ms per batch +CLAS +epoch train_loss valid_loss accuracy +1 0.712603 +Total time: 00:10 +Batch size torch.Size([32, 1445]) +Params = 22 MM +Training time is 156.0 ms per batch +LSTM +LM +epoch train_loss valid_loss accuracy +1 6.262911 +Total time: 00:21 +Batch size torch.Size([64, 70]) +Params = 37 MM +Training time is 143.0 ms per batch +CLAS +epoch train_loss valid_loss accuracy +1 0.706715 +Total time: 00:32 +Batch size torch.Size([32, 1445]) +Params = 37 MM +Training time is 467.0 ms per batch +``` \ No newline at end of file From 61ae344fda4ec7ddf6cd1e34b163459194ab686e Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Wed, 20 Feb 2019 17:02:56 -0300 Subject: [PATCH 6/7] Remove fastai warning --- results/time_benchmark/qrnn_benchmark.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/results/time_benchmark/qrnn_benchmark.py b/results/time_benchmark/qrnn_benchmark.py index 154ec9a..f6dab86 100644 --- a/results/time_benchmark/qrnn_benchmark.py +++ b/results/time_benchmark/qrnn_benchmark.py @@ -16,8 +16,8 @@ for mode in ['train', 'test']: for file in glob.glob(pattern): shutil.copy(file, tgt_path) -data_lm = TextLMDataBunch.from_folder(path) -data_clas = TextClasDataBunch.from_folder(path, bs=32, vocab=data_lm.train_ds.vocab) +data_lm = TextLMDataBunch.from_folder(path, valid='test') +data_clas = TextClasDataBunch.from_folder(path, bs=32, vocab=data_lm.train_ds.vocab, valid='test') print('Vocab size', len(data_lm.train_ds.vocab.itos)) From c5a93d19fea9088637ffe532ac7e309957aa1a44 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Wed, 20 Feb 2019 17:16:08 -0300 Subject: [PATCH 7/7] Changes moved to different branch --- ulmfit/train_clas.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py index 98980e1..e23688e 100644 --- a/ulmfit/train_clas.py +++ b/ulmfit/train_clas.py @@ -202,8 +202,6 @@ class CLSHyperParams(LMHyperParams): cls_name = f'{cls_name}noise{noise}' args = self.tokenizer_to_fastai_args(sp_data_func=lambda: trn_df[1], use_moses=use_moses) - args['text_cols'] = list(trn_df.columns.values)[1:] - args['mark_fields'] = True data_lm = self.lm_databunch('lm', train_df=lm_trn_df, valid_df=lm_val_df, bs=bs, force=force, **args) args['vocab'] = data_lm.vocab data_cls = self.cls_databunch(cls_name, train_df=trn_df, valid_df=val_df, bs=bs, force=force, **args)