diff --git a/results/time_benchmark/logs.md b/results/time_benchmark/logs.md new file mode 100644 index 0000000..d5ec78f --- /dev/null +++ b/results/time_benchmark/logs.md @@ -0,0 +1,50 @@ +# Results + +## Set-up. + - Num Tokens 15K + - GPU V100 + - LM BPTT = 70 + - LM BS = 64 + - CLAS BS = 32 + +| Model | LSTM | QRNN | +|----------------|-----------|-----------| +| LM ms/batch | 143ms | 71ms | +| CLAS ms/batch | 467ms | 156ms | + + +``` +> python results/time_benchmark/qrnn_benchmark.py + +Vocab size 14513 +QRNN +LM +epoch train_loss valid_loss accuracy +1 6.326089 +Total time: 00:11 +Batch size torch.Size([64, 70]) +Params = 22 MM +Training time is 71.0 ms per batch +CLAS +epoch train_loss valid_loss accuracy +1 0.712603 +Total time: 00:10 +Batch size torch.Size([32, 1445]) +Params = 22 MM +Training time is 156.0 ms per batch +LSTM +LM +epoch train_loss valid_loss accuracy +1 6.262911 +Total time: 00:21 +Batch size torch.Size([64, 70]) +Params = 37 MM +Training time is 143.0 ms per batch +CLAS +epoch train_loss valid_loss accuracy +1 0.706715 +Total time: 00:32 +Batch size torch.Size([32, 1445]) +Params = 37 MM +Training time is 467.0 ms per batch +``` \ No newline at end of file diff --git a/results/time_benchmark/qrnn_benchmark.py b/results/time_benchmark/qrnn_benchmark.py new file mode 100644 index 0000000..f6dab86 --- /dev/null +++ b/results/time_benchmark/qrnn_benchmark.py @@ -0,0 +1,52 @@ +import glob +import shutil +import time +from fastai.text import * + +orig_path = untar_data(URLs.IMDB) +path = Path('data') / 'imdb_small' +path.mkdir(parents=True, exist_ok=True) + +for mode in ['train', 'test']: + for label in ['pos', 'neg']: + tgt_path = path / mode / label + tgt_path.mkdir(parents=True, exist_ok=True) + # Keep just 10% of the files + pattern = str(orig_path / mode / label / '3*.txt') + for file in glob.glob(pattern): + shutil.copy(file, tgt_path) + +data_lm = TextLMDataBunch.from_folder(path, valid='test') +data_clas = TextClasDataBunch.from_folder(path, bs=32, vocab=data_lm.train_ds.vocab, valid='test') + +print('Vocab size', len(data_lm.train_ds.vocab.itos)) + + +def count_parameters(model, requires_grad): + return sum(p.numel() for p in model.parameters() if p.requires_grad == requires_grad) + + +def test(qrnn, func, config, data, arch=AWD_LSTM): + total = len(list(data.train_dl)) + config = config.copy() + config['qrnn'] = qrnn + + learn = func(data, AWD_LSTM, config=config, pretrained=False) + learn.unfreeze() + params = count_parameters(learn.model, True) + total = len(list(data.train_dl)) + start_time = time.clock() + learn.fit(1) + diff = time.clock() - start_time + + print('Batch size', data.one_batch()[0].shape) + print(f'Params = {params // 1000000} MM') + print(f'Training time is {1000 * diff // total} ms per batch') + + +for qrnn in [True, False]: + print('QRNN' if qrnn else 'LSTM') + print('LM') + test(qrnn, language_model_learner, config=awd_lstm_lm_config, data=data_lm) + print('CLAS') + test(qrnn, text_classifier_learner, config=awd_lstm_clas_config, data=data_clas) \ No newline at end of file