Merge pull request #37 from n-waves/qrnn_perf

QRNN Time Performance Benchark

results/time_benchmark/logs.md

@@ -0,0 +1,50 @@
+# Results
+
+## Set-up. 
+ - Num Tokens 15K
+ - GPU V100
+ - LM BPTT = 70
+ - LM BS = 64
+ - CLAS BS = 32
+
+| Model          |   LSTM    |   QRNN    |
+|----------------|-----------|-----------|
+| LM ms/batch    |   143ms   |   71ms    |
+| CLAS ms/batch  |   467ms   |   156ms   |
+
+
+```
+> python results/time_benchmark/qrnn_benchmark.py
+
+Vocab size 14513                                                                                                                                                                                                    
+QRNN
+LM
+epoch     train_loss  valid_loss  accuracy
+1         6.326089                                                                                                                                                                                                                        
+Total time: 00:11
+Batch size torch.Size([64, 70])
+Params = 22 MM
+Training time is 71.0 ms per batch
+CLAS
+epoch     train_loss  valid_loss  accuracy
+1         0.712603                                                                                                                                                                                                                      
+Total time: 00:10
+Batch size torch.Size([32, 1445])
+Params = 22 MM
+Training time is 156.0 ms per batch
+LSTM
+LM
+epoch     train_loss  valid_loss  accuracy
+1         6.262911                                                                                                                                                                                                                        
+Total time: 00:21
+Batch size torch.Size([64, 70])
+Params = 37 MM
+Training time is 143.0 ms per batch
+CLAS
+epoch     train_loss  valid_loss  accuracy
+1         0.706715                                                                                                                                                                                                                      
+Total time: 00:32
+Batch size torch.Size([32, 1445])
+Params = 37 MM
+Training time is 467.0 ms per batch
+```

results/time_benchmark/qrnn_benchmark.py

@@ -0,0 +1,52 @@
+import glob
+import shutil
+import time
+from fastai.text import * 
+
+orig_path = untar_data(URLs.IMDB)
+path = Path('data') / 'imdb_small'
+path.mkdir(parents=True, exist_ok=True)
+
+for mode in ['train', 'test']:
+    for label in ['pos', 'neg']:
+        tgt_path = path / mode / label
+        tgt_path.mkdir(parents=True, exist_ok=True)
+        # Keep just 10% of the files
+        pattern = str(orig_path / mode / label / '3*.txt')
+        for file in glob.glob(pattern):
+            shutil.copy(file, tgt_path)
+
+data_lm = TextLMDataBunch.from_folder(path, valid='test')
+data_clas = TextClasDataBunch.from_folder(path, bs=32, vocab=data_lm.train_ds.vocab, valid='test')
+
+print('Vocab size', len(data_lm.train_ds.vocab.itos))
+
+
+def count_parameters(model, requires_grad):
+    return sum(p.numel() for p in model.parameters() if p.requires_grad == requires_grad)
+
+
+def test(qrnn, func, config, data, arch=AWD_LSTM):
+    total = len(list(data.train_dl))
+    config = config.copy()
+    config['qrnn'] = qrnn
+    
+    learn = func(data, AWD_LSTM, config=config, pretrained=False)
+    learn.unfreeze()
+    params = count_parameters(learn.model, True)
+    total = len(list(data.train_dl))
+    start_time = time.clock()
+    learn.fit(1)
+    diff = time.clock() - start_time
+    
+    print('Batch size', data.one_batch()[0].shape)
+    print(f'Params = {params // 1000000} MM')
+    print(f'Training time is {1000 * diff // total} ms per batch')
+
+
+for qrnn in [True, False]:
+    print('QRNN' if qrnn else 'LSTM')
+    print('LM')
+    test(qrnn, language_model_learner, config=awd_lstm_lm_config, data=data_lm)
+    print('CLAS')
+    test(qrnn, text_classifier_learner, config=awd_lstm_clas_config, data=data_clas)