Fixes after mergin with master and updateing to newset fastai

fastai_contrib/utils.py

@@ -225,7 +225,7 @@ def read_imdb(dir_path, lang, split, spm_path=None) -> Tuple[List[List[str]], Li
         reader = csv.reader(f)
         for row in reader:
             label, text = row
-            lbls.append(label)
+            lbls.append(int(label))
             raw_tokens = mt.tokenize(text, return_str=True).split(' ')
             
             tokens = []

ulmfit/pretrain_lm.py

@@ -42,7 +42,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
     :param bidir: whether the language model is bidirectional
     """
     results = {}
-    model_dir = 'models' # removed from params, as it is absolute models location in train_clas and here it is relative
+
     if not torch.cuda.is_available():
         print('CUDA not available. Setting device=-1.')
         cuda_id = -1
@@ -50,7 +50,8 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
 
     dir_path = Path(dir_path)
     assert dir_path.exists()
-    (dir_path/model_dir).mkdir(exist_ok=True)
+    model_dir = dir_path / 'models'  # removed from params, as it is absolute models location in train_clas and here it is relative
+    model_dir.mkdir(exist_ok=True)
     print('Batch size:', bs)
     print('Max vocab:', max_vocab)
     model_name = 'qrnn' if qrnn else 'lstm'
@@ -93,11 +94,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
             itos.insert(1, PAD)  #  set pad id to 1 to conform to fast.ai standard
             assert UNK in itos, f'Unknown words are expected to have been replaced with {UNK} in the data.'
 
-            vocab = Vocab(itos)
-            stoi = vocab.stoi
-
             # save vocabulary
-
             print(f"Saving vocabulary as {itos_fname}")
             results['itos_fname'] = itos_fname
             with open(itos_fname, 'wb') as f:
@@ -139,7 +136,7 @@ def pretrain_lm(dir_path, lang='en', cuda_id=0, qrnn=True, subword=False, max_vo
 
     lm_learner = bilm_learner if bidir else language_model_learner
     learn = lm_learner(data_lm, bptt=bptt, emb_sz=emb_sz, nh=nh, nl=nl, pad_token=1,
-                       drop_mult=drop_mult, tie_weights=True, model_dir=model_dir,
+                       drop_mult=drop_mult, tie_weights=True, model_dir=model_dir.name,
                        bias=True, qrnn=qrnn, clip=0.12)
     # compared to standard Adam, we set beta_1 to 0.8
     learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))

ulmfit/train_clas.py

@@ -136,7 +136,7 @@ def new_train_clas(data_dir, lang='en', cuda_id=0, pretrain_name='wt103', model_
         print(f"Saving models at {learn.path / learn.model_dir}")
         learn.save(f'{model_name}_{name}')
 
-    results['accuracy'] = learn.metrics[-1][0]
+    results['accuracy'] = learn.recorder.metrics[-1][0]
     return results
 
 
@@ -149,7 +149,7 @@ def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_typ
         toks, lbls = read_clas_data(dataset_dir, dataset, lang)
 
         # create the vocabulary
-        counter = Counter(word for example in toks[TRN] for word in example)
+        counter = Counter(word for example in np.concatenate([toks[TRN],toks[TST],toks[VAL]]) for word in example)
         itos = [word for word, count in counter.most_common(n=max_vocab)]
         itos.insert(0, PAD)
         itos.insert(0, UNK)
@@ -178,12 +178,12 @@ def get_datasets(dataset, dataset_dir, bptt, bs, lang, max_vocab, ds_pct, lm_typ
         print(f"Making the dataset smaller {ds_pct}")
         for split in [TRN, VAL, TST]:
             ids[split] = ids[split][:int(len(ids[split]) * ds_pct)]
-    data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=ids[TRN],
+    data_lm = TextLMDataBunch.from_ids(path=tmp_dir, vocab=vocab, train_ids=np.concatenate([ids[TRN],ids[TST]]),
                                        valid_ids=ids[VAL], bs=bs, bptt=bptt, lm_type=lm_type)
     #  TODO TextClasDataBunch allows tst_ids as input, but not tst_lbls?
     data_clas = TextClasDataBunch.from_ids(
         path=tmp_dir, vocab=vocab, train_ids=ids[TRN], valid_ids=ids[VAL],
-        train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs)
+        train_lbls=lbls[TRN], valid_lbls=lbls[VAL], bs=bs, classes={l:l for l in lbls[VAL]})
     return data_clas, data_lm