fix dropout

fastai_contrib/utils.py

@@ -99,7 +99,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
         sp_params = [
             f"--input={raw_text_path}",
             f"--character_coverage={char_coverage}",
-            f"--unk_id={len(defaults.text_spec_tok)}",
+            f"--unk_id={len(special_cases)}",
             f"--pad_id=-1",
             f"--bos_id=-1",
             f"--eos_id=-1",

ulmfit/pretrain_lm.py

@@ -67,7 +67,8 @@ class LMHyperParams:
 
     # these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
     # for training on smaller datasets, more dropout is necessary
-    dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
+    # buggy dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
+    dps = dict(input_p=0.25, output_p=0.1, weight_p=0.2, embed_p=0.02, hidden_p=0.15) 
     clip: float = 0.12
     bptt: int = 70
     # alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()