diff --git a/fastai_contrib/learner.py b/fastai_contrib/learner.py
index 67e79cb..de5d26a 100644
--- a/fastai_contrib/learner.py
+++ b/fastai_contrib/learner.py
@@ -86,7 +86,13 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
if 'model' in wgts:
wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix)
else:
- dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight']
+ #dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight']
+ has_bias = prefix+'1.decoder.bias' in wgts
+ enc_wgts = wgts[prefix+'0.encoder.weight']
+ if has_bias:
+ dec_bias = wgts[prefix+'1.decoder.bias']
+ else:
+ dec_bias = enc_wgts.new_zeros((len(stoi_wgts),))
bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0)
new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_()
new_b = dec_bias.new_zeros((len(itos_new),)).zero_()
@@ -101,7 +107,8 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
wgts[prefix+'0.encoder.weight'] = new_w
wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone()
wgts[prefix+'1.decoder.weight'] = new_w.clone()
- wgts[prefix+'1.decoder.bias'] = new_b
+ if has_bias:
+ wgts[prefix+'1.decoder.bias'] = new_b
return wgts
#endregion
@@ -110,4 +117,4 @@ def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:
import fastai.text.learner
fastai.text.learner.convert_weights = convert_weights
-#endregion
\ No newline at end of file
+#endregion
diff --git a/fastai_contrib/models.py b/fastai_contrib/models.py
index 92ac7e4..8b5afdb 100644
--- a/fastai_contrib/models.py
+++ b/fastai_contrib/models.py
@@ -1,7 +1,7 @@
from fastai.torch_core import *
from fastai.layers import *
from fastai.text.models import *
-
+from fastai.text.learner import *
#region New code
class BiLMModel(nn.Module):
@@ -163,4 +163,4 @@ def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_s
model.reset()
return model
-#endregion
\ No newline at end of file
+#endregion
diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py
index ebd9674..f527596 100644
--- a/fastai_contrib/utils.py
+++ b/fastai_contrib/utils.py
@@ -80,7 +80,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
pre_rules = pre_rules if pre_rules is not None else defaults.text_pre_rules
post_rules = post_rules if post_rules is not None else defaults.text_post_rules
- special_cases = defaults.text_spec_tok
+ special_cases = defaults.text_spec_tok + ['', '', '', '', '']
if not os.path.isfile(cache_dir / 'spm.model') or not os.path.isfile(cache_dir / f'itos.pkl'):
# load the text from the train tokens file
text = load_text()
@@ -100,7 +100,7 @@ def get_sentencepiece(cache_dir:PathOrStr, load_text, pre_rules: ListRules=None,
sp_params = [
f"--input={raw_text_path}",
f"--character_coverage={char_coverage}",
- f"--unk_id={len(defaults.text_spec_tok)}",
+ f"--unk_id={len(special_cases)}",
f"--pad_id=-1",
f"--bos_id=-1",
f"--eos_id=-1",
diff --git a/ulmfit/pretrain_lm.py b/ulmfit/pretrain_lm.py
index 2a6f59f..e7f8c5c 100644
--- a/ulmfit/pretrain_lm.py
+++ b/ulmfit/pretrain_lm.py
@@ -39,7 +39,9 @@ def istitle(line):
return len(re.findall(r'^ ?= [^=]* = ?$', line)) != 0
def read_wiki_articles(filename):
+ return pd.read_csv(filename, sep="\t", header=None, names=["texts"])
articles = []
+
with open(filename, encoding='utf8') as f:
lines = f.readlines()
current_article = []
@@ -70,7 +72,8 @@ class LMHyperParams:
# these hyperparameters are for training on ~100M tokens (e.g. WikiText-103)
# for training on smaller datasets, more dropout is necessary
- dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
+ # buggy dps = dict(output_p=0.25, hidden_p=0.1, input_p=0.2, embed_p=0.02, weight_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
+ dps = dict(input_p=0.25, output_p=0.1, weight_p=0.2, embed_p=0.02, hidden_p=0.15) # consider removing dps & clip from the default hyperparams and put them to train
clip: float = 0.12
bptt: int = 70
# alpha and beta - defaults like in fastai/text/learner.py:RNNLearner()
@@ -171,10 +174,10 @@ class LMHyperParams:
with (self.model_dir / 'info.json').open("w") as fp: json.dump(vals, fp)
print("Saving info", self.model_dir / 'info.json')
- def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0):
+ def train_lm(self, num_epochs=20, data_lm=None, bs=70, true_wd=False, drop_mult=0.0, lr=5e-3, label_smoothing_eps=0.0, out_bias=True):
self.model_dir.mkdir(exist_ok=True, parents=True)
data_lm = self.load_wiki_data(bs=bs) if data_lm is None else data_lm
- learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps)
+ learn = self.create_lm_learner(data_lm, drop_mult=drop_mult, label_smoothing_eps=label_smoothing_eps, out_bias=out_bias)
print("Bptt", data_lm.bptt)
learn.true_wd = true_wd
if num_epochs > 0:
@@ -204,10 +207,10 @@ class LMHyperParams:
# do we need to return `learn'? it adds noise to Fire output
#return learn
- def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, **kwargs):
+ def create_lm_learner(self, data_lm, dps=None, label_smoothing_eps=0.0, out_bias=True, **kwargs):
assert self.bidir == False, "bidirectional model is not yet supported"
config = dict(emb_sz=self.emb_sz, n_hid=self.nh, n_layers=self.nl, pad_token=PAD_TOKEN_ID, qrnn=self.qrnn,
- tie_weights=True, out_bias=True)
+ tie_weights=True, out_bias=out_bias)
config.update(dps or self.dps)
trn_args = dict(clip=self.clip, alpha=self.rnn_alpha, beta=self.rnn_beta)
trn_args.update(kwargs)
diff --git a/ulmfit/train_clas.py b/ulmfit/train_clas.py
index da66aac..4396136 100644
--- a/ulmfit/train_clas.py
+++ b/ulmfit/train_clas.py
@@ -12,6 +12,12 @@ import fire
from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
+from sklearn.metrics import f1_score as f1s, precision_score, recall_score
+
+def f1_score(preds, targs):
+ preds = torch.max(preds, dim=1)[1].cpu().numpy()
+ targs = targs.cpu().numpy()
+ return torch.tensor(f1s(targs, preds))
class CLSHyperParams(LMHyperParams):
# dir_path -> data/imdb/
@@ -107,9 +113,12 @@ class CLSHyperParams(LMHyperParams):
if data_tst is None:
_, _, data_tst = self.load_cls_data(bs)
if learn is None:
- learn = self.create_cls_learner(data_tst, drop_mult=0.3)
+ learn = self.create_cls_learner(data_tst, drop_mult=0.3, metrics=[f1_score, accuracy])
learn.unfreeze()
learn.load(save_name)
+ print(data_tst.one_batch()[1])
+ print(data_tst.one_batch()[1])
+ print(data_tst.one_batch()[1])
results = learn.validate(data_tst.valid_dl)
print(f"Loss and accuracy using ({save_name}):", results)
return list(map(float, results))
@@ -141,7 +150,7 @@ class CLSHyperParams(LMHyperParams):
self.model_dir.mkdir(exist_ok=True, parents=True)
add_trn_to_lm = True
lang = self.lang
- use_moses = True
+ use_moses = False #True
if 'xnli' in str(self.dataset_dir):
NotImplementedError("Support for Xnli is not implemented yet")
if 'imdb' in self.dataset_dir.name: