mirror of
https://github.com/wassname/multifit.git
synced 2026-09-09 11:27:26 +08:00
Remove old bilm code that wasn't working
This commit is contained in:
Piotr Czapla
@@ -1,71 +0,0 @@
|
||||
"NLP data loading pipeline. Supports csv, folders, and preprocessed data."
|
||||
from fastai.text import *
|
||||
from fastai.torch_core import *
|
||||
from fastai.text.transform import *
|
||||
from fastai.basic_data import *
|
||||
from fastai.data_block import *
|
||||
|
||||
#region Modified fastai classes
|
||||
|
||||
LanguageModelType=Enum('LanguageModelType', 'FwdLM BwdLM BiLM')
|
||||
|
||||
class LanguageModelLoader(): # copy of the original LanguageModelLoader
|
||||
"Create a dataloader with bptt slightly changing."
|
||||
def __init__(self, dataset:LabelList, bs:int=64, bptt:int=70,
|
||||
lm_type:LanguageModelType=LanguageModelType.FwdLM, shuffle:bool=False,
|
||||
max_len:int=25, p_bptt:int=0.95):
|
||||
self.dataset,self.bs,self.bptt,self.lm_type,self.shuffle, self.p_bptt = dataset,bs,bptt,lm_type,shuffle,p_bptt
|
||||
self.first,self.i,self.iter = True,0,0
|
||||
self.n = len(np.concatenate(dataset.x.items)) // self.bs if len(dataset.x.items) > 0 else 0
|
||||
self.max_len,self.num_workers = max_len,0
|
||||
self.init_kwargs = dict(bs=bs, bptt=bptt, lm_type=lm_type, shuffle=shuffle, max_len=max_len, p_bptt=p_bptt)
|
||||
|
||||
def __iter__(self):
|
||||
if getattr(self.dataset, 'item', None) is not None:
|
||||
yield LongTensor(getattr(self.dataset, 'item'))[None],LongTensor([0])
|
||||
idx = np.random.permutation(len(self.dataset)) if self.shuffle else range(len(self.dataset))
|
||||
data = self.batchify(np.concatenate([self.dataset.x.items[i] for i in idx]))
|
||||
|
||||
pos, itr = 0,0
|
||||
while pos < self.n-1 and itr<len(self):
|
||||
if self.first and pos == 0: self.first,seq_len = False,self.bptt + self.max_len
|
||||
else:
|
||||
bptt = self.bptt if np.random.random() < self.p_bptt else self.bptt / 2.
|
||||
seq_len = max(5, int(np.random.normal(bptt, 5)))
|
||||
seq_len = min(seq_len, self.bptt + self.max_len)
|
||||
res = self.get_batch(data, pos, seq_len)
|
||||
pos += seq_len
|
||||
itr += 1
|
||||
yield res
|
||||
|
||||
def __len__(self) -> int: return int(math.ceil((self.n-1) / self.bptt)) # so that it is always at least 1
|
||||
def __getattr__(self,k:str)->Any: return getattr(self.dataset, k)
|
||||
|
||||
@property
|
||||
def batch_size(self): return self.bs
|
||||
@batch_size.setter
|
||||
def batch_size(self, v): self.bs = v
|
||||
|
||||
def batchify(self, data:np.ndarray) -> LongTensor:
|
||||
"Split the corpus `data` in batches."
|
||||
nb = data.shape[0] // self.bs
|
||||
data = np.array(data[:nb*self.bs]).reshape(self.bs, -1)
|
||||
if self.lm_type == LanguageModelType.BwdLM: data = data[:,::-1].copy()
|
||||
elif self.lm_type == LanguageModelType.BiLM: data = np.stack([data, data[:,::-1].copy()], axis=2)
|
||||
return LongTensor(data)
|
||||
|
||||
def get_batch(self, data:LongTensor, i:int, seq_len:int) -> Tuple[LongTensor, LongTensor]:
|
||||
"Create a batch at `i` of a given `seq_len`."
|
||||
seq_len = min(seq_len, data.shape[1] - 1 - i)
|
||||
x = data[:,i:i+seq_len]
|
||||
y = data[:,i+1:i+1+seq_len]
|
||||
y = y.contiguous().view(-1, 2) if self.lm_type == LanguageModelType.BiLM else y.contiguous().view(-1)
|
||||
return x,y
|
||||
|
||||
#endregion
|
||||
#region Replaces fastai classes
|
||||
|
||||
import fastai.text.data
|
||||
fastai.text.data.LanguageModelLoader = LanguageModelLoader # Replace original LanguageModelLoader with new verion
|
||||
|
||||
#endregion
|
||||
@@ -1,113 +0,0 @@
|
||||
from torch.nn import CrossEntropyLoss
|
||||
|
||||
from fastai import *
|
||||
from fastai.text import *
|
||||
|
||||
#region New code
|
||||
from fastai_contrib.models import *
|
||||
|
||||
|
||||
def bilm_learner(data:DataBunch, bptt:int=70, emb_sz:int=400, nh:int=1150, nl:int=3, pad_token:int=1,
|
||||
drop_mult:float=1., tie_weights:bool=True, bias:bool=True, qrnn:bool=False, pretrained_model=None,
|
||||
pretrained_fnames:OptStrTuple=None, **kwargs) -> 'LanguageLearner':
|
||||
"Create a `Learner` with a language model."
|
||||
dps = default_dropout['language'] * drop_mult
|
||||
vocab_size = len(data.vocab.itos)
|
||||
model = get_bilm(vocab_size, emb_sz, nh, nl, pad_token, input_p=dps[0], output_p=dps[1],
|
||||
weight_p=dps[2], embed_p=dps[3], hidden_p=dps[4], tie_weights=tie_weights, bias=bias, qrnn=qrnn)
|
||||
learn = LanguageLearner(data, model, bptt, split_func=bilm_split, **kwargs)
|
||||
if pretrained_model is not None:
|
||||
model_path = untar_data(pretrained_model, data=False)
|
||||
fnames = [list(model_path.glob(f'*.{ext}'))[0] for ext in ['pth', 'pkl']]
|
||||
learn.load_pretrained(*fnames)
|
||||
learn.freeze()
|
||||
if pretrained_fnames is not None:
|
||||
fnames = [learn.path/learn.model_dir/f'{fn}.{ext}' for fn,ext in zip(pretrained_fnames, ['pth', 'pkl'])]
|
||||
learn.load_pretrained(*fnames)
|
||||
learn.freeze()
|
||||
learn.loss_func = CrossEntropyLoss() # I'm not sure why fast ai is using CrossEntropyFlat but it breaks bilm
|
||||
return learn
|
||||
|
||||
def bilm_text_classifier_learner(data: DataBunch, bptt: int = 70, max_len: int = 70 * 20, emb_sz: int = 400,
|
||||
nh: int = 1150, nl: int = 3,
|
||||
lin_ftrs: Collection[int] = None, ps: Collection[float] = None, pad_token: int = 1,
|
||||
drop_mult: float = 1., qrnn: bool = False, bicls_head:str='BiPoolingLinearClassifier', **kwargs) -> 'TextClassifierLearner':
|
||||
"Create a RNN classifier."
|
||||
dps = default_dropout['classifier'] * drop_mult
|
||||
if lin_ftrs is None: lin_ftrs = [50]
|
||||
if ps is None: ps = [0.1]
|
||||
ds = data.train_ds
|
||||
vocab_size, n_class = len(data.vocab.itos), data.c
|
||||
if bicls_head == 'BiPoolingLinearClassifier':
|
||||
count = 3*2
|
||||
else:
|
||||
count = 3
|
||||
layers = [emb_sz * count] + lin_ftrs + [n_class]
|
||||
ps = [dps[4]] + ps
|
||||
model = get_birnn_classifier(bptt, max_len, n_class, vocab_size, emb_sz, nh, nl, pad_token,
|
||||
layers, ps, input_p=dps[0], weight_p=dps[1], embed_p=dps[2], hidden_p=dps[3],
|
||||
qrnn=qrnn, bicls_head=bicls_head)
|
||||
learn = RNNLearner(data, model, bptt, split_func=birnn_classifier_split, **kwargs)
|
||||
return learn
|
||||
|
||||
def bilm_split(model:nn.Module) -> List[nn.Module]:
|
||||
"Split a RNN `model` in groups for differential learning rates."
|
||||
|
||||
return [f+b for f,b in zip(lm_split(model.fwd_lm),lm_split(model.bwd_lm))]
|
||||
|
||||
def birnn_classifier_split(model:nn.Module) -> List[nn.Module]:
|
||||
"Split a RNN `model` in groups for differential learning rates."
|
||||
f_rnn,b_rnn = model[0].fwd_lm,model[0].bwd_lm
|
||||
groups = [[f_rnn.encoder, f_rnn.encoder_dp,b_rnn.encoder, b_rnn.encoder_dp]]
|
||||
groups += [a for a in zip(f_rnn.rnns, f_rnn.hidden_dps, b_rnn.rnns, b_rnn.hidden_dps, )]
|
||||
groups.append([model[1]])
|
||||
return groups
|
||||
|
||||
def accuracy_fwd(input, targs):
|
||||
return accuracy(input[...,0], targs[...,0])
|
||||
|
||||
def accuracy_bwd(input, targs):
|
||||
return accuracy(input[...,1], targs[...,1])
|
||||
|
||||
|
||||
#endregion
|
||||
#region Modified fastai code
|
||||
|
||||
def convert_weights(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str]) -> Weights:
|
||||
"Convert the model weights to go with a new vocabulary."
|
||||
if 'fwd_lm.0.encoder.weight' in wgts: #todo share embedding matrix computation
|
||||
wgts = convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='fwd_lm.')
|
||||
return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='bwd_lm.')
|
||||
else:
|
||||
return convert_weights_with_prefix(wgts, stoi_wgts, itos_new, prefix='')
|
||||
|
||||
def convert_weights_with_prefix(wgts:Weights, stoi_wgts:Dict[str,int], itos_new:Collection[str], prefix='') -> Weights:
|
||||
"Convert the model weights to go with a new vocabulary."
|
||||
if 'model' in wgts:
|
||||
wgts['model'] = convert_weights_with_prefix(wgts['model'], stoi_wgts, itos_new, prefix)
|
||||
else:
|
||||
dec_bias, enc_wgts = wgts[prefix+'1.decoder.bias'], wgts[prefix+'0.encoder.weight']
|
||||
bias_m, wgts_m = dec_bias.mean(0), enc_wgts.mean(0)
|
||||
new_w = enc_wgts.new_zeros((len(itos_new),enc_wgts.size(1))).zero_()
|
||||
new_b = dec_bias.new_zeros((len(itos_new),)).zero_()
|
||||
unk_tokens=[]
|
||||
for i,w in enumerate(itos_new):
|
||||
r = stoi_wgts[w] if w in stoi_wgts else -1
|
||||
if r < 0:
|
||||
unk_tokens.append(w)
|
||||
new_w[i] = enc_wgts[r] if r>=0 else wgts_m
|
||||
new_b[i] = dec_bias[r] if r>=0 else bias_m
|
||||
print(f"Unknown tokens {len(unk_tokens)}, first 100: {unk_tokens[:100]}")
|
||||
wgts[prefix+'0.encoder.weight'] = new_w
|
||||
wgts[prefix+'0.encoder_dp.emb.weight'] = new_w.clone()
|
||||
wgts[prefix+'1.decoder.weight'] = new_w.clone()
|
||||
wgts[prefix+'1.decoder.bias'] = new_b
|
||||
return wgts
|
||||
|
||||
#endregion
|
||||
#region Replace code in fastai
|
||||
|
||||
import fastai.text.learner
|
||||
fastai.text.learner.convert_weights = convert_weights
|
||||
|
||||
#endregion
|
||||
@@ -1,166 +0,0 @@
|
||||
from fastai.torch_core import *
|
||||
from fastai.layers import *
|
||||
from fastai.text.models import *
|
||||
|
||||
#region New code
|
||||
|
||||
class BiLMModel(nn.Module):
|
||||
|
||||
def __init__(self, fwd_lm:nn.Module, bwd_lm:nn.Module, squash_bs_sl=False):
|
||||
super().__init__()
|
||||
self.fwd_lm = fwd_lm
|
||||
self.bwd_lm = bwd_lm
|
||||
self.squash_bs_sl = squash_bs_sl
|
||||
|
||||
def __getitem__(self, idx):
|
||||
return BiLMModel(self.fwd_lm[idx], self.bwd_lm[idx])
|
||||
|
||||
def __len__(self):
|
||||
return len(self.fwd_lm)
|
||||
|
||||
def stack(self, fwd_o, bwd_o):
|
||||
if is_listy(fwd_o):
|
||||
return [self.stack(f, b) for f,b in zip(fwd_o,bwd_o)]
|
||||
else:
|
||||
return torch.stack([fwd_o, bwd_o], dim=len(fwd_o.shape))
|
||||
|
||||
def forward(self, input):
|
||||
if len(input.shape) == 3: # sl, bs, tracks
|
||||
f = input[..., 0]
|
||||
b = input[..., 1]
|
||||
elif len(input.shape) == 2: # sl, bs - support during classification mode
|
||||
f = input
|
||||
b = torch.flip(input, [1]) # todo test if we are duplicating the backward pass correctly
|
||||
else:
|
||||
raise AttributeError(f"Inorrect size of input, {input.shape}")
|
||||
fwd_o = self.fwd_lm(f)
|
||||
bwd_o = self.bwd_lm(b)
|
||||
|
||||
outs = self.stack(fwd_o, bwd_o)
|
||||
if self.squash_bs_sl:
|
||||
o = outs[0]
|
||||
o = o.view(o.shape[0]*o.shape[1],o.shape[2],o.shape[3])
|
||||
outs[0] = o
|
||||
return outs
|
||||
|
||||
def reset(self):
|
||||
"Reset the hidden states of underlaying lms."
|
||||
self.fwd_lm.reset()
|
||||
self.bwd_lm.reset()
|
||||
|
||||
class MultiBatchBiLMModel(BiLMModel):
|
||||
"Create a RNNCore module that can process a full sentence."
|
||||
|
||||
def __init__(self, bptt:int, max_seq:int, *args, **kwargs):
|
||||
self.max_seq,self.bptt = max_seq,bptt
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def concat(self, arrs:Collection[Tensor])->Tensor:
|
||||
"Concatenate the `arrs` along the batch dimension."
|
||||
return [torch.cat([l[si] for l in arrs], dim=1) for si in range_of(arrs[0])]
|
||||
|
||||
def forward(self, input:LongTensor)->Tuple[Tensor,Tensor]:
|
||||
bs,sl = input.size()
|
||||
self.reset()
|
||||
raw_outputs, outputs = [],[]
|
||||
for i in range(0, sl, self.bptt):
|
||||
r, o = super().forward(input[:,i: min(i+self.bptt, sl)])
|
||||
if i>(sl-self.max_seq):
|
||||
raw_outputs.append(r)
|
||||
outputs.append(o)
|
||||
return self.concat(raw_outputs), self.concat(outputs)
|
||||
#PoolingLinearClassifier
|
||||
class BiPoolingLinearClassifier(nn.Module):
|
||||
"Create a linear classifier with pooling."
|
||||
|
||||
def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]:
|
||||
raw_outputs, outputs = input
|
||||
output = outputs[-1]
|
||||
if len(output.size()) == 3:
|
||||
return super().forward(input)
|
||||
elif len(output.size()) == 4:
|
||||
bs, sl, em_sz, passes = output.size()
|
||||
|
||||
f_avgpool = self.pool(output[...,0], bs, False)
|
||||
f_mxpool = self.pool(output[...,0], bs, True)
|
||||
b_avgpool = self.pool(output[..., 1], bs, False)
|
||||
b_mxpool = self.pool(output[..., 1], bs, True)
|
||||
x = torch.cat([output[:,-1,..., 0], f_mxpool, f_avgpool,
|
||||
output[:,-1,..., 1], b_mxpool, b_avgpool,], 1)
|
||||
x = self.layers(x)
|
||||
return x, raw_outputs, outputs
|
||||
|
||||
|
||||
class AvgPoolingLinearClassifier(nn.Module):
|
||||
"Create a linear classifier with pooling."
|
||||
|
||||
def __init__(self, layers:Collection[int], drops:Collection[float]):
|
||||
super().__init__()
|
||||
mod_layers = []
|
||||
activs = [nn.ReLU(inplace=True)] * (len(layers) - 2) + [None]
|
||||
for n_in,n_out,p,actn in zip(layers[:-1],layers[1:], drops, activs):
|
||||
mod_layers += bn_drop_lin(n_in, n_out, p=p, actn=actn)
|
||||
self.layers = nn.Sequential(*mod_layers)
|
||||
|
||||
def pool(self, x:Tensor, bs:int, is_max:bool):
|
||||
"Pool the tensor along the seq_len dimension."
|
||||
f = F.adaptive_max_pool1d if is_max else F.adaptive_avg_pool1d
|
||||
return f(x.permute(1,2,0), (1,)).view(bs,-1)
|
||||
|
||||
def forward(self, input:Tuple[Tensor,Tensor])->Tuple[Tensor,Tensor,Tensor]:
|
||||
raw_outputs, outputs = input
|
||||
output = outputs[-1]
|
||||
if len(output.size()) == 3:
|
||||
sl,bs,_ = output.size()
|
||||
avgpool = self.pool(output, bs, False)
|
||||
mxpool = self.pool(output, bs, True)
|
||||
x = torch.cat([output[-1], mxpool, avgpool], 1)
|
||||
x = self.layers(x)
|
||||
return x, raw_outputs, outputs
|
||||
elif len(output.size()) == 4:
|
||||
sl, bs, em_sz, passes = output.size()
|
||||
|
||||
avgpool = (self.pool(output[...,0], bs, False) + self.pool(output[..., 1], bs, False))/2
|
||||
mxpool = (self.pool(output[...,0], bs, True) +self.pool(output[..., 1], bs, True))/2
|
||||
x = torch.cat([(output[-1][..., 0]+output[-1][..., 1])/2, mxpool, avgpool], 1)
|
||||
x = self.layers(x)
|
||||
return x, raw_outputs, outputs
|
||||
|
||||
|
||||
def get_bilm(vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int, pad_token:int, tie_weights:bool=True,
|
||||
qrnn:bool=False, bias:bool=True, bidir:bool=False, output_p:float=0.4, hidden_p:float=0.2, input_p:float=0.6,
|
||||
embed_p:float=0.1, weight_p:float=0.5)->nn.Module:
|
||||
"Create a two AWD-LSTM one for each direction "
|
||||
fwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir,
|
||||
hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
|
||||
bwd_rnn_enc = RNNCore(vocab_sz, emb_sz, n_hid=n_hid, n_layers=n_layers, pad_token=pad_token, qrnn=qrnn, bidir=bidir,
|
||||
hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
|
||||
enc = None
|
||||
if tie_weights:
|
||||
enc = fwd_rnn_enc.encoder
|
||||
fwd_rnn_enc.encoder.weight = enc.weight
|
||||
bwd_rnn_enc.encoder.weight = enc.weight
|
||||
|
||||
return BiLMModel(
|
||||
fwd_lm=SequentialRNN(fwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)),
|
||||
bwd_lm=SequentialRNN(bwd_rnn_enc, LinearDecoder(vocab_sz, emb_sz, output_p, tie_encoder=enc, bias=bias)),
|
||||
squash_bs_sl=True)
|
||||
|
||||
def get_birnn_classifier(bptt:int, max_seq:int, n_class:int, vocab_sz:int, emb_sz:int, n_hid:int, n_layers:int,
|
||||
pad_token:int, layers:Collection[int], drops:Collection[float], bidir:bool=False, qrnn:bool=False,
|
||||
hidden_p:float=0.2, input_p:float=0.6, embed_p:float=0.1, weight_p:float=0.5, bicls_head:str='BiPoolingLinearClassifier')->nn.Module:
|
||||
"Create a RNN classifier model."
|
||||
fwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,
|
||||
qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
|
||||
bwd_rnn_enc = MultiBatchRNNCore(bptt, max_seq, vocab_sz, emb_sz, n_hid, n_layers, pad_token=pad_token, bidir=bidir,
|
||||
qrnn=qrnn, hidden_p=hidden_p, input_p=input_p, embed_p=embed_p, weight_p=weight_p)
|
||||
|
||||
head = BiPoolingLinearClassifier
|
||||
if bicls_head == 'BiPoolingLinearClassifier': head = BiPoolingLinearClassifier
|
||||
elif bicls_head == 'AvgPoolingLinearClassifier': head = AvgPoolingLinearClassifier
|
||||
|
||||
model = SequentialRNN(BiLMModel(fwd_rnn_enc, bwd_rnn_enc), head(layers, drops))
|
||||
model.reset()
|
||||
return model
|
||||
|
||||
#endregion
|
||||
@@ -1,17 +1,16 @@
|
||||
"""
|
||||
Utility methods for data processing.
|
||||
"""
|
||||
import fire
|
||||
from fastai import *
|
||||
from fastai.text import *
|
||||
|
||||
import shutil
|
||||
import pathlib
|
||||
import shutil
|
||||
import tarfile
|
||||
from sklearn import model_selection
|
||||
from sacremoses import MosesTokenizer
|
||||
from typing import Dict, Tuple, List
|
||||
|
||||
import fire
|
||||
from sacremoses import MosesTokenizer
|
||||
|
||||
from fastai.text import *
|
||||
|
||||
EOS = 'xxeos' # fastai does not use eos, but we do
|
||||
SEP = 'xxsep' # special separator token for NLI
|
||||
|
||||
|
||||
@@ -136,51 +136,6 @@ def test_ulmfit_fastai_end_to_end_label_smoothing():
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, label_smoothing_eps=0.1 )
|
||||
|
||||
|
||||
def test_ulmfit_fastai_bidir_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-fastai'
|
||||
|
||||
exp = ulmfit.pretrain_lm.LMHyperParams(
|
||||
dataset_path=wt2,
|
||||
lang='en',
|
||||
cuda_id=cuda_id,
|
||||
qrnn=False,
|
||||
bidir=True,
|
||||
tokenizer='f',
|
||||
max_vocab=100,
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(str(test_data / 'imdb'), str(exp.model_dir))
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
def test_ulmfit_moses_fa_bidir_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
test_data, wt2 = get_test_data()
|
||||
lm_name = 'end-to-end-test-fastai'
|
||||
|
||||
exp = ulmfit.pretrain_lm.LMHyperParams(
|
||||
dataset_path=wt2,
|
||||
lang='en',
|
||||
cuda_id=cuda_id,
|
||||
qrnn=False,
|
||||
bidir=True,
|
||||
tokenizer='vf',
|
||||
max_vocab=100,
|
||||
name=lm_name,
|
||||
)
|
||||
exp.train_lm(num_epochs=1, bs=2)
|
||||
exp2 = ulmfit.train_clas.CLSHyperParams.from_lm(test_data / 'imdb', exp.model_dir)
|
||||
exp2.train_cls(num_lm_epochs=0, unfreeze=False, bs=4, )
|
||||
|
||||
# def test_classification_model_work_with_different_dropmul():
|
||||
# learn = self.create_cls_learner(data_clas, drop_mult=0.1)
|
||||
# learn = self.create_cls_learner(data_clas, drop_mult=0.0)
|
||||
|
||||
def test_ulmfit_sentencepiece_end_to_end():
|
||||
""" Test ulmfit with sentencepiece tokenizer on small wikipedia dataset.
|
||||
"""
|
||||
|
||||
+5
-26
@@ -4,25 +4,14 @@ expected to have been tokenized with Moses and processed with `postprocess_wikit
|
||||
That is, the data is expected to be white-space separated and numbers are expected
|
||||
to be split.
|
||||
"""
|
||||
from dataclasses import InitVar
|
||||
|
||||
import fastai
|
||||
import fire
|
||||
|
||||
from fastai import *
|
||||
from fastai.callbacks import CSVLogger, SaveModelCallback
|
||||
from fastai.callbacks import CSVLogger
|
||||
from fastai.text import *
|
||||
import torch
|
||||
from fastai_contrib.utils import read_file, read_whitespace_file, \
|
||||
validate, PAD, UNK, get_sentencepiece, read_clas_data, TRN, VAL, TST, PAD_TOKEN_ID, \
|
||||
from fastai_contrib.utils import read_whitespace_file, \
|
||||
validate, UNK, get_sentencepiece, PAD_TOKEN_ID, \
|
||||
replace_std_toks, MosesPreprocessingFunc
|
||||
from fastai_contrib.learner import bilm_learner, accuracy_fwd, accuracy_bwd, bilm_text_classifier_learner
|
||||
import pickle
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from collections import Counter
|
||||
import fastai_contrib.data as contrib_data
|
||||
|
||||
LM_BEST = "lm_best"
|
||||
ENC_BEST = "enc_best"
|
||||
@@ -62,9 +51,9 @@ def json_load(f):
|
||||
|
||||
@dataclass
|
||||
class LMHyperParams:
|
||||
dataset_path: str # data_dir
|
||||
dataset_path: Union[str, Path] # data_dir
|
||||
|
||||
base_lm_path: str = None
|
||||
base_lm_path: Union[str, Path] = None
|
||||
backwards: str = False
|
||||
bidir: bool =False
|
||||
qrnn: bool = True
|
||||
@@ -131,15 +120,6 @@ class LMHyperParams:
|
||||
@property
|
||||
def pretrained_fnames(self): return [self.base_lm_path / LM_BEST, self.base_lm_path / '../itos'] if self.base_lm_path else None
|
||||
|
||||
@property
|
||||
def lm_type(self):
|
||||
if self.bidir:
|
||||
return contrib_data.LanguageModelType.BiLM
|
||||
if self.backwards:
|
||||
return contrib_data.LanguageModelType.BwdLM
|
||||
else:
|
||||
return contrib_data.LanguageModelType.FwdLM
|
||||
|
||||
def tokenizer_to_fastai_args(self, sp_data_func, use_moses):
|
||||
moses_preproc = [MosesPreprocessingFunc(self.lang)] if use_moses else []
|
||||
if self.tokenizer is Tokenizers.SUBWORD or self.tokenizer is Tokenizers.BROKENSUBWORD:
|
||||
@@ -234,7 +214,6 @@ class LMHyperParams:
|
||||
learn.freeze()
|
||||
# compared to standard Adam, we set beta_1 to 0.8
|
||||
learn.opt_fn = partial(optim.Adam, betas=(0.8, 0.99))
|
||||
learn.metrics = [accuracy_fwd, accuracy_bwd] if self.bidir else [accuracy]
|
||||
learn.callback_fns += [partial(CSVLogger, filename=f"{learn.model_dir}/lm-history"),
|
||||
# partial(SaveModelCallback, every='improvement', name='lm') disabled due to Memory issues
|
||||
]
|
||||
|
||||
@@ -3,14 +3,12 @@ Train a classifier on top of a language model trained with `pretrain_lm.py`.
|
||||
Optionally fine-tune LM before.
|
||||
"""
|
||||
|
||||
from fastai.callbacks import CSVLogger
|
||||
from fastai.text import *
|
||||
|
||||
from fastai_contrib.utils import PAD_TOKEN_ID
|
||||
|
||||
import fire
|
||||
|
||||
from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST, json_save
|
||||
from fastai.callbacks import CSVLogger
|
||||
from fastai.text import *
|
||||
from fastai_contrib.utils import PAD_TOKEN_ID
|
||||
from ulmfit.pretrain_lm import LMHyperParams, ENC_BEST
|
||||
|
||||
|
||||
class CLSHyperParams(LMHyperParams):
|
||||
|
||||
Reference in New Issue
Block a user