From c4e0b7b3d652e8f788343b80f5f9df41a5afe0b0 Mon Sep 17 00:00:00 2001 From: "NAUSICAA\\Julian" Date: Sun, 25 Nov 2018 23:47:24 -0300 Subject: [PATCH] Lower memory footprint --- fastai_contrib/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fastai_contrib/utils.py b/fastai_contrib/utils.py index 2dd21f9..8ae01cd 100644 --- a/fastai_contrib/utils.py +++ b/fastai_contrib/utils.py @@ -70,11 +70,11 @@ def get_sentencepiece(path:PathOrStr, trn_path:Path, name:str, pre_rules:ListRul pre_rules = pre_rules if pre_rules is not None else [] post_rules = post_rules if post_rules is not None else [] - # load the text frmo the train tokens file - text = [line.rstrip('\n') for line in open(trn_path)] - text = list(filter(None, text)) if not os.path.isfile(path / 'models' / 'spm.model') or not os.path.isfile(path / 'models' / f'itos_{name}.pkl'): + # load the text frmo the train tokens file + text = [line.rstrip('\n') for line in open(trn_path)] + text = list(filter(None, text)) raw_text = reduce(lambda t, rule: rule(t), pre_rules, '\n'.join(text)) raw_text_path = path / cache_name / 'all_text.txt' with open(raw_text_path, 'w') as f: @@ -337,7 +337,7 @@ def read_file(file_path, outname): with open(file_path, encoding='utf8') as f: text = f.readlines() df = pd.DataFrame( - {'text': np.array(text), 'labels': np.zeros(len(text))}, + {'text': text, 'labels': np.zeros(len(text))}, columns=['labels', 'text']) df.to_csv(file_path.parent / f'{outname}.csv', header=False, index=False)