diff --git a/tests/test_text_data.py b/tests/test_text_data.py deleted file mode 100644 index 1b96079..0000000 --- a/tests/test_text_data.py +++ /dev/null @@ -1,56 +0,0 @@ -import pytest -import fastai.text - -from fastai import * -from fastai.text import * - -import fastai_contrib.data as contrib_data - -def text_df(labels): - data = [] - texts = ["fast ai is a cool project", "hello world"] * 20 - for ind, text in enumerate(texts): - sample = {} - sample["label"] = labels[ind%len(labels)] - sample["text"] = text - data.append(sample) - return pd.DataFrame(data) - -###################### UPDATED CODE -def test_should_load_backwards_lm(): - path = untar_data(URLs.IMDB_SAMPLE) - df = text_df(['neg','pos']) - - data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BwdLM) - lml = data.train_dl.dl - lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) - batch = lml.get_batch(lml.data, 0, 70) - - assert batch[0].shape == (lml.bs, 70) - assert batch[1].shape == (70*lml.bs,) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) - -def test_should_load_bi_lm(): - path = untar_data(URLs.IMDB_SAMPLE) - df = text_df(['neg', 'pos']) - - data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2, - lm_type=contrib_data.LanguageModelType.BiLM) - lml = data.train_dl.dl - lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))])) - batch = lml.get_batch(lml.data, 0, 70) - - assert batch[0].shape == (lml.bs, 70, 2) - assert batch[1].shape == (70*lml.bs, 2) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]] - np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split()) - - as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]] - np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool']) - -###################### NEW CODE - diff --git a/tests/test_text_train.py b/tests/test_text_train.py index ccc9bef..60fb1f9 100644 --- a/tests/test_text_train.py +++ b/tests/test_text_train.py @@ -55,49 +55,3 @@ def text_df(n_labels): def test_val_loss(learn): assert learn.validate()[1] > 0.5 - - -def test_bilm_classifier_loads_encoder(): - n_labels=1 - nl = 1 - emb_sz = 100 - path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp') - os.makedirs(path) - try: - df = text_df(n_labels=n_labels) - lmdf = df#[["text"]] - print(lmdf.head()) - lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer), - lm_type=contrib_data.LanguageModelType.BiLM) - learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) - learn.save_encoder("enc") - data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8) - classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False) - print(last_layer(classifier.model), ) - classifier.load_encoder("enc") - classifier.fit(1) - finally: - shutil.rmtree(path) - - -def test_bilm_lstm_can_be_trained(): - manual_seed() - path, df_trn, df_val = prep_human_numbers() - data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), - lm_type = contrib_data.LanguageModelType.BiLM) - - learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) - learn.metrics = [accuracy_fwd] - learn.fit_one_cycle(2, 5e-3) - assert learn.validate()[1] > 0.3 - - -def test_bwdlm_lstm_can_be_trained(): - manual_seed() - path, df_trn, df_val = prep_human_numbers() - data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer), - lm_type = contrib_data.LanguageModelType.BwdLM) - - learn = language_model_learner(data, AWD_LSTM, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False) - learn.fit_one_cycle(2, 5e-3) - assert learn.validate()[1] > 0.3