Removing more bi-directional test

2026-09-09 11:27:26 +08:00 · 2019-04-15 14:41:08 -03:00
parent 65397d7ae4
commit e4d0529a3d
2 changed files with 0 additions and 102 deletions
@@ -1,56 +0,0 @@
-import pytest
-import fastai.text
-
-from fastai import *
-from fastai.text import *
-
-import fastai_contrib.data as contrib_data
-
-def text_df(labels):
-    data = []
-    texts = ["fast ai is a cool project", "hello world"] * 20
-    for ind, text in enumerate(texts):
-        sample = {}
-        sample["label"] = labels[ind%len(labels)]
-        sample["text"] = text
-        data.append(sample)
-    return pd.DataFrame(data)
-
-###################### UPDATED CODE
-def test_should_load_backwards_lm():
-    path = untar_data(URLs.IMDB_SAMPLE)
-    df = text_df(['neg','pos'])
-
-    data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BwdLM)
-    lml = data.train_dl.dl
-    lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
-    batch = lml.get_batch(lml.data, 0, 70)
-
-    assert batch[0].shape == (lml.bs, 70)
-    assert batch[1].shape == (70*lml.bs,)
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
-
-def test_should_load_bi_lm():
-    path = untar_data(URLs.IMDB_SAMPLE)
-    df = text_df(['neg', 'pos'])
-
-    data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
-                                   lm_type=contrib_data.LanguageModelType.BiLM)
-    lml = data.train_dl.dl
-    lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
-    batch = lml.get_batch(lml.data, 0, 70)
-
-    assert batch[0].shape == (lml.bs, 70, 2)
-    assert batch[1].shape == (70*lml.bs, 2)
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]]
-    np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split())
-
-    as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]]
-    np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
-
-###################### NEW CODE
-
@@ -55,49 +55,3 @@ def text_df(n_labels):

 def test_val_loss(learn):
    assert learn.validate()[1] > 0.5
-
-
-def test_bilm_classifier_loads_encoder():
-    n_labels=1
-    nl = 1
-    emb_sz = 100
-    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'tmp')
-    os.makedirs(path)
-    try:
-        df = text_df(n_labels=n_labels)
-        lmdf = df#[["text"]]
-        print(lmdf.head())
-        lmdata = TextLMDataBunch.from_df(path, lmdf, lmdf, tokenizer=Tokenizer(BaseTokenizer),
-                                       lm_type=contrib_data.LanguageModelType.BiLM)
-        learn = bilm_learner(lmdata, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
-        learn.save_encoder("enc")
-        data = TextClasDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=list(range(n_labels)), text_cols=["text"], bs=8)
-        classifier = bilm_text_classifier_learner(data, emb_sz=emb_sz, nl=nl, drop_mult=0.1, qrnn=False)
-        print(last_layer(classifier.model), )
-        classifier.load_encoder("enc")
-        classifier.fit(1)
-    finally:
-        shutil.rmtree(path)
-
-
-def test_bilm_lstm_can_be_trained():
-    manual_seed()
-    path, df_trn, df_val = prep_human_numbers()
-    data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer),
-                                   lm_type = contrib_data.LanguageModelType.BiLM)
-
-    learn = bilm_learner(data, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
-    learn.metrics = [accuracy_fwd]
-    learn.fit_one_cycle(2, 5e-3)
-    assert learn.validate()[1] > 0.3
-
-
-def test_bwdlm_lstm_can_be_trained():
-    manual_seed()
-    path, df_trn, df_val = prep_human_numbers()
-    data = TextLMDataBunch.from_df(path, df_trn, df_val, tokenizer=Tokenizer(BaseTokenizer),
-                                   lm_type = contrib_data.LanguageModelType.BwdLM)
-
-    learn = language_model_learner(data, AWD_LSTM, emb_sz=100, nl=1, drop_mult=0.1, qrnn=False)
-    learn.fit_one_cycle(2, 5e-3)
-    assert learn.validate()[1] > 0.3