wassname/multifit
1
0
Fork 0
mirror of https://github.com/wassname/multifit.git synced 2026-09-09 11:27:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Adapted test_text_data to batch dimension being the first, and only xxbos token at the start

Browse Source
This commit is contained in:
Tomasz Pietruszka
2019-01-22 20:31:30 +01:00
parent 0e864fb291
commit 386fc49431
2 changed files with 13 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/cls_test_wt103_1_f.ipynb
+3 -3
View File
@@ -714,9 +714,9 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:fastaiv1]",
"display_name": "fastai-dev",
"language": "python",
"name": "conda-env-fastaiv1-py"
"name": "fastai-dev"
},
"language_info": {
"codemirror_mode": {
@@ -728,7 +728,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.0"
"version": "3.6.8"
}
},
"nbformat": 4,
tests/test_text_data.py
+10 -13
View File
@@ -22,38 +22,35 @@ def test_should_load_backwards_lm():
df = text_df(['neg','pos'])
data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
lm_type=contrib_data.LanguageModelType.BwdLM,
ld_cls=contrib_data.LanguageModelLoader)
lm_type=contrib_data.LanguageModelType.BwdLM)
lml = data.train_dl.dl
lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
batch = lml.get_batch(lml.data, 0, 70)
assert batch[0].shape == (70, lml.bs)
assert batch[0].shape == (lml.bs, 70)
assert batch[1].shape == (70*lml.bs,)
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0]]
np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',])
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0]]
np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
def test_should_load_bi_lm():
path = untar_data(URLs.IMDB_SAMPLE)
df = text_df(['neg', 'pos'])
data = TextLMDataBunch.from_df(path, train_df=df, valid_df=df, label_cols=0, text_cols=["text"], bs=2,
lm_type=contrib_data.LanguageModelType.BiLM,
ld_cls=contrib_data.LanguageModelLoader)
lm_type=contrib_data.LanguageModelType.BiLM)
lml = data.train_dl.dl
lml.data = lml.batchify(np.concatenate([lml.dataset.x.items[i] for i in range(len(lml.dataset))]))
batch = lml.get_batch(lml.data, 0, 70)
assert batch[0].shape == (70, lml.bs, 2)
assert batch[0].shape == (lml.bs, 70, 2)
assert batch[1].shape == (70*lml.bs, 2)
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:, 0, 0]]
np.testing.assert_array_equal(as_text[:7], "xxfld 1 fast ai is a cool".split())
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 0]]
np.testing.assert_array_equal(as_text[:7], "xxbos fast ai is a cool project".split())
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][:,0,1]]
np.testing.assert_array_equal(as_text[:5], ["world", "hello", '1', 'xxfld', 'project',])
as_text = [lml.dataset.vocab.itos[x] for x in batch[0][0, :, 1]]
np.testing.assert_array_equal(as_text[:5], ["world", "hello", 'xxbos', 'project', 'cool'])
###################### NEW CODE