wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-09-09 11:15:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

[fix] wait it pass?

Browse Source
This commit is contained in:
theblackcat102
2023-01-20 07:26:26 +00:00
parent 22e3ab1a89
commit aca3e9de89
1 changed files with 1 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/tests/test_datasets.py
+1 -5
View File
@@ -11,7 +11,7 @@ def test_all_datasets():
translation = ["dive_mt", "wmt2019_zh-en", "wmt2019_ru-en", "wmt2019_de-en", "ted_trans_de-ja", "ted_trans_nl-en"] translation = ["dive_mt", "wmt2019_zh-en", "wmt2019_ru-en", "wmt2019_de-en", "ted_trans_de-ja", "ted_trans_nl-en"]
config = Namespace(cache_dir=".cache") config = Namespace(cache_dir=".cache")
for dataset_name in translation: for dataset_name in translation + others + summarize_base + qa_base:
print(dataset_name) print(dataset_name)
train, eval = get_one_dataset(config, dataset_name) train, eval = get_one_dataset(config, dataset_name)
# sanity check # sanity check
@@ -49,7 +49,3 @@ def test_collate_fn():
dataloader = DataLoader(ConcatDataset(evals), collate_fn=collate_fn, batch_size=128) dataloader = DataLoader(ConcatDataset(evals), collate_fn=collate_fn, batch_size=128)
for batch in dataloader: for batch in dataloader:
assert batch["targets"].shape[1] <= 512 assert batch["targets"].shape[1] <= 512
if __name__ == "__main__":
test_all_datasets()