wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-12 11:40:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] added translation, rallio instruct tuning dataset, prosocial for safety, new summary dataset

Browse Source
This commit is contained in:
theblackcat102
2023-01-20 03:02:07 +00:00
parent da6a3b687e
commit 74cb9aaa5a
11 changed files with 399 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/tests/test_datasets.py
+4 -3
View File
@@ -7,10 +7,11 @@ from custom_datasets.dialogue_collator import DialogueDataCollator
def test_all_datasets():
qa_base = QA_DATASETS
summarize_base = SUMMARIZATION_DATASETS
others = ["prompt_dialogue", "webgpt", "soda", "joke"]
others = ["prompt_dialogue", "webgpt", "soda", "joke", "instruct_tuning"]
translation = ["dive_mt", "wmt2019_zh-en", "wmt2019_ru-en", "wmt2019_de-en", "ted_trans_de-ja", "ted_trans_nl-en"]
config = Namespace(cache_dir=".cache")
for dataset_name in others + qa_base + summarize_base:
for dataset_name in translation:
print(dataset_name)
train, eval = get_one_dataset(config, dataset_name)
# sanity check
@@ -51,4 +52,4 @@ def test_collate_fn():
if __name__ == "__main__":
test_collate_fn()
test_all_datasets()