wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-07-21 12:20:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] Add OA translated QA

Browse Source
This commit is contained in:
theblackcat102
2023-02-01 22:33:37 +00:00
parent f8eba68544
commit 9be4c921cd
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/custom_datasets/__init__.py
+4 -1
View File
@@ -2,7 +2,7 @@
High level functions for model training
"""
from custom_datasets.prompt_dialogue import InstructionTuning, PromptGeneratedDataset
from custom_datasets.qa_datasets import SODA, JokeExplaination, QADataset, SODADialogue, WebGPT
from custom_datasets.qa_datasets import SODA, JokeExplaination, QADataset, SODADialogue, TranslatedQA, WebGPT
from custom_datasets.summarization import SummarizationDataset
from custom_datasets.toxic_conversation import ProsocialDialogue, ProsocialDialogueExplaination
from custom_datasets.translation import WMT2019, DiveMT, TEDTalk
@@ -92,6 +92,9 @@ def get_one_dataset(conf, dataset_name):
elif dataset_name == "instruct_tuning":
dataset = InstructionTuning(conf.cache_dir)
train, eval = train_val_dataset(dataset, val_split=0.2)
elif dataset_name == "translate_qa":
dataset = TranslatedQA(conf.cache_dir)
train, eval = train_val_dataset(dataset, val_split=0.01)
else:
raise ValueError(f"Unknown dataset {dataset_name}")