wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-13 11:50:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactoring, now running

Browse Source
This commit is contained in:
Sotirios Anagnostidis
2023-01-11 22:42:04 +01:00
parent 5b77dd2e9f
commit 4a3ea0b033
5 changed files with 50 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/custom_datasets/__init__.py
+3 -1
View File
@@ -27,6 +27,8 @@ summarization_config_mapping = {
"reddit": (),
}
QA_DATASETS = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_noconext"]
SUMMARIZATION_DATASETS = ["xsum", "cnn_dailymail", "samsum", "multi_news"]
def index_squad_v2(example):
return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0]
@@ -157,4 +159,4 @@ def get_one_dataset(conf, dataset_name):
else:
raise ValueError(f"Unknown dataset {dataset_name}")
return train, eval
return train, eval