wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-16 11:14:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] add reddit eli5, asks, askh; bug fix

Browse Source
This commit is contained in:
theblackcat102
2023-01-29 03:05:56 +00:00
parent 994dce7c76
commit fdcc629678
3 changed files with 104 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/custom_datasets/summarization.py
+2
View File
@@ -57,6 +57,8 @@ def index_summary_merge(text, summary):
class SummarizationDataset(Dataset):
def __init__(self, dataset, cache_dir, split, max_words=512):
self.name = dataset
if summarization_config_mapping[dataset][0] in ["billsum", "tldr_news"] & split == "validation":
split = "test"
self.dataset = load_dataset(*summarization_config_mapping[dataset], cache_dir=cache_dir, split=split)
self.text_column, self.summary_column = summarization_name_mapping[dataset]
self.preprocess_fn = index_summary_merge if dataset == "scitldr" else index_summary_default