wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-09-09 11:15:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] Add mix conversation augmentation

Browse Source
This commit is contained in:
theblackcat102
2023-02-01 22:14:11 +00:00
parent 638d8c1572
commit f8eba68544
3 changed files with 95 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/custom_datasets/summarization.py
+1 -1
View File
@@ -57,7 +57,7 @@ def index_summary_merge(text, summary):
class SummarizationDataset(Dataset):
def __init__(self, dataset, cache_dir, split, max_words=512):
self.name = dataset
if summarization_config_mapping[dataset][0] in ["billsum", "tldr_news"] & split == "validation":
if (dataset in ["billsum", "tldr_news"]) and (split == "validation"):
split = "test"
self.dataset = load_dataset(*summarization_config_mapping[dataset], cache_dir=cache_dir, split=split)
self.text_column, self.summary_column = summarization_name_mapping[dataset]