wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-09-10 11:41:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] mix generation from different tasks

Browse Source
This commit is contained in:
theblackcat102
2023-02-03 00:15:29 +00:00
parent 9be4c921cd
commit 1041564db7
2 changed files with 44 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/tests/test_datasets.py
+7 -4
View File
@@ -27,7 +27,7 @@ def test_collate_fn():
config = Namespace(cache_dir=".cache", model_name="Salesforce/codegen-2B-multi")
tokenizer = get_tokenizer(config)
collate_fn = DialogueDataCollator(tokenizer, max_length=512)
collate_fn = DialogueDataCollator(tokenizer, max_length=620)
qa_base = QA_DATASETS
summarize_base = SUMMARIZATION_DATASETS
others = ["prompt_dialogue", "webgpt", "soda", "joke", "gsm8k"]
@@ -40,11 +40,14 @@ def test_collate_fn():
dataloader = DataLoader(ConcatDataset(trains), collate_fn=collate_fn, batch_size=128)
for batch in dataloader:
print(batch.keys())
print(batch["targets"].shape[0])
print(tokenizer.decode(batch["input_ids"][0]))
print("-----")
print(tokenizer.decode(batch["targets"][0][batch["label_masks"][0]]))
assert batch["targets"].shape[1] <= 512
assert batch["targets"].shape[1] <= 620
dataloader = DataLoader(ConcatDataset(evals), collate_fn=collate_fn, batch_size=128)
for batch in dataloader:
assert batch["targets"].shape[1] <= 512
assert batch["targets"].shape[1] <= 620
test_collate_fn()