wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-19 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] add initial version of anthropic dataset

Browse Source
This commit is contained in:
theblackcat102
2023-01-25 05:03:43 +00:00
parent 4146930eb9
commit 3215a7bbf8
4 changed files with 55 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/reward/instructor/trainer.py
+1 -1
View File
@@ -155,7 +155,7 @@ if __name__ == "__main__":
)
tokenizer = get_tokenizer(training_conf["tokenizer_name"])
train, evals = get_datasets(training_conf["datasets"])
train, evals = get_datasets(training_conf["datasets"], tokenizer)
if "rankgen" in model_name:
collate_fn = RankGenCollator(tokenizer, max_length=training_conf["max_length"])
else: