wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-13 11:50:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

testing rankgen integration into instructor trainer

Browse Source
This commit is contained in:
Bobak Hashemi
2023-01-01 23:30:12 -05:00
parent 8f0028bc44
commit 34ab948ade
6 changed files with 133 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/reward/instructor/utils.py
+5 -2
View File
@@ -4,7 +4,7 @@ import re
import yaml
from sklearn.model_selection import train_test_split
from torch.utils.data import Subset
from transformers import AutoTokenizer
from transformers import AutoTokenizer, T5Tokenizer
re_reference_remove = re.compile(r"\[([0-9])+\]|\[([0-9])+,([0-9])+\]")
@@ -26,7 +26,10 @@ def webgpt_return_format(row):
def get_tokenizer(tokenizer_name):
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
if "t5" in tokenizer_name: #rankgen
tokenizer = T5Tokenizer.from_pretrained(tokenizer_name, truncation_side="left")
else:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
if "galactica" in tokenizer_name:
tokenizer.add_special_tokens({"pad_token": "<pad>", "eos_token": "</s>"})