wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-07-31 12:00:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

pre-commits

Browse Source
This commit is contained in:
Sotirios Anagnostidis
2023-01-03 22:45:34 +01:00
parent 752eeb24f7
commit c20dfaad5b
4 changed files with 17 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/utils.py
+3 -4
View File
@@ -1,15 +1,14 @@
from pathlib import Path
import yaml
from custom_datasets import get_one_dataset
from custom_datasets import QA_SPECIAL_TOKENS, get_one_dataset
from custom_datasets.dialogue_collator import DialogueDataCollator
from losses import CrossEntropyLoss
from sklearn.model_selection import train_test_split
from torch.utils.data import ConcatDataset, Subset
from transformers import AutoModelForCausalLM, AutoTokenizer
from custom_datasets import QA_SPECIAL_TOKENS
SUPPORTED_MODELS = ["galactica"]
SUPPORTED_MODELS = ["galactica", "GPT-JT"] # deprecated ..
def get_tokenizer(conf):
@@ -20,7 +19,7 @@ def get_tokenizer(conf):
additional_special_tokens = (
[]
if not "additional_special_tokens" in tokenizer.special_tokens_map
if "additional_special_tokens" not in tokenizer.special_tokens_map
else tokenizer.special_tokens_map["additional_special_tokens"]
)
additional_special_tokens = list(set(additional_special_tokens + list(QA_SPECIAL_TOKENS.values())))