wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-12 11:40:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

pre commits

Browse Source
This commit is contained in:
Sotirios Anagnostidis
2023-01-06 22:09:24 +01:00
parent fb3266b759
commit d3952354e2
7 changed files with 106 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/utils.py
+1 -1
View File
@@ -4,10 +4,10 @@ import yaml
from custom_datasets import QA_SPECIAL_TOKENS, get_one_dataset
from custom_datasets.dialogue_collator import DialogueDataCollator
from losses import CrossEntropyLoss
from models import freeze_top_n_layers, get_specific_model
from sklearn.model_selection import train_test_split
from torch.utils.data import ConcatDataset, Subset
from transformers import AutoTokenizer
from models import get_specific_model, SUPPORTED_MODELS, freeze_top_n_layers
def get_tokenizer(conf):