wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-12 11:40:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] added GSM8k and code refactoring

Browse Source
This commit is contained in:
theblackcat102
2023-01-14 06:24:47 +00:00
parent 3966024871
commit 1546111094
5 changed files with 53 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/utils.py
+2 -1
View File
@@ -6,8 +6,9 @@ import nltk
import numpy as np
import transformers
import yaml
from custom_datasets import QA_DATASETS, QA_SPECIAL_TOKENS, SUMMARIZATION_DATASETS, get_one_dataset
from custom_datasets import QA_DATASETS, SUMMARIZATION_DATASETS, get_one_dataset
from custom_datasets.dialogue_collator import DialogueDataCollator
from custom_datasets.qa_datasets import QA_SPECIAL_TOKENS
from losses import CrossEntropyLoss, PolyLoss
from models import freeze_top_n_layers, get_specific_model
from sklearn.model_selection import train_test_split