mirror of
https://github.com/wassname/Open-Assistant.git
synced 2026-09-10 11:41:04 +08:00
Question-Answer special tokens
This commit is contained in:
@@ -2,6 +2,12 @@ from datasets import load_dataset
|
||||
from sklearn.model_selection import train_test_split
|
||||
from torch.utils.data import Dataset, Subset
|
||||
|
||||
QA_SPECIAL_TOKENS = {
|
||||
'Question': '<question>',
|
||||
'Answer': '<answer>'
|
||||
}
|
||||
|
||||
|
||||
|
||||
class SquadV2Dataset(Dataset):
|
||||
def __init__(self, cache_dir, split):
|
||||
|
||||
Reference in New Issue
Block a user