Question-Answer special tokens

This commit is contained in:
Sotirios Anagnostidis
2023-01-03 22:02:32 +01:00
parent e14c670cb0
commit 3a10e9412d
4 changed files with 38 additions and 12 deletions
@@ -2,6 +2,12 @@ from datasets import load_dataset
from sklearn.model_selection import train_test_split
from torch.utils.data import Dataset, Subset
QA_SPECIAL_TOKENS = {
'Question': '<question>',
'Answer': '<answer>'
}
class SquadV2Dataset(Dataset):
def __init__(self, cache_dir, split):