Add model utils tests

This commit is contained in:
Lewis Tunstall
2023-11-10 09:42:15 +00:00
parent 0af8011993
commit 2ed5a45d25
5 changed files with 152 additions and 7 deletions
+2 -2
View File
@@ -1,5 +1,5 @@
import re
from typing import List, Literal, Optional, Union
from typing import List, Literal, Optional
from datasets import DatasetDict, concatenate_datasets, load_dataset
@@ -67,7 +67,7 @@ def apply_chat_template(
def get_datasets(
data_config: Union[DataArguments, dict],
data_config: DataArguments | dict,
splits: List[str] = ["train", "test"],
shuffle: bool = True,
) -> DatasetDict: