diff --git a/src/dataset/create_dataset.py b/src/dataset/create_dataset.py index 8834c07..516e4cb 100644 --- a/src/dataset/create_dataset.py +++ b/src/dataset/create_dataset.py @@ -97,7 +97,9 @@ class ChunkedARD: # Load the dataset. streaming allows you to load one entry at a time, # so entries can be processed before the entire dataset has been saved. - iterable_data = load_dataset('StampyAI/alignment-research-dataset', 'aisafety.info', split='train', streaming=True) + # iterable_data = load_dataset('StampyAI/alignment-research-dataset', 'aisafety.info', split='train', streaming=True) + + iterable_data = load_dataset('StampyAI/alignment-research-dataset', 'all', split='train', streaming=True) for entry in tqdm(iterable_data): """Checks""" diff --git a/src/settings.py b/src/settings.py index 2362cc7..94abeb5 100644 --- a/src/settings.py +++ b/src/settings.py @@ -7,6 +7,4 @@ LEN_EMBEDDINGS = 1536 MAX_LEN_PROMPT = 4095 # This may be 8191, unsure. current_file_path = Path(__file__).resolve() -PATH_TO_RAW_DATA = str(current_file_path.parent / 'dataset' / 'data' / 'alignment_texts.jsonl') -PATH_TO_DATASET_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset.pkl') PATH_TO_DATASET_DICT_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset_dict.pkl') \ No newline at end of file