wassname/stampy-chat
1
0
Fork 0
mirror of https://github.com/wassname/stampy-chat.git synced 2026-09-12 13:00:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

fixed minor errors from testing

Browse Source
This commit is contained in:
Thomas Lemoine
2023-06-21 00:34:14 -04:00
parent a25f217530
commit e9ebb25f88
2 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/dataset/create_dataset.py
+3 -1
View File
@@ -97,7 +97,9 @@ class ChunkedARD:
# Load the dataset. streaming allows you to load one entry at a time,
# so entries can be processed before the entire dataset has been saved.
iterable_data = load_dataset('StampyAI/alignment-research-dataset', 'aisafety.info', split='train', streaming=True)
# iterable_data = load_dataset('StampyAI/alignment-research-dataset', 'aisafety.info', split='train', streaming=True)
iterable_data = load_dataset('StampyAI/alignment-research-dataset', 'all', split='train', streaming=True)
for entry in tqdm(iterable_data):
"""Checks"""
src/settings.py
-2
View File
@@ -7,6 +7,4 @@ LEN_EMBEDDINGS = 1536
MAX_LEN_PROMPT = 4095 # This may be 8191, unsure.
current_file_path = Path(__file__).resolve()
PATH_TO_RAW_DATA = str(current_file_path.parent / 'dataset' / 'data' / 'alignment_texts.jsonl')
PATH_TO_DATASET_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset.pkl')
PATH_TO_DATASET_DICT_PKL = str(current_file_path.parent / 'dataset' / 'data' / 'dataset_dict.pkl')