wassname/stampy-chat
1
0
Fork 0
mirror of https://github.com/wassname/stampy-chat.git synced 2026-09-10 12:40:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Chaining iterable_data calls

Browse Source
This commit is contained in:
henri123lemoine
2023-06-27 06:41:21 -04:00
parent 0204810aa5
commit abb506511e
1 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/dataset/update_dataset.py
+5 -4
View File
@@ -33,10 +33,11 @@ class ARDUpdater:
def update_source(self, source: str):
logger.info(f"Updating {source} entries...")
iterable_data = load_dataset(ARD_DATASET_NAME, source, split='train', streaming=True)
iterable_data = iterable_data.map(self.preprocess)
iterable_data = iterable_data.filter(lambda entry: entry is not None)
iterable_data = iterable_data.filter(lambda entry: self.sql_db.upsert_entry(entry))
iterable_data = load_dataset(
ARD_DATASET_NAME, source, split='train', streaming=True
).map(self.preprocess).filter(
lambda entry: entry is not None
).filter(lambda entry: self.sql_db.upsert_entry(entry))
for entry in tqdm(iterable_data):
try: