From abb506511e6fc99dfdd70a11eb3fe39839cbdddd Mon Sep 17 00:00:00 2001 From: henri123lemoine Date: Tue, 27 Jun 2023 06:41:21 -0400 Subject: [PATCH] Chaining iterable_data calls --- src/dataset/update_dataset.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/dataset/update_dataset.py b/src/dataset/update_dataset.py index 4c4dac5..d720815 100644 --- a/src/dataset/update_dataset.py +++ b/src/dataset/update_dataset.py @@ -33,10 +33,11 @@ class ARDUpdater: def update_source(self, source: str): logger.info(f"Updating {source} entries...") - iterable_data = load_dataset(ARD_DATASET_NAME, source, split='train', streaming=True) - iterable_data = iterable_data.map(self.preprocess) - iterable_data = iterable_data.filter(lambda entry: entry is not None) - iterable_data = iterable_data.filter(lambda entry: self.sql_db.upsert_entry(entry)) + iterable_data = load_dataset( + ARD_DATASET_NAME, source, split='train', streaming=True + ).map(self.preprocess).filter( + lambda entry: entry is not None + ).filter(lambda entry: self.sql_db.upsert_entry(entry)) for entry in tqdm(iterable_data): try: