From 4ab6be59e77492b5590d857024a8b9a40dad54d7 Mon Sep 17 00:00:00 2001 From: Juan Pablo Amoroso Date: Tue, 24 Sep 2019 18:34:44 -0300 Subject: [PATCH] Files deleted correctly after successful backup --- data_scraper/backup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/data_scraper/backup.py b/data_scraper/backup.py index 67ee892..77e4943 100644 --- a/data_scraper/backup.py +++ b/data_scraper/backup.py @@ -112,6 +112,7 @@ def _upload_folder(bucket, folder, data_path): if not os.path.isdir(folder): return + cleanup_files = [] for root, dirs, files in os.walk(folder): for file in files: file_path = os.path.join(root, file) @@ -129,7 +130,10 @@ def _upload_folder(bucket, folder, data_path): raise e else: logger.debug("Uploaded file %s to S3", file) - utils.remove_file(file_path, __name__) + cleanup_files.append(file_path) + + for file in cleanup_files: + utils.remove_file(file, __name__) def _key_exists(bucket, key):