diff --git a/data_scraper/__main__.py b/data_scraper/__main__.py index e3544a4..0e5de5f 100644 --- a/data_scraper/__main__.py +++ b/data_scraper/__main__.py @@ -6,24 +6,27 @@ from data_scraper import cboe, tiingo, backup parser = argparse.ArgumentParser(prog="data_scraper.py") parser.add_argument("-t", "--symbols", nargs="+", help="Symbols to fetch") -parser.add_argument( - "-s", - "--scraper", - choices=["cboe", "tiingo"], - default="cboe", - help="Scraper to use") -parser.add_argument( - "-v", "--verbose", action="store_true", help="Enable logging") -parser.add_argument( - "-a", - "--aggregate", - action="store_true", - help="Aggregate daily data files") -parser.add_argument( - "-b", "--backup", action="store_true", help="Backup files in S3 bucket") -parser.add_argument( - "-d","--backup_daily", action="store_true", help="Backup daily files in S3 bucket" -) +parser.add_argument("-s", + "--scraper", + choices=["cboe", "tiingo"], + default="cboe", + help="Scraper to use") +parser.add_argument("-v", + "--verbose", + action="store_true", + help="Enable logging") +parser.add_argument("-a", + "--aggregate", + action="store_true", + help="Aggregate daily data files") +parser.add_argument("-b", + "--backup", + action="store_true", + help="Backup files in S3 bucket") +parser.add_argument("-d", + "--backup-daily", + action="store_true", + help="Backup daily files in S3 bucket") args = parser.parse_args() module_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) diff --git a/data_scraper/backup.py b/data_scraper/backup.py index 58d8104..8ea3013 100644 --- a/data_scraper/backup.py +++ b/data_scraper/backup.py @@ -14,7 +14,7 @@ def backup_data(): """Uploads scraped files to S3 bucket. Set bucket name in environment variable $S3_BUCKET """ - bucket_name = get_bucket_name() + bucket_name = _get_bucket_name() s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) @@ -37,10 +37,14 @@ def backup_data(): for folder in os.listdir(tiingo_data) ] - done_cboe, fail_cboe = _upload_folders( - bucket, "cboe", cboe_folders, remove_files=False) - done_tiingo, fail_tiingo = _upload_folders( - bucket, "tiingo", tiingo_folders, remove_files=True) + done_cboe, fail_cboe = _upload_folders(bucket, + "cboe", + cboe_folders, + remove_files=False) + done_tiingo, fail_tiingo = _upload_folders(bucket, + "tiingo", + tiingo_folders, + remove_files=True) done = done_cboe + done_tiingo failed = fail_cboe + fail_tiingo @@ -51,11 +55,12 @@ def backup_data(): msg = "Unable to backup symbols: " + ", ".join(done) slack_notification(msg, __name__, status=Status.Warning) + def backup_cboe_data_daily(): """Uploads daily cboe scraped files to S3 bucket. Set bucket name in environment variable $S3_BUCKET """ - bucket_name = get_bucket_name() + bucket_name = _get_bucket_name() s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) @@ -70,8 +75,10 @@ def backup_cboe_data_daily(): if folder.endswith("daily") ] - done_cboe, fail_cboe = _upload_folders( - bucket, "cboe", cboe_folders, remove_files=False) + done_cboe, fail_cboe = _upload_folders(bucket, + "cboe", + cboe_folders, + remove_files=False) done = done_cboe failed = fail_cboe @@ -139,7 +146,7 @@ def _remove_old_files(bucket, prefix): file.delete() -def get_bucket_name(): +def _get_bucket_name(): try: bucket_name = utils.get_environment_var("S3_BUCKET") except EnvironmentError as e: