diff --git a/Makefile b/Makefile index 0e7ed9b..45da103 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,9 @@ aggregate: backup: pipenv run python -m data_scraper -b + +backup_daily: + pipenv run python -m data_scraper -d bench: pipenv run python backtester/test/run_benchmark.py diff --git a/data_scraper/__main__.py b/data_scraper/__main__.py index 3144d45..0e5de5f 100644 --- a/data_scraper/__main__.py +++ b/data_scraper/__main__.py @@ -6,21 +6,27 @@ from data_scraper import cboe, tiingo, backup parser = argparse.ArgumentParser(prog="data_scraper.py") parser.add_argument("-t", "--symbols", nargs="+", help="Symbols to fetch") -parser.add_argument( - "-s", - "--scraper", - choices=["cboe", "tiingo"], - default="cboe", - help="Scraper to use") -parser.add_argument( - "-v", "--verbose", action="store_true", help="Enable logging") -parser.add_argument( - "-a", - "--aggregate", - action="store_true", - help="Aggregate daily data files") -parser.add_argument( - "-b", "--backup", action="store_true", help="Backup files in S3 bucket") +parser.add_argument("-s", + "--scraper", + choices=["cboe", "tiingo"], + default="cboe", + help="Scraper to use") +parser.add_argument("-v", + "--verbose", + action="store_true", + help="Enable logging") +parser.add_argument("-a", + "--aggregate", + action="store_true", + help="Aggregate daily data files") +parser.add_argument("-b", + "--backup", + action="store_true", + help="Backup files in S3 bucket") +parser.add_argument("-d", + "--backup-daily", + action="store_true", + help="Backup daily files in S3 bucket") args = parser.parse_args() module_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) @@ -36,6 +42,8 @@ if args.aggregate: cboe.aggregate_monthly_data() elif args.backup: backup.backup_data() +elif args.backup_daily: + backup.backup_cboe_data_daily() else: if args.scraper == "tiingo": scraper = tiingo diff --git a/data_scraper/backup.py b/data_scraper/backup.py index 3baa1a8..8ea3013 100644 --- a/data_scraper/backup.py +++ b/data_scraper/backup.py @@ -14,13 +14,7 @@ def backup_data(): """Uploads scraped files to S3 bucket. Set bucket name in environment variable $S3_BUCKET """ - try: - bucket_name = utils.get_environment_var("S3_BUCKET") - except EnvironmentError as e: - logger.error(str(e)) - slack_notification("Backup failed. Set $S3_BUCKET env variable", - __name__) - raise e + bucket_name = _get_bucket_name() s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) @@ -43,10 +37,14 @@ def backup_data(): for folder in os.listdir(tiingo_data) ] - done_cboe, fail_cboe = _upload_folders( - bucket, "cboe", cboe_folders, remove_files=False) - done_tiingo, fail_tiingo = _upload_folders( - bucket, "tiingo", tiingo_folders, remove_files=True) + done_cboe, fail_cboe = _upload_folders(bucket, + "cboe", + cboe_folders, + remove_files=False) + done_tiingo, fail_tiingo = _upload_folders(bucket, + "tiingo", + tiingo_folders, + remove_files=True) done = done_cboe + done_tiingo failed = fail_cboe + fail_tiingo @@ -58,6 +56,40 @@ def backup_data(): slack_notification(msg, __name__, status=Status.Warning) +def backup_cboe_data_daily(): + """Uploads daily cboe scraped files to S3 bucket. + Set bucket name in environment variable $S3_BUCKET + """ + bucket_name = _get_bucket_name() + + s3 = boto3.resource("s3") + bucket = s3.Bucket(bucket_name) + + data_path = utils.get_save_data_path() + + cboe_data = os.path.join(data_path, "cboe") + cboe_folders = [] + if os.path.exists(cboe_data): + cboe_folders = [ + os.path.join(cboe_data, folder) for folder in os.listdir(cboe_data) + if folder.endswith("daily") + ] + + done_cboe, fail_cboe = _upload_folders(bucket, + "cboe", + cboe_folders, + remove_files=False) + + done = done_cboe + failed = fail_cboe + if len(done) > 0: + msg = "Successful backup of daily cboe symbols: " + ", ".join(done) + slack_notification(msg, __name__, status=Status.Success) + if len(failed) > 0: + msg = "Unable to backup daily symbols: " + ", ".join(done) + slack_notification(msg, __name__, status=Status.Warning) + + def _upload_folders(bucket, scraper, folders, remove_files=False): """Uploads folders to S3 bucket and (optionally) removes old files""" data_path = utils.get_save_data_path() @@ -105,10 +137,21 @@ def _key_exists(bucket, key): bucket.Object(key).load() except ClientError as e: return int(e.response["Error"]["Code"]) != 404 - return False + return True def _remove_old_files(bucket, prefix): old_files = bucket.objects.filter(Prefix=prefix) for file in old_files: file.delete() + + +def _get_bucket_name(): + try: + bucket_name = utils.get_environment_var("S3_BUCKET") + except EnvironmentError as e: + logger.error(str(e)) + slack_notification("Backup failed. Set $S3_BUCKET env variable", + __name__) + raise e + return bucket_name \ No newline at end of file diff --git a/docker/data_scraper/crontab b/docker/data_scraper/crontab index 4b98f1a..126b93b 100644 --- a/docker/data_scraper/crontab +++ b/docker/data_scraper/crontab @@ -1,4 +1,5 @@ 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=cboe 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=tiingo 0 0 1 * * root cd /finance && run-task make aggregate; run-task make backup +0 0 * * 6 root cd /finance && run-task make backup_daily