diff --git a/Makefile b/Makefile index 0e7ed9b..45da103 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,9 @@ aggregate: backup: pipenv run python -m data_scraper -b + +backup_daily: + pipenv run python -m data_scraper -d bench: pipenv run python backtester/test/run_benchmark.py diff --git a/data_scraper/__main__.py b/data_scraper/__main__.py index 3144d45..e3544a4 100644 --- a/data_scraper/__main__.py +++ b/data_scraper/__main__.py @@ -21,6 +21,9 @@ parser.add_argument( help="Aggregate daily data files") parser.add_argument( "-b", "--backup", action="store_true", help="Backup files in S3 bucket") +parser.add_argument( + "-d","--backup_daily", action="store_true", help="Backup daily files in S3 bucket" +) args = parser.parse_args() module_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) @@ -36,6 +39,8 @@ if args.aggregate: cboe.aggregate_monthly_data() elif args.backup: backup.backup_data() +elif args.backup_daily: + backup.backup_cboe_data_daily() else: if args.scraper == "tiingo": scraper = tiingo diff --git a/data_scraper/backup.py b/data_scraper/backup.py index 3baa1a8..58d8104 100644 --- a/data_scraper/backup.py +++ b/data_scraper/backup.py @@ -14,13 +14,7 @@ def backup_data(): """Uploads scraped files to S3 bucket. Set bucket name in environment variable $S3_BUCKET """ - try: - bucket_name = utils.get_environment_var("S3_BUCKET") - except EnvironmentError as e: - logger.error(str(e)) - slack_notification("Backup failed. Set $S3_BUCKET env variable", - __name__) - raise e + bucket_name = get_bucket_name() s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) @@ -57,6 +51,37 @@ def backup_data(): msg = "Unable to backup symbols: " + ", ".join(done) slack_notification(msg, __name__, status=Status.Warning) +def backup_cboe_data_daily(): + """Uploads daily cboe scraped files to S3 bucket. + Set bucket name in environment variable $S3_BUCKET + """ + bucket_name = get_bucket_name() + + s3 = boto3.resource("s3") + bucket = s3.Bucket(bucket_name) + + data_path = utils.get_save_data_path() + + cboe_data = os.path.join(data_path, "cboe") + cboe_folders = [] + if os.path.exists(cboe_data): + cboe_folders = [ + os.path.join(cboe_data, folder) for folder in os.listdir(cboe_data) + if folder.endswith("daily") + ] + + done_cboe, fail_cboe = _upload_folders( + bucket, "cboe", cboe_folders, remove_files=False) + + done = done_cboe + failed = fail_cboe + if len(done) > 0: + msg = "Successful backup of daily cboe symbols: " + ", ".join(done) + slack_notification(msg, __name__, status=Status.Success) + if len(failed) > 0: + msg = "Unable to backup daily symbols: " + ", ".join(done) + slack_notification(msg, __name__, status=Status.Warning) + def _upload_folders(bucket, scraper, folders, remove_files=False): """Uploads folders to S3 bucket and (optionally) removes old files""" @@ -105,10 +130,21 @@ def _key_exists(bucket, key): bucket.Object(key).load() except ClientError as e: return int(e.response["Error"]["Code"]) != 404 - return False + return True def _remove_old_files(bucket, prefix): old_files = bucket.objects.filter(Prefix=prefix) for file in old_files: file.delete() + + +def get_bucket_name(): + try: + bucket_name = utils.get_environment_var("S3_BUCKET") + except EnvironmentError as e: + logger.error(str(e)) + slack_notification("Backup failed. Set $S3_BUCKET env variable", + __name__) + raise e + return bucket_name \ No newline at end of file diff --git a/docker/data_scraper/crontab b/docker/data_scraper/crontab index 4b98f1a..126b93b 100644 --- a/docker/data_scraper/crontab +++ b/docker/data_scraper/crontab @@ -1,4 +1,5 @@ 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=cboe 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=tiingo 0 0 1 * * root cd /finance && run-task make aggregate; run-task make backup +0 0 * * 6 root cd /finance && run-task make backup_daily