From 145797d9f2f8a1980184b190f77defc4c710dff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Rodr=C3=ADguez=20Chatruc?= Date: Wed, 21 Aug 2019 18:38:25 -0300 Subject: [PATCH 1/2] Added backup of daily cboe --- Makefile | 3 +++ data_scraper/__main__.py | 5 ++++ data_scraper/backup.py | 52 +++++++++++++++++++++++++++++++------ docker/data_scraper/crontab | 1 + 4 files changed, 53 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 0e7ed9b..45da103 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,9 @@ aggregate: backup: pipenv run python -m data_scraper -b + +backup_daily: + pipenv run python -m data_scraper -d bench: pipenv run python backtester/test/run_benchmark.py diff --git a/data_scraper/__main__.py b/data_scraper/__main__.py index 3144d45..e3544a4 100644 --- a/data_scraper/__main__.py +++ b/data_scraper/__main__.py @@ -21,6 +21,9 @@ parser.add_argument( help="Aggregate daily data files") parser.add_argument( "-b", "--backup", action="store_true", help="Backup files in S3 bucket") +parser.add_argument( + "-d","--backup_daily", action="store_true", help="Backup daily files in S3 bucket" +) args = parser.parse_args() module_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) @@ -36,6 +39,8 @@ if args.aggregate: cboe.aggregate_monthly_data() elif args.backup: backup.backup_data() +elif args.backup_daily: + backup.backup_cboe_data_daily() else: if args.scraper == "tiingo": scraper = tiingo diff --git a/data_scraper/backup.py b/data_scraper/backup.py index 3baa1a8..58d8104 100644 --- a/data_scraper/backup.py +++ b/data_scraper/backup.py @@ -14,13 +14,7 @@ def backup_data(): """Uploads scraped files to S3 bucket. Set bucket name in environment variable $S3_BUCKET """ - try: - bucket_name = utils.get_environment_var("S3_BUCKET") - except EnvironmentError as e: - logger.error(str(e)) - slack_notification("Backup failed. Set $S3_BUCKET env variable", - __name__) - raise e + bucket_name = get_bucket_name() s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) @@ -57,6 +51,37 @@ def backup_data(): msg = "Unable to backup symbols: " + ", ".join(done) slack_notification(msg, __name__, status=Status.Warning) +def backup_cboe_data_daily(): + """Uploads daily cboe scraped files to S3 bucket. + Set bucket name in environment variable $S3_BUCKET + """ + bucket_name = get_bucket_name() + + s3 = boto3.resource("s3") + bucket = s3.Bucket(bucket_name) + + data_path = utils.get_save_data_path() + + cboe_data = os.path.join(data_path, "cboe") + cboe_folders = [] + if os.path.exists(cboe_data): + cboe_folders = [ + os.path.join(cboe_data, folder) for folder in os.listdir(cboe_data) + if folder.endswith("daily") + ] + + done_cboe, fail_cboe = _upload_folders( + bucket, "cboe", cboe_folders, remove_files=False) + + done = done_cboe + failed = fail_cboe + if len(done) > 0: + msg = "Successful backup of daily cboe symbols: " + ", ".join(done) + slack_notification(msg, __name__, status=Status.Success) + if len(failed) > 0: + msg = "Unable to backup daily symbols: " + ", ".join(done) + slack_notification(msg, __name__, status=Status.Warning) + def _upload_folders(bucket, scraper, folders, remove_files=False): """Uploads folders to S3 bucket and (optionally) removes old files""" @@ -105,10 +130,21 @@ def _key_exists(bucket, key): bucket.Object(key).load() except ClientError as e: return int(e.response["Error"]["Code"]) != 404 - return False + return True def _remove_old_files(bucket, prefix): old_files = bucket.objects.filter(Prefix=prefix) for file in old_files: file.delete() + + +def get_bucket_name(): + try: + bucket_name = utils.get_environment_var("S3_BUCKET") + except EnvironmentError as e: + logger.error(str(e)) + slack_notification("Backup failed. Set $S3_BUCKET env variable", + __name__) + raise e + return bucket_name \ No newline at end of file diff --git a/docker/data_scraper/crontab b/docker/data_scraper/crontab index 4b98f1a..126b93b 100644 --- a/docker/data_scraper/crontab +++ b/docker/data_scraper/crontab @@ -1,4 +1,5 @@ 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=cboe 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=tiingo 0 0 1 * * root cd /finance && run-task make aggregate; run-task make backup +0 0 * * 6 root cd /finance && run-task make backup_daily From a6aa6ea21d883f940737be7b79ca7078014d3982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Rodr=C3=ADguez=20Chatruc?= Date: Thu, 22 Aug 2019 17:41:57 -0300 Subject: [PATCH 2/2] Fixed some minor points --- data_scraper/__main__.py | 39 +++++++++++++++++++++------------------ data_scraper/backup.py | 25 ++++++++++++++++--------- 2 files changed, 37 insertions(+), 27 deletions(-) diff --git a/data_scraper/__main__.py b/data_scraper/__main__.py index e3544a4..0e5de5f 100644 --- a/data_scraper/__main__.py +++ b/data_scraper/__main__.py @@ -6,24 +6,27 @@ from data_scraper import cboe, tiingo, backup parser = argparse.ArgumentParser(prog="data_scraper.py") parser.add_argument("-t", "--symbols", nargs="+", help="Symbols to fetch") -parser.add_argument( - "-s", - "--scraper", - choices=["cboe", "tiingo"], - default="cboe", - help="Scraper to use") -parser.add_argument( - "-v", "--verbose", action="store_true", help="Enable logging") -parser.add_argument( - "-a", - "--aggregate", - action="store_true", - help="Aggregate daily data files") -parser.add_argument( - "-b", "--backup", action="store_true", help="Backup files in S3 bucket") -parser.add_argument( - "-d","--backup_daily", action="store_true", help="Backup daily files in S3 bucket" -) +parser.add_argument("-s", + "--scraper", + choices=["cboe", "tiingo"], + default="cboe", + help="Scraper to use") +parser.add_argument("-v", + "--verbose", + action="store_true", + help="Enable logging") +parser.add_argument("-a", + "--aggregate", + action="store_true", + help="Aggregate daily data files") +parser.add_argument("-b", + "--backup", + action="store_true", + help="Backup files in S3 bucket") +parser.add_argument("-d", + "--backup-daily", + action="store_true", + help="Backup daily files in S3 bucket") args = parser.parse_args() module_dir = os.path.join(os.getcwd(), os.path.dirname(__file__)) diff --git a/data_scraper/backup.py b/data_scraper/backup.py index 58d8104..8ea3013 100644 --- a/data_scraper/backup.py +++ b/data_scraper/backup.py @@ -14,7 +14,7 @@ def backup_data(): """Uploads scraped files to S3 bucket. Set bucket name in environment variable $S3_BUCKET """ - bucket_name = get_bucket_name() + bucket_name = _get_bucket_name() s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) @@ -37,10 +37,14 @@ def backup_data(): for folder in os.listdir(tiingo_data) ] - done_cboe, fail_cboe = _upload_folders( - bucket, "cboe", cboe_folders, remove_files=False) - done_tiingo, fail_tiingo = _upload_folders( - bucket, "tiingo", tiingo_folders, remove_files=True) + done_cboe, fail_cboe = _upload_folders(bucket, + "cboe", + cboe_folders, + remove_files=False) + done_tiingo, fail_tiingo = _upload_folders(bucket, + "tiingo", + tiingo_folders, + remove_files=True) done = done_cboe + done_tiingo failed = fail_cboe + fail_tiingo @@ -51,11 +55,12 @@ def backup_data(): msg = "Unable to backup symbols: " + ", ".join(done) slack_notification(msg, __name__, status=Status.Warning) + def backup_cboe_data_daily(): """Uploads daily cboe scraped files to S3 bucket. Set bucket name in environment variable $S3_BUCKET """ - bucket_name = get_bucket_name() + bucket_name = _get_bucket_name() s3 = boto3.resource("s3") bucket = s3.Bucket(bucket_name) @@ -70,8 +75,10 @@ def backup_cboe_data_daily(): if folder.endswith("daily") ] - done_cboe, fail_cboe = _upload_folders( - bucket, "cboe", cboe_folders, remove_files=False) + done_cboe, fail_cboe = _upload_folders(bucket, + "cboe", + cboe_folders, + remove_files=False) done = done_cboe failed = fail_cboe @@ -139,7 +146,7 @@ def _remove_old_files(bucket, prefix): file.delete() -def get_bucket_name(): +def _get_bucket_name(): try: bucket_name = utils.get_environment_var("S3_BUCKET") except EnvironmentError as e: