Added backup of daily cboe

Makefile

@@ -36,6 +36,9 @@ aggregate:
 
 backup:
 	pipenv run python -m data_scraper -b
+
+backup_daily:
+	pipenv run python -m data_scraper -d
 	
 bench:
 	pipenv run python backtester/test/run_benchmark.py

data_scraper/__main__.py

@@ -21,6 +21,9 @@ parser.add_argument(
     help="Aggregate daily data files")
 parser.add_argument(
     "-b", "--backup", action="store_true", help="Backup files in S3 bucket")
+parser.add_argument(
+    "-d","--backup_daily", action="store_true", help="Backup daily files in S3 bucket"
+)
 
 args = parser.parse_args()
 module_dir = os.path.join(os.getcwd(), os.path.dirname(__file__))
@@ -36,6 +39,8 @@ if args.aggregate:
         cboe.aggregate_monthly_data()
 elif args.backup:
     backup.backup_data()
+elif args.backup_daily:
+    backup.backup_cboe_data_daily()
 else:
     if args.scraper == "tiingo":
         scraper = tiingo

data_scraper/backup.py

@@ -14,13 +14,7 @@ def backup_data():
     """Uploads scraped files to S3 bucket.
     Set bucket name in environment variable $S3_BUCKET
     """
-    try:
-        bucket_name = utils.get_environment_var("S3_BUCKET")
-    except EnvironmentError as e:
-        logger.error(str(e))
-        slack_notification("Backup failed. Set $S3_BUCKET env variable",
-                           __name__)
-        raise e
+    bucket_name = get_bucket_name()
 
     s3 = boto3.resource("s3")
     bucket = s3.Bucket(bucket_name)
@@ -57,6 +51,37 @@ def backup_data():
         msg = "Unable to backup symbols: " + ", ".join(done)
         slack_notification(msg, __name__, status=Status.Warning)
 
+def backup_cboe_data_daily():
+    """Uploads daily cboe scraped files to S3 bucket.
+    Set bucket name in environment variable $S3_BUCKET
+    """
+    bucket_name = get_bucket_name()
+
+    s3 = boto3.resource("s3")
+    bucket = s3.Bucket(bucket_name)
+
+    data_path = utils.get_save_data_path()
+
+    cboe_data = os.path.join(data_path, "cboe")
+    cboe_folders = []
+    if os.path.exists(cboe_data):
+        cboe_folders = [
+            os.path.join(cboe_data, folder) for folder in os.listdir(cboe_data)
+            if folder.endswith("daily")
+        ]
+
+    done_cboe, fail_cboe = _upload_folders(
+        bucket, "cboe", cboe_folders, remove_files=False)
+
+    done = done_cboe
+    failed = fail_cboe
+    if len(done) > 0:
+        msg = "Successful backup of daily cboe symbols: " + ", ".join(done)
+        slack_notification(msg, __name__, status=Status.Success)
+    if len(failed) > 0:
+        msg = "Unable to backup daily symbols: " + ", ".join(done)
+        slack_notification(msg, __name__, status=Status.Warning)
+
 
 def _upload_folders(bucket, scraper, folders, remove_files=False):
     """Uploads folders to S3 bucket and (optionally) removes old files"""
@@ -105,10 +130,21 @@ def _key_exists(bucket, key):
         bucket.Object(key).load()
     except ClientError as e:
         return int(e.response["Error"]["Code"]) != 404
-    return False
+    return True
 
 
 def _remove_old_files(bucket, prefix):
     old_files = bucket.objects.filter(Prefix=prefix)
     for file in old_files:
         file.delete()
+
+
+def get_bucket_name():
+    try:
+        bucket_name = utils.get_environment_var("S3_BUCKET")
+    except EnvironmentError as e:
+        logger.error(str(e))
+        slack_notification("Backup failed. Set $S3_BUCKET env variable",
+                           __name__)
+        raise e
+    return bucket_name

docker/data_scraper/crontab

@@ -1,4 +1,5 @@
 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=cboe
 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=tiingo
 0 0 1 * * root cd /finance && run-task make aggregate; run-task make backup
+0 0 * * 6 root cd /finance && run-task make backup_daily