Merge pull request #47 from lambdaclass/Prueba

Added backup of daily cboe

Makefile

@@ -36,6 +36,9 @@ aggregate:
 
 backup:
 	pipenv run python -m data_scraper -b
+
+backup_daily:
+	pipenv run python -m data_scraper -d
 	
 bench:
 	pipenv run python backtester/test/run_benchmark.py

data_scraper/__main__.py

@@ -6,21 +6,27 @@ from data_scraper import cboe, tiingo, backup
 
 parser = argparse.ArgumentParser(prog="data_scraper.py")
 parser.add_argument("-t", "--symbols", nargs="+", help="Symbols to fetch")
-parser.add_argument(
-    "-s",
-    "--scraper",
-    choices=["cboe", "tiingo"],
-    default="cboe",
-    help="Scraper to use")
-parser.add_argument(
-    "-v", "--verbose", action="store_true", help="Enable logging")
-parser.add_argument(
-    "-a",
-    "--aggregate",
-    action="store_true",
-    help="Aggregate daily data files")
-parser.add_argument(
-    "-b", "--backup", action="store_true", help="Backup files in S3 bucket")
+parser.add_argument("-s",
+                    "--scraper",
+                    choices=["cboe", "tiingo"],
+                    default="cboe",
+                    help="Scraper to use")
+parser.add_argument("-v",
+                    "--verbose",
+                    action="store_true",
+                    help="Enable logging")
+parser.add_argument("-a",
+                    "--aggregate",
+                    action="store_true",
+                    help="Aggregate daily data files")
+parser.add_argument("-b",
+                    "--backup",
+                    action="store_true",
+                    help="Backup files in S3 bucket")
+parser.add_argument("-d",
+                    "--backup-daily",
+                    action="store_true",
+                    help="Backup daily files in S3 bucket")
 
 args = parser.parse_args()
 module_dir = os.path.join(os.getcwd(), os.path.dirname(__file__))
@@ -36,6 +42,8 @@ if args.aggregate:
         cboe.aggregate_monthly_data()
 elif args.backup:
     backup.backup_data()
+elif args.backup_daily:
+    backup.backup_cboe_data_daily()
 else:
     if args.scraper == "tiingo":
         scraper = tiingo

data_scraper/backup.py

@@ -14,13 +14,7 @@ def backup_data():
     """Uploads scraped files to S3 bucket.
     Set bucket name in environment variable $S3_BUCKET
     """
-    try:
-        bucket_name = utils.get_environment_var("S3_BUCKET")
-    except EnvironmentError as e:
-        logger.error(str(e))
-        slack_notification("Backup failed. Set $S3_BUCKET env variable",
-                           __name__)
-        raise e
+    bucket_name = _get_bucket_name()
 
     s3 = boto3.resource("s3")
     bucket = s3.Bucket(bucket_name)
@@ -43,10 +37,14 @@ def backup_data():
             for folder in os.listdir(tiingo_data)
         ]
 
-    done_cboe, fail_cboe = _upload_folders(
-        bucket, "cboe", cboe_folders, remove_files=False)
-    done_tiingo, fail_tiingo = _upload_folders(
-        bucket, "tiingo", tiingo_folders, remove_files=True)
+    done_cboe, fail_cboe = _upload_folders(bucket,
+                                           "cboe",
+                                           cboe_folders,
+                                           remove_files=False)
+    done_tiingo, fail_tiingo = _upload_folders(bucket,
+                                               "tiingo",
+                                               tiingo_folders,
+                                               remove_files=True)
 
     done = done_cboe + done_tiingo
     failed = fail_cboe + fail_tiingo
@@ -58,6 +56,40 @@ def backup_data():
         slack_notification(msg, __name__, status=Status.Warning)
 
 
+def backup_cboe_data_daily():
+    """Uploads daily cboe scraped files to S3 bucket.
+    Set bucket name in environment variable $S3_BUCKET
+    """
+    bucket_name = _get_bucket_name()
+
+    s3 = boto3.resource("s3")
+    bucket = s3.Bucket(bucket_name)
+
+    data_path = utils.get_save_data_path()
+
+    cboe_data = os.path.join(data_path, "cboe")
+    cboe_folders = []
+    if os.path.exists(cboe_data):
+        cboe_folders = [
+            os.path.join(cboe_data, folder) for folder in os.listdir(cboe_data)
+            if folder.endswith("daily")
+        ]
+
+    done_cboe, fail_cboe = _upload_folders(bucket,
+                                           "cboe",
+                                           cboe_folders,
+                                           remove_files=False)
+
+    done = done_cboe
+    failed = fail_cboe
+    if len(done) > 0:
+        msg = "Successful backup of daily cboe symbols: " + ", ".join(done)
+        slack_notification(msg, __name__, status=Status.Success)
+    if len(failed) > 0:
+        msg = "Unable to backup daily symbols: " + ", ".join(done)
+        slack_notification(msg, __name__, status=Status.Warning)
+
+
 def _upload_folders(bucket, scraper, folders, remove_files=False):
     """Uploads folders to S3 bucket and (optionally) removes old files"""
     data_path = utils.get_save_data_path()
@@ -105,10 +137,21 @@ def _key_exists(bucket, key):
         bucket.Object(key).load()
     except ClientError as e:
         return int(e.response["Error"]["Code"]) != 404
-    return False
+    return True
 
 
 def _remove_old_files(bucket, prefix):
     old_files = bucket.objects.filter(Prefix=prefix)
     for file in old_files:
         file.delete()
+
+
+def _get_bucket_name():
+    try:
+        bucket_name = utils.get_environment_var("S3_BUCKET")
+    except EnvironmentError as e:
+        logger.error(str(e))
+        slack_notification("Backup failed. Set $S3_BUCKET env variable",
+                           __name__)
+        raise e
+    return bucket_name

docker/data_scraper/crontab

@@ -1,4 +1,5 @@
 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=cboe
 0 19 * * 1-5 root cd /finance && run-task make scrape scraper=tiingo
 0 0 1 * * root cd /finance && run-task make aggregate; run-task make backup
+0 0 * * 6 root cd /finance && run-task make backup_daily