Fixed slack notifications for aggregate task. Now sends one notification for the total number of symbols failed

README.md

@@ -27,6 +27,7 @@ To use the data scraper the following environment variables need to be set:
 - `$S3_BUCKET`: name of the S3 bucket to backup data
 - `$AWS_ACCESS_KEY_ID`: AWS acces key id
 - `$AWS_SECRET_ACCESS_KEY`: AWS secret key
+- `$SLACK_WEBHOOK`: used to send Slack notifications 
 
 You can configure the data scraper by editing the configuration file `data_scraper.conf` (json-formated).  
 
@@ -36,9 +37,6 @@ Sample file:
 {
   "cboe": {
     "mute_notifications": ["BFB", "CBSA"]
-  },
-  "notifications": {
-      "slack_webhook": "https://hooks.slack.com/services/MY_WORKSPACE_WEBHOOK"
   }
 }
 ```

data_scraper/cboe.py

@@ -76,6 +76,8 @@ def aggregate_monthly_data(symbols=None):
     scraper_dir = os.path.join(save_data_path, "cboe")
 
     symbols = [symbol.upper() for symbol in symbols]
+    done = 0
+    failed = []
 
     for symbol in symbols:
         daily_dir = os.path.join(scraper_dir, symbol + "_daily")
@@ -109,9 +111,10 @@ def aggregate_monthly_data(symbols=None):
                 today = pd.Timestamp.today()
                 first_date = date_range[0]
                 if first_date.year != today.year or first_date.month != today.month:
-                    msg = "Some trading dates where missing for symbol {}".format(
-                        symbol)
-                    slack_notification(msg, __name__)
+                    msg = "Some trading dates where missing for symbol {} in period {}".format(
+                        symbol, month)
+                    logger.error(msg)
+                    failed.append(symbol)
                 continue
 
             if not os.path.exists(monthly_dir):
@@ -132,10 +135,13 @@ def aggregate_monthly_data(symbols=None):
                 continue
 
             logger.debug("Saved monthly data %s", monthly_file)
+            done += 1
 
             for file in daily_files:
                 utils.remove_file(file, logger)
 
+    send_report(done, failed, __name__, op="aggregate")
+
 
 def _get_all_listed_symbols():
     """Returns array of all listed symbols.

data_scraper/notifications.py

@@ -4,7 +4,7 @@ from enum import Enum
 
 import requests
 
-from .utils import get_module_config
+from .utils import get_module_config, get_environment_var
 
 logger = logging.getLogger(__name__)
 
@@ -12,8 +12,8 @@ Status = Enum("Status", "Success Warning Error")
 
 options = get_module_config("notifications")
 try:
-    webhook = options["slack_webhook"]
-except KeyError as e:
+    webhook = get_environment_var("SLACK_WEBHOOK")
+except EnvironmentError as e:
     logger.error("Missing slack webhook from configuration file")
     raise e
 
@@ -59,17 +59,17 @@ def slack_notification(text, scraper, status=Status.Error):
         logger.error(msg)
 
 
-def send_report(done, failed, scraper):
-    """Sends status report to Slack.
-    `done` is the count of successfully scraped symbols
-    `failed` is a list of symbol names that could not be scraped
+def send_report(done, failed, scraper, op="scrape"):
+    """Sends status report to Slack for given operation.
+    `done` is the count of successfully scraped/aggregated symbols
+    `failed` is a list of symbol names that could not be scraped/aggregated
     """
     if done > 0:
-        msg = "Successfully scraped " + _symbol_str(done)
+        msg = "Successfully {}d {}".format(op, _symbol_str(done))
         slack_notification(msg, scraper, status=Status.Success)
     if len(failed) > 0:
-        msg = "Failed to scrape {}: {}".format(_symbol_str(len(failed)),
-                                               ", ".join(failed))
+        msg = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)),
+                                           ", ".join(failed))
         slack_notification(msg, scraper, status=Status.Warning)