Merge pull request #19 from lambdaclass/data_scraper

Changed status report message

data_scraper/cboe.py

@@ -9,7 +9,7 @@ import requests
 import pandas as pd
 
 from . import utils, validation
-from .notifications import slack_notification, Status
+from .notifications import slack_notification, send_report
 
 logger = logging.getLogger(__name__)
 
@@ -39,7 +39,8 @@ def fetch_data(symbols=None):
     file_url = "http://www.cboe.com/delayedquote/quotedata.dat"
 
     symbols = [symbol.upper() for symbol in symbols]
-    done, failed = [], []
+    done = 0
+    failed = []
 
     for symbol in symbols:
         form_data["ctl00$ContentTop$C005$txtTicker"] = symbol
@@ -62,14 +63,9 @@ def fetch_data(symbols=None):
                 slack_notification(msg, __name__)
         else:
             _save_data(symbol, symbol_data)
-            done.append(symbol)
+            done += 1
 
-    if len(done) > 0:
-        msg = "Successfully scraped symbols: " + ", ".join(done)
-        slack_notification(msg, __name__, status=Status.Success)
-    if len(failed) > 0:
-        msg = "Failed to scrape symbols: " + ", ".join(failed)
-        slack_notification(msg, __name__, status=Status.Warning)
+    send_report(done, failed, __name__)
 
 
 def aggregate_monthly_data(symbols=None):

data_scraper/notifications.py

@@ -57,3 +57,21 @@ def slack_notification(text, scraper, status=Status.Error):
         msg = "Error connecting to Slack {}. Response is:\n{}".format(
             response.status_code, response.text)
         logger.error(msg)
+
+
+def send_report(done, failed, scraper):
+    """Sends status report to Slack.
+    `done` is the count of successfully scraped symbols
+    `failed` is a list of symbol names that could not be scraped
+    """
+    if done > 0:
+        msg = "Successfully scraped " + _symbol_str(done)
+        slack_notification(msg, scraper, status=Status.Success)
+    if len(failed) > 0:
+        msg = "Failed to scrape {}: {}".format(_symbol_str(len(failed)),
+                                               ", ".join(failed))
+        slack_notification(msg, scraper, status=Status.Warning)
+
+
+def _symbol_str(count):
+    return str(count) + " symbol" if count == 1 else str(count) + " symbols"

data_scraper/tiingo.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pandas_datareader as pdr
 
 from . import utils, validation
-from .notifications import slack_notification, Status
+from .notifications import slack_notification, send_report
 
 logger = logging.getLogger(__name__)
 
@@ -25,7 +25,8 @@ def fetch_data(symbols=assets):
     api_key = utils.get_environment_var("TIINGO_API_KEY")
 
     symbols = [symbol.upper() for symbol in symbols]
-    done, failed = [], []
+    done = 0
+    failed = []
 
     for symbol in symbols:
         try:
@@ -48,14 +49,9 @@ def fetch_data(symbols=assets):
             slack_notification(msg, __name__)
         else:
             _save_data(symbol, symbol_data.reset_index())
-            done.append(symbol)
+            done += 1
 
-    if len(done) > 0:
-        msg = "Successfully scraped symbols: " + ", ".join(done)
-        slack_notification(msg, __name__, status=Status.Success)
-    if len(failed) > 0:
-        msg = "Failed to scrape symbols: " + ", ".join(failed)
-        slack_notification(msg, __name__, status=Status.Warning)
+    send_report(done, failed, __name__)
 
 
 def _save_data(symbol, symbol_df):