changing notifications on scraper

data_scraper/cboe.py

@@ -9,7 +9,7 @@ import requests
 import pandas as pd
 
 from . import utils, validation
-from .notifications import slack_notification, send_report
+from .notifications import send_report
 
 logger = logging.getLogger(__name__)
 
@@ -20,19 +20,16 @@ def fetch_data(symbols=None):
     """Fetches options data for a given list of symbols"""
     symbols = symbols or _get_all_listed_symbols()
     options = utils.get_module_config("cboe")
-    mute_notifications = options.get("mute_notifications", [])
 
     try:
         form_data = _form_data()
     except requests.ConnectionError as ce:
         msg = "Connection error trying to reach {}".format(url)
         logger.error(msg)
-        slack_notification(msg, __name__)
         raise ce
     except Exception as e:
         msg = "Error parsing response"
         logger.error(msg, exc_info=True)
-        slack_notification(msg, __name__)
         raise e
 
     headers = {"Referer": url}
@@ -59,8 +56,6 @@ def fetch_data(symbols=None):
             failed.append(symbol)
             msg = "Error fetching symbol {} data".format(symbol)
             logger.error(msg, exc_info=True)
-            if symbol not in mute_notifications:
-                slack_notification(msg, __name__)
         else:
             _save_data(symbol, symbol_data)
             done += 1
@@ -84,7 +79,6 @@ def aggregate_monthly_data(symbols=None):
         if not os.path.exists(daily_dir):
             msg = "Error aggregating data. Dir {} not found.".format(daily_dir)
             logger.error(msg)
-            slack_notification(msg, __name__)
             continue
 
         monthly_dir = os.path.join(scraper_dir, symbol)
@@ -103,7 +97,6 @@ def aggregate_monthly_data(symbols=None):
             except Exception:
                 msg = "Error concatenating daily files for period " + month
                 logger.error(msg, exc_info=True)
-                slack_notification(msg, __name__)
                 continue
 
             date_range = pd.to_datetime(symbol_df["quotedate"].unique())
@@ -131,7 +124,6 @@ def aggregate_monthly_data(symbols=None):
                 msg = "Data in {} differs from the daily files".format(
                     monthly_file)
                 logger.error(msg)
-                slack_notification(msg, __name__)
                 continue
 
             logger.debug("Saved monthly data %s", monthly_file)

data_scraper/notifications.py

@@ -29,20 +29,9 @@ payload = {
 
 def slack_notification(text, scraper, status=Status.Error):
     """Post Slack notification"""
-
-    if status == Status.Error:
-        emoji = ":thumbsdown: "
-        title = "data_scraper error"
-        color = "#B22222"
-    else:
-        title = "data_scraper status report"
-        if status == Status.Success:
-            emoji = ":thumbsup: "
-            color = "#49C39E"
-        else:
-            emoji = ":warning: "
-            color = "#EDB625"
-    msg = emoji + text
+    title = "data_scraper status report"
+    color="#ff9906"
+    msg = text
 
     payload["attachments"][0]["fallback"] = msg
     payload["attachments"][0]["text"] = msg
@@ -65,13 +54,20 @@ def send_report(done, failed, scraper, op="scrape"):
     `failed` is a list of symbol names that could not be scraped/aggregated
     """
     if done > 0:
-        msg = "Successfully {}d {}".format(op, _symbol_str(done))
-        slack_notification(msg, scraper, status=Status.Success)
+        msg1 = "Successfully {}d {}".format(op, _symbol_str(done))
     if len(failed) > 0:
-        msg = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)),
+        msg2 = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)),
                                            ", ".join(failed))
-        slack_notification(msg, scraper, status=Status.Warning)
+    
+    msg= msg1 + " and " + msg2
+    slack_notification(msg, scraper, status=Status.Warning)
 
 
 def _symbol_str(count):
     return str(count) + " symbol" if count == 1 else str(count) + " symbols"
+
+## make init + make env ##
+##cambiar el channel
+##sacar notificaciones por cada error y armar un mensaje combinando todo
+##python
+##

data_scraper/tiingo.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pandas_datareader as pdr
 
 from . import utils, validation
-from .notifications import slack_notification, send_report
+from .notifications import send_report
 
 logger = logging.getLogger(__name__)
 
@@ -35,18 +35,15 @@ def fetch_data(symbols=assets):
             msg = "Unable to connect to api.tiingo.com when fetching symbol {}".format(
                 symbol)
             logger.error(msg, exc_info=True)
-            slack_notification(msg, __name__)
             raise ce
         except TypeError:
             # pandas_datareader raises TypeError when fetching invalid symbol
             failed.append(symbol)
             msg = "Attempted to fetch invalid symbol {}".format(symbol)
             logger.error(msg, exc_info=True)
-            slack_notification(msg, __name__)
         except Exception:
             msg = "Error fetching symbol {}".format(symbol)
             logger.error(msg, exc_info=True)
-            slack_notification(msg, __name__)
         else:
             _save_data(symbol, symbol_data.reset_index())
             done += 1
@@ -112,7 +109,6 @@ def _merge(symbol, symbol_df):
         msg = """Old data included dates not present in scraped file for symbol {}
             Merged new data with previous file.""".format(symbol)
         logger.error(msg)
-        slack_notification(msg, __name__)
         merged_df = pd.concat([symbol_df, old_df.loc[diffs]])
         merged_df.sort_index(inplace=True)
         return merged_df.reset_index()