diff --git a/data_scraper/cboe.py b/data_scraper/cboe.py index 38807f0..b01fdf1 100644 --- a/data_scraper/cboe.py +++ b/data_scraper/cboe.py @@ -9,7 +9,7 @@ import requests import pandas as pd from . import utils, validation -from .notifications import slack_notification, send_report +from .notifications import send_report logger = logging.getLogger(__name__) @@ -20,19 +20,16 @@ def fetch_data(symbols=None): """Fetches options data for a given list of symbols""" symbols = symbols or _get_all_listed_symbols() options = utils.get_module_config("cboe") - mute_notifications = options.get("mute_notifications", []) try: form_data = _form_data() except requests.ConnectionError as ce: msg = "Connection error trying to reach {}".format(url) logger.error(msg) - slack_notification(msg, __name__) raise ce except Exception as e: msg = "Error parsing response" logger.error(msg, exc_info=True) - slack_notification(msg, __name__) raise e headers = {"Referer": url} @@ -59,8 +56,6 @@ def fetch_data(symbols=None): failed.append(symbol) msg = "Error fetching symbol {} data".format(symbol) logger.error(msg, exc_info=True) - if symbol not in mute_notifications: - slack_notification(msg, __name__) else: _save_data(symbol, symbol_data) done += 1 @@ -84,7 +79,6 @@ def aggregate_monthly_data(symbols=None): if not os.path.exists(daily_dir): msg = "Error aggregating data. Dir {} not found.".format(daily_dir) logger.error(msg) - slack_notification(msg, __name__) continue monthly_dir = os.path.join(scraper_dir, symbol) @@ -103,7 +97,6 @@ def aggregate_monthly_data(symbols=None): except Exception: msg = "Error concatenating daily files for period " + month logger.error(msg, exc_info=True) - slack_notification(msg, __name__) continue date_range = pd.to_datetime(symbol_df["quotedate"].unique()) @@ -131,7 +124,6 @@ def aggregate_monthly_data(symbols=None): msg = "Data in {} differs from the daily files".format( monthly_file) logger.error(msg) - slack_notification(msg, __name__) continue logger.debug("Saved monthly data %s", monthly_file) diff --git a/data_scraper/notifications.py b/data_scraper/notifications.py index d544bf9..f1ef12a 100644 --- a/data_scraper/notifications.py +++ b/data_scraper/notifications.py @@ -29,20 +29,9 @@ payload = { def slack_notification(text, scraper, status=Status.Error): """Post Slack notification""" - - if status == Status.Error: - emoji = ":thumbsdown: " - title = "data_scraper error" - color = "#B22222" - else: - title = "data_scraper status report" - if status == Status.Success: - emoji = ":thumbsup: " - color = "#49C39E" - else: - emoji = ":warning: " - color = "#EDB625" - msg = emoji + text + title = "data_scraper status report" + color="#ff9906" + msg = text payload["attachments"][0]["fallback"] = msg payload["attachments"][0]["text"] = msg @@ -65,13 +54,20 @@ def send_report(done, failed, scraper, op="scrape"): `failed` is a list of symbol names that could not be scraped/aggregated """ if done > 0: - msg = "Successfully {}d {}".format(op, _symbol_str(done)) - slack_notification(msg, scraper, status=Status.Success) + msg1 = "Successfully {}d {}".format(op, _symbol_str(done)) if len(failed) > 0: - msg = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)), + msg2 = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)), ", ".join(failed)) - slack_notification(msg, scraper, status=Status.Warning) + + msg= msg1 + " and " + msg2 + slack_notification(msg, scraper, status=Status.Warning) def _symbol_str(count): return str(count) + " symbol" if count == 1 else str(count) + " symbols" + +## make init + make env ## +##cambiar el channel +##sacar notificaciones por cada error y armar un mensaje combinando todo +##python +## \ No newline at end of file diff --git a/data_scraper/tiingo.py b/data_scraper/tiingo.py index b720305..677adc9 100644 --- a/data_scraper/tiingo.py +++ b/data_scraper/tiingo.py @@ -6,7 +6,7 @@ import pandas as pd import pandas_datareader as pdr from . import utils, validation -from .notifications import slack_notification, send_report +from .notifications import send_report logger = logging.getLogger(__name__) @@ -35,18 +35,15 @@ def fetch_data(symbols=assets): msg = "Unable to connect to api.tiingo.com when fetching symbol {}".format( symbol) logger.error(msg, exc_info=True) - slack_notification(msg, __name__) raise ce except TypeError: # pandas_datareader raises TypeError when fetching invalid symbol failed.append(symbol) msg = "Attempted to fetch invalid symbol {}".format(symbol) logger.error(msg, exc_info=True) - slack_notification(msg, __name__) except Exception: msg = "Error fetching symbol {}".format(symbol) logger.error(msg, exc_info=True) - slack_notification(msg, __name__) else: _save_data(symbol, symbol_data.reset_index()) done += 1 @@ -112,7 +109,6 @@ def _merge(symbol, symbol_df): msg = """Old data included dates not present in scraped file for symbol {} Merged new data with previous file.""".format(symbol) logger.error(msg) - slack_notification(msg, __name__) merged_df = pd.concat([symbol_df, old_df.loc[diffs]]) merged_df.sort_index(inplace=True) return merged_df.reset_index()