diff --git a/README.md b/README.md index 6f8cde6..b83b022 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ To use the data scraper the following environment variables need to be set: - `$S3_BUCKET`: name of the S3 bucket to backup data - `$AWS_ACCESS_KEY_ID`: AWS acces key id - `$AWS_SECRET_ACCESS_KEY`: AWS secret key +- `$SLACK_WEBHOOK`: used to send Slack notifications You can configure the data scraper by editing the configuration file `data_scraper.conf` (json-formated). @@ -36,9 +37,6 @@ Sample file: { "cboe": { "mute_notifications": ["BFB", "CBSA"] - }, - "notifications": { - "slack_webhook": "https://hooks.slack.com/services/MY_WORKSPACE_WEBHOOK" } } ``` diff --git a/data_scraper/cboe.py b/data_scraper/cboe.py index 7cddca6..38807f0 100644 --- a/data_scraper/cboe.py +++ b/data_scraper/cboe.py @@ -76,6 +76,8 @@ def aggregate_monthly_data(symbols=None): scraper_dir = os.path.join(save_data_path, "cboe") symbols = [symbol.upper() for symbol in symbols] + done = 0 + failed = [] for symbol in symbols: daily_dir = os.path.join(scraper_dir, symbol + "_daily") @@ -109,9 +111,10 @@ def aggregate_monthly_data(symbols=None): today = pd.Timestamp.today() first_date = date_range[0] if first_date.year != today.year or first_date.month != today.month: - msg = "Some trading dates where missing for symbol {}".format( - symbol) - slack_notification(msg, __name__) + msg = "Some trading dates where missing for symbol {} in period {}".format( + symbol, month) + logger.error(msg) + failed.append(symbol) continue if not os.path.exists(monthly_dir): @@ -132,10 +135,13 @@ def aggregate_monthly_data(symbols=None): continue logger.debug("Saved monthly data %s", monthly_file) + done += 1 for file in daily_files: utils.remove_file(file, logger) + send_report(done, failed, __name__, op="aggregate") + def _get_all_listed_symbols(): """Returns array of all listed symbols. diff --git a/data_scraper/notifications.py b/data_scraper/notifications.py index 766ef43..d544bf9 100644 --- a/data_scraper/notifications.py +++ b/data_scraper/notifications.py @@ -4,7 +4,7 @@ from enum import Enum import requests -from .utils import get_module_config +from .utils import get_module_config, get_environment_var logger = logging.getLogger(__name__) @@ -12,8 +12,8 @@ Status = Enum("Status", "Success Warning Error") options = get_module_config("notifications") try: - webhook = options["slack_webhook"] -except KeyError as e: + webhook = get_environment_var("SLACK_WEBHOOK") +except EnvironmentError as e: logger.error("Missing slack webhook from configuration file") raise e @@ -59,17 +59,17 @@ def slack_notification(text, scraper, status=Status.Error): logger.error(msg) -def send_report(done, failed, scraper): - """Sends status report to Slack. - `done` is the count of successfully scraped symbols - `failed` is a list of symbol names that could not be scraped +def send_report(done, failed, scraper, op="scrape"): + """Sends status report to Slack for given operation. + `done` is the count of successfully scraped/aggregated symbols + `failed` is a list of symbol names that could not be scraped/aggregated """ if done > 0: - msg = "Successfully scraped " + _symbol_str(done) + msg = "Successfully {}d {}".format(op, _symbol_str(done)) slack_notification(msg, scraper, status=Status.Success) if len(failed) > 0: - msg = "Failed to scrape {}: {}".format(_symbol_str(len(failed)), - ", ".join(failed)) + msg = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)), + ", ".join(failed)) slack_notification(msg, scraper, status=Status.Warning)