diff --git a/data_scraper/cboe.py b/data_scraper/cboe.py index b01fdf1..54755e8 100644 --- a/data_scraper/cboe.py +++ b/data_scraper/cboe.py @@ -6,7 +6,9 @@ from itertools import groupby from bs4 import BeautifulSoup import requests -import pandas as pd +import pandas as pd +import tenacity +import time from . import utils, validation from .notifications import send_report @@ -59,10 +61,42 @@ def fetch_data(symbols=None): else: _save_data(symbol, symbol_data) done += 1 - + retry_failure(failed, done) send_report(done, failed, __name__) + +##if a symbol failes to scrape try again exponentialy +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300), stop = tenacity.stop_after_attempt(10), retry=tenacity.retry_if_exception_type(IOError)) +def retry_failure(failed, done): + local_time = time.ctime(time.time()) + form_data = _form_data() + headers = {"Referer": url} + file_url = "http://www.cboe.com/delayedquote/quotedata.dat" + for symbol in failed: + try: + response = requests.post(url, + data=form_data, + headers=headers, + allow_redirects=False) + + symbol_req = requests.get(file_url, + cookies=response.cookies, + headers=headers) + symbol_data = symbol_req.text + if symbol_data == "" or symbol_data.startswith(" 0: - msg1 = "Successfully {}d {}".format(op, _symbol_str(done)) + msg_success = "👍 Successfully {}d {}".format(op, _symbol_str(done)) if len(failed) > 0: - msg2 = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)), + msg_fail = "⚠️️ Failed to {} {}: {}".format(op, _symbol_str(len(failed)), ", ".join(failed)) - msg= msg1 + " and " + msg2 + msg= msg_success + '\n' + msg_fail slack_notification(msg, scraper, status=Status.Warning) diff --git a/data_scraper/test/test_cboe.py b/data_scraper/test/test_cboe.py index 72b9c28..eb61854 100644 --- a/data_scraper/test/test_cboe.py +++ b/data_scraper/test/test_cboe.py @@ -77,6 +77,16 @@ class TestCBOE(unittest.TestCase): spx_df = pd.read_csv(TestCBOE.spx_data_path) aggregate_df = pd.read_csv(aggregate_file) self.assertTrue(spx_df.equals(aggregate_df)) + + @patch("data_scraper.cboe.url", new="http://www.aldkfjaskldfjsa.com") + @patch("data_scraper.cboe.retry_failure", return_value=None) + def test_retry(self, mocked_retry): + """Raise ConnectionError and send notification when host is unreachable""" + with self.assertRaises(ConnectionError): + cboe.fetch_data(["SPX"]) + self.assertTrue(mocked_retry.called) + self.assertTrue(mocked_retry.call_count == 10) + @patch("data_scraper.cboe.utils.remove_file", return_value=None) @patch("data_scraper.cboe.slack_notification", return_value=None) @@ -95,7 +105,7 @@ class TestCBOE(unittest.TestCase): self.assertTrue(mocked_notification.called) self.assertFalse(mocked_remove.called) - def remove_files(file_path): + def remove_files(self, file_path): if os.path.exists(file_path): shutil.rmtree(file_path) diff --git a/data_scraper/test/test_tiingo.py b/data_scraper/test/test_tiingo.py index 34e1bc4..b1407b1 100644 --- a/data_scraper/test/test_tiingo.py +++ b/data_scraper/test/test_tiingo.py @@ -58,15 +58,23 @@ class TestTiingo(unittest.TestCase): @patch("data_scraper.tiingo.pdr.get_data_tiingo") # mock pandas_datareader @patch("data_scraper.tiingo.slack_notification", return_value=None) - def test_no_connection(self, mocked_notification, mocked_pdr): + def test_no_connection(self, mocked_notification): """Raise ConnectionError and send notification when host is unreachable""" - mocked_pdr.side_effect = ConnectionError("This is a test") - with self.assertRaises(ConnectionError): tiingo.fetch_data(["IBM"]) self.assertTrue(mocked_notification.called) + + @patch("data_scraper.tiingo.pdr.get_data_tiingo") # mock pandas_datareader + @patch("data_scraper.tiingo.retry_failure", return_value=None) + def test_retry(self, mocked_retry, mocked_pdr): + """Raise ConnectionError and retry when host is unreachable""" + mocked_pdr.side_effect = ConnectionError("This is a test") + with self.assertRaises(ConnectionError): + tiingo.fetch_data(["IBM"]) + self.assertTrue(mocked_retry.called) + self.assertTrue(mocked_retry.call_count == 10) - def remove_files(file_path): + def remove_files(self, file_path): if os.path.exists(file_path): shutil.rmtree(file_path) diff --git a/data_scraper/tiingo.py b/data_scraper/tiingo.py index 677adc9..a2197d6 100644 --- a/data_scraper/tiingo.py +++ b/data_scraper/tiingo.py @@ -4,6 +4,7 @@ from datetime import date import pandas as pd import pandas_datareader as pdr +import tenacity from . import utils, validation from .notifications import send_report @@ -47,10 +48,36 @@ def fetch_data(symbols=assets): else: _save_data(symbol, symbol_data.reset_index()) done += 1 - + retry_failure(failed, done) send_report(done, failed, __name__) +##if a symbol failes to scrape try again exponentialy +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300), stop = tenacity.stop_after_attempt(10), retry=tenacity.retry_if_exception_type(IOError)) +def retry_failure(failed, done): + api_key = utils.get_environment_var("TIINGO_API_KEY") + for symbol in failed: + try: + symbol_data = pdr.get_data_tiingo(symbol, api_key=api_key) + except ConnectionError as ce: + msg = "Unable to connect to api.tiingo.com when fetching symbol {}".format( + symbol) + logger.error(msg, exc_info=True) + raise ce + except TypeError: + # pandas_datareader raises TypeError when fetching invalid symbol + failed.append(symbol) + msg = "Attempted to fetch invalid symbol {}".format(symbol) + logger.error(msg, exc_info=True) + except Exception: + msg = "Error fetching symbol {}".format(symbol) + logger.error(msg, exc_info=True) + else: + _save_data(symbol, symbol_data) + done+=1 + failed.remove(symbol) + + def _save_data(symbol, symbol_df): """Saves the contents of `symbol_df` to `$SAVE_DATA_PATH/tiingo/{symbol}/{symbol}_{%date}.csv`"""