adding retry function with tests

data_scraper/cboe.py

@@ -6,7 +6,9 @@ from itertools import groupby
 
 from bs4 import BeautifulSoup
 import requests
-import pandas as pd
+import pandas as pd 
+import tenacity
+import time
 
 from . import utils, validation
 from .notifications import send_report
@@ -59,10 +61,42 @@ def fetch_data(symbols=None):
         else:
             _save_data(symbol, symbol_data)
             done += 1
-
+    retry_failure(failed, done)
     send_report(done, failed, __name__)
 
 
+
+##if a symbol failes to scrape try again exponentialy
+@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300), stop = tenacity.stop_after_attempt(10), retry=tenacity.retry_if_exception_type(IOError))
+def retry_failure(failed, done):
+    local_time = time.ctime(time.time())
+    form_data = _form_data()
+    headers = {"Referer": url}
+    file_url = "http://www.cboe.com/delayedquote/quotedata.dat"
+    for symbol in failed:
+        try:
+            response = requests.post(url,
+                                     data=form_data,
+                                     headers=headers,
+                                     allow_redirects=False)
+            
+            symbol_req = requests.get(file_url,
+                                      cookies=response.cookies,
+                                      headers=headers)
+            symbol_data = symbol_req.text
+            if symbol_data == "" or symbol_data.startswith("<!DOCTYPE"):
+                raise Exception
+        except Exception:
+            msg = "error fetching symbol {} data".format(symbol)
+            logger.error(msg, exc_info = True)
+            return local_time
+        else:
+            _save_data(symbol, symbol_data)
+            done+=1
+            failed.remove(symbol)
+
+
+
 def aggregate_monthly_data(symbols=None):
     """Aggregate daily snapshots into monthly files and validate data"""
     symbols = symbols or _get_all_listed_symbols()

data_scraper/notifications.py

@@ -54,12 +54,12 @@ def send_report(done, failed, scraper, op="scrape"):
     `failed` is a list of symbol names that could not be scraped/aggregated
     """
     if done > 0:
-        msg1 = "Successfully {}d {}".format(op, _symbol_str(done))
+        msg_success = "👍 Successfully {}d {}".format(op, _symbol_str(done))
     if len(failed) > 0:
-        msg2 = "Failed to {} {}: {}".format(op, _symbol_str(len(failed)),
+        msg_fail = "⚠️️ Failed to {} {}: {}".format(op, _symbol_str(len(failed)),
                                            ", ".join(failed))
     
-    msg= msg1 + " and " + msg2
+    msg= msg_success + '\n' + msg_fail
     slack_notification(msg, scraper, status=Status.Warning)
 
 

data_scraper/test/test_cboe.py

@@ -77,6 +77,16 @@ class TestCBOE(unittest.TestCase):
             spx_df = pd.read_csv(TestCBOE.spx_data_path)
             aggregate_df = pd.read_csv(aggregate_file)
             self.assertTrue(spx_df.equals(aggregate_df))
+    
+    @patch("data_scraper.cboe.url", new="http://www.aldkfjaskldfjsa.com")
+    @patch("data_scraper.cboe.retry_failure", return_value=None)
+    def test_retry(self, mocked_retry):
+        """Raise ConnectionError and send notification when host is unreachable"""
+        with self.assertRaises(ConnectionError):
+            cboe.fetch_data(["SPX"])
+            self.assertTrue(mocked_retry.called)
+            self.assertTrue(mocked_retry.call_count == 10)
+
 
     @patch("data_scraper.cboe.utils.remove_file", return_value=None)
     @patch("data_scraper.cboe.slack_notification", return_value=None)
@@ -95,7 +105,7 @@ class TestCBOE(unittest.TestCase):
         self.assertTrue(mocked_notification.called)
         self.assertFalse(mocked_remove.called)
 
-    def remove_files(file_path):
+    def remove_files(self, file_path):
         if os.path.exists(file_path):
             shutil.rmtree(file_path)
 

data_scraper/test/test_tiingo.py

@@ -58,15 +58,23 @@ class TestTiingo(unittest.TestCase):
 
     @patch("data_scraper.tiingo.pdr.get_data_tiingo")  # mock pandas_datareader
     @patch("data_scraper.tiingo.slack_notification", return_value=None)
-    def test_no_connection(self, mocked_notification, mocked_pdr):
+    def test_no_connection(self, mocked_notification):
         """Raise ConnectionError and send notification when host is unreachable"""
-        mocked_pdr.side_effect = ConnectionError("This is a test")
-
         with self.assertRaises(ConnectionError):
             tiingo.fetch_data(["IBM"])
             self.assertTrue(mocked_notification.called)
+    
+    @patch("data_scraper.tiingo.pdr.get_data_tiingo")  # mock pandas_datareader
+    @patch("data_scraper.tiingo.retry_failure", return_value=None)
+    def test_retry(self, mocked_retry, mocked_pdr):
+        """Raise ConnectionError and retry when host is unreachable"""
+        mocked_pdr.side_effect = ConnectionError("This is a test")
+        with self.assertRaises(ConnectionError):
+            tiingo.fetch_data(["IBM"])
+            self.assertTrue(mocked_retry.called)
+            self.assertTrue(mocked_retry.call_count == 10)
 
-    def remove_files(file_path):
+    def remove_files(self, file_path):
         if os.path.exists(file_path):
             shutil.rmtree(file_path)
 

data_scraper/tiingo.py

@@ -4,6 +4,7 @@ from datetime import date
 
 import pandas as pd
 import pandas_datareader as pdr
+import tenacity
 
 from . import utils, validation
 from .notifications import send_report
@@ -47,10 +48,36 @@ def fetch_data(symbols=assets):
         else:
             _save_data(symbol, symbol_data.reset_index())
             done += 1
-
+    retry_failure(failed, done)
     send_report(done, failed, __name__)
 
 
+##if a symbol failes to scrape try again exponentialy
+@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300), stop = tenacity.stop_after_attempt(10), retry=tenacity.retry_if_exception_type(IOError))
+def retry_failure(failed, done):
+    api_key = utils.get_environment_var("TIINGO_API_KEY")
+    for symbol in failed:
+        try:
+            symbol_data = pdr.get_data_tiingo(symbol, api_key=api_key)
+        except ConnectionError as ce:
+            msg = "Unable to connect to api.tiingo.com when fetching symbol {}".format(
+                symbol)
+            logger.error(msg, exc_info=True)
+            raise ce
+        except TypeError:
+            # pandas_datareader raises TypeError when fetching invalid symbol
+            failed.append(symbol)
+            msg = "Attempted to fetch invalid symbol {}".format(symbol)
+            logger.error(msg, exc_info=True)
+        except Exception:
+            msg = "Error fetching symbol {}".format(symbol)
+            logger.error(msg, exc_info=True)
+        else:
+            _save_data(symbol, symbol_data)
+            done+=1
+            failed.remove(symbol)
+
+
 def _save_data(symbol, symbol_df):
     """Saves the contents of `symbol_df` to
     `$SAVE_DATA_PATH/tiingo/{symbol}/{symbol}_{%date}.csv`"""