From d36abd14e5ed914b133974c257688a772ab5056d Mon Sep 17 00:00:00 2001 From: Catalina Syddall Date: Fri, 2 Aug 2019 17:54:06 -0300 Subject: [PATCH] changing expected columns --- data_scraper/tiingo.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/data_scraper/tiingo.py b/data_scraper/tiingo.py index b7e8a5f..27e14fb 100644 --- a/data_scraper/tiingo.py +++ b/data_scraper/tiingo.py @@ -50,8 +50,10 @@ def fetch_data(symbols=assets): send_report(done, failed, __name__) -##if a symbol failes to scrape try again exponentialy -@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300), stop = tenacity.stop_after_attempt(10), retry=tenacity.retry_if_exception_type(IOError)) +# if a symbol failes to scrape try again exponentialy +@tenacity.retry(wait=tenacity.wait_exponential(multiplier=300), + stop=tenacity.stop_after_attempt(10), + retry=tenacity.retry_if_exception_type(IOError)) def retry_failure(failed, done): api_key = utils.get_environment_var("TIINGO_API_KEY") for symbol in failed: @@ -72,7 +74,7 @@ def retry_failure(failed, done): logger.error(msg, exc_info=True) else: _save_data(symbol, symbol_data) - done+=1 + done += 1 failed.remove(symbol) @@ -94,9 +96,9 @@ def _save_data(symbol, symbol_df): logger.debug("File %s already downloaded", file_path) else: expected_columns = [ - "symbol", "date", "adjClose", "adjHigh", "adjLow", "adjOpen", - "adjVolume", "close", "divCash", "high", "low", "open", - "splitFactor", "volume" + "symbol", "date", "close", "high", "low", "open", "volume", + "adjClose", "adjHigh", "adjLow", "adjOpen", "adjVolume", "divCash", + "splitFactor" ] if validation.validate_historical_dates( @@ -109,6 +111,7 @@ def _save_data(symbol, symbol_df): merged_df.to_csv(file_path, index=False) logger.debug("Saved symbol data as %s", file_path) + def _merge(symbol, symbol_df): """Merge `symbol_df` with previous data file.""" save_data_path = utils.get_save_data_path()