From e1c2f40ab9f61700987262aa7bd9250cdb40f6be Mon Sep 17 00:00:00 2001 From: fredfortier Date: Thu, 12 Oct 2017 14:06:47 -0400 Subject: [PATCH] Making some adjustments to the ingestion method after discussion with Victor --- .../buy_low_sell_high_neo_with_interface.py | 36 ++++++++-------- catalyst/exchange/bitfinex/bitfinex.py | 43 +++++++++---------- catalyst/exchange/bundle_utils.py | 17 +++++++- catalyst/exchange/exchange.py | 1 + catalyst/exchange/exchange_bundle.py | 2 + tests/exchange/test_bitfinex.py | 4 ++ tests/exchange/test_bundle.py | 4 +- 7 files changed, 64 insertions(+), 43 deletions(-) diff --git a/catalyst/examples/buy_low_sell_high_neo_with_interface.py b/catalyst/examples/buy_low_sell_high_neo_with_interface.py index 8cb1a5a6..b70e451a 100644 --- a/catalyst/examples/buy_low_sell_high_neo_with_interface.py +++ b/catalyst/examples/buy_low_sell_high_neo_with_interface.py @@ -36,7 +36,7 @@ def _handle_data(context, data): prices = data.history( context.asset, fields='price', - bar_count=50, + bar_count=1, frequency='1m' ) rsi = talib.RSI(prices.values, timeperiod=14)[-1] @@ -148,27 +148,27 @@ def analyze(context, stats): pass -run_algorithm( - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - live=True, - algo_namespace=algo_namespace, - base_currency='btc', - live_graph=False -) - -# Backtest # run_algorithm( -# capital_base=250, -# start=pd.to_datetime('2017-09-08', utc=True), -# end=pd.to_datetime('2017-09-15', utc=True), -# data_frequency='minute', # initialize=initialize, # handle_data=handle_data, # analyze=analyze, # exchange_name='bitfinex', +# live=True, # algo_namespace=algo_namespace, -# base_currency='btc' +# base_currency='btc', +# live_graph=False # ) + +# Backtest +run_algorithm( + capital_base=250, + start=pd.to_datetime('2017-10-01', utc=True), + end=pd.to_datetime('2017-10-15', utc=True), + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=algo_namespace, + base_currency='btc' +) diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index a4179a82..ac1af89e 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -26,7 +26,6 @@ from catalyst.protocol import Account from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ download_exchange_symbols - # Trying to account for REST api instability # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request requests.adapters.DEFAULT_RETRIES = 20 @@ -50,7 +49,9 @@ class Bitfinex(Exchange): self._portfolio = portfolio self.minute_writer = None self.minute_reader = None - self.num_candles_limit = 10000 + + # The candle limit for each request + self.num_candles_limit = 1000 # Max is 90 but playing it safe # https://www.bitfinex.com/posts/188 @@ -576,7 +577,7 @@ class Bitfinex(Exchange): response = self._request('symbols', None) for symbol in response.json(): - if(source_dates): + if (source_dates): start_date = self.get_symbol_start_date(symbol) else: try: @@ -587,25 +588,26 @@ class Bitfinex(Exchange): try: end_daily = cached_symbols[symbol]['end_daily'] except KeyError as e: - end_daily ='N/A' + end_daily = 'N/A' try: end_minute = cached_symbols[symbol]['end_minute'] except KeyError as e: end_minute = 'N/A' - symbol_map[symbol]= dict( - symbol = symbol[:-3]+'_'+symbol[-3:], - start_date = start_date, - end_daily = end_daily, - end_minute = end_minute, + symbol_map[symbol] = dict( + symbol=symbol[:-3] + '_' + symbol[-3:], + start_date=start_date, + end_daily=end_daily, + end_minute=end_minute, ) - if(filename is None): + if (filename is None): filename = get_exchange_symbols_filename(self.name) - with open(filename,'w') as f: - json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':')) + with open(filename, 'w') as f: + json.dump(symbol_map, f, sort_keys=True, indent=2, + separators=(',', ':')) def get_symbol_start_date(self, symbol): @@ -634,10 +636,10 @@ class Bitfinex(Exchange): arbitrarily set the ref. date to 15 days ago to be safe with +/- 31 days """ - if(len(response.json())): + if (len(response.json())): startmonth = response.json()[-1][0] else: - startmonth = int((time.time()-15*24*3600)*1000) + startmonth = int((time.time() - 15 * 24 * 3600) * 1000) """ Query again with daily resolution setting the start and end around @@ -646,8 +648,9 @@ class Bitfinex(Exchange): url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format( url=self.url, symbol=symbol_v2, - start=startmonth - 3600 *24 *31 *1000, - end=min(startmonth + 3600 *24 *31 *1000, int(time.time()*1000)) + start=startmonth - 3600 * 24 * 31 * 1000, + end=min(startmonth + 3600 * 24 * 31 * 1000, + int(time.time() * 1000)) ) try: @@ -656,9 +659,5 @@ class Bitfinex(Exchange): except Exception as e: raise ExchangeRequestError(error=e) - return time.strftime('%Y-%m-%d', time.gmtime(int(response.json()[-1][0]/1000))) - - - - - + return time.strftime('%Y-%m-%d', + time.gmtime(int(response.json()[-1][0] / 1000))) diff --git a/catalyst/exchange/bundle_utils.py b/catalyst/exchange/bundle_utils.py index d9b34ac8..2f87b6c9 100644 --- a/catalyst/exchange/bundle_utils.py +++ b/catalyst/exchange/bundle_utils.py @@ -29,6 +29,21 @@ def get_seconds_from_date(date): return int((date - epoch).total_seconds()) +def get_bcolz_chunk(exchange_name, data_frequency, symbol, period_a, period_b): + """ + + :param exchange_name: + :param data_frequency: + :param symbol: + :param period_a: + Example: 2017 + :param period_b: + Example: 10 + + Note: + Filename: bitfinex-daily-neo_eth-2017-10.tar.gz + :return: + """ def get_history(exchange_name, data_frequency, symbol, start=None, end=None): """ History API provides OHLCV data for any of the supported exchanges up to yesterday. @@ -55,7 +70,7 @@ def get_history(exchange_name, data_frequency, symbol, start=None, end=None): Notes ===== - Using milliseconds for the start and end dates for ease of use in the + Using seconds for the start and end dates for ease of use in the function query parameters. Sometimes, one minute goes by without completing a trade of the given diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 3f03be8a..882aeb2b 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -559,6 +559,7 @@ class Exchange: if freq_match: candle_size = int(freq_match.group(1)) unit = freq_match.group(2) + else: raise InvalidHistoryFrequencyError(frequency) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 134657bc..15022384 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -13,6 +13,7 @@ from catalyst.exchange.bundle_utils import get_ffill_candles, get_start_dt, \ get_periods, range_in_bundle from catalyst.exchange.exchange_utils import get_exchange_folder from catalyst.utils.cli import maybe_show_progress +from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import ensure_directory @@ -208,6 +209,7 @@ class ExchangeBundle: return missing_assets + @deprecated def ingest_chunk(self, bar_count, end_dt, data_frequency, asset, writer, previous_candle=dict()): """ diff --git a/tests/exchange/test_bitfinex.py b/tests/exchange/test_bitfinex.py index 7efd4884..c8f969fd 100644 --- a/tests/exchange/test_bitfinex.py +++ b/tests/exchange/test_bitfinex.py @@ -47,6 +47,10 @@ class BitfinexTestCase(BaseExchangeTestCase): def test_get_candles(self): log.info('retrieving candles') + ohlcv_neo = self.exchange.get_candles( + data_frequency='1m', + assets=self.exchange.get_asset('neo_btc') + ) pass def test_tickers(self): diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index 43895198..47ebf30d 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -14,8 +14,8 @@ class ExchangeBundleTestCase: exchange_name = 'bitfinex' # start = pd.to_datetime('2017-09-01', utc=True) - end = pd.Timestamp.utcnow() - timedelta(minutes=5) - start = end - timedelta(minutes=30) + start = pd.to_datetime('2017-10-01', utc=True) + end = pd.to_datetime('2017-10-06', utc=True) exchange_bundle = ExchangeBundle(get_exchange(exchange_name))