diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index 0397c2d8..cc34d436 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -48,7 +48,12 @@ class Bitfinex(Exchange): self._portfolio = portfolio self.minute_writer = None self.minute_reader = None - self.num_candles_limit = 100 + self.num_candles_limit = 1000 + + # Max is 90 but playing it safe + # https://www.bitfinex.com/posts/188 + self.max_requests_per_minute = 20 + self.request_cpt = dict() def _request(self, operation, data, version='v1'): payload_object = { @@ -176,6 +181,7 @@ class Bitfinex(Exchange): def get_balances(self): log.debug('retrieving wallets balances') try: + self.ask_request() response = self._request('balances', None) balances = response.json() except Exception as e: @@ -295,6 +301,7 @@ class Bitfinex(Exchange): url += '/last' try: + self.ask_request() response = requests.get(url) except Exception as e: raise ExchangeRequestError(error=e) @@ -377,6 +384,7 @@ class Bitfinex(Exchange): date = pd.Timestamp.utcnow() try: + self.ask_request() response = self._request('order/new', req) order_status = response.json() except Exception as e: @@ -418,6 +426,7 @@ class Bitfinex(Exchange): orders for this asset. """ try: + self.ask_request() response = self._request('orders', None) order_statuses = response.json() except Exception as e: @@ -452,6 +461,7 @@ class Bitfinex(Exchange): The order object. """ try: + self.ask_request() response = self._request( 'order/status', {'order_id': int(order_id)}) order_status = response.json() @@ -477,6 +487,7 @@ class Bitfinex(Exchange): if isinstance(order_param, Order) else order_param try: + self.ask_request() response = self._request('order/cancel', {'order_id': order_id}) status = response.json() except Exception as e: @@ -501,6 +512,7 @@ class Bitfinex(Exchange): log.debug('fetching tickers {}'.format(symbols)) try: + self.ask_request() response = requests.get( '{url}/v2/tickers?symbols={symbols}'.format( url=self.url, diff --git a/catalyst/exchange/bitfinex/symbols.json b/catalyst/exchange/bitfinex/symbols.json index bd6951d1..ab0f38f9 100644 --- a/catalyst/exchange/bitfinex/symbols.json +++ b/catalyst/exchange/bitfinex/symbols.json @@ -2,7 +2,7 @@ "neobtc": { "symbol": "neo_btc", "start_date": "2017-09-07", - "precision":5 + "precision": 5 }, "neousd": { "symbol": "neo_usd", @@ -30,19 +30,19 @@ }, "ethusd": { "symbol": "eth_usd", - "start_date": "2010-01-01" + "start_date": "2017-01-01" }, "ethbtc": { "symbol": "eth_btc", - "start_date": "2010-01-01" + "start_date": "2017-01-01" }, "etcbtc": { "symbol": "etc_btc", - "start_date": "2010-01-01" + "start_date": "2017-01-01" }, "etcusd": { "symbol": "etc_usd", - "start_date": "2010-01-01" + "start_date": "2017-01-01" }, "rrtusd": { "symbol": "rrt_usd", diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index 3593e89c..86f0db88 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -26,6 +26,11 @@ class Bittrex(Exchange): self.base_currency = base_currency self._portfolio = portfolio + # Not sure what the rate limit is but trying to play it safe + # https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit + self.max_requests_per_minute = 60 + self.request_cpt = dict() + self.minute_writer = None self.minute_reader = None @@ -62,6 +67,7 @@ class Bittrex(Exchange): """ symbol_map = dict() + self.ask_request() markets = self.api.getmarkets() for market in markets: exchange_symbol = market['MarketName'] @@ -79,6 +85,7 @@ class Bittrex(Exchange): def get_balances(self): try: log.debug('retrieving wallet balances') + self.ask_request() balances = self.api.getbalances() except Exception as e: raise ExchangeRequestError(error=e) @@ -99,6 +106,7 @@ class Bittrex(Exchange): price = style.get_limit_price(is_buy) try: + self.ask_request() if is_buy: order_status = self.api.buylimit(exchange_symbol, amount, price) @@ -139,6 +147,7 @@ class Bittrex(Exchange): def get_open_orders(self, asset): symbol = self.get_symbol(asset) try: + self.ask_request() open_orders = self.api.getopenorders(symbol) except Exception as e: raise ExchangeRequestError(error=e) @@ -182,6 +191,7 @@ class Bittrex(Exchange): def get_order(self, order_id): log.info('retrieving order {}'.format(order_id)) try: + self.ask_request() order_status = self.api.getorder(order_id) except Exception as e: raise ExchangeRequestError(error=e) @@ -197,6 +207,7 @@ class Bittrex(Exchange): log.info('cancelling order {}'.format(order_id)) try: + self.ask_request() status = self.api.cancel(order_id) except Exception as e: raise ExchangeRequestError(error=e) @@ -208,7 +219,8 @@ class Bittrex(Exchange): error=status['message'] ) - def get_candles(self, data_frequency, assets, bar_count=None, start_date=None): + def get_candles(self, data_frequency, assets, bar_count=None, + start_date=None): """ Supported Intervals ------------------- @@ -299,6 +311,7 @@ class Bittrex(Exchange): for asset in assets: symbol = self.get_symbol(asset) try: + self.ask_request() ticker = self.api.getticker(symbol) except Exception as e: raise ExchangeRequestError(error=e) diff --git a/catalyst/exchange/data_portal_exchange.py b/catalyst/exchange/data_portal_exchange.py index 3c1dd01d..c9dc5bd0 100644 --- a/catalyst/exchange/data_portal_exchange.py +++ b/catalyst/exchange/data_portal_exchange.py @@ -12,21 +12,25 @@ # limitations under the License. import abc +import os from time import sleep -import os +import collections import pandas as pd from catalyst.assets._assets import TradingPair from logbook import Logger +from catalyst.data.bundles.core import load, from_bundle_ingest_dirname, \ + BundleData, minute_path, five_minute_path, daily_path from catalyst.data.data_portal import DataPortal +from catalyst.data.five_minute_bars import BcolzFiveMinuteBarReader from catalyst.data.minute_bars import BcolzMinuteBarReader +from catalyst.data.us_equity_pricing import BcolzDailyBarReader from catalyst.exchange.exchange_errors import ( ExchangeRequestError, - ExchangeBarDataError -) -from catalyst.data.bundles.core import load -from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root + ExchangeBarDataError, + BundleNotFoundError) +from catalyst.utils.paths import data_path log = Logger('DataPortalExchange') @@ -145,8 +149,8 @@ class DataPortalExchangeBase(DataPortal): try: if isinstance(assets, TradingPair): exchange = self.exchanges[assets.exchange] - return exchange.get_spot_value( - assets, field, dt, data_frequency) + return self.get_exchange_spot_value( + exchange, assets, field, dt, data_frequency) else: exchange_assets = dict() @@ -156,20 +160,29 @@ class DataPortalExchangeBase(DataPortal): exchange_assets[asset.exchange].append(asset) - spot_values = [] - for exchange_name in exchange_assets: - exchange = self.exchanges[exchange_name] - assets = exchange_assets[exchange_name] - exchange_spot_values = self.get_exchange_spot_value( - exchange, - assets, - field, - dt, - data_frequency - ) - spot_values += exchange_spot_values + if len(exchange_assets.keys()) == 1: + exchange = self.exchanges[exchange_assets.keys()[0]] + return self.get_exchange_spot_value( + exchange, assets, field, dt, data_frequency) - return spot_values + else: + spot_values = [] + for exchange_name in exchange_assets: + exchange = self.exchanges[exchange_name] + assets = exchange_assets[exchange_name] + exchange_spot_values = self.get_exchange_spot_value( + exchange, + assets, + field, + dt, + data_frequency + ) + if len(assets) == 1: + spot_values.append(exchange_spot_values) + else: + spot_values += exchange_spot_values + + return spot_values except ExchangeRequestError as e: log.warn( @@ -239,10 +252,53 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs) - self.minute_readers = dict() + self.daily_bar_readers = dict() + self.minute_bar_readers = dict() + self.five_minute_bar_readers = dict() for exchange_name in self.exchanges: - root = get_exchange_minute_writer_root(exchange_name) - self.minute_readers[exchange_name] = BcolzMinuteBarReader(root) + name = 'exchange_{}'.format(exchange_name) + time_folder = \ + DataPortalExchangeBacktest.find_most_recent_time(name) + + if time_folder is None: + raise BundleNotFoundError(exchange=exchange_name) + + self.daily_bar_readers[exchange_name] = \ + BcolzDailyBarReader( + daily_path(name, time_folder), + ) + + self.five_minute_bar_readers[exchange_name] = \ + BcolzFiveMinuteBarReader( + five_minute_path(name, time_folder), + ) + + self.minute_bar_readers[exchange_name] = \ + BcolzMinuteBarReader( + minute_path(name, time_folder), + ) + + @staticmethod + def find_most_recent_time(bundle_name): + try: + bundle_folders = os.listdir( + data_path([bundle_name]), + ) + except OSError: + return None + + most_recent_bundle = dict() + for folder in bundle_folders: + date = from_bundle_ingest_dirname(folder) + if not most_recent_bundle or date > \ + most_recent_bundle[most_recent_bundle.keys()[0]]: + most_recent_bundle = dict() + most_recent_bundle[folder] = date + + if most_recent_bundle: + return most_recent_bundle.keys()[0] + else: + return None def get_exchange_history_window(self, exchange, @@ -267,7 +323,11 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): data_frequency): if data_frequency == 'minute': - reader = self.minute_readers[exchange.name] + reader = self.minute_bar_readers[exchange.name] + elif data_frequency == '5-minute': + reader = self.five_minute_bar_readers[exchange.name] + elif data_frequency == 'daily': + reader = self.daily_bar_readers[exchange.name] else: raise ValueError('Unsupported frequency') @@ -284,4 +344,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): log.warn('minute data not found: {}'.format(e)) values.append(None) - return values + if len(assets) == 1: + return values[0] + else: + return values diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index cf26c9ff..361413a5 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -2,6 +2,7 @@ import abc import collections import random from abc import ABCMeta, abstractmethod, abstractproperty +from datetime import timedelta from time import sleep import numpy as np @@ -10,9 +11,6 @@ from catalyst.assets._assets import TradingPair from logbook import Logger from catalyst.data.data_portal import BASE_FIELDS -from catalyst.errors import ( - SymbolNotFound, -) from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \ @@ -35,7 +33,10 @@ class Exchange: self.minute_writer = None self.minute_reader = None self.base_currency = None - self.num_candles_limit = 100 + + self.num_candles_limit = None + self.max_requests_per_minute = None + self.request_cpt = None @property def positions(self): @@ -64,6 +65,50 @@ class Exchange: def time_skew(self): pass + def ask_request(self): + """ + Asks permission to issue a request to the exchange. + The primary purpose is to avoid hitting rate limits. + + The application will pause if the maximum requests per minute + permitted by the exchange is exceeded. + + :return boolean: + + """ + now = pd.Timestamp.utcnow() + if not self.request_cpt: + self.request_cpt = dict() + self.request_cpt[now] = 0 + return True + + cpt_date = self.request_cpt.keys()[0] + cpt = self.request_cpt[cpt_date] + + if now > cpt_date + timedelta(minutes=1): + self.request_cpt = dict() + self.request_cpt[now] = 0 + return True + + if cpt >= self.max_requests_per_minute: + delta = now - cpt_date + + sleep_period = 60 - delta.total_seconds() + + # log.debug( + # 'max requests {} reached, sleeping for {} seconds'.format( + # self.max_requests_per_minute, + # sleep_period + # )) + sleep(sleep_period) + + now = pd.Timestamp.utcnow() + self.request_cpt = dict() + self.request_cpt[now] = 0 + return True + else: + self.request_cpt[cpt_date] += 1 + def get_symbol(self, asset): """ Get the exchange specific symbol of the given asset. diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 1605f279..2b4c6435 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -1,9 +1,17 @@ from datetime import timedelta -import pandas as pd -from logbook import Logger +from time import sleep -from catalyst.data.minute_bars import BcolzMinuteOverlappingData +import os +import pandas as pd +from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle + +from catalyst import get_calendar +from logbook import Logger, INFO + +from catalyst.data.five_minute_bars import BcolzFiveMinuteOverlappingData +from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ + BcolzMinuteBarReader from catalyst.exchange.bitfinex.bitfinex import Bitfinex from catalyst.exchange.bittrex.bittrex import Bittrex from catalyst.exchange.exchange_errors import ExchangeNotFoundError @@ -32,15 +40,111 @@ def fetch_candles_chunk(exchange, assets, data_frequency, end_dt, bar_count): asset_candles = candles[asset] asset_df = pd.DataFrame(asset_candles) - asset_df.set_index('last_traded', inplace=True, drop=True) - asset_df.sort_index(inplace=True) + if not asset_df.empty: + asset_df.set_index('last_traded', inplace=True, drop=True) + asset_df.sort_index(inplace=True) - series[asset] = asset_df + series[asset] = asset_df return series -def exchange_bundle(exchange_name, symbols, start=None, end=None): +def process_bar_data(exchange, assets, writer, data_frequency, + show_progress, start, end): + open_calendar = get_calendar('OPEN') + + writer.calendar = open_calendar + writer.minutes_per_day = 1440 + writer.write_metadata = True + + delta = end - start + if data_frequency == 'minute': + delta_periods = delta.total_seconds() / 60 + frequency = '1m' + + elif data_frequency == '5-minute': + delta_periods = delta.total_seconds() / 60 / 5 + frequency = '5m' + + elif data_frequency == 'daily': + delta_periods = delta.total_seconds() / 60 / 60 / 24 + frequency = '1d' + + else: + raise ValueError('frequency not supported') + + if delta_periods > exchange.num_candles_limit: + bar_count = exchange.num_candles_limit + + chunks = [] + last_chunk_date = end + while last_chunk_date > start + timedelta(minutes=bar_count): + # TODO: account for the partial last bar + chunk = dict(end=last_chunk_date, bar_count=bar_count) + chunks.append(chunk) + + last_chunk_date = \ + last_chunk_date - timedelta(minutes=(bar_count + 1)) + + chunks.reverse() + + else: + chunks = [dict(end=end, bar_count=delta_periods)] + + with maybe_show_progress( + chunks, + show_progress, + label='Fetching {exchange} {frequency} candles: '.format( + exchange=exchange.name, + frequency=data_frequency + )) as it: + + for chunk in it: + assets_candles_dict = fetch_candles_chunk( + exchange=exchange, + assets=assets, + data_frequency=frequency, + end_dt=chunk['end'], + bar_count=chunk['bar_count'] + ) + log.debug('requests counter {}'.format(exchange.request_cpt)) + + if not assets_candles_dict.keys(): + log.debug( + 'no data: {symbols} on {exchange}, date {end}'.format( + symbols=assets, + exchange=exchange.name, + end=chunk['end'] + ) + ) + continue + + data = [] + for asset in assets_candles_dict: + df = assets_candles_dict[asset] + sid = asset.sid + data.append((sid, df)) + + try: + log.debug( + 'writing chunk {start} to {end}'.format( + start=chunk['end'] - timedelta( + minutes=chunk['bar_count']), + end=chunk['end'] + ) + ) + writer.write( + data=data, + show_progress=False, + invalid_data_behavior='raise' + ) + except (BcolzMinuteOverlappingData, + BcolzFiveMinuteOverlappingData) as e: + log.warn('chunk already exists {}: {}'.format(chunk, e)) + + +def exchange_bundle(exchange_name, symbols=None, start=None, end=None, + log_level=INFO): """Create a data bundle ingest function for the specified exchange. Parameters @@ -68,24 +172,27 @@ def exchange_bundle(exchange_name, symbols, start=None, end=None): .. code-block:: python from catalyst.data.bundles import register + from catalyst.exchange.exchange_bundle import exchange_bundle symbols = ( + 'btc_usd', 'eth_btc', 'etc_btc', 'neo_btc', ) - register('bitfinex_bundle', exchange_bundle('bitfinex', symbols)) + register('exchange_bitfinex', exchange_bundle('bitfinex', symbols)) Notes ----- The sids for each symbol will be the index into the symbols sequence. """ # strict this in memory so that we can reiterate over it - symbols = tuple(symbols) + log.level = log_level def ingest(environ, asset_db_writer, - minute_bar_writer, # unused + minute_bar_writer, + five_minute_bar_writer, daily_bar_writer, adjustment_writer, calendar, @@ -93,16 +200,23 @@ def exchange_bundle(exchange_name, symbols, start=None, end=None): end_session, cache, show_progress, + is_compile, output_dir, - # pass these as defaults to make them 'nonlocal' in py2 start=start, end=end): + log.info('ingesting bundle {}'.format(output_dir)) + # TODO: I don't understand this session vs dates idea if start is None: start = start_session if end is None: - end = None + end = end_session + + now = pd.Timestamp.utcnow() + if end > now: + log.info('adjusting the end date to now {}'.format(now)) + end = now log.info('ingesting data from {} to {}'.format(start, end)) @@ -124,61 +238,59 @@ def exchange_bundle(exchange_name, symbols, start=None, end=None): else: raise ExchangeNotFoundError(exchange_name=exchange_name) - assets = exchange.get_assets(symbols) - - delta = end - start - delta_minutes = delta.total_seconds() / 60 - if delta_minutes > exchange.num_candles_limit: - bar_count = exchange.num_candles_limit - - chunks = [] - last_chunk_date = end - while last_chunk_date > start + timedelta(minutes=bar_count): - # TODO: account for the partial last bar - chunk = dict(end=last_chunk_date, bar_count=bar_count) - chunks.append(chunk) - - last_chunk_date = \ - last_chunk_date - timedelta(minutes=(bar_count + 1)) - - chunks.reverse() - + if symbols is not None: + assets = exchange.get_assets(symbols) else: - chunks = [dict(end=end, bar_count=delta_minutes)] + assets = exchange.assets - with maybe_show_progress( - chunks, - show_progress, - label='Fetching {} candles: '.format(exchange_name)) as it: + earliest_trade = None + for asset in assets: + if earliest_trade is None or earliest_trade > asset.start_date: + earliest_trade = asset.start_date - for chunk in it: - asset_df = fetch_candles_chunk( - exchange=exchange, - assets=assets, - data_frequency='1m', - end_dt=chunk['end'], - bar_count=chunk['bar_count'] - ) + if earliest_trade > start: + log.info( + 'adjusting start date to earliest trade date found {}'.format( + earliest_trade + )) + start = earliest_trade - data = [] - for asset in asset_df: - df = asset_df[asset] - sid = asset.sid - data.append((sid, df)) + if start >= end: + raise ValueError('start date cannot be after end date') - try: - log.debug( - 'writing chunk: {sid} start: {start} end: {end}'.format( - sid=sid, - start=chunk['end'] - timedelta( - minutes=chunk['bar_count']), - end=chunk['end'] - ) - ) - minute_bar_writer.write(data, show_progress=show_progress) - except KeyError: - minute_bar_writer.write(data, show_progress=show_progress) - except BcolzMinuteOverlappingData as e: - log.warn('Unable to write chunk {}: {}'.format(chunk, e)) + if daily_bar_writer is not None: + process_bar_data( + exchange=exchange, + assets=assets, + writer=daily_bar_writer, + data_frequency='daily', + show_progress=show_progress, + start=start, + end=end + ) + + if five_minute_bar_writer is not None: + process_bar_data( + exchange=exchange, + assets=assets, + writer=five_minute_bar_writer, + data_frequency='5-minute', + show_progress=show_progress, + start=start, + end=end + ) + + if minute_bar_writer is not None: + process_bar_data( + exchange=exchange, + assets=assets, + writer=minute_bar_writer, + data_frequency='minute', + show_progress=show_progress, + start=start, + end=end + ) return ingest + + diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index a0a137f9..d823cd87 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -140,3 +140,10 @@ class SymbolNotFoundOnExchange(ZiplineError): """ msg = ('Symbol {symbol} not found on exchange {exchange}. ' 'Choose from: {supported_symbols}').strip() + + +class BundleNotFoundError(ZiplineError): + msg = ('Unable to find bundle data for exchange {exchange}. ' + 'Please ingest data using the command ' + '`catalyst ingest -b exchange_{exchange}`. ' + 'See catalyst documentation for details.').strip() diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index 784aecea..7cf236a2 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -1,14 +1,13 @@ -from datetime import timedelta - import os +from datetime import timedelta +from logging import Logger, DEBUG + import pandas as pd -from logging import Logger from catalyst import get_calendar - from catalyst.data.minute_bars import BcolzMinuteBarWriter from catalyst.exchange.exchange_bundle import exchange_bundle -from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root +from catalyst.utils.paths import ensure_directory, data_root log = Logger('test_exchange_bundle') @@ -17,44 +16,52 @@ class ExchangeBundleTestCase: def test_ingest(self): exchange_name = 'bitfinex' - start = pd.Timestamp.utcnow() - timedelta(days=2) + start = pd.Timestamp.utcnow() - timedelta(days=365) end = pd.Timestamp.utcnow() open_calendar = get_calendar('OPEN') - root = get_exchange_minute_writer_root(exchange_name) - filename = os.path.join(root, 'metadata.json') + root = data_root(os.environ) + output_dir = '{root}/exchange_{exchange}/test'.format( + root=root, + exchange=exchange_name + ) + ensure_directory(output_dir) + filename = os.path.join(output_dir, 'metadata.json') + + start_session = start.floor('1d') if os.path.isfile(filename): - minute_bar_writer = BcolzMinuteBarWriter.open(root, end) + minute_bar_writer = BcolzMinuteBarWriter.open(output_dir, end) else: # TODO: need to be able to write more precise numbers minute_bar_writer = BcolzMinuteBarWriter( - rootdir=root, + rootdir=output_dir, calendar=open_calendar, minutes_per_day=1440, - start_session=start.floor('1d'), + start_session=start_session, end_session=end, write_metadata=True ) ingest = exchange_bundle( exchange_name=exchange_name, - symbols=['btc_usd'] + symbols=['eth_btc'], + log_level=DEBUG ) - ingest( - environ=os.environ, - asset_db_writer=None, # TODO: nice to have - minute_bar_writer=minute_bar_writer, - daily_bar_writer=None, # TODO: add later - adjustment_writer=None, # Not applicable to crypto - calendar=open_calendar, - start_session=start, - end_session=end, - cache=dict(), - show_progress=True, - output_dir=exchange_name, # TODO: not sure - start=start, - end=end - ) + ingest(environ=os.environ, + asset_db_writer=None, + minute_bar_writer=minute_bar_writer, + five_minute_bar_writer=None, + daily_bar_writer=None, + adjustment_writer=None, + calendar=open_calendar, + start_session=start_session, + end_session=end, + cache=dict(), + show_progress=True, + is_compile=False, + output_dir=output_dir, + start=start, + end=end) pass diff --git a/tests/exchange/test_data_portal.py b/tests/exchange/test_data_portal.py index 08980162..0ea2b8e9 100644 --- a/tests/exchange/test_data_portal.py +++ b/tests/exchange/test_data_portal.py @@ -1,5 +1,6 @@ from datetime import timedelta +import os import pandas as pd from catalyst import get_calendar from logbook import Logger @@ -10,7 +11,9 @@ from catalyst.exchange.bitfinex.bitfinex import Bitfinex from catalyst.exchange.bittrex.bittrex import Bittrex from catalyst.exchange.data_portal_exchange import DataPortalExchangeBacktest, \ DataPortalExchangeLive +from catalyst.exchange.exchange_bundle import exchange_bundle from catalyst.exchange.exchange_utils import get_exchange_auth +from catalyst.utils.run_algo import load_extensions log = Logger('test_bitfinex') @@ -44,13 +47,14 @@ class ExchangeDataPortalTestCase: first_trading_day=pd.to_datetime('today', utc=True) ) self.data_portal_backtest = DataPortalExchangeBacktest( - exchanges=dict(bitfinex=self.bitfinex, bittrex=self.bittrex), + exchanges=dict(bitfinex=self.bitfinex), asset_finder=asset_finder, trading_calendar=open_calendar, first_trading_day=pd.to_datetime('today', utc=True) ) def test_get_history_window_live(self): + asset_finder = self.data_portal_live.asset_finder assets = [ @@ -79,13 +83,14 @@ class ExchangeDataPortalTestCase: pass def test_get_spot_value_backtest(self): + asset_finder = self.data_portal_backtest.asset_finder assets = [ - asset_finder.lookup_symbol('btc_usd', self.bitfinex), + asset_finder.lookup_symbol('neo_btc', self.bitfinex), ] - date = pd.Timestamp.utcnow() - timedelta(hours=2) + date = pd.Timestamp.utcnow() - timedelta(hours=8) value = self.data_portal_backtest.get_spot_value( assets, 'close', date, 'minute') pass