diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 0a48c0b4..1c675014 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -112,6 +112,11 @@ class CCXT(Exchange): @staticmethod def find_exchanges(features=None, is_authenticated=False): + ccxt_features = [] + for feature in features: + if not feature.endswith('Bundle'): + ccxt_features.append(feature) + exchange_names = [] for exchange_name in ccxt.exchanges: if is_authenticated: @@ -126,13 +131,13 @@ class CCXT(Exchange): log.debug('loading exchange: {}'.format(exchange_name)) exchange = getattr(ccxt, exchange_name)() - if features is None: + if ccxt_features is None: has_feature = True else: try: has_feature = all( - [exchange.has[feature] for feature in features] + [exchange.has[feature] for feature in ccxt_features] ) except Exception: @@ -158,13 +163,20 @@ class CCXT(Exchange): def time_skew(self): return None - def get_candle_frequencies(self): + def get_candle_frequencies(self, data_frequency=None): frequencies = [] try: for timeframe in self.api.timeframes: - frequencies.append( - CCXT.get_frequency(timeframe, raise_error=False) - ) + freq = CCXT.get_frequency(timeframe, raise_error=False) + + # TODO: support all frequencies + if data_frequency == 'minute' and not freq.endswith('T'): + continue + + elif data_frequency == 'daily' and not freq.endswith('D'): + continue + + frequencies.append(freq) except Exception as e: log.warn( diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 2b213226..41e361b5 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -18,7 +18,7 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ PricingDataNotLoadedError, \ NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError from catalyst.exchange.exchange_utils import get_exchange_symbols, \ - get_frequency, resample_history_df + get_frequency, resample_history_df, has_bundle log = Logger('Exchange', level=LOG_LEVEL) @@ -47,6 +47,9 @@ class Exchange: def time_skew(self): pass + def has_bundle(self, data_frequency): + return has_bundle(self.name, data_frequency) + def is_open(self, dt): """ Is the exchange open diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index 5af11a2c..ac8d8f4b 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -9,7 +9,7 @@ from catalyst.exchange.exchange_errors import ExchangeRequestError, \ ExchangePortfolioDataError, ExchangeTransactionError from catalyst.finance.blotter import Blotter from catalyst.finance.commission import CommissionModel -from catalyst.finance.order import ORDER_STATUS, Order +from catalyst.finance.order import ORDER_STATUS from catalyst.finance.slippage import SlippageModel from catalyst.finance.transaction import create_transaction, Transaction from catalyst.utils.input_validation import expect_types @@ -67,7 +67,6 @@ class TradingPairFeeSchedule(CommissionModel): or (order.amount < 0 and order.limit > transaction.price)) \ and order.limit_reached else taker - # Assuming just the taker fee for now fee = cost * multiplier return fee diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 20bd09e8..fc87233a 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -140,7 +140,7 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): filename = get_exchange_symbols_filename(exchange_name, is_local) if not is_local and (not os.path.isfile(filename) or pd.Timedelta( - pd.Timestamp('now', tz='UTC') - last_modified_time( + pd.Timestamp('now', tz='UTC') - last_modified_time( filename)).days > 1): download_exchange_symbols(exchange_name, environ) @@ -435,6 +435,15 @@ def get_exchange_bundles_folder(exchange_name, environ=None): return temp_bundles +def has_bundle(exchange_name, data_frequency, environ=None): + exchange_folder = get_exchange_folder(exchange_name, environ) + + folder_name = '{}_bundle'.format(data_frequency.lower()) + folder = os.path.join(exchange_folder, folder_name) + + return os.path.isdir(folder) + + def symbols_serial(obj): """ JSON serializer for objects not serializable by default json code diff --git a/catalyst/exchange/factory.py b/catalyst/exchange/factory.py index 6f7dd0ac..7bc72f7a 100644 --- a/catalyst/exchange/factory.py +++ b/catalyst/exchange/factory.py @@ -1,11 +1,10 @@ import os -import ccxt from logbook import Logger from catalyst.constants import LOG_LEVEL -from catalyst.exchange.exchange import Exchange from catalyst.exchange.ccxt.ccxt_exchange import CCXT +from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_errors import ExchangeAuthEmpty from catalyst.exchange.exchange_utils import get_exchange_auth, \ get_exchange_folder, is_blacklist @@ -57,6 +56,10 @@ def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False, features: str The list of features. + skip_blacklist: bool + is_authenticated: bool + base_currency: bool + Returns ------- list[Exchange] @@ -74,6 +77,15 @@ def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False, skip_init=True, base_currency=base_currency, ) + + if 'dailyBundle' in features \ + and not exchange.has_bundle('daily'): + continue + + elif 'minuteBundle' in features \ + and not exchange.has_bundle('minute'): + continue + exchanges.append(exchange) return exchanges diff --git a/catalyst/exchange/test_utils.py b/catalyst/exchange/test_utils.py new file mode 100644 index 00000000..331d0807 --- /dev/null +++ b/catalyst/exchange/test_utils.py @@ -0,0 +1,83 @@ +import os +import random + +import tempfile +from catalyst.assets._assets import TradingPair + +from catalyst.exchange.exchange_utils import get_exchange_folder +from catalyst.exchange.factory import find_exchanges +from catalyst.utils.paths import ensure_directory + + +def handle_exchange_error(exchange, e): + try: + message = '{}: {}'.format( + e.__class__, e.message.decode('ascii', 'ignore') + ) + except Exception: + message = 'unexpected error' + + folder = get_exchange_folder(exchange.name) + filename = os.path.join(folder, 'blacklist.txt') + with open(filename, 'wt') as handle: + handle.write(message) + + +def select_random_exchanges(population=3, features=None, + is_authenticated=False, base_currency=None): + all_exchanges = find_exchanges( + features=features, + is_authenticated=is_authenticated, + base_currency=base_currency, + ) + + if population is not None: + if len(all_exchanges) < population: + population = len(all_exchanges) + + exchanges = random.sample(all_exchanges, population) + + else: + exchanges = all_exchanges + + return exchanges + + +def select_random_assets(all_assets, population=3): + assets = random.sample(all_assets, population) + return assets + + +def output_df(df, assets, name=None): + """ + Outputs a price DataFrame to a temp folder. + + Parameters + ---------- + df: pd.DataFrame + assets + name + + Returns + ------- + + """ + if isinstance(assets, TradingPair): + exchange_folder = assets.exchange + asset_folder = assets.symbol + else: + exchange_folder = ','.join([asset.exchange for asset in assets]) + asset_folder = ','.join([asset.symbol for asset in assets]) + + folder = os.path.join( + tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder + ) + ensure_directory(folder) + + if name is None: + name = 'output' + + path = os.path.join(folder, '{}.csv'.format(name)) + df.to_csv(path) + + return path diff --git a/catalyst/exchange/validator.py b/catalyst/exchange/validator.py deleted file mode 100644 index 5cd0b1e2..00000000 --- a/catalyst/exchange/validator.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -import tempfile - -import pandas as pd -import six -from catalyst.assets._assets import TradingPair, get_calendar -from logbook import Logger -from pandas.util.testing import assert_frame_equal - -from catalyst.constants import LOG_LEVEL -from catalyst.exchange.asset_finder_exchange import AssetFinderExchange -from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest -from catalyst.exchange.factory import get_exchanges -from catalyst.utils.paths import ensure_directory - -log = Logger('Validator', level=LOG_LEVEL) - - -def output_df(df, assets, name=None): - """ - Outputs a price DataFrame to a temp folder. - - Parameters - ---------- - df: pd.DataFrame - assets - name - - Returns - ------- - - """ - if isinstance(assets, TradingPair): - exchange_folder = assets.exchange - asset_folder = assets.symbol - else: - exchange_folder = ','.join([asset.exchange for asset in assets]) - asset_folder = ','.join([asset.symbol for asset in assets]) - - folder = os.path.join( - tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder - ) - ensure_directory(folder) - - if name is None: - name = 'output' - - path = os.path.join(folder, '{}.csv'.format(name)) - df.to_csv(path) - - return path - - -class Validator(object): - def __init__(self, data_portal): - self.data_portal = data_portal - - def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count, - sample_minutes): - """ - Creates DataFrames from the bundle and exchange for the specified - data set. - - Parameters - ---------- - exchange: Exchange - assets - end_dt - bar_count - sample_minutes - - Returns - ------- - - """ - freq = '{}T'.format(sample_minutes) - - log.info('creating data sample from bundle') - df1 = self.data_portal.get_history_window( - assets=assets, - end_dt=end_dt, - bar_count=bar_count, - frequency=freq, - field='close', - data_frequency='minute' - ) - path = output_df(df1, assets, '{}_resampled'.format(freq)) - log.info('saved resampled bundle candles: {}\n{}'.format( - path, df1.tail(10)) - ) - - log.info('creating data sample from exchange api') - candles = exchange.get_candles( - end_dt=end_dt, - freq='{}T'.format(sample_minutes), - assets=assets, - bar_count=bar_count - ) - - series = dict() - for asset in assets: - series[asset] = pd.Series( - data=[candle['close'] for candle in candles[asset]], - index=[candle['last_traded'] for candle in candles[asset]] - ) - - df2 = pd.DataFrame(series) - path = output_df(df2, assets, '{}_api'.format(freq)) - log.info('saved exchange api candles: {}\n{}'.format( - path, df2.tail(10)) - ) - - try: - assert_frame_equal(df1, df2) - return True - except: - log.warn('differences found in dataframes') - return False - - -if __name__ == '__main__': - exchanges = get_exchanges(['poloniex']) - exchange = six.next(six.itervalues(exchanges)) - assets = exchange.get_assets(symbols=['eth_btc']) - - open_calendar = get_calendar('OPEN') - asset_finder = AssetFinderExchange() - data_portal = DataPortalExchangeBacktest( - exchanges=exchanges, - asset_finder=asset_finder, - trading_calendar=open_calendar, - first_trading_day=None # will set dynamically based on assets - ) - validator = Validator(data_portal=data_portal) - - validator.compare_bundle_with_exchange( - exchange=exchange, - assets=assets, - end_dt=pd.to_datetime('2017-11-10 1:00', utc=True), - bar_count=200, - sample_minutes=30 - ) diff --git a/tests/exchange/test_suite_bundle.py b/tests/exchange/test_suite_bundle.py new file mode 100644 index 00000000..0e649425 --- /dev/null +++ b/tests/exchange/test_suite_bundle.py @@ -0,0 +1,126 @@ +import random + +from logbook import Logger +from pandas.util.testing import assert_frame_equal + +import pandas as pd + +from catalyst import get_calendar +from catalyst.exchange.asset_finder_exchange import AssetFinderExchange +from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest +from catalyst.exchange.test_utils import select_random_exchanges, output_df, \ + select_random_assets + +log = Logger('TestSuiteExchange') + + +class TestSuiteBundle: + @staticmethod + def get_data_portal(exchange_names): + open_calendar = get_calendar('OPEN') + asset_finder = AssetFinderExchange() + + data_portal = DataPortalExchangeBacktest( + exchange_names=exchange_names, + asset_finder=asset_finder, + trading_calendar=open_calendar, + first_trading_day=None # will set dynamically based on assets + ) + return data_portal + + def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count, + freq, data_portal): + """ + Creates DataFrames from the bundle and exchange for the specified + data set. + + Parameters + ---------- + exchange: Exchange + assets + end_dt + bar_count + sample_minutes + + Returns + ------- + + """ + log.info('creating data sample from bundle') + df1 = data_portal.get_history_window( + assets=assets, + end_dt=end_dt, + bar_count=bar_count, + frequency=freq, + field='close', + data_frequency='minute' + ) + path = output_df(df1, assets, '{}_resampled'.format(freq)) + log.info('saved resampled bundle candles: {}\n{}'.format( + path, df1.tail(10)) + ) + + log.info('creating data sample from exchange api') + candles = exchange.get_candles( + end_dt=end_dt, + freq=freq, + assets=assets, + bar_count=bar_count + ) + + series = dict() + for asset in assets: + series[asset] = pd.Series( + data=[candle['close'] for candle in candles[asset]], + index=[candle['last_traded'] for candle in candles[asset]] + ) + + df2 = pd.DataFrame(series) + path = output_df(df2, assets, '{}_api'.format(freq)) + log.info('saved exchange api candles: {}\n{}'.format( + path, df2.tail(10)) + ) + + try: + assert_frame_equal(df1, df2) + return True + except: + log.warn('differences found in dataframes') + return False + + def test_validate_bundles(self): + exchange_population = 3 + asset_population = 3 + data_frequency = random.choice(['minute', 'daily']) + + bundle = 'dailyBundle' if data_frequency == 'daily' else 'minuteBundle' + exchanges = select_random_exchanges( + population=exchange_population, + features=[bundle], + ) # Type: list[Exchange] + + data_portal = TestSuiteBundle.get_data_portal( + [exchange.name for exchange in exchanges] + ) + for exchange in exchanges: + exchange.init() + + frequencies = exchange.get_candle_frequencies(data_frequency) + freq = random.sample(frequencies, 1)[0] + + bar_count = random.randint(1, 10) + end_dt = pd.Timestamp.utcnow().floor('1T') + dt_range = pd.date_range( + end=end_dt, periods=bar_count, freq=freq + ) + assets = select_random_assets( + exchange.assets, asset_population + ) + self.compare_bundle_with_exchange( + exchange=exchange, + assets=assets, + end_dt=dt_range[-1], + bar_count=bar_count, + freq=freq, + data_portal=data_portal, + ) diff --git a/tests/exchange/test_suite_exchange.py b/tests/exchange/test_suite_exchange.py index 75d5b660..239ee939 100644 --- a/tests/exchange/test_suite_exchange.py +++ b/tests/exchange/test_suite_exchange.py @@ -5,69 +5,16 @@ from logging import Logger from time import sleep import pandas as pd -from ccxt import AuthenticationError from catalyst.exchange.exchange_errors import ExchangeRequestError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.exchange_utils import get_exchange_folder -from catalyst.exchange.factory import find_exchanges +from catalyst.exchange.test_utils import select_random_exchanges, \ + handle_exchange_error, select_random_assets log = Logger('TestSuiteExchange') -def handle_exchange_error(exchange, e): - is_blacklist = False - - if isinstance(e, AuthenticationError): - is_blacklist = True - - elif isinstance(e, ValueError) or isinstance(e, ExchangeRequestError): - is_blacklist = True - - else: - log.warn('unexpected error: {}'.format(e)) - is_blacklist = True - - if is_blacklist: - try: - message = '{}: {}'.format( - e.__class__, e.message.decode('ascii', 'ignore') - ) - except Exception: - message = 'unexpected error' - - folder = get_exchange_folder(exchange.name) - filename = os.path.join(folder, 'blacklist.txt') - with open(filename, 'wt') as handle: - handle.write(message) - - -def select_random_exchanges(population=3, features=None, - is_authenticated=False, base_currency=None): - all_exchanges = find_exchanges( - features=features, - is_authenticated=is_authenticated, - base_currency=base_currency, - ) - - if population is not None: - if len(all_exchanges) < population: - population = len(all_exchanges) - - exchanges = random.sample(all_exchanges, population) - - else: - exchanges = all_exchanges - - return exchanges - - -def select_random_assets(all_assets, population=3): - assets = random.sample(all_assets, population) - return assets - - -# TODO: convert to Nosetest class TestSuiteExchange: def _test_markets_exchange(self, exchange, attempts=0): assets = None