From bd4b0d2756beae4f758c1fafbfe5b8b9b39747d9 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 4 Jan 2018 02:46:10 -0500 Subject: [PATCH 1/8] DOC: Fixed old release header --- docs/source/releases.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index a23bfe40..ede098c0 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -39,7 +39,7 @@ Build - Added market orders in live mode (:issue:`81`) Version 0.3.10 -^^^^^^^^^^^^^ +~~~~~~~~~~~~~~ **Release Date**: 2017-11-28 Bug Fixes From a818d10d406de39ecb742f2fc7fca46cf2305d21 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 4 Jan 2018 21:06:40 -0500 Subject: [PATCH 2/8] BLD: improving unit tests --- tests/exchange/{__init.py => __init__.py} | 0 tests/exchange/test_ccxt.py | 6 +- tests/exchange/test_suites/__init__.py | 0 .../{ => test_suites}/test_suite_bundle.py | 0 .../{ => test_suites}/test_suite_exchange.py | 65 ++++++++++++------- 5 files changed, 44 insertions(+), 27 deletions(-) rename tests/exchange/{__init.py => __init__.py} (100%) create mode 100644 tests/exchange/test_suites/__init__.py rename tests/exchange/{ => test_suites}/test_suite_bundle.py (100%) rename tests/exchange/{ => test_suites}/test_suite_exchange.py (72%) diff --git a/tests/exchange/__init.py b/tests/exchange/__init__.py similarity index 100% rename from tests/exchange/__init.py rename to tests/exchange/__init__.py diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 7be111d1..7ef939b1 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -19,16 +19,16 @@ class TestCCXT(BaseExchangeTestCase): exchange_name=exchange_name, key=auth['key'], secret=auth['secret'], - base_currency='eth', + base_currency='bnb', ) self.exchange.init() def test_order(self): log.info('creating order') - asset = self.exchange.get_asset('neo_eth') + asset = self.exchange.get_asset('neo_bnb') order_id = self.exchange.order( asset=asset, - style=ExchangeLimitOrder(limit_price=0.7), + style=ExchangeLimitOrder(limit_price=10), amount=1, ) log.info('order created {}'.format(order_id)) diff --git a/tests/exchange/test_suites/__init__.py b/tests/exchange/test_suites/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/exchange/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py similarity index 100% rename from tests/exchange/test_suite_bundle.py rename to tests/exchange/test_suites/test_suite_bundle.py diff --git a/tests/exchange/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py similarity index 72% rename from tests/exchange/test_suite_exchange.py rename to tests/exchange/test_suites/test_suite_exchange.py index cf92845e..3eb3b38b 100644 --- a/tests/exchange/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -1,21 +1,25 @@ import json import os import random -from logging import Logger +from logging import Logger, WARNING from time import sleep import pandas as pd +from catalyst.assets._assets import TradingPair +from logbook import TestHandler from catalyst.exchange.exchange_errors import ExchangeRequestError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.utils.exchange_utils import get_exchange_folder from catalyst.exchange.utils.test_utils import select_random_exchanges, \ handle_exchange_error, select_random_assets +from catalyst.testing import ZiplineTestCase +from catalyst.testing.fixtures import WithLogger log = Logger('TestSuiteExchange') -class TestSuiteExchange: +class TestSuiteExchange(WithLogger, ZiplineTestCase): def _test_markets_exchange(self, exchange, attempts=0): assets = None try: @@ -156,34 +160,47 @@ class TestSuiteExchange: base_currency=quote_currency, ) # Type: list[Exchange] - for exchange in exchanges: - exchange.init() + log_catcher = TestHandler() + with log_catcher: + for exchange in exchanges: + exchange.init() - assets = exchange.get_assets(quote_currency=quote_currency) - asset = select_random_assets(assets, 1)[0] - assert asset + assets = exchange.get_assets(quote_currency=quote_currency) + asset = select_random_assets(assets, 1)[0] + self.assertIsInstance(asset, TradingPair) - tickers = exchange.tickers([asset]) - price = tickers[asset]['last_price'] + tickers = exchange.tickers([asset]) + price = tickers[asset]['last_price'] - amount = order_amount / price + amount = order_amount / price - limit_price = price * 0.8 - style = ExchangeLimitOrder(limit_price=limit_price) + limit_price = price * 0.8 + style = ExchangeLimitOrder(limit_price=limit_price) - order = exchange.order( - asset=asset, - amount=amount, - style=style, - ) - sleep(1) + order = exchange.order( + asset=asset, + amount=amount, + style=style, + ) + sleep(1) - open_order, _ = exchange.get_order(order.id, asset) - assert open_order.status == 0 + open_order, _ = exchange.get_order(order.id, asset) + self.assertEqual(0, open_order.status) - exchange.cancel_order(open_order, asset) - sleep(1) + exchange.cancel_order(open_order, asset) + sleep(1) - canceled_order, _ = exchange.get_order(open_order.id, asset) - assert canceled_order.status == 2 + canceled_order, _ = exchange.get_order(open_order.id, asset) + warnings = [record for record in log_catcher.records if + record.level == WARNING] + + self.assertEqual(0, len(warnings)) + self.assertEqual(2, canceled_order.status) + print( + 'tested {exchange} / {symbol}, order: {order}'.format( + exchange=exchange.name, + symbol=asset.symbol, + order=order.id, + ) + ) pass From b4e7629e8bdc99d4536e11864518e53c2d8d48c1 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 4 Jan 2018 21:07:15 -0500 Subject: [PATCH 3/8] BLD: upgraded CCXT --- etc/python2.7-environment.yml | 2 +- etc/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index b8b62360..d2c3da70 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -20,7 +20,7 @@ dependencies: - bcolz==0.12.1 - bottleneck==1.2.1 - chardet==3.0.4 - - ccxt==1.10.283 + - ccxt==1.10.565 - click==6.7 - contextlib2==0.5.5 - cycler==0.10.0 diff --git a/etc/requirements.txt b/etc/requirements.txt index cc009908..f01a54b5 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -81,6 +81,6 @@ empyrical==0.2.1 tables==3.3.0 #Catalyst dependencies -ccxt==1.10.283 +ccxt==1.10.565 boto3==1.4.8 redo==1.6 From 79e4854973ad0f5e0b5fe7dddd8350e9de449e15 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 4 Jan 2018 22:46:21 -0500 Subject: [PATCH 4/8] BUG: fixed potential issue with refreshing the stats in live mode --- catalyst/examples/mean_reversion_simple.py | 4 ++-- catalyst/exchange/exchange_algorithm.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index dd4bd305..14f7ec99 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -37,7 +37,7 @@ def initialize(context): context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 50 + context.RSI_OVERSOLD = 55 context.RSI_OVERBOUGHT = 65 context.CANDLE_SIZE = '5T' @@ -244,7 +244,7 @@ def analyze(context=None, perf=None): if __name__ == '__main__': # The execution mode: backtest or live - live = False + live = True if live: run_algorithm( diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 491a9ff1..df30b368 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -680,7 +680,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if new_orders != self._last_orders: self.performance_needs_update = True - self._last_orders = new_orders + self._last_orders = copy.deepcopy(new_orders) if self.performance_needs_update: self.perf_tracker.update_performance() From 56481bbbe00656970130c6cec87188a9941696f6 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 5 Jan 2018 00:09:51 -0500 Subject: [PATCH 5/8] BUG: rolled back run_algo functions splitting to quickly resolve issue #137. We'll merge it more carefully in the next release. --- catalyst/__main__.py | 12 +- catalyst/utils/run_algo.py | 483 +++++++++++++++++-------------------- 2 files changed, 227 insertions(+), 268 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 82febb0b..5f8f9136 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -282,6 +282,7 @@ def run(ctx, exchange=exchange_name, algo_namespace=algo_namespace, base_currency=base_currency, + analyze_live=None, live_graph=False, simulate_orders=True, stats_output=None, @@ -312,11 +313,11 @@ def catalyst_magic(line, cell=None): '--algotext', cell, '--output', os.devnull, # don't write the results by default ] + ([ - # these options are set when running in line magic mode - # set a non None algo text to use the ipython user_ns - '--algotext', '', - '--local-namespace', - ] if cell is None else []) + line.split(), + # these options are set when running in line magic mode + # set a non None algo text to use the ipython user_ns + '--algotext', '', + '--local-namespace', + ] if cell is None else []) + line.split(), '%s%%catalyst' % ((cell or '') and '%'), # don't use system exit and propogate errors to the caller standalone_mode=False, @@ -470,6 +471,7 @@ def live(ctx, algo_namespace=algo_namespace, base_currency=base_currency, live_graph=live_graph, + analyze_live=None, simulate_orders=simulate_orders, stats_output=None, ) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index d0129f28..015f91e4 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -70,7 +70,36 @@ class _RunAlgoError(click.ClickException, ValueError): return self.pyfunc_msg -def _build_namespace(algotext, local_namespace, defines): +def _run(handle_data, + initialize, + before_trading_start, + analyze, + algofile, + algotext, + defines, + data_frequency, + capital_base, + data, + bundle, + bundle_timestamp, + start, + end, + output, + print_algo, + local_namespace, + environ, + live, + exchange, + algo_namespace, + base_currency, + live_graph, + analyze_live, + simulate_orders, + stats_output): + """Run a backtest for the given algorithm. + + This is shared between the cli and :func:`catalyst.run_algo`. + """ if algotext is not None: if local_namespace: ip = get_ipython() # noqa @@ -84,197 +113,173 @@ def _build_namespace(algotext, local_namespace, defines): except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % - assign) + assign, + ) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( - 'failed to execute definition for name %r: %s' % (name, e)) + 'failed to execute definition for name %r: %s' % (name, e), + ) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', - "cannot pass '-D' / '--define' without '-t' / '--algotext'") + "cannot pass '-D' / '--define' without '-t' / '--algotext'", + ) else: namespace = {} + if algofile is not None: + algotext = algofile.read() - return namespace + if print_algo: + if PYGMENTS: + highlight( + algotext, + PythonLexer(), + TerminalFormatter(), + outfile=sys.stdout, + ) + else: + click.echo(algotext) + mode = 'paper-trading' if simulate_orders else 'live-trading' \ + if live else 'backtest' + log.info('running algo in {mode} mode'.format(mode=mode)) -def _mode(simulate_orders, live): - if not live: - return 'backtest' - elif simulate_orders: - return 'paper-trading' - else: - return 'live-trading' - - -def _build_exchanges_dict(exchange, live, simulate_orders, base_currency): exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] - exchanges = {exchange_name: get_exchange( - exchange_name=exchange_name, - base_currency=base_currency, - must_authenticate=(live and not simulate_orders)) - for exchange_name in exchange_list} - - return exchanges - - -def _pretty_print_code(algotext): - if PYGMENTS: - highlight( - algotext, - PythonLexer(), - TerminalFormatter(), - outfile=sys.stdout) - else: - click.echo(algotext) - - -def _choose_loader(data_frequency, column): - bound_cols = TradingPairPricing.columns - if column in bound_cols: - return ExchangePricingLoader(data_frequency) - raise ValueError( - "No PipelineLoader registered for column %s." % column) - - -def _get_live_time_range(): - start = pd.Timestamp.utcnow() - # TODO: fix the end data. - end = start + timedelta(hours=8760) - return start, end - - -def _data_for_live_trading(sim_params, exchanges, env, open_calendar): - data = DataPortalExchangeLive( - exchanges=exchanges, - asset_finder=env.asset_finder, - trading_calendar=open_calendar, - first_trading_day=pd.to_datetime('today', utc=True)) - - return data - - -# TODO use proper retry here -def _fetch_capital_base(base_currency, exchange_name, exchange, - attempt_index=0): - """ - Fetch the base currency amount required to bootstrap - the algorithm against the exchange. - - The algorithm cannot continue without this value. - - :param exchange: the targeted exchange - :param attempt_index: - :return capital_base: the amount of base currency available for - trading - """ - try: - log.debug('retrieving capital base in {} to bootstrap ' - 'exchange {}'.format(base_currency, exchange_name)) - balances = exchange.get_balances() - except ExchangeRequestError as e: - if attempt_index < 20: - log.warn( - 'could not retrieve balances on {}: {}'.format( - exchange.name, e)) - sleep(5) - return _fetch_capital_base(base_currency, exchange_name, exchange, - attempt_index + 1) - - else: - raise ExchangeRequestErrorTooManyAttempts( - attempts=attempt_index, - error=e) - - if base_currency in balances: - base_currency_available = balances[base_currency]['free'] - log.info( - 'base currency available in the account: {} {}'.format( - base_currency_available, base_currency)) - - return base_currency_available - else: - raise BaseCurrencyNotFoundError( + exchanges = dict() + for exchange_name in exchange_list: + exchanges[exchange_name] = get_exchange( + exchange_name=exchange_name, base_currency=base_currency, - exchange=exchange_name) + must_authenticate=(live and not simulate_orders), + skip_init=True, + ) + open_calendar = get_calendar('OPEN') -def _algorithm_class_for_live(algo_namespace, live_graph, stats_output, - analyze_live, base_currency, simulate_orders, - exchanges, capital_base): - if not simulate_orders: - for exchange_name in exchanges: - exchange = exchanges[exchange_name] - balance = _fetch_capital_base(base_currency, exchange_name, - exchange) + env = TradingEnvironment( + load=partial( + load_crypto_market_data, + environ=environ, + start_dt=start, + end_dt=end + ), + environ=environ, + exchange_tz='UTC', + asset_db_path=None # We don't need an asset db, we have exchanges + ) + env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) - if balance < capital_base: - raise NotEnoughCapitalError( - exchange=exchange_name, - base_currency=base_currency, - balance=balance, - capital_base=capital_base) - - algorithm_class = partial( - ExchangeTradingAlgorithmLive, - exchanges=exchanges, - algo_namespace=algo_namespace, - live_graph=live_graph, - simulate_orders=simulate_orders, - stats_output=stats_output, - analyze_live=analyze_live,) - - return algorithm_class - - -def _bundle_trading_environment(bundle_data, environ): - prefix, connstr = re.split( - r'sqlite:///', - str(bundle_data.asset_finder.engine.url), - maxsplit=1) - if prefix: + def choose_loader(column): + bound_cols = TradingPairPricing.columns + if column in bound_cols: + return ExchangePricingLoader(data_frequency) raise ValueError( - "invalid url %r, must begin with 'sqlite:///'" % - str(bundle_data.asset_finder.engine.url)) + "No PipelineLoader registered for column %s." % column + ) - return TradingEnvironment(asset_db_path=connstr, environ=environ) + if live: + start = pd.Timestamp.utcnow() + # TODO: fix the end data. + end = start + timedelta(hours=8760) -def _build_live_algo_and_data(sim_params, exchanges, env, open_calendar, - simulate_orders, algo_namespace, capital_base, - live_graph, stats_output, analyze_live, - base_currency, namespace, choose_loader, - algorithm_class_kwargs): - sim_params._arena = 'live' # TODO: use the constructor instead + data = DataPortalExchangeLive( + exchanges=exchanges, + asset_finder=env.asset_finder, + trading_calendar=open_calendar, + first_trading_day=pd.to_datetime('today', utc=True) + ) - data = _data_for_live_trading(sim_params, exchanges, env, open_calendar) + def fetch_capital_base(exchange, attempt_index=0): + """ + Fetch the base currency amount required to bootstrap + the algorithm against the exchange. - algorithm_class = _algorithm_class_for_live( - algo_namespace, live_graph, stats_output, analyze_live, - base_currency, simulate_orders, exchanges, capital_base) + The algorithm cannot continue without this value. - return data, algorithm_class( - namespace=namespace, - env=env, - get_pipeline_loader=choose_loader, - sim_params=sim_params, - **algorithm_class_kwargs) + :param exchange: the targeted exchange + :param attempt_index: + :return capital_base: the amount of base currency available for + trading + """ + try: + log.debug('retrieving capital base in {} to bootstrap ' + 'exchange {}'.format(base_currency, exchange_name)) + balances = exchange.get_balances() + except ExchangeRequestError as e: + if attempt_index < 20: + log.warn( + 'could not retrieve balances on {}: {}'.format( + exchange.name, e + ) + ) + sleep(5) + return fetch_capital_base(exchange, attempt_index + 1) + else: + raise ExchangeRequestErrorTooManyAttempts( + attempts=attempt_index, + error=e + ) -def _build_backtest_algo_and_data( - exchanges, bundle, env, environ, bundle_timestamp, open_calendar, - start, end, namespace, choose_loader, sim_params, - algorithm_class_kwargs): - if exchanges: + if base_currency in balances: + base_currency_available = balances[base_currency]['free'] + log.info( + 'base currency available in the account: {} {}'.format( + base_currency_available, base_currency + ) + ) + + return base_currency_available + else: + raise BaseCurrencyNotFoundError( + base_currency=base_currency, + exchange=exchange_name + ) + + if not simulate_orders: + for exchange_name in exchanges: + exchange = exchanges[exchange_name] + balance = fetch_capital_base(exchange) + + if balance < capital_base: + raise NotEnoughCapitalError( + exchange=exchange_name, + base_currency=base_currency, + balance=balance, + capital_base=capital_base, + ) + + sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=capital_base, + emission_rate='minute', + data_frequency='minute' + ) + + # TODO: use the constructor instead + sim_params._arena = 'live' + + algorithm_class = partial( + ExchangeTradingAlgorithmLive, + exchanges=exchanges, + algo_namespace=algo_namespace, + live_graph=live_graph, + simulate_orders=simulate_orders, + stats_output=stats_output, + analyze_live=analyze_live, + ) + elif exchanges: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. @@ -288,19 +293,41 @@ def _build_backtest_algo_and_data( asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, - last_available_session=end) + last_available_session=end + ) + + sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=capital_base, + data_frequency=data_frequency, + emission_rate=data_frequency, + ) algorithm_class = partial( ExchangeTradingAlgorithmBacktest, - exchanges=exchanges) + exchanges=exchanges + ) + elif bundle is not None: - # TODO This branch should probably be removed or fixed: it doesn't even - # build `algorithm_class`, so it will break when trying to instantiate - # it. - bundle_data = load(bundle, environ, bundle_timestamp) + bundle_data = load( + bundle, + environ, + bundle_timestamp, + ) - env = _bundle_trading_environment(bundle_data, environ) + prefix, connstr = re.split( + r'sqlite:///', + str(bundle_data.asset_finder.engine.url), + maxsplit=1, + ) + if prefix: + raise ValueError( + "invalid url %r, must begin with 'sqlite:///'" % + str(bundle_data.asset_finder.engine.url), + ) + env = TradingEnvironment(asset_db_path=connstr, environ=environ) first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day @@ -309,103 +336,27 @@ def _build_backtest_algo_and_data( first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, - adjustment_reader=bundle_data.adjustment_reader) + adjustment_reader=bundle_data.adjustment_reader, + ) - return data, algorithm_class( + perf = algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, - **algorithm_class_kwargs) - - -def _build_algo_and_data(handle_data, initialize, before_trading_start, - analyze, algofile, algotext, defines, data_frequency, - capital_base, data, bundle, bundle_timestamp, start, - end, output, print_algo, local_namespace, environ, - live, exchange, algo_namespace, base_currency, - live_graph, analyze_live, simulate_orders, - stats_output): - namespace = _build_namespace(algotext, local_namespace, defines) - if algotext is not None: - algotext = algofile.read() - - if print_algo: - _pretty_print_code(algotext) - - mode = _mode(simulate_orders, live) - log.info('running algo in {mode} mode'.format(mode=mode)) - - exchanges = _build_exchanges_dict(exchange, live, simulate_orders, - base_currency) - - open_calendar = get_calendar('OPEN') - - env = TradingEnvironment( - load=partial(load_crypto_market_data, environ=environ, start_dt=start, - end_dt=end), - environ=environ, - exchange_tz='UTC', - asset_db_path=None) # We don't need an asset db, we have exchanges - - env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) - - choose_loader = partial(_choose_loader, data_frequency) - - if live: - start, end = _get_live_time_range() - data_frequency = 'minute' # TODO double check if this is the desired behavior - - sim_params = create_simulation_parameters( - start=start, - end=end, - capital_base=capital_base, - emission_rate=data_frequency, - data_frequency=data_frequency) - - if algotext is None: - algorithm_class_kwargs = {'initialize': initialize, - 'handle_data': handle_data, - 'before_trading_start': before_trading_start, - 'analyze': analyze} - else: - algorithm_class_kwargs = {'algo_filename': getattr(algofile, 'name', - ''), - 'script': algotext} - - if live: - return _build_live_algo_and_data( - sim_params, exchanges, env, open_calendar, simulate_orders, - algo_namespace, capital_base, live_graph, stats_output, - analyze_live, base_currency, namespace, choose_loader, - algorithm_class_kwargs) - else: - return _build_backtest_algo_and_data( - exchanges, bundle, env, environ, bundle_timestamp, open_calendar, - start, end, namespace, choose_loader, sim_params, - algorithm_class_kwargs) - - -def _run(handle_data, initialize, before_trading_start, analyze, algofile, - algotext, defines, data_frequency, capital_base, data, bundle, - bundle_timestamp, start, end, output, print_algo, local_namespace, - environ, live, exchange, algo_namespace, base_currency, live_graph, - analyze_live, simulate_orders, stats_output): - """Run an algorithm in backtest, - paper-trading or live-trading mode. - - This is shared between the cli and :func:`catalyst.run_algo`. - """ - - data, algorithm = _build_algo_and_data( - handle_data, initialize, before_trading_start, analyze, algofile, - algotext, defines, data_frequency, capital_base, data, bundle, - bundle_timestamp, start, end, output, print_algo, local_namespace, - environ, live, exchange, algo_namespace, base_currency, live_graph, - analyze_live, simulate_orders, stats_output) - perf = algorithm.run( + **{ + 'initialize': initialize, + 'handle_data': handle_data, + 'before_trading_start': before_trading_start, + 'analyze': analyze, + } if algotext is None else { + 'algo_filename': getattr(algofile, 'name', ''), + 'script': algotext, + } + ).run( data, - overwrite_sim_params=False) + overwrite_sim_params=False, + ) if output == '-': click.echo(str(perf)) @@ -462,7 +413,8 @@ def load_extensions(default, extensions, strict, environ, reload=False): # without `strict` we should just log the failure warnings.warn( 'Failed to load extension: %r\n%s' % (ext, e), - stacklevel=2) + stacklevel=2 + ) else: _loaded_extensions.add(ext) @@ -561,7 +513,8 @@ def run_algorithm(initialize, catalyst.data.bundles.bundles : The available data bundles. """ load_extensions( - default_extension, extensions, strict_extensions, environ) + default_extension, extensions, strict_extensions, environ + ) if capital_base is None: raise ValueError( @@ -569,7 +522,8 @@ def run_algorithm(initialize, 'amount of base currency available for trading. For example, ' 'if the `capital_base` is 5ETH, the ' '`order_target_percent(asset, 1)` command will order 5ETH worth ' - 'of the specified asset.') + 'of the specified asset.' + ) # I'm not sure that we need this since the modified DataPortal # does not require extensions to be explicitly loaded. @@ -587,11 +541,13 @@ def run_algorithm(initialize, elif len(non_none_data) != 1: raise ValueError( 'must specify one of `data`, `data_portal`, or `bundle`,' - ' got: %r' % non_none_data) + ' got: %r' % non_none_data, + ) elif 'bundle' not in non_none_data and bundle_timestamp is not None: raise ValueError( - 'cannot specify `bundle_timestamp` without passing `bundle`') + 'cannot specify `bundle_timestamp` without passing `bundle`', + ) return _run( handle_data=handle_data, initialize=initialize, @@ -618,4 +574,5 @@ def run_algorithm(initialize, live_graph=live_graph, analyze_live=analyze_live, simulate_orders=simulate_orders, - stats_output=stats_output) + stats_output=stats_output + ) From 9515d10cefffab83e981b6d3ac4ddb220765c891 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 5 Jan 2018 00:11:04 -0500 Subject: [PATCH 6/8] DOC: updated release notes --- docs/source/releases.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index ede098c0..7ca55863 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,6 +2,14 @@ Release Notes ============= +Version 0.4.3 +^^^^^^^^^^^^^ +**Release Date**: 2017-01-05 + +Bug Fixes +~~~~~~~~~ +- Fixed CLI issue (:issue:`137`) + Version 0.4.2 ^^^^^^^^^^^^^ **Release Date**: 2017-01-03 From 595bd82234d50cb46fab0d38224dad27c94f263d Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 5 Jan 2018 00:13:18 -0500 Subject: [PATCH 7/8] BLD: improving unit tests --- .../exchange/test_suites/test_suite_bundle.py | 83 +++++++++---------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index 94952b9c..e490fc22 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -1,7 +1,7 @@ import random import pandas as pd -from logbook import Logger +from logbook import TestHandler from pandas.util.testing import assert_frame_equal from catalyst import get_calendar @@ -11,8 +11,7 @@ from catalyst.exchange.utils.exchange_utils import get_candles_df from catalyst.exchange.utils.factory import get_exchange from catalyst.exchange.utils.test_utils import output_df, \ select_random_assets - -log = Logger('TestSuiteExchange') +from catalyst.testing.fixtures import WithLogger, ZiplineTestCase pd.set_option('display.expand_frame_repr', False) pd.set_option('precision', 8) @@ -20,7 +19,7 @@ pd.set_option('display.width', 1000) pd.set_option('display.max_colwidth', 1000) -class TestSuiteBundle: +class TestSuiteBundle(WithLogger, ZiplineTestCase): @staticmethod def get_data_portal(exchange_names): open_calendar = get_calendar('OPEN') @@ -54,46 +53,46 @@ class TestSuiteBundle: """ data = dict() - log.info('creating data sample from bundle') - data['bundle'] = data_portal.get_history_window( - assets=assets, - end_dt=end_dt, - bar_count=bar_count, - frequency=freq, - field='close', - data_frequency=data_frequency, - ) - log.info('bundle data:\n{}'.format( - data['bundle'].tail(10)) - ) + log_catcher = TestHandler() + with log_catcher: + data['bundle'] = data_portal.get_history_window( + assets=assets, + end_dt=end_dt, + bar_count=bar_count, + frequency=freq, + field='close', + data_frequency=data_frequency, + ) + print('bundle data:\n{}'.format( + data['bundle'].tail(10)) + ) - log.info('creating data sample from exchange api') - candles = exchange.get_candles( - end_dt=end_dt, - freq=freq, - assets=assets, - bar_count=bar_count, - ) - data['exchange'] = get_candles_df( - candles=candles, - field='close', - freq=freq, - bar_count=bar_count, - end_dt=end_dt, - ) - log.info('exchange data:\n{}'.format( - data['exchange'].tail(10)) - ) - for source in data: - df = data[source] - path = output_df(df, assets, '{}_{}'.format(freq, source)) - log.info('saved {}:\n{}'.format(source, path)) + candles = exchange.get_candles( + end_dt=end_dt, + freq=freq, + assets=assets, + bar_count=bar_count, + ) + data['exchange'] = get_candles_df( + candles=candles, + field='close', + freq=freq, + bar_count=bar_count, + end_dt=end_dt, + ) + print('exchange data:\n{}'.format( + data['exchange'].tail(10)) + ) + for source in data: + df = data[source] + path = output_df(df, assets, '{}_{}'.format(freq, source)) + print('saved {}:\n{}'.format(source, path)) - assert_frame_equal( - right=data['bundle'], - left=data['exchange'], - check_less_precise=True, - ) + assert_frame_equal( + right=data['bundle'], + left=data['exchange'], + check_less_precise=True, + ) def test_validate_bundles(self): # exchange_population = 3 From 4d8d1e33d03f3fed38aff1f79073fe1f6f6b955a Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 5 Jan 2018 00:15:19 -0500 Subject: [PATCH 8/8] DOC: updated release notes --- docs/source/releases.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 7ca55863..e095d0b4 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -9,6 +9,7 @@ Version 0.4.3 Bug Fixes ~~~~~~~~~ - Fixed CLI issue (:issue:`137`) +- Upgraded CCXT Version 0.4.2 ^^^^^^^^^^^^^