diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 7dfe91b1..7e8cdd25 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -28,9 +28,9 @@ except NameError: '--strict-extensions/--non-strict-extensions', is_flag=True, help='If --strict-extensions is passed then catalyst will not run if it' - ' cannot load all of the specified extensions. If this is not passed or' - ' --non-strict-extensions is passed then the failure will be logged but' - ' execution will continue.', + ' cannot load all of the specified extensions. If this is not passed or' + ' --non-strict-extensions is passed then the failure will be logged but' + ' execution will continue.', ) @click.option( '--default-extension/--no-default-extension', @@ -64,6 +64,7 @@ def extract_option_object(option): option_object : click.Option The option object that this decorator will create. """ + @option def opt(): pass @@ -95,7 +96,9 @@ def ipython_only(option): def _(*args, **kwargs): kwargs[argname] = None return f(*args, **kwargs) + return _ + return d @@ -117,9 +120,9 @@ def ipython_only(option): '--define', multiple=True, help="Define a name to be bound in the namespace before executing" - " the algotext. For example '-Dname=value'. The value may be any python" - " expression. These are evaluated in order so they may refer to previously" - " defined names.", + " the algotext. For example '-Dname=value'. The value may be any python" + " expression. These are evaluated in order so they may refer to previously" + " defined names.", ) @click.option( '--data-frequency', @@ -149,7 +152,7 @@ def ipython_only(option): default=pd.Timestamp.utcnow(), show_default=False, help='The date to lookup data on or before.\n' - '[default: ]' + '[default: ]' ) @click.option( '-s', @@ -170,7 +173,7 @@ def ipython_only(option): metavar='FILENAME', show_default=True, help="The location to write the perf data. If this is '-' the perf will" - " be written to stdout.", + " be written to stdout.", ) @click.option( '--print-algo/--no-print-algo', @@ -184,6 +187,29 @@ def ipython_only(option): default=None, help='Should the algorithm methods be resolved in the local namespace.' )) +@click.option( + '--live/--no-live', + is_flag=True, + default=False, + help='Enable live trading.', +) +@click.option( + '-x', + '--exchange-name', + type=click.Choice({'bitfinex'}), + help='The name of the exchange (supported: bitfinex).', +) +@click.option( + '-n', + '--algo-name', + help='A label assigned to the algorithm for tracking purposes.', +) +@click.option( + '-c', + '--reference-currency', + help='The reference currency used to calculate statistics ' + '(e.g. usd, btc, eth).', +) @click.pass_context def run(ctx, algofile, @@ -197,21 +223,37 @@ def run(ctx, end, output, print_algo, - local_namespace): + local_namespace, + live, + exchange_name, + algo_namespace, + base_currency): """Run a backtest for the given algorithm. """ - # check that the start and end dates are passed correctly - if start is None and end is None: - # check both at the same time to avoid the case where a user - # does not pass either of these and then passes the first only - # to be told they need to pass the second argument also - ctx.fail( - "must specify dates with '-s' / '--start' and '-e' / '--end'", - ) - if start is None: - ctx.fail("must specify a start date with '-s' / '--start'") - if end is None: - ctx.fail("must specify an end date with '-e' / '--end'") + + if live: + if exchange_name is None: + ctx.fail("must specify an exchange name '-x' in live execution " + "mode '--live'") + if algo_namespace is None: + ctx.fail("must specify an algorithm name '-n' in live execution " + "mode '--live'") + if base_currency is None: + ctx.fail("must specify a reference currency '-c' in live " + "execution mode '--live'") + else: + # check that the start and end dates are passed correctly + if start is None and end is None: + # check both at the same time to avoid the case where a user + # does not pass either of these and then passes the first only + # to be told they need to pass the second argument also + ctx.fail( + "must specify dates with '-s' / '--start' and '-e' / '--end'", + ) + if start is None: + ctx.fail("must specify a start date with '-s' / '--start'") + if end is None: + ctx.fail("must specify an end date with '-e' / '--end'") if (algotext is not None) == (algofile is not None): ctx.fail( @@ -238,6 +280,10 @@ def run(ctx, print_algo=print_algo, local_namespace=local_namespace, environ=os.environ, + live=live, + exchange=exchange_name, + algo_namespace=algo_namespace, + base_currency=base_currency ) if output == '-': @@ -265,11 +311,11 @@ def catalyst_magic(line, cell=None): '--algotext', cell, '--output', os.devnull, # don't write the results by default ] + ([ - # these options are set when running in line magic mode - # set a non None algo text to use the ipython user_ns - '--algotext', '', - '--local-namespace', - ] if cell is None else []) + line.split(), + # these options are set when running in line magic mode + # set a non None algo text to use the ipython user_ns + '--algotext', '', + '--local-namespace', + ] if cell is None else []) + line.split(), '%s%%catalyst' % ((cell or '') and '%'), # don't use system exit and propogate errors to the caller standalone_mode=False, diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 40b8f32f..3b559726 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -9,7 +9,7 @@ from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \ from catalyst.utils.paths import data_root, ensure_directory SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'live-trading/catalyst/exchange/symbols/{exchange}.json' + 'exchange-trading/catalyst/exchange/{exchange}/symbols.json' def get_exchange_folder(exchange_name, environ=None): diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index d63c314a..e48e4b64 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -3,12 +3,18 @@ import re from runpy import run_path import sys import warnings +from time import sleep +from datetime import timedelta + +import pandas as pd import click + try: from pygments import highlight from pygments.lexers import PythonLexer from pygments.formatters import TerminalFormatter + PYGMENTS = True except: PYGMENTS = False @@ -29,6 +35,21 @@ from catalyst.utils.calendars import get_calendar from catalyst.utils.factory import create_simulation_parameters import catalyst.utils.paths as pth +from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm +from catalyst.exchange.data_portal_exchange import DataPortalExchange +from catalyst.exchange.bitfinex.bitfinex import Bitfinex +from catalyst.exchange.asset_finder_exchange import AssetFinderExchange +from catalyst.exchange.exchange_portfolio import ExchangePortfolio +from catalyst.exchange.exchange_errors import ( + ExchangeRequestError, + ExchangeRequestErrorTooManyAttempts +) +from catalyst.exchange.exchange_utils import get_exchange_auth, \ + get_algo_object +from logbook import Logger + +log = Logger('run_algo') + class _RunAlgoError(click.ClickException, ValueError): """Signal an error that should have a different message if invoked from @@ -68,7 +89,11 @@ def _run(handle_data, output, print_algo, local_namespace, - environ): + environ, + live, + exchange, + algo_namespace, + base_currency): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. @@ -117,6 +142,18 @@ def _run(handle_data, else: click.echo(algotext) + if exchange is not None: + start = pd.Timestamp.utcnow() + end = start + timedelta(minutes=1439) + + open_calendar = get_calendar('OPEN') + sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=capital_base, + data_frequency=data_frequency, + emission_rate=data_frequency, + ) if bundle is not None: bundles = bundle.split(',') @@ -146,8 +183,6 @@ def _run(handle_data, str(bundle_data.asset_finder.engine.url), ) - open_calendar = get_calendar('OPEN') - env = TradingEnvironment( load=partial(load_crypto_market_data, environ=environ), bm_symbol='USDT_BTC', @@ -179,16 +214,16 @@ def _run(handle_data, if b == 'poloniex': return CryptoPricingLoader( - bundle_data, - data_frequency, - CryptoPricing, - ) + bundle_data, + data_frequency, + CryptoPricing, + ) elif b == 'quandl': return USEquityPricingLoader( - bundle_data, - data_frequency, - USEquityPricing, - ) + bundle_data, + data_frequency, + USEquityPricing, + ) raise ValueError( "No PipelineLoader registered for bundle %s." % b ) @@ -205,20 +240,65 @@ def _run(handle_data, ) else: - env = TradingEnvironment(environ=environ) - choose_loader = None + if live and exchange is not None: + env = TradingEnvironment( + environ=environ, + exchange_tz="UTC", + asset_db_path=None + ) + env.asset_finder = AssetFinderExchange(exchange) - perf = TradingAlgorithm( + data = DataPortalExchange( + exchange=exchange, + asset_finder=env.asset_finder, + trading_calendar=open_calendar, + first_trading_day=pd.to_datetime('today', utc=True) + ) + choose_loader = None + + def update_portfolio(attempt_index=0): + """ + Fetch the portfolio for the exchange + We can't continue on error because it is required to bootstrap + the algorithm. + :param attempt_index: + :return: + """ + try: + exchange.update_portfolio() + return exchange.portfolio + except ExchangeRequestError as e: + if attempt_index < 20: + sleep(5) + return update_portfolio(attempt_index + 1) + else: + raise ExchangeRequestErrorTooManyAttempts( + attempts=attempt_index, + error=e + ) + + portfolio = update_portfolio() + sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=portfolio.starting_cash, + emission_rate='minute', + data_frequency='minute' + ) + else: + env = TradingEnvironment(environ=environ) + choose_loader = None + + TradingAlgorithmClass = ( + partial(ExchangeTradingAlgorithm, exchange=exchange, + algo_namespace=algo_namespace) + if live and exchange else TradingAlgorithm) + + perf = TradingAlgorithmClass( namespace=namespace, env=env, get_pipeline_loader=choose_loader, - sim_params=create_simulation_parameters( - start=start, - end=end, - capital_base=capital_base, - data_frequency=data_frequency, - emission_rate=data_frequency, - ), + sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, @@ -294,10 +374,10 @@ def load_extensions(default, extensions, strict, environ, reload=False): _loaded_extensions.add(ext) -def run_algorithm(start, - end, - initialize, - capital_base, +def run_algorithm(initialize, + capital_base=None, + start=None, + end=None, handle_data=None, before_trading_start=None, analyze=None, @@ -308,7 +388,11 @@ def run_algorithm(start, default_extension=True, extensions=(), strict_extensions=True, - environ=os.environ): + environ=os.environ, + live=False, + exchange_name=None, + base_currency=None, + algo_namespace=None): """Run a trading algorithm. Parameters @@ -362,6 +446,12 @@ def run_algorithm(start, environ : mapping[str -> str], optional The os environment to use. Many extensions use this to get parameters. This defaults to ``os.environ``. + live: execute live trading + exchange_conn: The exchange connection parameters + + Supported Exchanges + ------------------- + bitfinex Returns ------- @@ -372,26 +462,53 @@ def run_algorithm(start, -------- catalyst.data.bundles.bundles : The available data bundles. """ + mode = 'live' if live else 'backtest' + log.info('running algo in {mode} mode'.format(mode=mode)) load_extensions(default_extension, extensions, strict_extensions, environ) - non_none_data = valfilter(bool, { - 'data': data is not None, - 'bundle': bundle is not None, - }) - if not non_none_data: - # if neither data nor bundle are passed use 'quantopian-quandl' - bundle = 'quantopian-quandl' + exchange = None + if mode == 'backtest': + non_none_data = valfilter(bool, { + 'data': data is not None, + 'bundle': bundle is not None, + }) + if not non_none_data: + # if neither data nor bundle are passed use 'quantopian-quandl' + bundle = 'quantopian-quandl' - elif len(non_none_data) != 1: - raise ValueError( - 'must specify one of `data`, `data_portal`, or `bundle`,' - ' got: %r' % non_none_data, - ) + elif len(non_none_data) != 1: + raise ValueError( + 'must specify one of `data`, `data_portal`, or `bundle`,' + ' got: %r' % non_none_data, + ) - elif 'bundle' not in non_none_data and bundle_timestamp is not None: - raise ValueError( - 'cannot specify `bundle_timestamp` without passing `bundle`', - ) + elif 'bundle' not in non_none_data and bundle_timestamp is not None: + raise ValueError( + 'cannot specify `bundle_timestamp` without passing `bundle`', + ) + else: + if exchange_name is not None: + portfolio = get_algo_object( + algo_name=algo_namespace, + key='portfolio_{}'.format(exchange_name), + environ=environ + ) + if portfolio is None: + portfolio = ExchangePortfolio( + start_date=pd.Timestamp.utcnow() + ) + + exchange_auth = get_exchange_auth(exchange_name) + if exchange_name == 'bitfinex': + exchange = Bitfinex( + key=exchange_auth['key'], + secret=exchange_auth['secret'].encode('UTF-8'), + base_currency=base_currency, + portfolio=portfolio + ) + else: + raise NotImplementedError( + 'exchange not supported: %s' % exchange_name) return _run( handle_data=handle_data, @@ -412,4 +529,8 @@ def run_algorithm(start, print_algo=False, local_namespace=False, environ=environ, + live=live, + exchange=exchange, + algo_namespace=algo_namespace, + base_currency=base_currency )