From 8a3cc7e5faf0888f8cbdc35b6b328b9f0e1e82f7 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Thu, 1 Mar 2018 01:30:18 +0200 Subject: [PATCH] BLD: added a cmd for running on the cloud (WIP) --- catalyst/__main__.py | 174 ++++ catalyst/assets/_assets.pyx | 69 +- catalyst/constants.py | 3 +- catalyst/examples/dual_moving_average.py | 44 +- catalyst/examples/mean_reversion_simple.py | 14 +- catalyst/exchange/ccxt/ccxt_exchange.py | 421 +++++--- catalyst/exchange/exchange.py | 96 +- catalyst/exchange/exchange_algorithm.py | 81 +- catalyst/exchange/exchange_bundle.py | 87 +- catalyst/exchange/exchange_errors.py | 14 - catalyst/exchange/utils/ccxt_utils.py | 307 ------ catalyst/exchange/utils/exchange_utils.py | 164 +++- catalyst/exchange/utils/factory.py | 53 +- .../exchange/utils/serialization_utils.py | 29 +- catalyst/exchange/utils/stats_utils.py | 6 +- docs/source/beginner-tutorial.rst | 158 +-- docs/source/example-algos.rst | 898 +----------------- docs/source/install.rst | 11 +- docs/source/live-trading.rst | 9 + etc/python2.7-environment.yml | 2 + etc/python3.6-environment.yml | 37 +- tests/exchange/test_config.py | 8 - .../exchange/test_suites/test_suite_bundle.py | 2 +- .../test_suites/test_suite_exchange.py | 43 +- 24 files changed, 879 insertions(+), 1851 deletions(-) delete mode 100644 catalyst/exchange/utils/ccxt_utils.py delete mode 100644 tests/exchange/test_config.py diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 1b08f69d..6df67fde 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -14,6 +14,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.utils.exchange_utils import delete_algo_folder from catalyst.utils.cli import Date, Timestamp from catalyst.utils.run_algo import _run, load_extensions +from catalyst.utils.run_server import run_server try: __IPYTHON__ @@ -505,6 +506,179 @@ def live(ctx, return perf +@main.command(name='serve-live') +@click.option( + '-f', + '--algofile', + default=None, + type=click.File('r'), + help='The file that contains the algorithm to run.', +) +@click.option( + '--capital-base', + type=float, + show_default=True, + help='The amount of capital (in base_currency) allocated to trading.', +) +@click.option( + '-t', + '--algotext', + help='The algorithm script to run.', +) +@click.option( + '-D', + '--define', + multiple=True, + help="Define a name to be bound in the namespace before executing" + " the algotext. For example '-Dname=value'. The value may be" + " any python expression. These are evaluated in order so they" + " may refer to previously defined names.", +) +@click.option( + '-o', + '--output', + default='-', + metavar='FILENAME', + show_default=True, + help="The location to write the perf data. If this is '-' the perf will" + " be written to stdout.", +) +@click.option( + '--print-algo/--no-print-algo', + is_flag=True, + default=False, + help='Print the algorithm to stdout.', +) +@ipython_only(click.option( + '--local-namespace/--no-local-namespace', + is_flag=True, + default=None, + help='Should the algorithm methods be resolved in the local namespace.' +)) +@click.option( + '-x', + '--exchange-name', + help='The name of the targeted exchange.', +) +@click.option( + '-n', + '--algo-namespace', + help='A label assigned to the algorithm for data storage purposes.' +) +@click.option( + '-c', + '--base-currency', + help='The base currency used to calculate statistics ' + '(e.g. usd, btc, eth).', +) +@click.option( + '-e', + '--end', + type=Date(tz='utc', as_timestamp=True), + help='An optional end date at which to stop the execution.', +) +@click.option( + '--live-graph/--no-live-graph', + is_flag=True, + default=False, + help='Display live graph.', +) +@click.option( + '--simulate-orders/--no-simulate-orders', + is_flag=True, + default=True, + help='Simulating orders enable the paper trading mode. No orders will be ' + 'sent to the exchange unless set to false.', +) +@click.option( + '--auth-aliases', + default=None, + help='Authentication file aliases for the specified exchanges. By default,' + 'each exchange uses the "auth.json" file in the exchange folder. ' + 'Specifying an "auth2" alias would use "auth2.json". It should be ' + 'specified like this: "[exchange_name],[alias],..." For example, ' + '"binance,auth2" or "binance,auth2,bittrex,auth2".', +) +@click.pass_context +def serve_live(ctx, + algofile, + capital_base, + algotext, + define, + output, + print_algo, + local_namespace, + exchange_name, + algo_namespace, + base_currency, + end, + live_graph, + auth_aliases, + simulate_orders): + """Trade live with the given algorithm on the server. + """ + if (algotext is not None) == (algofile is not None): + ctx.fail( + "must specify exactly one of '-f' / '--algofile' or" + " '-t' / '--algotext'", + ) + + if exchange_name is None: + ctx.fail("must specify an exchange name '-x'") + + if algo_namespace is None: + ctx.fail("must specify an algorithm name '-n' in live execution mode") + + if base_currency is None: + ctx.fail("must specify a base currency '-c' in live execution mode") + + if capital_base is None: + ctx.fail("must specify a capital base with '--capital-base'") + + if simulate_orders: + click.echo('Running in paper trading mode.', sys.stdout) + + else: + click.echo('Running in live trading mode.', sys.stdout) + + perf = run_server( + initialize=None, + handle_data=None, + before_trading_start=None, + analyze=None, + algofile=algofile, + algotext=algotext, + defines=define, + data_frequency=None, + capital_base=capital_base, + data=None, + bundle=None, + bundle_timestamp=None, + start=None, + end=end, + output=output, + print_algo=print_algo, + local_namespace=local_namespace, + environ=os.environ, + live=True, + exchange=exchange_name, + algo_namespace=algo_namespace, + base_currency=base_currency, + live_graph=live_graph, + analyze_live=None, + simulate_orders=simulate_orders, + auth_aliases=auth_aliases, + stats_output=None, + ) + + if output == '-': + click.echo(str(perf), sys.stdout) + elif output != os.devnull: # make the catalyst magic not write any data + perf.to_pickle(output) + + return perf + + @main.command(name='ingest-exchange') @click.option( '-x', diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index f79a5b46..f80fbf4a 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -433,7 +433,7 @@ cdef class TradingPair(Asset): 'taker', 'trading_state', 'data_source', - 'decimals', + 'decimals' }) def __init__(self, object symbol, @@ -455,7 +455,7 @@ cdef class TradingPair(Asset): float taker=0.0025, float lot=0, int decimals = 8, - int trading_state=1, + int trading_state=0, object data_source='catalyst'): """ Replicates the Asset constructor with some built-in conventions @@ -600,51 +600,14 @@ cdef class TradingPair(Asset): cpdef to_dict(self): """ Convert to a python dict. - - Repeat constructor params: - object symbol, - object exchange, - object start_date=None, - object asset_name=None, - int sid=0, - float leverage=1.0, - object end_daily=None, - object end_minute=None, - object end_date=None, - object exchange_symbol=None, - object first_traded=None, - object auto_close_date=None, - object exchange_full=None, - float min_trade_size=0.0001, - float max_trade_size=1000000, - float maker=0.0015, - float taker=0.0025, - float lot=0, - int decimals = 8, - int trading_state=1, - object data_source='catalyst', """ - trading_pair_dict = dict( - symbol=self.symbol, - exchange=self.exchange, - start_date=self.start_date, - asset_name=self.asset_name, - leverage=self.leverage, - end_daily=self.end_daily, - end_minute=self.end_minute, - end_date=self.end_date, - exchange_symbol=self.exchange_symbol, - exchange_full=self.exchange_full, - min_trade_size=self.min_trade_size, - max_trade_size=self.max_trade_size, - maker=self.maker, - taker=self.taker, - lot=self.lot, - decimals=self.decimals, - trading_state=self.trading_state, - data_source=self.data_source, - ) - return trading_pair_dict + #TODO: missing fields + super_dict = super(TradingPair, self).to_dict() + super_dict['end_daily'] = self.end_daily + super_dict['end_minute'] = self.end_minute + super_dict['leverage'] = self.leverage + super_dict['min_trade_size'] = self.min_trade_size + return super_dict def is_exchange_open(self, dt_minute): """ @@ -660,16 +623,6 @@ cdef class TradingPair(Asset): #TODO: make more dymanic to catch holds return True - def set_end_date(self, dt, data_frequency): - if data_frequency == 'minute': - self.end_minute = dt - - else: - self.end_daily = dt - - def set_start_date(self, dt): - self.start_date = dt - cpdef __reduce__(self): """ Function used by pickle to determine how to serialize/deserialize this @@ -693,9 +646,7 @@ cdef class TradingPair(Asset): self.lot, self.decimals, self.taker, - self.maker, - self.trading_state, - self.data_source)) + self.maker)) def make_asset_array(int size, Asset asset): cdef np.ndarray out = np.empty([size], dtype=object) diff --git a/catalyst/constants.py b/catalyst/constants.py index b1b6200d..912d3894 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -11,8 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' -EXCHANGE_CONFIG_URL = 'https://s3.amazonaws.com/enigmaco/exchanges/' \ - '{exchange}/config.json' + DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_FORMAT = '%Y-%m-%d' diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index 00eb01af..f11e6b77 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -4,8 +4,7 @@ import pandas as pd from logbook import Logger from catalyst import run_algorithm -from catalyst.api import (record, symbol, order_target_percent, - get_open_orders) +from catalyst.api import (record, symbol, order_target_percent,) from catalyst.exchange.utils.stats_utils import extract_transactions NAMESPACE = 'dual_moving_average' @@ -20,8 +19,8 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages - short_window = 2 - long_window = 3 + short_window = 50 + long_window = 200 # Skip as many bars as long_window to properly compute the average context.i += 1 @@ -63,7 +62,7 @@ def handle_data(context, data): # Since we are using limit orders, some orders may not execute immediately # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) + orders = context.blotter.open_orders if len(orders) > 0: return @@ -150,27 +149,16 @@ def analyze(context, perf): if __name__ == '__main__': + run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - simulate_orders=True, - live=True, - ) - # run_algorithm( - # capital_base=1000, - # data_frequency='minute', - # initialize=initialize, - # handle_data=handle_data, - # analyze=analyze, - # exchange_name='bitfinex', - # algo_namespace=NAMESPACE, - # base_currency='usd', - # start=pd.to_datetime('2017-9-22', utc=True), - # end=pd.to_datetime('2017-9-23', utc=True), - # ) + capital_base=1000, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=NAMESPACE, + base_currency='usd', + start=pd.to_datetime('2017-9-22', utc=True), + end=pd.to_datetime('2017-9-23', utc=True), + ) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index d2652f8c..c697a88a 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -33,12 +33,12 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - context.market = symbol('eth_btc') + context.market = symbol('bnb_eth') context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 55 - context.RSI_OVERBOUGHT = 60 + context.RSI_OVERSOLD = 60 + context.RSI_OVERBOUGHT = 70 context.CANDLE_SIZE = '15T' context.start_time = time.time() @@ -248,14 +248,14 @@ if __name__ == '__main__': if live: run_algorithm( - capital_base=0.03, + capital_base=0.1, initialize=initialize, handle_data=handle_data, analyze=analyze, - exchange_name='poloniex', + exchange_name='binance', live=True, algo_namespace=NAMESPACE, - base_currency='btc', + base_currency='eth', live_graph=False, simulate_orders=False, stats_output=None, @@ -274,7 +274,7 @@ if __name__ == '__main__': # -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \ # --data-frequency minute --capital-base 10000 run_algorithm( - capital_base=0.1, + capital_base=0.035, data_frequency='minute', initialize=initialize, handle_data=handle_data, diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 2a35b7e4..875661e9 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1,33 +1,34 @@ +import json +import os import re from collections import defaultdict import ccxt import pandas as pd import six -from catalyst.assets._assets import TradingPair -from redo import retry +from ccxt import InvalidOrder, NetworkError, \ + ExchangeError +from logbook import Logger +from six import string_types from catalyst.algorithm import MarketOrder +from catalyst.assets._assets import TradingPair from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ - UnsupportedHistoryFrequencyError, \ ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \ - MarketsNotFoundError, InvalidMarketError + UnsupportedHistoryFrequencyError from catalyst.exchange.exchange_execution import ExchangeLimitOrder -from catalyst.exchange.utils.ccxt_utils import get_exchange_config +from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ + get_exchange_folder, get_catalyst_symbol, \ + get_exchange_auth from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \ get_epoch, \ get_periods_range -from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol from catalyst.finance.order import Order, ORDER_STATUS from catalyst.finance.transaction import Transaction -from ccxt import InvalidOrder, NetworkError, \ - ExchangeError -from logbook import Logger -from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -43,7 +44,7 @@ SUPPORTED_EXCHANGES = dict( class CCXT(Exchange): def __init__(self, exchange_name, key, - secret, password, base_currency, config=None): + secret, password, base_currency): log.debug( 'finding {} in CCXT exchanges:\n{}'.format( exchange_name, ccxt.exchanges @@ -63,8 +64,6 @@ class CCXT(Exchange): 'password': password, }) self.api.enableRateLimit = True - self.has = self.api.has - self.fees = self.api.fees except Exception: raise ExchangeNotFoundError(exchange_name=exchange_name) @@ -72,7 +71,6 @@ class CCXT(Exchange): self._symbol_maps = [None, None] self.name = exchange_name - self.assets = [] self.base_currency = base_currency self.transactions = defaultdict(list) @@ -84,123 +82,97 @@ class CCXT(Exchange): self._common_symbols = dict() self.bundle = ExchangeBundle(self.name) + self.markets = None self._is_init = False - self._config = config def init(self): if self._is_init: return - if self._config is None: - self._config = get_exchange_config(self.name) - log.debug( - 'got exchange config {}:\n{}'.format( - self.name, self._config + exchange_folder = get_exchange_folder(self.name) + filename = os.path.join(exchange_folder, 'cctx_markets.json') + + if os.path.exists(filename): + timestamp = os.path.getmtime(filename) + dt = pd.to_datetime(timestamp, unit='s', utc=True) + + if dt >= pd.Timestamp.utcnow().floor('1D'): + with open(filename) as f: + self.markets = json.load(f) + + log.debug('loaded markets for {}'.format(self.name)) + + if self.markets is None: + try: + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) ) - ) + + self.markets = self.api.fetch_markets() + with open(filename, 'w+') as f: + json.dump(self.markets, f, indent=4) + + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch markets {}: {}'.format( + self.name, e + ) + ) + raise ExchangeRequestError(error=e) self.load_assets() self._is_init = True - def load_assets(self): - if self._config is None: - raise ValueError('Exchange config not available.') + @staticmethod + def find_exchanges(features=None, is_authenticated=False): + ccxt_features = [] + if features is not None: + for feature in features: + if not feature.endswith('Bundle'): + ccxt_features.append(feature) - self.assets = [] - for asset_dict in self._config['assets']: - asset = TradingPair(**asset_dict) - self.assets.append(asset) + exchange_names = [] + for exchange_name in ccxt.exchanges: + if is_authenticated: + exchange_auth = get_exchange_auth(exchange_name) - def _fetch_markets(self): - markets_symbols = self.api.load_markets() - log.debug( - 'fetching {} markets:\n{}'.format( - self.name, markets_symbols - ) - ) - try: - markets = self.api.fetch_markets() + has_auth = (exchange_auth['key'] != '' + and exchange_auth['secret'] != '') - except NetworkError as e: - raise ExchangeRequestError(error=e) + if not has_auth: + continue - if not markets: - raise MarketsNotFoundError( - exchange=self.name, - ) + log.debug('loading exchange: {}'.format(exchange_name)) + exchange = getattr(ccxt, exchange_name)() - for market in markets: - if 'id' not in market: - raise InvalidMarketError( - exchange=self.name, - market=market, - ) - return markets + if ccxt_features is None: + has_feature = True - def create_exchange_config(self): - config = dict( - name=self.name, - features=[feature for feature in self.has if self.has[feature]] - ) - markets = retry( - action=self._fetch_markets, - attempts=5, - sleeptime=5, - retry_exceptions=(ExchangeRequestError,), - cleanup=lambda: log.warn( - 'fetching markets again for {}'.format(self.name) - ), - ) + else: + try: + has_feature = all( + [exchange.has[feature] for feature in ccxt_features] + ) - config['assets'] = [] - for market in markets: - asset = self.create_trading_pair(market=market) - config['assets'].append(asset) + except Exception: + has_feature = False - return config + if has_feature: + try: + log.info('initializing {}'.format(exchange_name)) + exchange_names.append(exchange_name) - def create_trading_pair(self, market, start_dt=None, end_dt=None, - leverage=1, end_daily=None, end_minute=None): - """ - Creating a TradingPair from market and asset data. + except Exception as e: + log.warn( + 'unable to initialize exchange {}: {}'.format( + exchange_name, e + ) + ) - Parameters - ---------- - market: dict[str, Object] - start_dt - end_dt - leverage - end_daily - end_minute - - Returns - ------- - - """ - params = dict( - exchange=self.name, - data_source='catalyst', - exchange_symbol=market['id'], - symbol=get_catalyst_symbol(market), - start_date=start_dt, - end_date=end_dt, - leverage=leverage, - asset_name=market['symbol'], - end_daily=end_daily, - end_minute=end_minute, - ) - self.apply_conditional_market_params(params, market) - - return TradingPair(**params) - - def load_assets(self): - if self._config is None or 'error' in self._config: - raise ValueError('Exchange config not available.') - - self.assets = [] - for asset_dict in self._config['assets']: - asset = TradingPair(**asset_dict) - self.assets.append(asset) + return exchange_names def account(self): return None @@ -232,6 +204,42 @@ class CCXT(Exchange): return frequencies + def get_market(self, symbol): + """ + The CCXT market. + + Parameters + ---------- + symbol: + The CCXT symbol. + + Returns + ------- + dict[str, Object] + + """ + s = self.get_symbol(symbol) + market = next( + (market for market in self.markets if market['symbol'] == s), + None, + ) + return market + + def substitute_currency_code(self, currency, source='catalyst'): + if source == 'catalyst': + currency = currency.upper() + + key = self.api.common_currency_code(currency) + self._common_symbols[key] = currency.lower() + return key + + else: + if currency in self._common_symbols: + return self._common_symbols[currency] + + else: + return currency.lower() + def get_symbol(self, asset_or_symbol, source='catalyst'): """ The CCXT symbol. @@ -249,7 +257,13 @@ class CCXT(Exchange): if source == 'ccxt': if isinstance(asset_or_symbol, string_types): parts = asset_or_symbol.split('/') - return '{}_{}'.format(parts[0].lower(), parts[1].lower()) + base_currency = self.substitute_currency_code( + parts[0], source + ) + quote_currency = self.substitute_currency_code( + parts[1], source + ) + return '{}_{}'.format(base_currency, quote_currency) else: return asset_or_symbol.symbol @@ -260,7 +274,13 @@ class CCXT(Exchange): ) else asset_or_symbol.symbol parts = symbol.split('_') - return '{}/{}'.format(parts[0].upper(), parts[1].upper()) + base_currency = self.substitute_currency_code( + parts[0], source + ) + quote_currency = self.substitute_currency_code( + parts[1], source + ) + return '{}/{}'.format(base_currency, quote_currency) @staticmethod def map_frequency(value, source='ccxt', raise_error=True): @@ -460,53 +480,144 @@ class CCXT(Exchange): except ExchangeSymbolsNotFound: return None - def apply_conditional_market_params(self, params, market): + def get_asset_defs(self, market): """ - Applies a CCXT market dict to parameters of TradingPair init. + The local and Catalyst definitions of the specified market. Parameters ---------- - params: dict[Object] - market: dict[Object] + market: dict[str, Object] + The CCXT market dicts. + + Returns + ------- + dict[str, Object] + The asset definition. + + """ + asset_defs = [] + + for is_local in (False, True): + asset_def = self.get_asset_def(market, is_local) + asset_defs.append((asset_def, is_local)) + + return asset_defs + + def get_asset_def(self, market, is_local=False): + """ + The asset definition (in symbols.json files) corresponding + to the the specified market. + + Parameters + ---------- + market: dict[str, Object] + The CCXT market dict. + is_local + Whether to search in local or Catalyst asset definitions. + + Returns + ------- + dict[str, Object] + The asset definition. + + """ + exchange_symbol = market['id'] + + symbol_map = self._fetch_symbol_map(is_local) + if symbol_map is not None: + assets_lower = {k.lower(): v for k, v in symbol_map.items()} + key = exchange_symbol.lower() + + asset = assets_lower[key] if key in assets_lower else None + if asset is not None: + return asset + + else: + return None + + else: + return None + + def create_trading_pair(self, market, asset_def=None, is_local=False): + """ + Creating a TradingPair from market and asset data. + + Parameters + ---------- + market: dict[str, Object] + asset_def: dict[str, Object] + is_local: bool Returns ------- """ - # TODO: make this more externalized / configurable - # Consider representing in some type of JSON structure - if 'active' in market: - params['trading_state'] = 1 if market['active'] else 0 + data_source = 'local' if is_local else 'catalyst' + params = dict( + exchange=self.name, + data_source=data_source, + exchange_symbol=market['id'], + ) + mixin_market_params(self.name, params, market) + + if asset_def is not None: + params['symbol'] = asset_def['symbol'] + + params['start_date'] = asset_def['start_date'] \ + if 'start_date' in asset_def else None + + params['end_date'] = asset_def['end_date'] \ + if 'end_date' in asset_def else None + + params['leverage'] = asset_def['leverage'] \ + if 'leverage' in asset_def else 1.0 + + params['asset_name'] = asset_def['asset_name'] \ + if 'asset_name' in asset_def else None + + params['end_daily'] = asset_def['end_daily'] \ + if 'end_daily' in asset_def \ + and asset_def['end_daily'] != 'N/A' else None + + params['end_minute'] = asset_def['end_minute'] \ + if 'end_minute' in asset_def \ + and asset_def['end_minute'] != 'N/A' else None else: - params['trading_state'] = 1 + params['symbol'] = get_catalyst_symbol(market) + # TODO: add as an optional column + params['leverage'] = 1.0 - if 'lot' in market: - params['min_trade_size'] = market['lot'] - params['lot'] = market['lot'] + return TradingPair(**params) - if self.name == 'bitfinex': - params['maker'] = 0.001 - params['taker'] = 0.002 + def load_assets(self): + log.debug('loading assets for {}'.format(self.name)) + self.assets = [] - elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None \ - and market['taker'] is not None: - params['maker'] = market['maker'] - params['taker'] = market['taker'] + for market in self.markets: + if 'id' not in market: + log.warn('invalid market: {}'.format(market)) + continue - else: - # TODO: default commission, make configurable - params['maker'] = 0.0015 - params['taker'] = 0.0025 + asset_defs = self.get_asset_defs(market) - info = market['info'] if 'info' in market else None - if info: - if 'minimum_order_size' in info: - params['min_trade_size'] = float(info['minimum_order_size']) + asset = None + for asset_def in asset_defs: + if asset_def[0] is not None or not asset_defs[1]: + try: + asset = self.create_trading_pair( + market=market, + asset_def=asset_def[0], + is_local=asset_def[1] + ) + self.assets.append(asset) - if 'lot' not in params: - params['lot'] = params['min_trade_size'] + except TypeError as e: + log.warn('unable to add asset: {}'.format(e)) + + if asset is None: + asset = self.create_trading_pair(market=market) + self.assets.append(asset) def get_balances(self): try: @@ -644,14 +755,18 @@ class CCXT(Exchange): side = 'buy' if amount > 0 else 'sell' if hasattr(self.api, 'amount_to_lots'): - adj_amount = self.api.amount_to_lots( - symbol=symbol, - amount=abs(amount), - ) - if adj_amount != abs(amount): - log.info( - 'adjusted order amount {} to {} based on lot size'.format( - abs(amount), adj_amount, + # TODO: is this right? + if self.api.markets is None: + self.api.load_markets() + + # https://github.com/ccxt/ccxt/issues/1483 + adj_amount = round(abs(amount), asset.decimals) + market = self.api.markets[symbol] + if 'lots' in market and market['lots'] > amount: + raise CreateOrderError( + exchange=self.name, + e='order amount lower than the smallest lot: {}'.format( + amount ) ) @@ -865,7 +980,8 @@ class CCXT(Exchange): ) raise ExchangeRequestError(error=e) - def cancel_order(self, order_param, asset_or_symbol=None): + def cancel_order(self, order_param, + asset_or_symbol=None, params={}): order_id = order_param.id \ if isinstance(order_param, Order) else order_param @@ -877,7 +993,8 @@ class CCXT(Exchange): try: symbol = self.get_symbol(asset_or_symbol) \ if asset_or_symbol is not None else None - self.api.cancel_order(id=order_id, symbol=symbol) + self.api.cancel_order(id=order_id, + symbol=symbol, params= params) except (ExchangeError, NetworkError) as e: log.warn( diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 28dfdbfe..310d12fc 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -1,12 +1,11 @@ import abc +import pytz from abc import ABCMeta, abstractmethod, abstractproperty from datetime import timedelta from time import sleep import numpy as np import pandas as pd -from logbook import Logger - from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -18,8 +17,9 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ from catalyst.exchange.utils.datetime_utils import get_delta, \ get_periods_range, \ get_periods, get_start_dt, get_frequency -from catalyst.exchange.utils.exchange_utils import \ +from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ resample_history_df, has_bundle +from logbook import Logger log = Logger('Exchange', level=LOG_LEVEL) @@ -291,6 +291,16 @@ class Exchange: log.debug('found asset: {}'.format(asset)) return asset + def fetch_symbol_map(self, is_local=False): + index = 1 if is_local else 0 + if self._symbol_maps[index] is not None: + return self._symbol_maps[index] + + else: + symbol_map = get_exchange_symbols(self.name, is_local) + self._symbol_maps[index] = symbol_map + return symbol_map + @abstractmethod def init(self): """ @@ -302,13 +312,24 @@ class Exchange: """ @abstractmethod - def create_exchange_config(self): + def load_assets(self, is_local=False): """ - Fetch the exchange market data and generate a config object - Returns - ------- + Populate the 'assets' attribute with a dictionary of Assets. + The key of the resulting dictionary is the exchange specific + currency pair symbol. The universal symbol is contained in the + 'symbol' attribute of each asset. + + Notes + ----- + The sid of each asset is calculated based on a numeric hash of the + universal symbol. This simple approach avoids maintaining a mapping + of sids. + + This method can be omerridden if an exchange offers equivalent data + via its api. """ + pass def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): """ @@ -494,32 +515,37 @@ class Exchange: series = dict() for asset in candles: - first_candle = candles[asset][0] - asset_series = self.get_series_from_candles( - candles=candles[asset], - start_dt=first_candle['last_traded'], - end_dt=end_dt, - data_frequency=frequency, - field=field, - ) - - delta_candle_size = candle_size * 60 if unit == 'H' else candle_size - # Checking to make sure that the dates match - delta = get_delta(delta_candle_size, data_frequency) - adj_end_dt = end_dt - delta - last_traded = asset_series.index[-1] - - if last_traded < adj_end_dt: - raise LastCandleTooEarlyError( - last_traded=last_traded, - end_dt=adj_end_dt, - exchange=self.name, + if candles[asset]: + first_candle = candles[asset][0] + asset_series = self.get_series_from_candles( + candles=candles[asset], + start_dt=first_candle['last_traded'], + end_dt=end_dt, + data_frequency=frequency, + field=field, ) + delta_candle_size = candle_size * 60 if unit == 'H' else candle_size + # Checking to make sure that the dates match + delta = get_delta(delta_candle_size, data_frequency) + adj_end_dt = end_dt - delta + last_traded = asset_series.index[-1] + + if last_traded < adj_end_dt: + raise LastCandleTooEarlyError( + last_traded=last_traded, + end_dt=adj_end_dt, + exchange=self.name, + ) + else: # empty candle received + # because other assets are tz-aware, we need its tz to be set as well + asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc)) + + series[asset] = asset_series df = pd.DataFrame(series) - df.dropna(inplace=True) + #df.dropna(inplace=True) # commented out due to issue 236 return df @@ -636,20 +662,16 @@ class Exchange: return df - def _check_low_balance(self, currency, balances, amount, open_orders=None): + def _check_low_balance(self, currency, balances, amount): free = balances[currency]['free'] if currency in balances else 0.0 - if open_orders: - # TODO: make sure that this works - free += sum([order.amount for order in open_orders]) - if free < amount: return free, True else: return free, False - def sync_positions(self, positions, open_orders=None, cash=None, + def sync_positions(self, positions, cash=None, check_balances=False): """ Update the portfolio cash and position balances based on the @@ -679,7 +701,7 @@ class Exchange: balances=balances, amount=cash, ) - if is_lower and not open_orders: + if is_lower: raise NotEnoughCashError( currency=self.base_currency, exchange=self.name, @@ -906,7 +928,8 @@ class Exchange: """ @abstractmethod - def cancel_order(self, order_param, symbol_or_asset=None): + def cancel_order(self, order_param, + symbol_or_asset=None, params={}): """Cancel an open order. Parameters @@ -915,6 +938,7 @@ class Exchange: The order_id or order object to cancel. symbol_or_asset: str|TradingPair The catalyst symbol, some exchanges need this + params: """ pass diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 92c307d3..b9c319f1 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -18,11 +18,9 @@ from datetime import timedelta from os import listdir from os.path import isfile, join, exists +import catalyst.protocol as zp import logbook import pandas as pd -from redo import retry - -import catalyst.protocol as zp from catalyst.algorithm import TradingAlgorithm from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange_blotter import ExchangeBlotter @@ -52,6 +50,7 @@ from catalyst.utils.api_support import api_method from catalyst.utils.input_validation import error_keywords, ensure_upper_case from catalyst.utils.math_utils import round_nearest from catalyst.utils.preprocess import preprocess +from redo import retry log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL) @@ -376,19 +375,30 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if error: log.warning(error) - self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats') + # in order to save paper & live files separately + self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live' - self.custom_signals_stats = \ - get_algo_df(self.algo_namespace, 'custom_signals_stats') + self.pnl_stats = get_algo_df( + self.algo_namespace, + 'pnl_stats_{}'.format(self.mode_name), + ) - self.exposure_stats = \ - get_algo_df(self.algo_namespace, 'exposure_stats') + self.custom_signals_stats = get_algo_df( + self.algo_namespace, + 'custom_signals_stats_{}'.format(self.mode_name) + ) + + self.exposure_stats = get_algo_df( + self.algo_namespace, + 'exposure_stats_{}'.format(self.mode_name) + ) self.is_running = True self.stats_minutes = 1 self._last_orders = [] + self._last_open_orders = [] self.trading_client = None super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs) @@ -515,7 +525,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): """ self.state = get_algo_object( algo_name=self.algo_namespace, - key='context.state', + key='context.state_{}'.format(self.mode_name), ) if self.state is None: self.state = {} @@ -538,7 +548,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # Unpacking the perf_tracker and positions if available cum_perf = get_algo_object( algo_name=self.algo_namespace, - key='cumulative_performance', + key='cumulative_performance_{}'.format(self.mode_name), ) if cum_perf is not None: tracker.cumulative_performance = cum_perf @@ -549,7 +559,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): todays_perf = get_algo_object( algo_name=self.algo_namespace, key=today.strftime('%Y-%m-%d'), - rel_path='daily_performance', + rel_path='daily_performance_{}'.format(self.mode_name), ) if todays_perf is not None: # Ensure single common position tracker @@ -641,7 +651,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): required_cash = self.portfolio.cash if not orders else None cash, positions_value = exchange.sync_positions( positions=exchange_positions, - open_orders=orders, check_balances=check_balances, cash=required_cash, ) @@ -687,7 +696,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): ) self.pnl_stats = pd.concat([self.pnl_stats, df]) - save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats) + save_algo_df( + self.algo_namespace, + 'pnl_stats_{}'.format(self.mode_name), + self.pnl_stats, + ) def add_custom_signals_stats(self, period_stats): """ @@ -708,8 +721,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): ) self.custom_signals_stats = pd.concat([self.custom_signals_stats, df]) - save_algo_df(self.algo_namespace, 'custom_signals_stats', - self.custom_signals_stats) + save_algo_df( + self.algo_namespace, + 'custom_signals_stats_{}'.format(self.mode_name), + self.custom_signals_stats, + ) def add_exposure_stats(self, period_stats): """ @@ -736,7 +752,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.exposure_stats = pd.concat([self.exposure_stats, df]) save_algo_df( - self.algo_namespace, 'exposure_stats', self.exposure_stats + self.algo_namespace, + 'exposure_stats_{}'.format(self.mode_name), + self.exposure_stats ) def nullify_frame_stats(self, now): @@ -760,6 +778,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): obj=self.frame_stats, rel_path='frame_stats' ) + error = remove_old_files( algo_name=self.algo_namespace, today=now, @@ -792,12 +811,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.nullify_frame_stats(now=data.current_dt) self.performance_needs_update = False - orders = list(self.perf_tracker.todays_performance.orders_by_id.keys()) - if orders != self._last_orders: + last_orders_list = list(self.blotter.orders.keys()) + open_orders_list = list(self.blotter.open_orders.keys()) + + if last_orders_list != self._last_orders or \ + open_orders_list != self._last_open_orders: self.performance_needs_update = True - # Saving current orders to detect changes in the next frame - self._last_orders = copy.deepcopy(orders) + # Saving current order positions + # to detect changes in the next frame + self._last_orders = copy.deepcopy(last_orders_list) + self._last_open_orders = copy.deepcopy(open_orders_list) if self.performance_needs_update: self.perf_tracker.update_performance() @@ -839,7 +863,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): log.debug('saving cumulative performance object') save_algo_object( algo_name=self.algo_namespace, - key='cumulative_performance', + key='cumulative_performance_{}'.format(self.mode_name), obj=self.perf_tracker.cumulative_performance, ) log.debug('saving todays performance object') @@ -847,12 +871,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): algo_name=self.algo_namespace, key=today.strftime('%Y-%m-%d'), obj=self.perf_tracker.todays_performance, - rel_path='daily_performance' + rel_path='daily_performance_{}'.format(self.mode_name) ) log.debug('saving context.state object') save_algo_object( algo_name=self.algo_namespace, - key='context.state', + key='context.state_{}'.format(self.mode_name), obj=self.state) def _process_stats(self, data): @@ -908,6 +932,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): csv_bytes = stats_to_algo_folder( stats=self.frame_stats, algo_namespace=self.algo_namespace, + folder_name='stats_{}'.format(self.mode_name), recorded_cols=recorded_cols, ) except Exception as e: @@ -1012,13 +1037,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): args=(order_id,)) @api_method - def cancel_order(self, order_param, exchange_name): + def cancel_order(self, order_param, exchange_name, + symbol=None, params={}): """Cancel an open order. Parameters ---------- order_param : str or Order The order_id or order object to cancel. + + exchange_name: name of exchange from + which you want to cancel the order + symbol: + params: """ exchange = self.exchanges[exchange_name] @@ -1032,4 +1063,4 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): sleeptime=self.attempts['retry_sleeptime'], retry_exceptions=(ExchangeRequestError,), cleanup=lambda: log.warn('cancelling order again.'), - args=(order_id,)) + args=(order_id, symbol, params)) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 1acb7087..569fa6e5 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -1,4 +1,3 @@ -import copy import os import shutil from datetime import timedelta @@ -9,12 +8,8 @@ from operator import is_not import numpy as np import pandas as pd import pytz -from catalyst.assets._assets import TradingPair -from logbook import Logger -from pytz import UTC -from six import itervalues - from catalyst import get_calendar +from catalyst.assets._assets import TradingPair from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST from catalyst.constants import LOG_LEVEL from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ @@ -27,11 +22,15 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \ PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ get_bcolz_chunk, get_df_from_arrays, get_assets -from catalyst.exchange.utils.datetime_utils import get_start_dt, \ +from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \ get_period_label, get_month_start_end, get_year_start_end -from catalyst.exchange.utils.exchange_utils import get_exchange_folder +from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ + save_exchange_symbols, mixin_market_params, get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory +from logbook import Logger +from pytz import UTC +from six import itervalues log = Logger('exchange_bundle', level=LOG_LEVEL) @@ -626,13 +625,13 @@ class ExchangeBundle: key=lambda chunk: pd.to_datetime(chunk['period']) ) with maybe_show_progress( - all_chunks, - show_progress, - label='Ingesting {frequency} price data on ' - '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, - )) as it: + all_chunks, + show_progress, + label='Ingesting {frequency} price data on ' + '{exchange}'.format( + exchange=self.exchange_name, + frequency=data_frequency, + )) as it: for chunk in it: problems += self.ingest_ctable( asset=chunk['asset'], @@ -701,36 +700,42 @@ class ExchangeBundle: for symbol in symbols: start_dt = df.index.get_level_values(1).min() end_dt = df.index.get_level_values(1).max() + end_dt_key = 'end_{}'.format(data_frequency) - try: - asset = self.exchange.get_asset(symbol, is_local=True) - except: - asset = copy.deepcopy(self.exchange.get_asset(symbol)) + market = self.exchange.get_market(symbol) + if market is None: + raise ValueError('symbol not available in the exchange.') - if asset.data_source == 'local': - asset.start_date = asset.start_date \ - if asset.start_date < start_dt else start_dt + params = dict( + exchange=self.exchange.name, + data_source='local', + exchange_symbol=market['id'], + ) + mixin_market_params(self.exchange_name, params, market) - if data_frequency == 'daily': - asset.end_date = asset.end_daily = asset.end_daily \ - if asset.end_daily > end_dt else end_dt + asset_def = self.exchange.get_asset_def(market, True) + if asset_def is not None: + params['symbol'] = asset_def['symbol'] - else: - asset.end_date = asset.end_minute = asset.end_minute \ - if asset.end_minute > end_dt else end_dt + params['start_date'] = asset_def['start_date'] \ + if asset_def['start_date'] < start_dt else start_dt + + params['end_date'] = asset_def[end_dt_key] \ + if asset_def[end_dt_key] > end_dt else end_dt + + params['end_daily'] = end_dt \ + if data_frequency == 'daily' else asset_def['end_daily'] + + params['end_minute'] = end_dt \ + if data_frequency == 'minute' else asset_def['end_minute'] else: - asset.data_source = 'local' - asset.start_date = start_dt - asset.end_dt = end_dt + params['symbol'] = get_catalyst_symbol(market) - if data_frequency == 'daily': - asset.end_daily = end_dt - asset.end_minute = None - - else: - asset.end_daily = None - asset.end_minute = end_dt + params['end_daily'] = end_dt \ + if data_frequency == 'daily' else 'N/A' + params['end_minute'] = end_dt \ + if data_frequency == 'minute' else 'N/A' if min_start_dt is None or start_dt < min_start_dt: min_start_dt = start_dt @@ -738,9 +743,11 @@ class ExchangeBundle: if max_end_dt is None or end_dt > max_end_dt: max_end_dt = end_dt - assets[symbol] = asset + asset = TradingPair(**params) + assets[market['id']] = asset + + save_exchange_symbols(self.exchange_name, assets, True) - # TODO: update config.json writer = self.get_writer( start_dt=min_start_dt.replace(hour=00, minute=00), end_dt=max_end_dt.replace(hour=23, minute=59), diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 91fba0da..d5af87c4 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -322,17 +322,3 @@ class BalanceTooLowError(ZiplineError): 'add positions to hold a free amount greater than {amount}, or clean ' 'the state of this algo and restart.' ).strip() - - -class MarketsNotFoundError(ZiplineError): - msg = ( - 'Exchange {exchange} contains no valid market so it is unusable in ' - 'Catalyst.' - ).strip() - - -class InvalidMarketError(ZiplineError): - msg = ( - 'Exchange {exchange} contains at least one incorrectly structured ' - 'market: {market}, so it is unusable in Catalyst.' - ).strip() diff --git a/catalyst/exchange/utils/ccxt_utils.py b/catalyst/exchange/utils/ccxt_utils.py deleted file mode 100644 index 964f966c..00000000 --- a/catalyst/exchange/utils/ccxt_utils.py +++ /dev/null @@ -1,307 +0,0 @@ -import json -import os -import pandas as pd -from six.moves.urllib import request - -from catalyst.assets._assets import TradingPair -from ccxt import NetworkError -from catalyst.constants import LOG_LEVEL, EXCHANGE_CONFIG_URL -from catalyst.exchange.exchange_errors import MarketsNotFoundError, \ - InvalidMarketError -from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol, \ - get_exchange_folder, get_exchange_auth -from catalyst.exchange.utils.serialization_utils import ExchangeJSONDecoder, \ - ExchangeJSONEncoder -from logbook import Logger -from redo import retry -from ccxt.base.exchange import Exchange -from catalyst.utils.paths import last_modified_time, data_root, \ - ensure_directory -import ccxt - -log = Logger('ccxt_utils', level=LOG_LEVEL) - - -def scan_exchange_configs(features=None, history=None, is_authenticated=False, - path=None): - """ - Finding exchanges from their config files - - Parameters - ---------- - features - is_authenticated - - Returns - ------- - - """ - for exchange_name in ccxt.exchanges: - config = get_exchange_config(exchange_name, path) - if not config or 'error' in config: - log.info( - 'skipping invalid exchange {}'.format(exchange_name) - ) - - # Check if the exchange has an auth.json file - if is_authenticated: - exchange_auth = get_exchange_auth(exchange_name) - has_auth = (exchange_auth['key'] != '' - and exchange_auth['secret'] != '') - - if not has_auth: - continue - - if features is None: - has_features = True - - else: - try: - supported_features = [ - feature for feature in features if - feature in config['features'] - ] - has_features = len(supported_features) > 0 - except Exception: - has_features = False - - # TODO: filter by history - if has_features: - yield config - - -def get_exchange_config(exchange_name, path=None, environ=None, - expiry='2H'): - """ - The de-serialized content of the exchange's config.json. - Parameters - ---------- - exchange_name: str - The exchange name - filename: str - The target file - environ: - - Returns - ------- - config: dict[srt, Object] - The config dictionary. - - """ - try: - if path is None: - root = data_root(environ) - path = os.path.join(root, 'exchanges') - - folder = os.path.join(path, exchange_name) - ensure_directory(folder) - - filename = os.path.join(folder, 'config.json') - url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name) - if os.path.isfile(filename): - # If the file exists, only update periodically to avoid - # unnecessary calls - now = pd.Timestamp.utcnow() - limit = pd.Timedelta(expiry) - if pd.Timedelta(now - last_modified_time(filename)) > limit: - try: - request.urlretrieve(url=url, filename=filename) - except Exception as e: - log.warn( - 'unable to update config {} => {}: {}'.format( - url, filename, e - ) - ) - - else: - request.urlretrieve(url=url, filename=filename) - - with open(filename) as data_file: - data = json.load(data_file, cls=ExchangeJSONDecoder) - return data - - except Exception as e: - log.warn( - 'unable to download {} config: {}'.format( - exchange_name, e - ) - ) - return dict(error=e) - - -def save_exchange_config(config, filename=None, environ=None): - """ - Save assets into an exchange_config file. - Parameters - ---------- - exchange_name: str - config - environ - Returns - ------- - """ - if filename is None: - name = 'config.json' - exchange_folder = get_exchange_folder(config['id'], environ) - filename = os.path.join(exchange_folder, name) - - with open(filename, 'w+') as handle: - json.dump(config, handle, indent=4, cls=ExchangeJSONEncoder) - - -def fetch_markets(ccxt_exchange): - """ - Fetches CCXT market objects. - - Parameters - ---------- - ccxt_exchange: Exchange - - Returns - ------- - - """ - markets_symbols = ccxt_exchange.load_markets() - log.debug( - 'fetching {} markets:\n{}'.format( - ccxt_exchange.name, markets_symbols - ) - ) - markets = ccxt_exchange.fetch_markets() - - if not markets: - raise MarketsNotFoundError( - exchange=ccxt_exchange.name, - ) - - for market in markets: - if 'id' not in market: - raise InvalidMarketError( - exchange=ccxt_exchange.name, - market=market, - ) - return markets - - -def create_exchange_config(ccxt_exchange): - """ - Creates an exchange config structure. - - Parameters - ---------- - ccxt_exchange: Exchange - - Returns - ------- - - """ - exchange_name = ccxt_exchange.__class__.__name__ - config = dict( - id=exchange_name, - name=ccxt_exchange.name, - features=[ - feature for feature in ccxt_exchange.has if - ccxt_exchange.has[feature] - ] - ) - markets = retry( - action=fetch_markets, - attempts=5, - sleeptime=5, - retry_exceptions=(NetworkError,), - cleanup=lambda: log.warn( - 'fetching markets again for {}'.format(exchange_name) - ), - args=(ccxt_exchange,) - ) - - config['assets'] = [] - for market in markets: - asset = create_trading_pair(exchange_name, market) - config['assets'].append(asset) - - return config - - -def create_trading_pair(exchange_name, market, start_dt=None, end_dt=None, - leverage=1, end_daily=None, end_minute=None): - """ - Creating a TradingPair from market and asset data. - - Parameters - ---------- - market: dict[str, Object] - start_dt - end_dt - leverage - end_daily - end_minute - - Returns - ------- - - """ - params = dict( - exchange=exchange_name, - data_source='catalyst', - exchange_symbol=market['id'], - symbol=get_catalyst_symbol(market), - start_date=start_dt, - end_date=end_dt, - leverage=leverage, - asset_name=market['symbol'], - end_daily=end_daily, - end_minute=end_minute, - ) - apply_conditional_market_params(exchange_name, params, market) - - return TradingPair(**params) - - -def apply_conditional_market_params(exchange_name, params, market): - """ - Applies a CCXT market dict to parameters of TradingPair init. - - Parameters - ---------- - params: dict[Object] - market: dict[Object] - - Returns - ------- - - """ - # TODO: make this more externalized / configurable - # Consider representing in some type of JSON structure - if 'active' in market: - params['trading_state'] = 1 if market['active'] else 0 - - else: - params['trading_state'] = 1 - - if 'lot' in market: - params['min_trade_size'] = market['lot'] - params['lot'] = market['lot'] - - if exchange_name == 'bitfinex': - params['maker'] = 0.001 - params['taker'] = 0.002 - - elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None \ - and market['taker'] is not None: - params['maker'] = market['maker'] - params['taker'] = market['taker'] - - else: - # TODO: default commission, make configurable - params['maker'] = 0.0015 - params['taker'] = 0.0025 - - info = market['info'] if 'info' in market else None - if info: - if 'minimum_order_size' in info: - params['min_trade_size'] = float(info['minimum_order_size']) - - if 'lot' not in params: - params['lot'] = params['min_trade_size'] diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 49d347b1..658a7002 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -11,14 +11,11 @@ from six import string_types from six.moves.urllib import request from catalyst.constants import DATE_FORMAT, SYMBOLS_URL -from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \ - InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias +from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ - ExchangeJSONDecoder, ConfigJSONEncoder + ExchangeJSONDecoder from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time -from six import string_types -from six.moves.urllib import request def get_sid(symbol): @@ -72,7 +69,7 @@ def is_blacklist(exchange_name, environ=None): return os.path.exists(filename) -def get_exchange_config_filename(exchange_name, environ=None): +def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): """ The absolute path of the exchange's symbol.json file. @@ -86,12 +83,12 @@ def get_exchange_config_filename(exchange_name, environ=None): str """ - name = 'config.json' + name = 'symbols.json' if not is_local else 'symbols_local.json' exchange_folder = get_exchange_folder(exchange_name, environ) return os.path.join(exchange_folder, name) -def download_exchange_config(exchange_name, filename, environ=None): +def download_exchange_symbols(exchange_name, environ=None): """ Downloads the exchange's symbols.json from the repository. @@ -105,13 +102,15 @@ def download_exchange_config(exchange_name, filename, environ=None): str """ - url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name) - request.urlretrieve(url=url, filename=filename) + filename = get_exchange_symbols_filename(exchange_name) + url = SYMBOLS_URL.format(exchange=exchange_name) + response = request.urlretrieve(url=url, filename=filename) + return response -def get_exchange_config(exchange_name, filename=None, environ=None): +def get_exchange_symbols(exchange_name, is_local=False, environ=None): """ - The de-serialized content of the exchange's config.json. + The de-serialized content of the exchange's symbols.json. Parameters ---------- @@ -124,47 +123,55 @@ def get_exchange_config(exchange_name, filename=None, environ=None): Object """ - if filename is None: - filename = get_exchange_config_filename(exchange_name) + filename = get_exchange_symbols_filename(exchange_name, is_local) + + if not is_local and (not os.path.isfile(filename) or pd.Timedelta( + pd.Timestamp('now', tz='UTC') - last_modified_time( + filename)).days > 1): + try: + download_exchange_symbols(exchange_name, environ) + except Exception: + pass if os.path.isfile(filename): - now = pd.Timestamp.utcnow() - limit = pd.Timedelta('2H') - if pd.Timedelta(now - last_modified_time(filename)) > limit: - download_exchange_config(exchange_name, filename, environ) + with open(filename) as data_file: + try: + data = json.load(data_file, cls=ExchangeJSONDecoder) + return data + except ValueError: + return dict() else: - download_exchange_config(exchange_name, filename, environ) + raise ExchangeSymbolsNotFound( + exchange=exchange_name, + filename=filename + ) - with open(filename) as data_file: - try: - data = json.load(data_file, cls=ExchangeJSONDecoder) - return data - except ValueError: - return dict() - -def save_exchange_config(exchange_name, config, filename=None, environ=None): +def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None): """ - Save assets into an exchange_config file. + Save assets into an exchange_symbols file. Parameters ---------- exchange_name: str - config + assets: list[dict[str, object]] + is_local: bool environ Returns ------- """ - if filename is None: - name = 'config.json' - exchange_folder = get_exchange_folder(exchange_name, environ) - filename = os.path.join(exchange_folder, name) + asset_dicts = dict() + for symbol in assets: + asset_dicts[symbol] = assets[symbol].to_dict() - with open(filename, 'w+') as handle: - json.dump(config, handle, indent=4, cls=ConfigJSONEncoder) + filename = get_exchange_symbols_filename( + exchange_name, is_local, environ + ) + with open(filename, 'wt') as handle: + json.dump(asset_dicts, handle, indent=4, default=symbols_serial) def get_symbols_string(assets): @@ -413,7 +420,7 @@ def clear_frame_stats_directory(algo_name): return error -def remove_old_files(algo_name, today, rel_path): +def remove_old_files(algo_name, today, rel_path, environ=None): """ remove old files from a directory to avoid overloading the disk @@ -423,27 +430,31 @@ def remove_old_files(algo_name, today, rel_path): algo_name: str today: Timestamp rel_path: str + environ: Returns ------- error: str """ + error = None - algo_folder = get_algo_folder(algo_name) + algo_folder = get_algo_folder(algo_name, environ) folder = os.path.join(algo_folder, rel_path) + ensure_directory(folder) # run on all files in the folder for f in os.listdir(folder): - creation_unix = os.path.getctime(f) - creation_time = pd.to_datetime(creation_unix, unit='s', ) + try: + file_path = os.path.join(folder, f) + creation_unix = os.path.getctime(file_path) + creation_time = pd.to_datetime(creation_unix, unit='s', utc=True) - # if the file is older than 30 days erase it - if today - pd.DateOffset(30) > creation_time: - try: - os.unlink(f) - except OSError: - error = 'unable to erase files in {}'.format(folder) + # if the file is older than 30 days erase it + if today - pd.DateOffset(30) > creation_time: + os.unlink(file_path) + except OSError: + error = 'unable to erase files in {}'.format(folder) return error @@ -501,6 +512,25 @@ def has_bundle(exchange_name, data_frequency, environ=None): return os.path.isdir(folder) +def symbols_serial(obj): + """ + JSON serializer for objects not serializable by default json code + + Parameters + ---------- + obj: Object + + Returns + ------- + str + + """ + if isinstance(obj, (datetime, date)): + return obj.floor('1D').strftime(DATE_FORMAT) + + raise TypeError("Type %s not serializable" % type(obj)) + + def perf_serial(obj): """ JSON serializer for objects not serializable by default json code @@ -590,12 +620,46 @@ def resample_history_df(df, freq, field, start_dt=None): return resampled_df -def from_ms_timestamp(ms): - return pd.to_datetime(ms, unit='ms', utc=True) +def mixin_market_params(exchange_name, params, market): + """ + Applies a CCXT market dict to parameters of TradingPair init. + Parameters + ---------- + params: dict[Object] + market: dict[Object] -def get_epoch(): - return pd.to_datetime('1970-1-1', utc=True) + Returns + ------- + + """ + # TODO: make this more externalized / configurable + if 'lot' in market: + params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] + + if exchange_name == 'bitfinex': + params['maker'] = 0.001 + params['taker'] = 0.002 + + elif 'maker' in market and 'taker' in market and \ + market['maker'] is not None and market['taker'] is not None: + + params['maker'] = market['maker'] + params['taker'] = market['taker'] + + else: + # TODO: default commission, make configurable + params['maker'] = 0.0015 + params['taker'] = 0.0025 + + info = market['info'] if 'info' in market else None + if info: + if 'minimum_order_size' in info: + params['min_trade_size'] = float(info['minimum_order_size']) + + if 'lot' not in params: + params['lot'] = params['min_trade_size'] def group_assets_by_exchange(assets): @@ -662,12 +726,14 @@ def get_candles_df(candles, field, freq, bar_count, end_dt, values = [candle[field] for candle in candles[asset]] series = pd.Series(values, index=dates) + """ series = series.reindex( periods, method='ffill', fill_value=previous_value, ) series.sort_index(inplace=True) + """ all_series[asset] = series df = pd.DataFrame(all_series) diff --git a/catalyst/exchange/utils/factory.py b/catalyst/exchange/utils/factory.py index 4465b8ca..67294f95 100644 --- a/catalyst/exchange/utils/factory.py +++ b/catalyst/exchange/utils/factory.py @@ -4,9 +4,8 @@ from catalyst.constants import LOG_LEVEL from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_errors import ExchangeAuthEmpty -from catalyst.exchange.utils.ccxt_utils import scan_exchange_configs from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \ - get_exchange_folder + get_exchange_folder, is_blacklist from logbook import Logger log = Logger('factory', level=LOG_LEVEL) @@ -14,7 +13,7 @@ exchange_cache = dict() def get_exchange(exchange_name, base_currency=None, must_authenticate=False, - skip_init=False, auth_alias=None, config=None): + skip_init=False, auth_alias=None): key = (exchange_name, base_currency) if key in exchange_cache: return exchange_cache[key] @@ -37,7 +36,6 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False, password=exchange_auth['password'] if 'password' in exchange_auth.keys() else '', base_currency=base_currency, - config=config, ) exchange_cache[key] = exchange @@ -55,8 +53,8 @@ def get_exchanges(exchange_names): return exchanges -def find_exchanges(features=None, history=None, skip_blacklist=True, path=None, - is_authenticated=False, base_currency=None): +def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False, + base_currency=None): """ Find exchanges filtered by a list of feature. @@ -74,33 +72,28 @@ def find_exchanges(features=None, history=None, skip_blacklist=True, path=None, list[Exchange] """ + exchange_names = CCXT.find_exchanges(features, is_authenticated) - return list( - scan_exchanges( - features, - history, - skip_blacklist, - path, - is_authenticated, - base_currency - ) - ) - - -def scan_exchanges(features=None, history=None, skip_blacklist=True, path=None, - is_authenticated=False, base_currency=None): - for config in scan_exchange_configs( - features=features, - history=history, - is_authenticated=is_authenticated, - path=path, - ): - if skip_blacklist and (config is None or 'error' in config): + exchanges = [] + for exchange_name in exchange_names: + if skip_blacklist and is_blacklist(exchange_name): continue - yield get_exchange( - exchange_name=config['id'], + exchange = get_exchange( + exchange_name=exchange_name, skip_init=True, base_currency=base_currency, - config=config, ) + + if features is not None: + if 'dailyBundle' in features \ + and not exchange.has_bundle('daily'): + continue + + elif 'minuteBundle' in features \ + and not exchange.has_bundle('minute'): + continue + + exchanges.append(exchange) + + return exchanges diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py index 4f849c3b..62ab74d5 100644 --- a/catalyst/exchange/utils/serialization_utils.py +++ b/catalyst/exchange/utils/serialization_utils.py @@ -3,42 +3,15 @@ import re from json import JSONEncoder import pandas as pd +from catalyst.constants import DATE_TIME_FORMAT from six import string_types -from datetime import date, datetime -from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT -from catalyst.assets._assets import TradingPair - - -class ConfigJSONEncoder(json.JSONEncoder): - def default(self, obj): - """ - JSON serializer for objects not serializable by default json code - - Parameters - ---------- - obj: Object - - Returns - ------- - str - - """ - if isinstance(obj, (datetime, date)): - return obj.floor('1D').strftime(DATE_FORMAT) - - elif isinstance(obj, TradingPair): - return obj.to_dict() - class ExchangeJSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, pd.Timestamp): return obj.strftime(DATE_TIME_FORMAT) - elif isinstance(obj, TradingPair): - return obj.to_dict() - # Let the base class default method raise the TypeError return JSONEncoder.default(self, obj) diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py index 6e2aab0b..3db79d3b 100644 --- a/catalyst/exchange/utils/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -396,7 +396,8 @@ def email_error(algo_name, dt, e, environ=None): )}) -def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): +def stats_to_algo_folder(stats, algo_namespace, + folder_name, recorded_cols=None): """ Saves the performance stats to the algo local folder. @@ -404,6 +405,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): ---------- stats: list[Object] algo_namespace: str + folder_name: str recorded_cols: list[str] Returns @@ -416,7 +418,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): timestr = time.strftime('%Y%m%d') folder = get_algo_folder(algo_namespace) - stats_folder = os.path.join(folder, 'stats') + stats_folder = os.path.join(folder, folder_name) ensure_directory(stats_folder) filename = os.path.join(stats_folder, '{}.csv'.format(timestr)) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 41afe3c0..291dd205 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -580,162 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in the ``examples`` directory: `dual_moving_average.py `_. -.. code-block:: python - - import numpy as np - import pandas as pd - from logbook import Logger - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order, record, symbol, order_target_percent, - get_open_orders) - from catalyst.exchange.utils.stats_utils import extract_transactions - - NAMESPACE = 'dual_moving_average' - log = Logger(NAMESPACE) - - def initialize(context): - context.i = 0 - context.asset = symbol('ltc_usd') - context.base_price = None - - - def handle_data(context, data): - # define the windows for the moving averages - short_window = 50 - long_window = 200 - - # Skip as many bars as long_window to properly compute the average - context.i += 1 - if context.i < long_window: - return - - # Compute moving averages calling data.history() for each - # moving average with the appropriate parameters. We choose to use - # minute bars for this simulation -> freq="1m" - # Returns a pandas dataframe. - short_mavg = data.history(context.asset, 'price', - bar_count=short_window, frequency="1m").mean() - long_mavg = data.history(context.asset, 'price', - bar_count=long_window, frequency="1m").mean() - - # Let's keep the price of our asset in a more handy variable - price = data.current(context.asset, 'price') - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - price_change = (price - context.base_price) / context.base_price - - # Save values for later inspection - record(price=price, - cash=context.portfolio.cash, - price_change=price_change, - short_mavg=short_mavg, - long_mavg=long_mavg) - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.asset): - return - - # We check what's our position on our portfolio and trade accordingly - pos_amount = context.portfolio.positions[context.asset].amount - - # Trading logic - if short_mavg > long_mavg and pos_amount == 0: - # we buy 100% of our portfolio for this asset - order_target_percent(context.asset, 1) - elif short_mavg < long_mavg and pos_amount > 0: - # we sell all our positions for this asset - order_target_percent(context.asset, 0) - - - def analyze(context, perf): - - # Get the base_currency that was passed as a parameter to the simulation - exchange = list(context.exchanges.values())[0] - base_currency = exchange.base_currency.upper() - - # First chart: Plot portfolio value using base_currency - ax1 = plt.subplot(411) - perf.loc[:, ['portfolio_value']].plot(ax=ax1) - ax1.legend_.remove() - ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) - start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Second chart: Plot asset price, moving averages and buys/sells - ax2 = plt.subplot(412, sharex=ax1) - perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price') - ax2.legend_.remove() - ax2.set_ylabel('{asset}\n({base})'.format( - asset = context.asset.symbol, - base = base_currency - )) - start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index, 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index, 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - # Third chart: Compare percentage change between our portfolio - # and the price of the asset - ax3 = plt.subplot(413, sharex=ax1) - perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3) - ax3.legend_.remove() - ax3.set_ylabel('Percent Change') - start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Fourth chart: Plot our cash - ax4 = plt.subplot(414, sharex=ax1) - perf.cash.plot(ax=ax4) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) - - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/dual_moving_average.py + :language: python In order to run the code above, you have to ingest the needed data first: diff --git a/docs/source/example-algos.rst b/docs/source/example-algos.rst index 0136b899..ec5b74a0 100644 --- a/docs/source/example-algos.rst +++ b/docs/source/example-algos.rst @@ -52,35 +52,8 @@ Buy BTC Simple Algorithm Source code: `examples/buy_btc_simple.py `_ -.. code-block:: python - - ''' - Run this example, by executing the following from your terminal: - catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle - - If you want to run this code using another exchange, make sure that - the asset is available on that exchange. For example, if you were to run - it for exchange Poloniex, you would need to edit the following line: - - context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd' - - and specify exchange poloniex as follows: - catalyst ingest-exchange -x poloniex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle - - To see which assets are available on each exchange, visit: - https://www.enigma.co/catalyst/status - ''' - - from catalyst.api import order, record, symbol - - def initialize(context): - context.asset = symbol('btc_usd') - - def handle_data(context, data): - order(context.asset, 1) - record(btc = data.current(context.asset, 'price')) +.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py + :language: python This simple algorithm does not produce any output nor displays any chart. @@ -90,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart. Buy and Hodl Algorithm ~~~~~~~~~~~~~~~~~~~~~~ -Source code: `examples/buy_and_hodl.py `_ - First ingest the historical pricing data needed to run this algorithm: .. code-block:: bash @@ -119,157 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an earlier date, you'll get an error), and the most recent date you can choose is one day prior to the current date. +Source code: `examples/buy_and_hodl.py `_ -.. code-block:: python - - #!/usr/bin/env python - # - # Copyright 2017 Enigma MPC, Inc. - # Copyright 2015 Quantopian, Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - import pandas as pd - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order_target_value, symbol, record, - cancel_order, get_open_orders, ) - - - def initialize(context): - context.ASSET_NAME = 'btc_usd' - context.TARGET_HODL_RATIO = 0.8 - context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO - - context.is_buying = True - context.asset = symbol(context.ASSET_NAME) - - context.i = 0 - - - def handle_data(context, data): - context.i += 1 - - starting_cash = context.portfolio.starting_cash - target_hodl_value = context.TARGET_HODL_RATIO * starting_cash - reserve_value = context.RESERVE_RATIO * starting_cash - - # Cancel any outstanding orders - orders = get_open_orders(context.asset) or [] - for order in orders: - cancel_order(order) - - # Stop buying after passing the reserve threshold - cash = context.portfolio.cash - if cash <= reserve_value: - context.is_buying = False - - # Retrieve current asset price from pricing data - price = data.current(context.asset, 'price') - - # Check if still buying and could (approximately) afford another purchase - if context.is_buying and cash > price: - print('buying') - # Place order to make position in asset equal to target_hodl_value - order_target_value( - context.asset, - target_hodl_value, - limit_price=price * 1.1, - ) - - record( - price=price, - volume=data.current(context.asset, 'volume'), - cash=cash, - starting_cash=context.portfolio.starting_cash, - leverage=context.account.leverage, - ) - - - def analyze(context=None, results=None): - - # Plot the portfolio and asset data. - ax1 = plt.subplot(611) - results[['portfolio_value']].plot(ax=ax1) - ax1.set_ylabel('Portfolio Value (USD)') - - ax2 = plt.subplot(612, sharex=ax1) - ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME)) - results[['price']].plot(ax=ax2) - - trans = results.ix[[t != [] for t in results.transactions]] - buys = trans.ix[ - [t[0]['amount'] > 0 for t in trans.transactions] - ] - ax2.scatter( - buys.index.to_pydatetime(), - results.price[buys.index], - marker='^', - s=100, - c='g', - label='' - ) - - ax3 = plt.subplot(613, sharex=ax1) - results[['leverage', 'alpha', 'beta']].plot(ax=ax3) - ax3.set_ylabel('Leverage ') - - ax4 = plt.subplot(614, sharex=ax1) - results[['starting_cash', 'cash']].plot(ax=ax4) - ax4.set_ylabel('Cash (USD)') - - results[[ - 'treasury', - 'algorithm', - 'benchmark', - ]] = results[[ - 'treasury_period_return', - 'algorithm_period_return', - 'benchmark_period_return', - ]] - - ax5 = plt.subplot(615, sharex=ax1) - results[[ - 'treasury', - 'algorithm', - 'benchmark', - ]].plot(ax=ax5) - ax5.set_ylabel('Percent Change') - - ax6 = plt.subplot(616, sharex=ax1) - results[['volume']].plot(ax=ax6) - ax6.set_ylabel('Volume (mCoins/5min)') - - plt.legend(loc=3) - - # Show the plot. - plt.gcf().set_size_inches(18, 8) - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=10000, - data_frequency='daily', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace='buy_and_hodl', - base_currency='usd', - start=pd.to_datetime('2015-03-01', utc=True), - end=pd.to_datetime('2017-10-31', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png @@ -278,166 +102,13 @@ one day prior to the current date. Dual Moving Average Crossover ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Source Code: `examples/dual_moving_average.py `_ - This strategy is covered in detail in the last part of `this tutorial `_. -.. code-block:: python +Source Code: `examples/dual_moving_average.py `_ - import numpy as np - import pandas as pd - from logbook import Logger - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order, record, symbol, order_target_percent, - get_open_orders) - from catalyst.exchange.stats_utils import extract_transactions - - NAMESPACE = 'dual_moving_average' - log = Logger(NAMESPACE) - - def initialize(context): - context.i = 0 - context.asset = symbol('ltc_usd') - context.base_price = None - - - def handle_data(context, data): - # define the windows for the moving averages - short_window = 50 - long_window = 200 - - # Skip as many bars as long_window to properly compute the average - context.i += 1 - if context.i < long_window: - return - - # Compute moving averages calling data.history() for each - # moving average with the appropriate parameters. We choose to use - # minute bars for this simulation -> freq="1m" - # Returns a pandas dataframe. - short_mavg = data.history(context.asset, 'price', - bar_count=short_window, frequency="1m").mean() - long_mavg = data.history(context.asset, 'price', - bar_count=long_window, frequency="1m").mean() - - # Let's keep the price of our asset in a more handy variable - price = data.current(context.asset, 'price') - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - price_change = (price - context.base_price) / context.base_price - - # Save values for later inspection - record(price=price, - cash=context.portfolio.cash, - price_change=price_change, - short_mavg=short_mavg, - long_mavg=long_mavg) - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.asset): - return - - # We check what's our position on our portfolio and trade accordingly - pos_amount = context.portfolio.positions[context.asset].amount - - # Trading logic - if short_mavg > long_mavg and pos_amount == 0: - # we buy 100% of our portfolio for this asset - order_target_percent(context.asset, 1) - elif short_mavg < long_mavg and pos_amount > 0: - # we sell all our positions for this asset - order_target_percent(context.asset, 0) - - - def analyze(context, perf): - - # Get the base_currency that was passed as a parameter to the simulation - base_currency = context.exchanges.values()[0].base_currency.upper() - - # First chart: Plot portfolio value using base_currency - ax1 = plt.subplot(411) - perf.loc[:, ['portfolio_value']].plot(ax=ax1) - ax1.legend_.remove() - ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) - start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Second chart: Plot asset price, moving averages and buys/sells - ax2 = plt.subplot(412, sharex=ax1) - perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price') - ax2.legend_.remove() - ax2.set_ylabel('{asset}\n({base})'.format( - asset = context.asset.symbol, - base = base_currency - )) - start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index, 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index, 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - # Third chart: Compare percentage change between our portfolio - # and the price of the asset - ax3 = plt.subplot(413, sharex=ax1) - perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3) - ax3.legend_.remove() - ax3.set_ylabel('Percent Change') - start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Fourth chart: Plot our cash - ax4 = plt.subplot(414, sharex=ax1) - perf.cash.plot(ax=ax4) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) - - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/dual_moving_average.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png @@ -447,8 +118,6 @@ This strategy is covered in detail in the last part of Mean Reversion Algorithm ~~~~~~~~~~~~~~~~~~~~~~~~ -Source code: `examples/mean_reversion_simple.py `_ - This algorithm is based on a simple momentum strategy. When the cryptoasset goes up quickly, we're going to buy; when it goes down quickly, we're going to sell. Hopefully, we'll ride the waves. @@ -469,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type: python mean_reversion_simple.py -.. code-block:: python +Source code: `examples/mean_reversion_simple.py `_ - import os - import tempfile - import time - - import numpy as np - import pandas as pd - import talib - from logbook import Logger - - from catalyst import run_algorithm - from catalyst.api import symbol, record, order_target_percent, get_open_orders - from catalyst.exchange.stats_utils import extract_transactions - # We give a name to the algorithm which Catalyst will use to persist its state. - # In this example, Catalyst will create the `.catalyst/data/live_algos` - # directory. If we stop and start the algorithm, Catalyst will resume its - # state using the files included in the folder. - from catalyst.utils.paths import ensure_directory - - NAMESPACE = 'mean_reversion_simple' - log = Logger(NAMESPACE) - - - # To run an algorithm in Catalyst, you need two functions: initialize and - # handle_data. - - def initialize(context): - # This initialize function sets any data or variables that you'll use in - # your algorithm. For instance, you'll want to define the trading pair (or - # trading pairs) you want to backtest. You'll also want to define any - # parameters or values you're going to use. - - # In our example, we're looking at Neo in USD. - context.neo_eth = symbol('neo_usd') - context.base_price = None - context.current_day = None - - context.RSI_OVERSOLD = 30 - context.RSI_OVERBOUGHT = 80 - context.CANDLE_SIZE = '15T' - - context.start_time = time.time() - - - def handle_data(context, data): - # This handle_data function is where the real work is done. Our data is - # minute-level tick data, and each minute is called a frame. This function - # runs on each frame of the data. - - # We flag the first period of each day. - # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle - # would only execute once. This method works with minute and daily - # frequencies. - today = data.current_dt.floor('1D') - if today != context.current_day: - context.traded_today = False - context.current_day = today - - # We're computing the volume-weighted-average-price of the security - # defined above, in the context.neo_eth variable. For this example, we're - # using three bars on the 15 min bars. - - # The frequency attribute determine the bar size. We use this convention - # for the frequency alias: - # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases - prices = data.history( - context.neo_eth, - fields='close', - bar_count=50, - frequency=context.CANDLE_SIZE - ) - - # Ta-lib calculates various technical indicator based on price and - # volume arrays. - - # In this example, we are comp - rsi = talib.RSI(prices.values, timeperiod=14) - - # We need a variable for the current price of the security to compare to - # the average. Since we are requesting two fields, data.current() - # returns a DataFrame with - current = data.current(context.neo_eth, fields=['close', 'volume']) - price = current['close'] - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - - price_change = (price - context.base_price) / context.base_price - cash = context.portfolio.cash - - # Now that we've collected all current data for this frame, we use - # the record() method to save it. This data will be available as - # a parameter of the analyze() function for further analysis. - record( - price=price, - volume=current['volume'], - price_change=price_change, - rsi=rsi[-1], - cash=cash - ) - - # We are trying to avoid over-trading by limiting our trades to - # one per day. - if context.traded_today: - return - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.neo_eth) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.neo_eth): - return - - # Another powerful built-in feature of the Catalyst backtester is the - # portfolio object. The portfolio object tracks your positions, cash, - # cost basis of specific holdings, and more. In this line, we calculate - # how long or short our position is at this minute. - pos_amount = context.portfolio.positions[context.neo_eth].amount - - if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: - log.info( - '{}: buying - price: {}, rsi: {}'.format( - data.current_dt, price, rsi[-1] - ) - ) - # Set a style for limit orders, - limit_price = price * 1.005 - order_target_percent( - context.neo_eth, 1, limit_price=limit_price - ) - context.traded_today = True - - elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0: - log.info( - '{}: selling - price: {}, rsi: {}'.format( - data.current_dt, price, rsi[-1] - ) - ) - limit_price = price * 0.995 - order_target_percent( - context.neo_eth, 0, limit_price=limit_price - ) - context.traded_today = True - - - def analyze(context=None, perf=None): - end = time.time() - log.info('elapsed time: {}'.format(end - context.start_time)) - - import matplotlib.pyplot as plt - # The base currency of the algo exchange - base_currency = context.exchanges.values()[0].base_currency.upper() - - # Plot the portfolio value over time. - ax1 = plt.subplot(611) - perf.loc[:, 'portfolio_value'].plot(ax=ax1) - ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency)) - - # Plot the price increase or decrease over time. - ax2 = plt.subplot(612, sharex=ax1) - perf.loc[:, 'price'].plot(ax=ax2, label='Price') - - ax2.set_ylabel('{asset}\n({base})'.format( - asset=context.neo_eth.symbol, base=base_currency - )) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index.floor('1 min'), 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index.floor('1 min'), 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - ax4 = plt.subplot(613, sharex=ax1) - perf.loc[:, 'cash'].plot( - ax=ax4, label='Base Currency ({})'.format(base_currency) - ) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - - perf['algorithm'] = perf.loc[:, 'algorithm_period_return'] - - ax5 = plt.subplot(614, sharex=ax1) - perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5) - ax5.set_ylabel('Percent\nChange') - - ax6 = plt.subplot(615, sharex=ax1) - perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI') - ax6.set_ylabel('RSI') - ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod') - ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod') - - if not transaction_df.empty: - ax6.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index.floor('1 min'), 'rsi'], - marker='^', - s=100, - c='green', - label='' - ) - ax6.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index.floor('1 min'), 'rsi'], - marker='v', - s=100, - c='red', - label='' - ) - plt.legend(loc=3) - start, end = ax6.get_ylim() - ax6.yaxis.set_ticks(np.arange(0, end, end/5)) - - # Show the plot. - plt.gcf().set_size_inches(18, 8) - plt.show() - pass - - - if __name__ == '__main__': - # The execution mode: backtest or live - MODE = 'backtest' - - if MODE == 'backtest': - folder = os.path.join( - tempfile.gettempdir(), 'catalyst', NAMESPACE - ) - ensure_directory(folder) - - timestr = time.strftime('%Y%m%d-%H%M%S') - out = os.path.join(folder, '{}.p'.format(timestr)) - # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000 - run_algorithm( - capital_base=10000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-10-01', utc=True), - end=pd.to_datetime('2017-11-10', utc=True), - output=out - ) - log.info('saved perf stats: {}'.format(out)) - - elif MODE == 'live': - run_algorithm( - capital_base=0.5, - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bittrex', - live=True, - algo_namespace=NAMESPACE, - base_currency='usd', - live_graph=False - ) +.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png @@ -763,8 +158,6 @@ strategy. Simple Universe ~~~~~~~~~~~~~~~ -Source code: `examples/simple_universe.py `_ - This example aims to provide an easy way for users to learn how to collect data from any given exchange and select a subset of the available currency pairs for trading. You simply need to specify the exchange and @@ -791,142 +184,10 @@ of the file: catalyst ingest-exchange -x bitfinex -f minute -.. code-block:: bash - - python simple_universe.py - -Credits: This code was originally submitted by `Abner Ayala-Acevedo -`_. Thank you! - -.. code-block:: python - - from datetime import timedelta - - import numpy as np - import pandas as pd - - from catalyst import run_algorithm - from catalyst.exchange.utils.exchange_utils import get_exchange_symbols - from catalyst.api import (symbols, ) - - - def initialize(context): - context.i = -1 # minute counter - context.exchange = context.exchanges.values()[0].name.lower() - context.base_currency = context.exchanges.values()[0].base_currency.lower() - - - def handle_data(context, data): - context.i += 1 - lookback_days = 7 # 7 days - - # current date & time in each iteration formatted into a string - now = data.current_dt - date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ') - lookback_date = now - timedelta(days=lookback_days) - # keep only the date as a string, discard the time - lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] - - one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute' - # update universe everyday at midnight - if not context.i % one_day_in_minutes: - context.universe = universe(context, lookback_date, date) - - # get data every 30 minutes - minutes = 30 - # get lookback_days of history data: that is 'lookback' number of bins - lookback = one_day_in_minutes / minutes * lookback_days - if not context.i % minutes and context.universe: - # we iterate for every pair in the current universe - for coin in context.coins: - pair = str(coin.symbol) - - # Get 30 minute interval OHLCV data. This is the standard data - # required for candlestick or indicators/signals. Return Pandas - # DataFrames. 30T means 30-minute re-sampling of one minute data. - # Adjust it to your desired time interval as needed. - opened = fill(data.history(coin, 'open', - bar_count=lookback, frequency='30T')).values - high = fill(data.history(coin, 'high', - bar_count=lookback, frequency='30T')).values - low = fill(data.history(coin, 'low', - bar_count=lookback, frequency='30T')).values - close = fill(data.history(coin, 'price', - bar_count=lookback, frequency='30T')).values - volume = fill(data.history(coin, 'volume', - bar_count=lookback, frequency='30T')).values - - # close[-1] is the last value in the set, which is the equivalent - # to current price (as in the most recent value) - # displays the minute price for each pair every 30 minutes - print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format( - now=now, - pair=pair, - o=opened[-1], - h=high[-1], - l=low[-1], - c=close[-1], - v=volume[-1], - )) - - # ------------------------------------------------------------- - # --------------- Insert Your Strategy Here ------------------- - # ------------------------------------------------------------- - - - def analyze(context=None, results=None): - pass - - - # Get the universe for a given exchange and a given base_currency market - # Example: Poloniex BTC Market - def universe(context, lookback_date, current_date): - # get all the pairs for the given exchange - json_symbols = get_exchange_symbols(context.exchange) - # convert into a DataFrame for easier processing - df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) - df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1) - df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1) - - # Filter all the pairs to get only the ones for a given base_currency - df = df[df['base_currency'] == context.base_currency] - - # Filter all the pairs to ensure that pair existed in the current date range - df = df[df.start_date < lookback_date] - df = df[df.end_daily >= current_date] - context.coins = symbols(*df.symbol) # convert all the pairs to symbols - - return df.symbol.tolist() - - - # Replace all NA, NAN or infinite values with its nearest value - def fill(series): - if isinstance(series, pd.Series): - return series.replace([np.inf, -np.inf], np.nan).ffill().bfill() - elif isinstance(series, np.ndarray): - return pd.Series(series).replace( - [np.inf, -np.inf], np.nan - ).ffill().bfill().values - else: - return series - - - if __name__ == '__main__': - start_date = pd.to_datetime('2017-11-10', utc=True) - end_date = pd.to_datetime('2017-11-13', utc=True) - - performance = run_algorithm(start=start_date, end=end_date, - capital_base=100.0, # amount of base_currency - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - data_frequency='minute', - base_currency='btc', - live=False, - live_graph=False, - algo_namespace='simple_universe') +Source code: `examples/simple_universe.py `_ +.. literalinclude:: ../../catalyst/examples/simple_universe.py + :language: python .. _portfolio_optimization: @@ -940,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used in writting the following article: `Markowitz Portfolio Optimization for Cryptocurrencies `_. -.. code-block:: python +Source code: `examples/simple_universe.py `_ - ''' - You can run this code using the Python interpreter: - - $ python portfolio_optimization.py - ''' - - from __future__ import division - import os - import pytz - import numpy as np - import pandas as pd - from scipy.optimize import minimize - import matplotlib.pyplot as plt - from datetime import datetime - - from catalyst.api import record, symbol, symbols, order_target_percent - from catalyst.utils.run_algo import run_algorithm - - np.set_printoptions(threshold='nan', suppress=True) - - - def initialize(context): - # Portfolio assets list - context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt', - 'xmr_usdt') - context.nassets = len(context.assets) - # Set the time window that will be used to compute expected return - # and asset correlations - context.window = 180 - # Set the number of days between each portfolio rebalancing - context.rebalance_period = 30 - context.i = 0 - - - def handle_data(context, data): - # Only rebalance at the beggining of the algorithm execution and - # every multiple of the rebalance period - if context.i == 0 or context.i%context.rebalance_period == 0: - n = context.window - prices = data.history(context.assets, fields='price', - bar_count=n+1, frequency='1d') - pr = np.asmatrix(prices) - t_prices = prices.iloc[1:n+1] - t_val = t_prices.values - tminus_prices = prices.iloc[0:n] - tminus_val = tminus_prices.values - # Compute daily returns (r) - r = np.asmatrix(t_val/tminus_val-1) - # Compute the expected returns of each asset with the average - # daily return for the selected time window - m = np.asmatrix(np.mean(r, axis=0)) - # ### - stds = np.std(r, axis=0) - # Compute excess returns matrix (xr) - xr = r - m - # Matrix algebra to get variance-covariance matrix - cov_m = np.dot(np.transpose(xr),xr)/n - # Compute asset correlation matrix (informative only) - corr_m = cov_m/np.dot(np.transpose(stds),stds) - - # Define portfolio optimization parameters - n_portfolios = 50000 - results_array = np.zeros((3+context.nassets,n_portfolios)) - for p in xrange(n_portfolios): - weights = np.random.random(context.nassets) - weights /= np.sum(weights) - w = np.asmatrix(weights) - p_r = np.sum(np.dot(w,np.transpose(m)))*365 - p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365) - - #store results in results array - results_array[0,p] = p_r - results_array[1,p] = p_std - #store Sharpe Ratio (return / volatility) - risk free rate element - #excluded for simplicity - results_array[2,p] = results_array[0,p] / results_array[1,p] - i = 0 - for iw in weights: - results_array[3+i,p] = weights[i] - i += 1 - - #convert results array to Pandas DataFrame - results_frame = pd.DataFrame(np.transpose(results_array), - columns=['r','stdev','sharpe']+context.assets) - #locate position of portfolio with highest Sharpe Ratio - max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()] - #locate positon of portfolio with minimum standard deviation - min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()] - - #order optimal weights for each asset - for asset in context.assets: - if data.can_trade(asset): - order_target_percent(asset, max_sharpe_port[asset]) - - #create scatter plot coloured by Sharpe Ratio - plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn') - plt.xlabel('Volatility') - plt.ylabel('Returns') - plt.colorbar() - #plot red star to highlight position of portfolio with highest Sharpe Ratio - plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200) - #plot green star to highlight position of minimum variance portfolio - plt.show() - print(max_sharpe_port) - record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m) - context.i += 1 - - - def analyze(context=None, results=None): - # Form DataFrame with selected data - data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']] - - # Save results in CSV file - filename = os.path.splitext(os.path.basename(__file__))[0] - data.to_csv(filename + '.csv') - - - # Bitcoin data is available from 2015-3-2. Dates vary for other tokens. - start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc) - end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) - results = run_algorithm(initialize=initialize, - handle_data=handle_data, - analyze=analyze, - start=start, - end=end, - exchange_name='poloniex', - capital_base=100000, ) +.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py + :language: python .. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ. :align: center diff --git a/docs/source/install.rst b/docs/source/install.rst index 5c24159f..6ef366b5 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -89,7 +89,7 @@ Once either Conda or MiniConda has been set up you can install Catalyst: .. code-block:: bash - conda env create -f python2.7-environment.yml + conda env create -f python2.7-environment.yml 4. Activate the environment (which you need to do every time you start a new session to run Catalyst): @@ -132,10 +132,19 @@ with the following steps: conda env remove --name catalyst 2. Create the environment: + + for python 2.7: .. code-block:: bash conda create --name catalyst python=2.7 scipy zlib + + or for python 3.6: + + .. code-block:: bash + + conda create --name catalyst python=3.6 scipy zlib + 3. Activate the environment: diff --git a/docs/source/live-trading.rst b/docs/source/live-trading.rst index f2b6873a..7d5f2394 100644 --- a/docs/source/live-trading.rst +++ b/docs/source/live-trading.rst @@ -184,5 +184,14 @@ Here is the breakdown of the new arguments: essentially sleep and when the predefined time comes, it would start executing. + +The `catalyst live` command offers additional parameters. +You can learn more by running the following from the command line: + +.. code-block:: bash + + catalyst live --help + + Here is a complete algorithm for reference: `Buy Low and Sell High `_ diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 2037515b..b2e2486c 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -1,9 +1,11 @@ name: catalyst channels: - defaults +- conda-forge dependencies: - certifi=2016.2.28=py27_0 - mkl=2017.0.3 +- matplotlib=2.1.2=py36_0 - numpy=1.13.1=py27_0 - openssl=1.0.2l - pip=9.0.1=py27_1 diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml index 446198e0..c93f3c82 100644 --- a/etc/python3.6-environment.yml +++ b/etc/python3.6-environment.yml @@ -1,29 +1,24 @@ name: catalyst channels: - defaults +- conda-forge dependencies: -- ca-certificates=2017.08.26=ha1e5d58_0 -- certifi=2018.1.18=py36_0 -- intel-openmp=2018.0.0=h8158457_8 -- libcxx=4.0.1=h579ed51_0 -- libcxxabi=4.0.1=hebd6815_0 -- libedit=3.1=hb4e282d_0 -- libffi=3.2.1=h475c297_4 -- libgfortran=3.0.1=h93005f0_2 -- mkl=2018.0.1=hfbd8650_4 -- ncurses=6.0=hd04f020_2 -- numpy=1.14.0=py36h8a80b8c_1 -- openssl=1.0.2n=hdbc3d79_0 -- pip=9.0.1=py36h1555ced_4 -- python=3.6.4=hc167b69_1 -- readline=7.0=hc1231fa_4 -- scipy=1.0.0=py36h1de22e9_0 +- ca-certificates=2017.08.26 +- certifi=2018.1.18 +- intel-openmp=2018.0.0 +- mkl=2018.0.1 +- numpy=1.14.0 +- openssl=1.0.2n +- matplotlib=2.1.2=py36_0 +- pip=9.0.1 +- python=3.6.4 +- scipy=1.0.0 - setuptools=38.4.0=py36_0 -- sqlite=3.22.0=h3efe00b_0 -- tk=8.6.7=h35a86e2_3 -- wheel=0.30.0=py36h5eb2c71_1 -- xz=5.2.3=h0278029_2 -- zlib=1.2.11=hf3cbc9b_2 +- sqlite=3.22.0 +- tk=8.6.7 +- wheel=0.30.0 +- xz=5.2.3 +- zlib=1.2.11 - pip: - aiodns==1.1.1 - aiohttp==3.0.1 diff --git a/tests/exchange/test_config.py b/tests/exchange/test_config.py deleted file mode 100644 index f13f526b..00000000 --- a/tests/exchange/test_config.py +++ /dev/null @@ -1,8 +0,0 @@ -from catalyst.exchange.utils.factory import get_exchange - - -class TestConfig: - def test_create_config(self): - exchange = get_exchange('binance', skip_init=True) - config = exchange.create_exchange_config() - pass diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index b546ff34..d338045d 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -197,7 +197,7 @@ class TestSuiteBundle: # population=exchange_population, # features=[bundle], # ) # Type: list[Exchange] - exchanges = [get_exchange('binance', skip_init=True)] + exchanges = [get_exchange('poloniex', skip_init=True)] data_portal = TestSuiteBundle.get_data_portal(exchanges) for exchange in exchanges: diff --git a/tests/exchange/test_suites/test_suite_exchange.py b/tests/exchange/test_suites/test_suite_exchange.py index 28ceaa7d..79f87a2a 100644 --- a/tests/exchange/test_suites/test_suite_exchange.py +++ b/tests/exchange/test_suites/test_suite_exchange.py @@ -5,28 +5,63 @@ from logging import Logger, WARNING from time import sleep import pandas as pd +from catalyst.assets._assets import TradingPair from logbook import TestHandler -from catalyst.assets._assets import TradingPair +from catalyst.exchange.exchange_errors import ExchangeRequestError from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.utils.exchange_utils import get_exchange_folder -from catalyst.exchange.utils.factory import get_exchanges, get_exchange from catalyst.exchange.utils.test_utils import select_random_exchanges, \ - select_random_assets + handle_exchange_error, select_random_assets from catalyst.testing import ZiplineTestCase from catalyst.testing.fixtures import WithLogger +from catalyst.exchange.utils.factory import get_exchanges, get_exchange log = Logger('TestSuiteExchange') class TestSuiteExchange(WithLogger, ZiplineTestCase): + def _test_markets_exchange(self, exchange, attempts=0): + assets = None + try: + exchange.init() + + # Verify that the assets and markets are populated + if not exchange.markets: + raise ValueError( + 'no markets found' + ) + if not exchange.assets: + raise ValueError( + 'no assets derived from markets' + ) + assets = exchange.assets + + except ExchangeRequestError as e: + sleep(5) + + if attempts > 5: + handle_exchange_error(exchange, e) + + else: + print( + 're-trying an exchange request {} {}'.format( + exchange.name, attempts + ) + ) + self._test_markets_exchange(exchange, attempts + 1) + + except Exception as e: + handle_exchange_error(exchange, e) + + return assets + def test_markets(self): population = 3 results = dict() exchanges = select_random_exchanges(population) # Type: list[Exchange] for exchange in exchanges: - exchange.init() assets = self._test_markets_exchange(exchange) if assets is not None: