diff --git a/Dockerfile b/Dockerfile index b36313c9..340b2883 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,13 +11,13 @@ # # https://127.0.0.1 # -# default password is jupyter. to provide another, see: +# Default password is 'jupyter'. To provide another, see: # http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password # -# once generated, you can pass the new value via `docker run --env` the first time +# Once generated, you can pass the new value via `docker run --env` the first time # you start the container. # -# You can also run an algo using the docker exec command. For example: +# You can also run an algo using the docker exec command. For example: # # docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle # diff --git a/Dockerfile-dev b/Dockerfile-dev index 4a729651..a44d3bf0 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -5,7 +5,7 @@ # # Note: the dev build requires a quantopian/catalyst image, which you can build as follows: # -# docker build -t quantopian/catalyst -f Dockerfile +# docker build -t quantopian/catalyst -f Dockerfile . # # To run the container: # @@ -15,13 +15,13 @@ # # https://127.0.0.1 # -# default password is jupyter. to provide another, see: +# Default password is 'jupyter'. To provide another, see: # http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password # -# once generated, you can pass the new value via `docker run --env` the first time +# Once generated, you can pass the new value via `docker run --env` the first time # you start the container. # -# You can also run an algo using the docker exec command. For example: +# You can also run an algo using the docker exec command. For example: # # docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle # diff --git a/catalyst/__main__.py b/catalyst/__main__.py index db37f24d..82febb0b 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -9,7 +9,7 @@ from six import text_type from catalyst.data import bundles as bundles_module from catalyst.exchange.exchange_bundle import ExchangeBundle -from catalyst.exchange.exchange_utils import delete_algo_folder +from catalyst.exchange.utils.exchange_utils import delete_algo_folder from catalyst.utils.cli import Date, Timestamp from catalyst.utils.run_algo import _run, load_extensions diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index 07e7e813..f80fbf4a 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -17,6 +17,7 @@ """ Cythonized Asset object. """ + import hashlib cimport cython @@ -38,7 +39,7 @@ from numpy cimport int64_t import warnings cimport numpy as np -from catalyst.exchange.exchange_utils import get_sid +from catalyst.exchange.utils.exchange_utils import get_sid from catalyst.utils.calendars import get_calendar from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError diff --git a/catalyst/constants.py b/catalyst/constants.py index c4111fdd..8cfd3a6b 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -7,7 +7,7 @@ import logbook For example, if you want to see the DEBUG messages, run: $ export CATALYST_LOG_LEVEL=10 ''' -LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) +LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.DEBUG)) SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ '{exchange}/symbols.json' diff --git a/catalyst/curate/poloniex.py b/catalyst/curate/poloniex.py index d09afcc6..b58d0677 100644 --- a/catalyst/curate/poloniex.py +++ b/catalyst/curate/poloniex.py @@ -1,16 +1,16 @@ -import os -import time -import shutil -import json import csv +import json +import os +import shutil +import time from datetime import datetime +import logbook import pandas as pd import requests -import logbook - -from catalyst.exchange.exchange_utils import get_exchange_symbols_filename +from catalyst.exchange.utils.exchange_utils import \ + get_exchange_symbols_filename DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())) DT_END = pd.to_datetime('today').value // 10 ** 9 @@ -193,7 +193,8 @@ class PoloniexCurator(object): for this currencyPair ''' try: - if('end_file' in locals() and end_file + 3600 < end): + if(temp is not None + or ('end_file' in locals() and end_file + 3600 < end)): if (temp is None): temp = os.tmpfile() tempcsv = csv.writer(temp) @@ -261,7 +262,7 @@ class PoloniexCurator(object): vol = df['total'].to_frame('volume') # set Vol aside df.drop('total', axis=1, inplace=True) # Drop volume data ohlc = df.resample('T').ohlc() # Resample OHLC 1min - ohlc.cols = ohlc.cols.map(lambda t: t[1]) # Raname cols + ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Rename cols closes = ohlc['close'].fillna(method='pad') # Pad fwd missing close ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill NA w/ last close vol = vol.resample('T').sum().fillna(0) # Add volumes by bin diff --git a/catalyst/data/loader.py b/catalyst/data/loader.py index b9227b3e..bfe6c701 100644 --- a/catalyst/data/loader.py +++ b/catalyst/data/loader.py @@ -22,6 +22,7 @@ from pandas_datareader.data import DataReader from six import iteritems from six.moves.urllib_error import HTTPError +from catalyst.constants import LOG_LEVEL from catalyst.utils.calendars import get_calendar from . import treasuries, treasuries_can from .benchmarks import get_benchmark_returns @@ -31,8 +32,6 @@ from ..utils.paths import ( data_root, ) -from catalyst.constants import LOG_LEVEL - logger = logbook.Logger('Loader', level=LOG_LEVEL) # Mapping from index symbol to appropriate bond data @@ -143,7 +142,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None, if exchange is None: # This is exceptional, since placing the import at the module scope # breaks things and it's only needed here - from catalyst.exchange.factory import get_exchange + from catalyst.exchange.utils.factory import get_exchange exchange = get_exchange( exchange_name='poloniex', base_currency='usdt' ) diff --git a/catalyst/examples/arbitrage_with_interface.py b/catalyst/examples/arbitrage_with_interface.py index 67459e9a..ce2c7e13 100644 --- a/catalyst/examples/arbitrage_with_interface.py +++ b/catalyst/examples/arbitrage_with_interface.py @@ -6,7 +6,7 @@ from catalyst.api import ( symbol, get_open_orders ) -from catalyst.exchange.stats_utils import get_pretty_stats +from catalyst.exchange.utils.stats_utils import get_pretty_stats from catalyst.utils.run_algo import run_algorithm algo_namespace = 'arbitrage_eth_btc' diff --git a/catalyst/examples/buy_low_sell_high.py b/catalyst/examples/buy_low_sell_high.py index f4c53973..51f965b4 100644 --- a/catalyst/examples/buy_low_sell_high.py +++ b/catalyst/examples/buy_low_sell_high.py @@ -1,19 +1,7 @@ -''' -This algorithm requires an additional library (ta-lib) beyond those -required by catalyst. Install it first by running: -$ pip install TA-Lib - -If you get build errors like: - "fatal error: ta-lib/ta_libc.h: No such file or directory" -it typically means that it can't find the underlying TA-Lib library and it -needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for -instructions on how to install the required dependencies. -''' - import talib +import pandas as pd from logbook import Logger -from catalyst import run_algorithm from catalyst.api import ( order, order_target_percent, @@ -21,59 +9,52 @@ from catalyst.api import ( record, get_open_orders, ) -from catalyst.exchange.stats_utils import get_pretty_stats -import pandas as pd +from catalyst.exchange.utils.stats_utils import get_pretty_stats +from catalyst.utils.run_algo import run_algorithm -algo_namespace = 'buy_low_sell_high_xrp' -log = Logger(algo_namespace) +algo_namespace = 'buy_the_dip_live' +log = Logger('buy low sell high') def initialize(context): log.info('initializing algo') - context.ASSET_NAME = 'XRP_USDT' + context.ASSET_NAME = 'btc_usdt' context.asset = symbol(context.ASSET_NAME) - context.TARGET_POSITIONS = 5000 + context.TARGET_POSITIONS = 30 context.PROFIT_TARGET = 0.1 - context.SLIPPAGE_ALLOWED = 0.05 - - context.retry_check_open_orders = 10 - context.retry_update_portfolio = 10 - context.retry_order = 5 - - context.swallow_errors = True + context.SLIPPAGE_ALLOWED = 0.02 context.errors = [] pass def _handle_data(context, data): + price = data.current(context.asset, 'price') + log.info('got price {price}'.format(price=price)) + prices = data.history( context.asset, fields='price', bar_count=20, - frequency='15m' + frequency='1D' ) - rsi = talib.RSI(prices.values, timeperiod=14)[-1] log.info('got rsi: {}'.format(rsi)) # Buying more when RSI is low, this should lower our cost basis if rsi <= 30: - buy_increment = 50 + buy_increment = 1 elif rsi <= 40: - buy_increment = 20 + buy_increment = 0.5 elif rsi <= 70: - buy_increment = 5 + buy_increment = 0.2 else: - buy_increment = None + buy_increment = 0.1 cash = context.portfolio.cash log.info('base currency available: {cash}'.format(cash=cash)) - price = data.current(context.asset, 'price') - log.info('got price {price}'.format(price=price)) - record( price=price, rsi=rsi, @@ -141,11 +122,11 @@ def _handle_data(context, data): def handle_data(context, data): log.info('handling bar {}'.format(data.current_dt)) - try: - _handle_data(context, data) - except Exception as e: - log.warn('aborting the bar on error {}'.format(e)) - context.errors.append(e) + # try: + _handle_data(context, data) + # except Exception as e: + # log.warn('aborting the bar on error {}'.format(e)) + # context.errors.append(e) log.info('completed bar {}, total execution errors {}'.format( data.current_dt, @@ -162,15 +143,29 @@ def analyze(context, stats): if __name__ == '__main__': - run_algorithm( - capital_base=10000, - data_frequency='daily', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='poloniex', - algo_namespace='buy_and_hodl', - base_currency='usd', - start=pd.to_datetime('2015-03-01', utc=True), - end=pd.to_datetime('2017-10-31', utc=True), - ) + live = False + if live: + run_algorithm( + capital_base=0.001, + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='binance', + live=True, + algo_namespace=algo_namespace, + base_currency='btc', + simulate_orders=True, + ) + else: + run_algorithm( + capital_base=10000, + data_frequency='daily', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace='buy_and_hodl', + base_currency='usdt', + start=pd.to_datetime('2015-03-01', utc=True), + end=pd.to_datetime('2017-10-31', utc=True), + ) diff --git a/catalyst/examples/buy_low_sell_high_live.py b/catalyst/examples/buy_low_sell_high_live.py deleted file mode 100644 index 34b1e5f6..00000000 --- a/catalyst/examples/buy_low_sell_high_live.py +++ /dev/null @@ -1,160 +0,0 @@ -import talib -from logbook import Logger - -import pandas as pd -from catalyst.api import ( - order, - order_target_percent, - symbol, - record, - get_open_orders, -) -from catalyst.exchange.stats_utils import get_pretty_stats -from catalyst.utils.run_algo import run_algorithm - -algo_namespace = 'buy_the_dip_live' -log = Logger('buy low sell high') - - -def initialize(context): - log.info('initializing algo') - context.ASSET_NAME = 'btc_usdt' - context.asset = symbol(context.ASSET_NAME) - - context.TARGET_POSITIONS = 30 - context.PROFIT_TARGET = 0.1 - context.SLIPPAGE_ALLOWED = 0.02 - - context.retry_check_open_orders = 10 - context.retry_update_portfolio = 10 - context.retry_order = 5 - - context.errors = [] - pass - - -def _handle_data(context, data): - price = data.current(context.asset, 'price') - log.info('got price {price}'.format(price=price)) - - prices = data.history( - context.asset, - fields='price', - bar_count=20, - frequency='1D' - ) - rsi = talib.RSI(prices.values, timeperiod=14)[-1] - log.info('got rsi: {}'.format(rsi)) - - # Buying more when RSI is low, this should lower our cost basis - if rsi <= 30: - buy_increment = 1 - elif rsi <= 40: - buy_increment = 0.5 - elif rsi <= 70: - buy_increment = 0.2 - else: - buy_increment = 0.1 - - cash = context.portfolio.cash - log.info('base currency available: {cash}'.format(cash=cash)) - - record( - price=price, - rsi=rsi, - ) - - orders = get_open_orders(context.asset) - if orders: - log.info('skipping bar until all open orders execute') - return - - is_buy = False - cost_basis = None - if context.asset in context.portfolio.positions: - position = context.portfolio.positions[context.asset] - - cost_basis = position.cost_basis - log.info( - 'found {amount} positions with cost basis {cost_basis}'.format( - amount=position.amount, - cost_basis=cost_basis - ) - ) - - if position.amount >= context.TARGET_POSITIONS: - log.info('reached positions target: {}'.format(position.amount)) - return - - if price < cost_basis: - is_buy = True - elif (position.amount > 0 - and price > cost_basis * (1 + context.PROFIT_TARGET)): - profit = (price * position.amount) - (cost_basis * position.amount) - log.info('closing position, taking profit: {}'.format(profit)) - order_target_percent( - asset=context.asset, - target=0, - limit_price=price * (1 - context.SLIPPAGE_ALLOWED), - ) - else: - log.info('no buy or sell opportunity found') - else: - is_buy = True - - if is_buy: - if buy_increment is None: - log.info('the rsi is too high to consider buying {}'.format(rsi)) - return - - if price * buy_increment > cash: - log.info('not enough base currency to consider buying') - return - - log.info( - 'buying position cheaper than cost basis {} < {}'.format( - price, - cost_basis - ) - ) - order( - asset=context.asset, - amount=buy_increment, - limit_price=price * (1 + context.SLIPPAGE_ALLOWED) - ) - - -def handle_data(context, data): - log.info('handling bar {}'.format(data.current_dt)) - # try: - _handle_data(context, data) - # except Exception as e: - # log.warn('aborting the bar on error {}'.format(e)) - # context.errors.append(e) - - log.info('completed bar {}, total execution errors {}'.format( - data.current_dt, - len(context.errors) - )) - - if len(context.errors) > 0: - log.info('the errors:\n{}'.format(context.errors)) - - -def analyze(context, stats): - log.info('the daily stats:\n{}'.format(get_pretty_stats(stats))) - pass - - -if __name__ == '__main__': - run_algorithm( - capital_base=0.001, - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='binance', - live=True, - algo_namespace=algo_namespace, - base_currency='btc', - simulate_orders=True, - ) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index f54d91b6..3a0a3f1f 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -1,12 +1,12 @@ +import matplotlib.pyplot as plt import numpy as np import pandas as pd from logbook import Logger -import matplotlib.pyplot as plt from catalyst import run_algorithm from catalyst.api import (record, symbol, order_target_percent, get_open_orders) -from catalyst.exchange.stats_utils import extract_transactions +from catalyst.exchange.utils.stats_utils import extract_transactions NAMESPACE = 'dual_moving_average' log = Logger(NAMESPACE) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index b3fb7934..0b96a7ed 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -12,8 +12,7 @@ from logbook import Logger from catalyst import run_algorithm from catalyst.api import symbol, record, order_target_percent, get_open_orders -from catalyst.exchange.stats_utils import extract_transactions - +from catalyst.exchange.utils.stats_utils import extract_transactions # We give a name to the algorithm which Catalyst will use to persist its state. # In this example, Catalyst will create the `.catalyst/data/live_algos` # directory. If we stop and start the algorithm, Catalyst will resume its @@ -34,12 +33,12 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - context.market = symbol('neo_eth') + context.market = symbol('eth_btc') context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 30 - context.RSI_OVERBOUGHT = 80 + context.RSI_OVERSOLD = 50 + context.RSI_OVERBOUGHT = 65 context.CANDLE_SIZE = '5T' context.start_time = time.time() @@ -245,9 +244,24 @@ def analyze(context=None, perf=None): if __name__ == '__main__': # The execution mode: backtest or live - MODE = 'backtest' + live = True - if MODE == 'backtest': + if live: + run_algorithm( + capital_base=0.03, + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + live=True, + algo_namespace=NAMESPACE, + base_currency='btc', + live_graph=False, + simulate_orders=False, + stats_output=None, + ) + + else: folder = os.path.join( tempfile.gettempdir(), 'catalyst', NAMESPACE ) @@ -272,18 +286,3 @@ if __name__ == '__main__': output=out ) log.info('saved perf stats: {}'.format(out)) - - elif MODE == 'live': - run_algorithm( - capital_base=0.05, - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='binance', - live=True, - algo_namespace=NAMESPACE, - base_currency='eth', - live_graph=False, - simulate_orders=True, - stats_output=None - ) diff --git a/catalyst/examples/simple_loop.py b/catalyst/examples/simple_loop.py index 51ea435c..1e639264 100644 --- a/catalyst/examples/simple_loop.py +++ b/catalyst/examples/simple_loop.py @@ -1,23 +1,26 @@ -import talib import pandas as pd +import talib +from logbook import Logger, INFO from catalyst import run_algorithm from catalyst.api import symbol, record -from catalyst.exchange.stats_utils import get_pretty_stats, \ +from catalyst.exchange.utils.stats_utils import get_pretty_stats, \ extract_transactions +log = Logger('simple_loop', level=INFO) + def initialize(context): - print('initializing') + log.info('initializing') context.asset = symbol('eth_btc') context.base_price = None def handle_data(context, data): - print('handling bar: {}'.format(data.current_dt)) + log.info('handling bar: {}'.format(data.current_dt)) price = data.current(context.asset, 'close') - print('got price {price}'.format(price=price)) + log.info('got price {price}'.format(price=price)) prices = data.history( context.asset, @@ -26,10 +29,10 @@ def handle_data(context, data): frequency='30T' ) last_traded = prices.index[-1] - print('last candle date: {}'.format(last_traded)) + log.info('last candle date: {}'.format(last_traded)) rsi = talib.RSI(prices.values, timeperiod=14)[-1] - print('got rsi: {}'.format(rsi)) + log.info('got rsi: {}'.format(rsi)) # If base_price is not set, we use the current value. This is the # price at the first bar which we reference to calculate price_change. @@ -51,7 +54,7 @@ def handle_data(context, data): def analyze(context, perf): import matplotlib.pyplot as plt - print('the stats: {}'.format(get_pretty_stats(perf))) + log.info('the stats: {}'.format(get_pretty_stats(perf))) # The base currency of the algo exchange base_currency = context.exchanges.values()[0].base_currency.upper() @@ -111,15 +114,31 @@ def analyze(context, perf): if __name__ == '__main__': - run_algorithm( - capital_base=1, - initialize=initialize, - handle_data=handle_data, - analyze=None, - exchange_name='poloniex', - live=True, - algo_namespace='simple_loop', - base_currency='eth', - live_graph=False, - simulate_orders=True - ) + mode = 'backtest' + + if mode == 'backtest': + run_algorithm( + capital_base=1, + initialize=initialize, + handle_data=handle_data, + analyze=None, + exchange_name='poloniex', + algo_namespace='simple_loop', + base_currency='eth', + data_frequency='minute', + start=pd.to_datetime('2017-9-1', utc=True), + end=pd.to_datetime('2017-12-1', utc=True), + ) + else: + run_algorithm( + capital_base=1, + initialize=initialize, + handle_data=handle_data, + analyze=None, + exchange_name='binance', + live=True, + algo_namespace='simple_loop', + base_currency='eth', + live_graph=False, + simulate_orders=True + ) diff --git a/catalyst/examples/simple_universe.py b/catalyst/examples/simple_universe.py index fec0c340..e781281f 100644 --- a/catalyst/examples/simple_universe.py +++ b/catalyst/examples/simple_universe.py @@ -35,8 +35,8 @@ import numpy as np import pandas as pd from catalyst import run_algorithm -from catalyst.exchange.exchange_utils import get_exchange_symbols from catalyst.api import (symbols, ) +from catalyst.exchange.utils.exchange_utils import get_exchange_symbols def initialize(context): diff --git a/catalyst/examples/talib_simple.py b/catalyst/examples/talib_simple.py index 5129dda9..cd9eee59 100644 --- a/catalyst/examples/talib_simple.py +++ b/catalyst/examples/talib_simple.py @@ -23,7 +23,7 @@ from catalyst.api import ( order_target_percent, symbol, ) -from catalyst.exchange.stats_utils import get_pretty_stats +from catalyst.exchange.utils.stats_utils import get_pretty_stats algo_namespace = 'talib_sample' log = Logger(algo_namespace) diff --git a/catalyst/exchange/asset_finder_exchange.py b/catalyst/exchange/asset_finder_exchange.py deleted file mode 100644 index 2cf3aa4e..00000000 --- a/catalyst/exchange/asset_finder_exchange.py +++ /dev/null @@ -1,99 +0,0 @@ -from logbook import Logger - -from catalyst.constants import LOG_LEVEL - -log = Logger('AssetFinderExchange', level=LOG_LEVEL) - - -class AssetFinderExchange(object): - def __init__(self): - self._asset_cache = {} - - @property - def sids(self): - """ - This seems to be used to pre-fetch assets. - I don't think that we need this for live-trading. - Leaving the list empty. - """ - return list() - - def retrieve_all(self, sids, default_none=False): - """ - Retrieve all assets in `sids`. - - Parameters - ---------- - sids : iterable of int - Assets to retrieve. - default_none : bool - If True, return None for failed lookups. - If False, raise `SidsNotFound`. - - Returns - ------- - assets : list[Asset or None] - A list of the same length as `sids` containing Assets (or Nones) - corresponding to the requested sids. - - Raises - ------ - SidsNotFound - When a requested sid is not found and default_none=False. - """ - # for sid in sids: - # if sid in self._asset_cache: - # log.debug('got asset from cache: {}'.format(sid)) - # else: - # log.debug('fetching asset: {}'.format(sid)) - return list() - - def lookup_symbol(self, symbol, exchange, data_frequency=None, - as_of_date=None, fuzzy=False): - """Lookup an asset by symbol. - - Parameters - ---------- - symbol : str - The ticker symbol to resolve. - as_of_date : datetime or None - Look up the last owner of this symbol as of this datetime. - If ``as_of_date`` is None, then this can only resolve the equity - if exactly one equity has ever owned the ticker. - fuzzy : bool, optional - Should fuzzy symbol matching be used? Fuzzy symbol matching - attempts to resolve differences in representations for - shareclasses. For example, some people may represent the ``A`` - shareclass of ``BRK`` as ``BRK.A``, where others could write - ``BRK_A``. - - Returns - ------- - equity : Asset - The equity that held ``symbol`` on the given ``as_of_date``, or the - only equity to hold ``symbol`` if ``as_of_date`` is None. - - Raises - ------ - SymbolNotFound - Raised when no equity has ever held the given symbol. - MultipleSymbolsFound - Raised when no ``as_of_date`` is given and more than one equity - has held ``symbol``. This is also raised when ``fuzzy=True`` and - there are multiple candidates for the given ``symbol`` on the - ``as_of_date``. - """ - log.debug('looking up symbol: {} {}'.format(symbol, exchange.name)) - - if data_frequency is not None: - key = ','.join([exchange.name, symbol, data_frequency]) - - else: - key = ','.join([exchange.name, symbol]) - - if key in self._asset_cache: - return self._asset_cache[key] - else: - asset = exchange.get_asset(symbol, data_frequency) - self._asset_cache[key] = asset - return asset diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py deleted file mode 100644 index 591e8486..00000000 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ /dev/null @@ -1,709 +0,0 @@ -import base64 -import datetime -import hashlib -import hmac -import json -import re -import time - -import numpy as np -import pandas as pd -import pytz -import requests -import six -from catalyst.assets._assets import TradingPair -from logbook import Logger - -from catalyst.constants import LOG_LEVEL -from catalyst.exchange.exchange import Exchange -from catalyst.exchange.exchange_bundle import ExchangeBundle -from catalyst.exchange.exchange_errors import ( - ExchangeRequestError, - InvalidHistoryFrequencyError, - InvalidOrderStyle, OrderCancelError) -from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ - ExchangeStopLimitOrder, ExchangeStopOrder -from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ - download_exchange_symbols, get_symbols_string -from catalyst.finance.order import Order, ORDER_STATUS -from catalyst.protocol import Account - -# Trying to account for REST api instability -# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request -from catalyst.utils.deprecate import deprecated - -requests.adapters.DEFAULT_RETRIES = 20 - -BITFINEX_URL = 'https://api.bitfinex.com' - -log = Logger('Bitfinex', level=LOG_LEVEL) -warning_logger = Logger('AlgoWarning') - - -@deprecated -class Bitfinex(Exchange): - def __init__(self, key, secret, base_currency, portfolio=None): - self.url = BITFINEX_URL - self.key = key - self.secret = secret.encode('UTF-8') - self.name = 'bitfinex' - self.color = 'green' - - self.assets = dict() - self.load_assets() - - self.local_assets = dict() - self.load_assets(is_local=True) - - self.base_currency = base_currency - self._portfolio = portfolio - self.minute_writer = None - self.minute_reader = None - - # The candle limit for each request - self.num_candles_limit = 1000 - - # Max is 90 but playing it safe - # https://www.bitfinex.com/posts/188 - self.max_requests_per_minute = 80 - self.request_cpt = dict() - - self.bundle = ExchangeBundle(self.name) - - def _request(self, operation, data, version='v1'): - payload_object = { - 'request': '/{}/{}'.format(version, operation), - 'nonce': '{0:f}'.format(time.time() * 1000000), - # convert to string - 'options': {} - } - - if data is None: - payload_dict = payload_object - else: - payload_dict = payload_object.copy() - payload_dict.update(data) - - payload_json = json.dumps(payload_dict) - if six.PY3: - payload = base64.b64encode(bytes(payload_json, 'utf-8')) - else: - payload = base64.b64encode(payload_json) - - m = hmac.new(self.secret, payload, hashlib.sha384) - m = m.hexdigest() - - # headers - headers = { - 'X-BFX-APIKEY': self.key, - 'X-BFX-PAYLOAD': payload, - 'X-BFX-SIGNATURE': m - } - - if data is None: - request = requests.get( - '{url}/{version}/{operation}'.format( - url=self.url, - version=version, - operation=operation - ), data={}, - headers=headers) - else: - request = requests.post( - '{url}/{version}/{operation}'.format( - url=self.url, - version=version, - operation=operation - ), - headers=headers) - - return request - - def _get_v2_symbol(self, asset): - pair = asset.symbol.split('_') - symbol = 't' + pair[0].upper() + pair[1].upper() - return symbol - - def _get_v2_symbols(self, assets): - """ - Workaround to support Bitfinex v2 - TODO: Might require a separate asset dictionary - - :param assets: - :return: - """ - - v2_symbols = [] - for asset in assets: - v2_symbols.append(self._get_v2_symbol(asset)) - - return v2_symbols - - def _create_order(self, order_status): - """ - Create a Catalyst order object from a Bitfinex order dictionary - :param order_status: - :return: Order - """ - if order_status['is_cancelled']: - status = ORDER_STATUS.CANCELLED - elif not order_status['is_live']: - log.info('found executed order {}'.format(order_status)) - status = ORDER_STATUS.FILLED - else: - status = ORDER_STATUS.OPEN - - amount = float(order_status['original_amount']) - filled = float(order_status['executed_amount']) - - if order_status['side'] == 'sell': - amount = -amount - filled = -filled - - price = float(order_status['price']) - order_type = order_status['type'] - - stop_price = None - limit_price = None - - # TODO: is this comprehensive enough? - if order_type.endswith('limit'): - limit_price = price - elif order_type.endswith('stop'): - stop_price = price - - executed_price = float(order_status['avg_execution_price']) - - # TODO: bitfinex does not specify comission. - # I could calculate it but not sure if it's worth it. - commission = None - - date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) - date = pytz.utc.localize(date) - order = Order( - dt=date, - asset=self.assets[order_status['symbol']], - amount=amount, - stop=stop_price, - limit=limit_price, - filled=filled, - id=str(order_status['id']), - commission=commission - ) - order.status = status - - return order, executed_price - - def get_balances(self): - log.debug('retrieving wallets balances') - try: - self.ask_request() - response = self._request('balances', None) - balances = response.json() - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'message' in balances: - raise ExchangeRequestError( - error='unable to fetch balance {}'.format(balances['message']) - ) - - std_balances = dict() - for balance in balances: - currency = balance['currency'].lower() - std_balances[currency] = float(balance['available']) - - return std_balances - - @property - def account(self): - account = Account() - - account.settled_cash = None - account.accrued_interest = None - account.buying_power = None - account.equity_with_loan = None - account.total_positions_value = None - account.total_positions_exposure = None - account.regt_equity = None - account.regt_margin = None - account.initial_margin_requirement = None - account.maintenance_margin_requirement = None - account.available_funds = None - account.excess_liquidity = None - account.cushion = None - account.day_trades_remaining = None - account.leverage = None - account.net_leverage = None - account.net_liquidation = None - - return account - - @property - def time_skew(self): - # TODO: research the time skew conditions - return pd.Timedelta('0s') - - def get_account(self): - # TODO: fetch account data and keep in cache - return None - - def get_candles(self, freq, assets, bar_count=None, - start_dt=None, end_dt=None): - """ - Retrieve OHLVC candles from Bitfinex - - :param data_frequency: - :param assets: - :param bar_count: - :return: - - Available Frequencies - --------------------- - '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D', - '1M' - """ - log.debug( - 'retrieving {bars} {freq} candles on {exchange} from ' - '{end_dt} for markets {symbols}, '.format( - bars=bar_count, - freq=freq, - exchange=self.name, - end_dt=end_dt, - symbols=get_symbols_string(assets) - ) - ) - - allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T', - '360T', '720T', '1D', '7D', '14D', '30D'] - if freq not in allowed_frequencies: - raise InvalidHistoryFrequencyError(frequency=freq) - - freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I) - if freq_match: - number = int(freq_match.group(1)) - unit = freq_match.group(2) - - if unit == 'T': - if number in [60, 180, 360, 720]: - number = number / 60 - converted_unit = 'h' - else: - converted_unit = 'm' - else: - converted_unit = unit - - frequency = '{}{}'.format(number, converted_unit) - - else: - raise InvalidHistoryFrequencyError(frequency=freq) - - # Making sure that assets are iterable - asset_list = [assets] if isinstance(assets, TradingPair) else assets - ohlc_map = dict() - for asset in asset_list: - symbol = self._get_v2_symbol(asset) - url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format( - url=self.url, - frequency=frequency, - symbol=symbol - ) - - if bar_count: - is_list = True - url += '/hist?limit={}'.format(int(bar_count)) - - def get_ms(date): - epoch = datetime.datetime.utcfromtimestamp(0) - epoch = epoch.replace(tzinfo=pytz.UTC) - - return (date - epoch).total_seconds() * 1000.0 - - if start_dt is not None: - start_ms = get_ms(start_dt) - url += '&start={0:f}'.format(start_ms) - - if end_dt is not None: - end_ms = get_ms(end_dt) - url += '&end={0:f}'.format(end_ms) - - else: - is_list = False - url += '/last' - - try: - self.ask_request() - response = requests.get(url) - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'error' in response.content: - raise ExchangeRequestError( - error='Unable to retrieve candles: {}'.format( - response.content) - ) - - candles = response.json() - - def ohlc_from_candle(candle): - last_traded = pd.Timestamp.utcfromtimestamp( - candle[0] / 1000.0) - last_traded = last_traded.replace(tzinfo=pytz.UTC) - ohlc = dict( - open=np.float64(candle[1]), - high=np.float64(candle[3]), - low=np.float64(candle[4]), - close=np.float64(candle[2]), - volume=np.float64(candle[5]), - price=np.float64(candle[2]), - last_traded=last_traded - ) - return ohlc - - if is_list: - ohlc_bars = [] - # We can to list candles from old to new - for candle in reversed(candles): - ohlc = ohlc_from_candle(candle) - ohlc_bars.append(ohlc) - - ohlc_map[asset] = ohlc_bars - - else: - ohlc = ohlc_from_candle(candles) - ohlc_map[asset] = ohlc - - return ohlc_map[assets] \ - if isinstance(assets, TradingPair) else ohlc_map - - def create_order(self, asset, amount, is_buy, style): - """ - Creating order on the exchange. - - :param asset: - :param amount: - :param is_buy: - :param style: - :return: - """ - exchange_symbol = self.get_symbol(asset) - if isinstance(style, ExchangeLimitOrder) \ - or isinstance(style, ExchangeStopLimitOrder): - price = style.get_limit_price(is_buy) - order_type = 'limit' - - elif isinstance(style, ExchangeStopOrder): - price = style.get_stop_price(is_buy) - order_type = 'stop' - - else: - raise InvalidOrderStyle(exchange=self.name, - style=style.__class__.__name__) - - req = dict( - symbol=exchange_symbol, - amount=str(float(abs(amount))), - price="{:.20f}".format(float(price)), - side='buy' if is_buy else 'sell', - type='exchange ' + order_type, # TODO: support margin trades - exchange=self.name, - is_hidden=False, - is_postonly=False, - use_all_available=0, - ocoorder=False, - buy_price_oco=0, - sell_price_oco=0 - ) - - date = pd.Timestamp.utcnow() - try: - self.ask_request() - response = self._request('order/new', req) - order_status = response.json() - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'message' in order_status: - raise ExchangeRequestError( - error='unable to create Bitfinex order {}'.format( - order_status['message']) - ) - - order_id = str(order_status['id']) - order = Order( - dt=date, - asset=asset, - amount=amount, - stop=style.get_stop_price(is_buy), - limit=style.get_limit_price(is_buy), - id=order_id - ) - - return order - - def get_open_orders(self, asset=None): - """Retrieve all of the current open orders. - - Parameters - ---------- - asset : Asset - If passed and not None, return only the open orders for the given - asset instead of all open orders. - - Returns - ------- - open_orders : dict[list[Order]] or list[Order] - If no asset is passed this will return a dict mapping Assets - to a list containing all the open orders for the asset. - If an asset is passed then this will return a list of the open - orders for this asset. - """ - try: - self.ask_request() - response = self._request('orders', None) - order_statuses = response.json() - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'message' in order_statuses: - raise ExchangeRequestError( - error='Unable to retrieve open orders: {}'.format( - order_statuses['message']) - ) - - orders = [] - for order_status in order_statuses: - order, executed_price = self._create_order(order_status) - if asset is None or asset == order.sid: - orders.append(order) - - return orders - - def get_order(self, order_id): - """Lookup an order based on the order id returned from one of the - order functions. - - Parameters - ---------- - order_id : str - The unique identifier for the order. - - Returns - ------- - order : Order - The order object. - """ - try: - self.ask_request() - response = self._request( - 'order/status', {'order_id': int(order_id)}) - order_status = response.json() - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'message' in order_status: - raise ExchangeRequestError( - error='Unable to retrieve order status: {}'.format( - order_status['message']) - ) - return self._create_order(order_status) - - def cancel_order(self, order_param): - """Cancel an open order. - - Parameters - ---------- - order_param : str or Order - The order_id or order object to cancel. - """ - order_id = order_param.id \ - if isinstance(order_param, Order) else order_param - - try: - self.ask_request() - response = self._request('order/cancel', {'order_id': order_id}) - status = response.json() - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'message' in status: - raise OrderCancelError( - order_id=order_id, - exchange=self.name, - error=status['message'] - ) - - def tickers(self, assets): - """ - Fetch ticket data for assets - https://docs.bitfinex.com/v2/reference#rest-public-tickers - - :param assets: - :return: - """ - symbols = self._get_v2_symbols(assets) - log.debug('fetching tickers {}'.format(symbols)) - - try: - self.ask_request() - response = requests.get( - '{url}/v2/tickers?symbols={symbols}'.format( - url=self.url, - symbols=','.join(symbols), - ) - ) - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'error' in response.content: - raise ExchangeRequestError( - error='Unable to retrieve tickers: {}'.format( - response.content) - ) - - try: - tickers = response.json() - except Exception as e: - raise ExchangeRequestError(error=e) - - ticks = dict() - for index, ticker in enumerate(tickers): - if not len(ticker) == 11: - raise ExchangeRequestError( - error='Invalid ticker in response: {}'.format(ticker) - ) - - ticks[assets[index]] = dict( - timestamp=pd.Timestamp.utcnow(), - bid=ticker[1], - ask=ticker[3], - last_price=ticker[7], - low=ticker[10], - high=ticker[9], - volume=ticker[8], - ) - - log.debug('got tickers {}'.format(ticks)) - return ticks - - def generate_symbols_json(self, filename=None, source_dates=False): - symbol_map = {} - - if not source_dates: - fn, r = download_exchange_symbols(self.name) - with open(fn) as data_file: - cached_symbols = json.load(data_file) - - response = self._request('symbols', None) - - for symbol in response.json(): - if (source_dates): - start_date = self.get_symbol_start_date(symbol) - else: - try: - start_date = cached_symbols[symbol]['start_date'] - except KeyError: - start_date = time.strftime('%Y-%m-%d') - - try: - end_daily = cached_symbols[symbol]['end_daily'] - except KeyError: - end_daily = 'N/A' - - try: - end_minute = cached_symbols[symbol]['end_minute'] - except KeyError: - end_minute = 'N/A' - - symbol_map[symbol] = dict( - symbol=symbol[:-3] + '_' + symbol[-3:], - start_date=start_date, - end_daily=end_daily, - end_minute=end_minute, - ) - - if (filename is None): - filename = get_exchange_symbols_filename(self.name) - - with open(filename, 'w') as f: - json.dump(symbol_map, f, sort_keys=True, indent=2, - separators=(',', ':')) - - def get_symbol_start_date(self, symbol): - - print(symbol) - symbol_v2 = 't' + symbol.upper() - - """ - For each symbol we retrieve candles with Monhtly resolution - We get the first month, and query again with daily resolution - around that date, and we get the first date - """ - url = '{url}/v2/candles/trade:1M:{symbol}/hist'.format( - url=self.url, - symbol=symbol_v2 - ) - - try: - self.ask_request() - response = requests.get(url) - except Exception as e: - raise ExchangeRequestError(error=e) - - """ - If we don't get any data back for our monthly-resolution query - it means that symbol started trading less than a month ago, so - arbitrarily set the ref. date to 15 days ago to be safe with - +/- 31 days - """ - if (len(response.json())): - startmonth = response.json()[-1][0] - else: - startmonth = int((time.time() - 15 * 24 * 3600) * 1000) - - """ - Query again with daily resolution setting the start and end around - the startmonth we got above. Avoid end dates greater than - now: time.time() - """ - url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}' - '&end={end}').format( - url=self.url, - symbol=symbol_v2, - start=startmonth - 3600 * 24 * 31 * 1000, - end=min(startmonth + 3600 * 24 * 31 * 1000, - int(time.time() * 1000))) - - try: - self.ask_request() - response = requests.get(url) - except Exception as e: - raise ExchangeRequestError(error=e) - - return time.strftime('%Y-%m-%d', - time.gmtime(int(response.json()[-1][0] / 1000))) - - def get_orderbook(self, asset, order_type='all', limit=100): - exchange_symbol = asset.exchange_symbol - try: - self.ask_request() - # TODO: implement limit - response = self._request( - 'book/{}'.format(exchange_symbol), None) - data = response.json() - - except Exception as e: - raise ExchangeRequestError(error=e) - - # TODO: filter by type - result = dict() - for order_type in data: - result[order_type] = [] - - for entry in data[order_type]: - result[order_type].append(dict( - rate=float(entry['price']), - quantity=float(entry['amount']) - )) - - return result diff --git a/catalyst/exchange/bitfinex/symbols.json b/catalyst/exchange/bitfinex/symbols.json deleted file mode 100644 index ab0f38f9..00000000 --- a/catalyst/exchange/bitfinex/symbols.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "neobtc": { - "symbol": "neo_btc", - "start_date": "2017-09-07", - "precision": 5 - }, - "neousd": { - "symbol": "neo_usd", - "start_date": "2017-09-07" - }, - "neoeth": { - "symbol": "neo_eth", - "start_date": "2017-09-07" - }, - "btcusd": { - "symbol": "btc_usd", - "start_date": "2010-01-01" - }, - "bchusd": { - "symbol": "bch_usd", - "start_date": "2010-01-01" - }, - "ltcusd": { - "symbol": "ltc_usd", - "start_date": "2010-01-01" - }, - "ltcbtc": { - "symbol": "ltc_btc", - "start_date": "2010-01-01" - }, - "ethusd": { - "symbol": "eth_usd", - "start_date": "2017-01-01" - }, - "ethbtc": { - "symbol": "eth_btc", - "start_date": "2017-01-01" - }, - "etcbtc": { - "symbol": "etc_btc", - "start_date": "2017-01-01" - }, - "etcusd": { - "symbol": "etc_usd", - "start_date": "2017-01-01" - }, - "rrtusd": { - "symbol": "rrt_usd", - "start_date": "2010-01-01" - }, - "rrtbtc": { - "symbol": "rrt_btc", - "start_date": "2010-01-01" - }, - "zecusd": { - "symbol": "zec_usd", - "start_date": "2010-01-01" - }, - "zecbtc": { - "symbol": "zec_btc", - "start_date": "2010-01-01" - }, - "xmrusd": { - "symbol": "xmr_usd", - "start_date": "2010-01-01" - }, - "xmrbtc": { - "symbol": "xmr_btc", - "start_date": "2010-01-01" - }, - "dshusd": { - "symbol": "dsh_usd", - "start_date": "2010-01-01" - }, - "dshbtc": { - "symbol": "dsh_btc", - "start_date": "2010-01-01" - }, - "bccbtc": { - "symbol": "bcc_btc", - "start_date": "2010-01-01" - }, - "bcubtc": { - "symbol": "bcu_btc", - "start_date": "2010-01-01" - }, - "bccusd": { - "symbol": "bcc_usd", - "start_date": "2010-01-01" - }, - "bcuusd": { - "symbol": "bcu_usd", - "start_date": "2010-01-01" - }, - "xrpusd": { - "symbol": "xrp_usd", - "start_date": "2010-01-01" - }, - "xrpbtc": { - "symbol": "xrp_btc", - "start_date": "2010-01-01" - }, - "iotusd": { - "symbol": "iot_usd", - "start_date": "2010-01-01" - }, - "iotbtc": { - "symbol": "iot_btc", - "start_date": "2010-01-01" - }, - "ioteth": { - "symbol": "iot_eth", - "start_date": "2010-01-01" - }, - "eosusd": { - "symbol": "eos_usd", - "start_date": "2010-01-01" - }, - "eosbtc": { - "symbol": "eos_btc", - "start_date": "2010-01-01" - }, - "eoseth": { - "symbol": "eos_eth", - "start_date": "2010-01-01" - } -} \ No newline at end of file diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py deleted file mode 100644 index 3057d501..00000000 --- a/catalyst/exchange/bittrex/bittrex.py +++ /dev/null @@ -1,417 +0,0 @@ -import json -import time - -import pandas as pd -from catalyst.assets._assets import TradingPair -from logbook import Logger -from six.moves import urllib - -from catalyst.constants import LOG_LEVEL -from catalyst.exchange.bittrex.bittrex_api import Bittrex_api -from catalyst.exchange.exchange import Exchange -from catalyst.exchange.exchange_bundle import ExchangeBundle -from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ - ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \ - CreateOrderError -from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ - download_exchange_symbols, get_symbols_string -from catalyst.finance.execution import LimitOrder, StopLimitOrder -from catalyst.finance.order import Order, ORDER_STATUS - -# TODO: consider using this: https://github.com/mondeja/bittrex_v2 -from catalyst.utils.deprecate import deprecated - -log = Logger('Bittrex', level=LOG_LEVEL) - -URL2 = 'https://bittrex.com/Api/v2.0' - - -@deprecated -class Bittrex(Exchange): - def __init__(self, key, secret, base_currency, portfolio=None): - self.api = Bittrex_api(key=key, secret=secret) - self.name = 'bittrex' - self.color = 'blue' - self.base_currency = base_currency - self._portfolio = portfolio - - self.num_candles_limit = 2000 - - # Not sure what the rate limit is but trying to play it safe - # https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit - self.max_requests_per_minute = 60 - self.request_cpt = dict() - - self.minute_writer = None - self.minute_reader = None - - self.assets = dict() - self.load_assets() - - self.local_assets = dict() - self.load_assets(is_local=True) - - self.bundle = ExchangeBundle(self.name) - - @property - def account(self): - pass - - @property - def time_skew(self): - # TODO: research the time skew conditions - return pd.Timedelta('0s') - - def sanitize_curency_symbol(self, exchange_symbol): - """ - Helper method used to build the universal pair. - Include any symbol mapping here if appropriate. - - :param exchange_symbol: - :return universal_symbol: - """ - return exchange_symbol.lower() - - def get_balances(self): - balances = self.api.getbalances() - try: - log.debug('retrieving wallet balances') - self.ask_request() - - except Exception as e: - raise ExchangeRequestError(error=e) - - std_balances = dict() - try: - for balance in balances: - currency = balance['Currency'].lower() - std_balances[currency] = balance['Available'] - - except TypeError: - raise ExchangeRequestError(error=balances) - - return std_balances - - def create_order(self, asset, amount, is_buy, style): - log.info('creating {} order'.format('buy' if is_buy else 'sell')) - exchange_symbol = self.get_symbol(asset) - - if isinstance(style, LimitOrder) or isinstance(style, StopLimitOrder): - if isinstance(style, StopLimitOrder): - log.warn('{} will ignore the stop price'.format(self.name)) - - price = style.get_limit_price(is_buy) - try: - self.ask_request() - if is_buy: - order_status = self.api.buylimit(exchange_symbol, amount, - price) - else: - order_status = self.api.selllimit(exchange_symbol, - abs(amount), price) - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'uuid' in order_status: - order_id = order_status['uuid'] - order = Order( - dt=pd.Timestamp.utcnow(), - asset=asset, - amount=amount, - stop=style.get_stop_price(is_buy), - limit=style.get_limit_price(is_buy), - id=order_id - ) - return order - else: - if order_status == 'INSUFFICIENT_FUNDS': - log.warn('not enough funds to create order') - return None - elif order_status == 'DUST_TRADE_DISALLOWED_MIN_VALUE_50K_SAT': - log.warn('Your order is too small, order at least 50K' - ' Satoshi') - return None - else: - raise CreateOrderError( - exchange=self.name, - error=order_status - ) - else: - raise InvalidOrderStyle(exchange=self.name, - style=style.__class__.__name__) - - def get_open_orders(self, asset): - symbol = self.get_symbol(asset) - try: - self.ask_request() - open_orders = self.api.getopenorders(symbol) - except Exception as e: - raise ExchangeRequestError(error=e) - - orders = list() - for order_status in open_orders: - order = self._create_order(order_status) - orders.append(order) - - return orders - - def _create_order(self, order_status): - log.info( - 'creating catalyst order from Bittrex {}'.format(order_status)) - if order_status['CancelInitiated']: - status = ORDER_STATUS.CANCELLED - elif order_status['Closed'] is not None: - status = ORDER_STATUS.FILLED - else: - status = ORDER_STATUS.OPEN - - date = pd.to_datetime(order_status['Opened'], utc=True) - amount = order_status['Quantity'] - filled = amount - order_status['QuantityRemaining'] - order = Order( - dt=date, - asset=self.assets[order_status['Exchange']], - amount=amount, - stop=None, # Not yet supported by Bittrex - limit=order_status['Limit'], - filled=filled, - id=order_status['OrderUuid'], - commission=order_status['CommissionPaid'] - ) - order.status = status - - executed_price = order_status['PricePerUnit'] - - return order, executed_price - - def get_order(self, order_id): - log.info('retrieving order {}'.format(order_id)) - try: - self.ask_request() - order_status = self.api.getorder(order_id) - except Exception as e: - raise ExchangeRequestError(error=e) - - if order_status is None: - raise OrderNotFound(order_id=order_id, exchange=self.name) - - return self._create_order(order_status) - - def cancel_order(self, order_param): - order_id = order_param.id \ - if isinstance(order_param, Order) else order_param - log.info('cancelling order {}'.format(order_id)) - - try: - self.ask_request() - status = self.api.cancel(order_id) - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'message' in status: - raise OrderCancelError( - order_id=order_id, - exchange=self.name, - error=status['message'] - ) - - def get_candles(self, freq, assets, bar_count=None, - start_dt=None, end_dt=None): - """ - Supported Intervals - ------------------- - day, oneMin, fiveMin, thirtyMin, hour - - :param freq: - :param assets: - :param bar_count: - :param start_dt - :param end_dt - :return: - """ - - # TODO: this has no effect at the moment - if end_dt is None: - end_dt = pd.Timestamp.utcnow() - - log.debug( - 'retrieving {bars} {freq} candles on {exchange} from ' - '{end_dt} for markets {symbols}, '.format( - bars=bar_count, - freq=freq, - exchange=self.name, - end_dt=end_dt, - symbols=get_symbols_string(assets) - ) - ) - - if freq == '1T': - frequency = 'oneMin' - elif freq == '5T': - frequency = 'fiveMin' - elif freq == '30T': - frequency = 'thirtyMin' - elif freq == '60T': - frequency = 'hour' - elif freq == '1D': - frequency = 'day' - else: - raise InvalidHistoryFrequencyError(frequency=freq) - - # Making sure that assets are iterable - asset_list = [assets] if isinstance(assets, TradingPair) else assets - for asset in asset_list: - end = int(time.mktime(end_dt.timetuple())) - url = '{url}/pub/market/GetTicks?marketName={symbol}' \ - '&tickInterval={frequency}&_={end}'.format( - url=URL2, - symbol=self.get_symbol(asset), - frequency=frequency, - end=end, ) - - try: - data = json.loads(urllib.request.urlopen(url).read().decode()) - except Exception as e: - raise ExchangeRequestError(error=e) - - if data['message']: - raise ExchangeRequestError( - error='Unable to fetch candles {}'.format(data['message']) - ) - - candles = data['result'] - - def ohlc_from_candle(candle): - ohlc = dict( - open=candle['O'], - high=candle['H'], - low=candle['L'], - close=candle['C'], - volume=candle['V'], - price=candle['C'], - last_traded=pd.to_datetime(candle['T'], utc=True) - ) - return ohlc - - ordered_candles = list(reversed(candles)) - ohlc_map = dict() - if bar_count is None: - ohlc_map[asset] = ohlc_from_candle(ordered_candles[0]) - else: - # TODO: optimize - ohlc_bars = [] - for candle in ordered_candles[:bar_count]: - ohlc = ohlc_from_candle(candle) - ohlc_bars.append(ohlc) - - ohlc_map[asset] = ohlc_bars - - return ohlc_map[assets] \ - if isinstance(assets, TradingPair) else ohlc_map - - def tickers(self, assets): - """ - As of v1.1, Bittrex only allows one ticker at the time. - So we have to make multiple calls to fetch multiple assets. - - :param assets: - :return: - """ - log.info('retrieving tickers') - - ticks = dict() - for asset in assets: - symbol = self.get_symbol(asset) - try: - self.ask_request() - ticker = self.api.getticker(symbol) - except Exception as e: - raise ExchangeRequestError(error=e) - - # TODO: catch invalid ticker - ticks[asset] = dict( - timestamp=pd.Timestamp.utcnow(), - bid=ticker['Bid'], - ask=ticker['Ask'], - last_price=ticker['Last'] - ) - - log.debug('got tickers {}'.format(ticks)) - return ticks - - def get_account(self): - log.info('retrieving account data') - pass - - def generate_symbols_json(self, filename=None): - symbol_map = {} - - fn, r = download_exchange_symbols(self.name) - with open(fn) as data_file: - cached_symbols = json.load(data_file) - - markets = self.api.getmarkets() - for market in markets: - exchange_symbol = market['MarketName'] - symbol = '{market}_{base}'.format( - market=self.sanitize_curency_symbol(market['MarketCurrency']), - base=self.sanitize_curency_symbol(market['BaseCurrency']) - ) - - try: - end_daily = cached_symbols[exchange_symbol]['end_daily'] - except KeyError: - end_daily = 'N/A' - - try: - end_minute = cached_symbols[exchange_symbol]['end_minute'] - except KeyError: - end_minute = 'N/A' - - symbol_map[exchange_symbol] = dict( - symbol=symbol, - start_date=pd.to_datetime(market['Created'], - utc=True).strftime("%Y-%m-%d"), - end_daily=end_daily, - end_minute=end_minute, - ) - - if (filename is None): - filename = get_exchange_symbols_filename(self.name) - - with open(filename, 'w') as f: - json.dump(symbol_map, f, sort_keys=True, indent=2, - separators=(',', ':')) - - def get_orderbook(self, asset, order_type='all', limit=100): - if order_type == 'all': - order_type = 'both' - elif order_type == 'bid': - order_type = 'buy' - elif order_type == 'ask': - order_type = 'sell' - else: - raise ValueError('invalid type') - - exchange_symbol = asset.exchange_symbol - data = self.api.getorderbook( - market=exchange_symbol, - type=order_type, - depth=100 - ) - - result = dict() - for exchange_type in data: - if exchange_type == 'buy': - order_type = 'bids' - elif exchange_type == 'sell': - order_type = 'asks' - - result[order_type] = [] - for entry in data[exchange_type]: - result[order_type].append(dict( - rate=entry['Rate'], - quantity=entry['Quantity'] - )) - - return result diff --git a/catalyst/exchange/bittrex/bittrex_api.py b/catalyst/exchange/bittrex/bittrex_api.py deleted file mode 100644 index ca5f9c4a..00000000 --- a/catalyst/exchange/bittrex/bittrex_api.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env python -import json -import time -import hmac -import hashlib -import ssl - -# Workaround for backwards compatibility -# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7 -from six.moves import urllib - -urlopen = urllib.request.urlopen - - -class Bittrex_api(object): - def __init__(self, key, secret): - self.key = key - self.secret = secret - self.public = ['getmarkets', 'getcurrencies', 'getticker', - 'getmarketsummaries', 'getmarketsummary', - 'getorderbook', 'getmarkethistory'] - self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket', - 'cancel', 'getopenorders'] - self.account = ['getbalances', 'getbalance', 'getdepositaddress', - 'withdraw', 'getorder', 'getorderhistory', - 'getwithdrawalhistory', 'getdeposithistory'] - - def query(self, method, values={}): - if method in self.public: - url = 'https://bittrex.com/api/v1.1/public/' - elif method in self.market: - url = 'https://bittrex.com/api/v1.1/market/' - elif method in self.account: - url = 'https://bittrex.com/api/v1.1/account/' - else: - return 'Something went wrong, sorry.' - - url += method + '?' + urllib.parse.urlencode(values) - - if method not in self.public: - url += '&apikey=' + self.key - url += '&nonce=' + str(int(time.time())) - - signature = hmac.new(self.secret.encode('utf-8'), - url.encode('utf-8'), - hashlib.sha512).hexdigest() - headers = {'apisign': signature} - else: - headers = {} - - req = urllib.request.Request(url, headers=headers) - response = json.loads(urlopen( - req, context=ssl._create_unverified_context()).read()) - - if response["result"]: - return response["result"] - else: - return response["message"] - - def getmarkets(self): - return self.query('getmarkets') - - def getcurrencies(self): - return self.query('getcurrencies') - - def getticker(self, market): - return self.query('getticker', {'market': market}) - - def getmarketsummaries(self): - return self.query('getmarketsummaries') - - def getmarketsummary(self, market): - return self.query('getmarketsummary', {'market': market}) - - def getorderbook(self, market, type, depth=20): - return self.query('getorderbook', - {'market': market, 'type': type, 'depth': depth}) - - def getmarkethistory(self, market, count=20): - return self.query('getmarkethistory', - {'market': market, 'count': count}) - - def buylimit(self, market, quantity, rate): - return self.query('buylimit', {'market': market, 'quantity': quantity, - 'rate': rate}) - - def buymarket(self, market, quantity): - return self.query('buymarket', - {'market': market, 'quantity': quantity}) - - def selllimit(self, market, quantity, rate): - return self.query('selllimit', {'market': market, 'quantity': quantity, - 'rate': rate}) - - def sellmarket(self, market, quantity): - return self.query('sellmarket', - {'market': market, 'quantity': quantity}) - - def cancel(self, uuid): - return self.query('cancel', {'uuid': uuid}) - - def getopenorders(self, market): - return self.query('getopenorders', {'market': market}) - - def getbalances(self): - return self.query('getbalances') - - def getbalance(self, currency): - return self.query('getbalance', {'currency': currency}) - - def getdepositaddress(self, currency): - return self.query('getdepositaddress', {'currency': currency}) - - def withdraw(self, currency, quantity, address): - return self.query('withdraw', - {'currency': currency, 'quantity': quantity, - 'address': address}) - - def getorder(self, uuid): - return self.query('getorder', {'uuid': uuid}) - - def getorderhistory(self, market, count): - return self.query('getorderhistory', - {'market': market, 'count': count}) - - def getwithdrawalhistory(self, currency, count): - return self.query('getwithdrawalhistory', - {'currency': currency, 'count': count}) - - def getdeposithistory(self, currency, count): - return self.query('getdeposithistory', - {'currency': currency, 'count': count}) diff --git a/catalyst/exchange/bittrex/extensions-example.py b/catalyst/exchange/bittrex/extensions-example.py deleted file mode 100644 index 4b087899..00000000 --- a/catalyst/exchange/bittrex/extensions-example.py +++ /dev/null @@ -1,7 +0,0 @@ -from catalyst.data.bundles import register -from catalyst.exchange.exchange_bundle import exchange_bundle - -symbols = ( - 'neo_btc', -) -register('exchange_bitfinex', exchange_bundle('bitfinex', symbols)) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 97b29cb1..f1107ade 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -1,24 +1,27 @@ +import json +import os import re from collections import defaultdict import ccxt import pandas as pd import six +from catalyst.assets._assets import TradingPair from ccxt import ExchangeNotAvailable, InvalidOrder from logbook import Logger from six import string_types from catalyst.algorithm import MarketOrder -from catalyst.assets._assets import TradingPair from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ - ExchangeNotFoundError, CreateOrderError + ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError from catalyst.exchange.exchange_execution import ExchangeLimitOrder -from catalyst.exchange.exchange_utils import mixin_market_params, \ - from_ms_timestamp, get_epoch +from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ + from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \ + get_exchange_auth from catalyst.finance.order import Order, ORDER_STATUS log = Logger('CCXT', level=LOG_LEVEL) @@ -58,26 +61,103 @@ class CCXT(Exchange): self._symbol_maps = [None, None] - try: - markets_symbols = self.api.load_markets() - log.debug('the markets:\n{}'.format(markets_symbols)) - - except ExchangeNotAvailable as e: - raise ExchangeRequestError(error=e) - self.name = exchange_name - self.markets = self.api.fetch_markets() - self.load_assets() - self.base_currency = base_currency self.transactions = defaultdict(list) self.num_candles_limit = 2000 self.max_requests_per_minute = 60 + self.low_balance_threshold = 0.1 self.request_cpt = dict() self.bundle = ExchangeBundle(self.name) + self.markets = None + self._is_init = False + + def init(self): + if self._is_init: + return + + exchange_folder = get_exchange_folder(self.name) + filename = os.path.join(exchange_folder, 'cctx_markets.json') + + if os.path.exists(filename): + timestamp = os.path.getmtime(filename) + dt = pd.to_datetime(timestamp, unit='s', utc=True) + + if dt >= pd.Timestamp.utcnow().floor('1D'): + with open(filename) as f: + self.markets = json.load(f) + + log.debug('loaded markets for {}'.format(self.name)) + + if self.markets is None: + try: + markets_symbols = self.api.load_markets() + log.debug( + 'fetching {} markets:\n{}'.format( + self.name, markets_symbols + ) + ) + + self.markets = self.api.fetch_markets() + with open(filename, 'w+') as f: + json.dump(self.markets, f, indent=4) + + except ExchangeNotAvailable as e: + raise ExchangeRequestError(error=e) + + self.load_assets() + self._is_init = True + + @staticmethod + def find_exchanges(features=None, is_authenticated=False): + ccxt_features = [] + if features is not None: + for feature in features: + if not feature.endswith('Bundle'): + ccxt_features.append(feature) + + exchange_names = [] + for exchange_name in ccxt.exchanges: + if is_authenticated: + exchange_auth = get_exchange_auth(exchange_name) + + has_auth = (exchange_auth['key'] != '' + and exchange_auth['secret'] != '') + + if not has_auth: + continue + + log.debug('loading exchange: {}'.format(exchange_name)) + exchange = getattr(ccxt, exchange_name)() + + if ccxt_features is None: + has_feature = True + + else: + try: + has_feature = all( + [exchange.has[feature] for feature in ccxt_features] + ) + + except Exception: + has_feature = False + + if has_feature: + try: + log.info('initializing {}'.format(exchange_name)) + exchange_names.append(exchange_name) + + except Exception as e: + log.warn( + 'unable to initialize exchange {}: {}'.format( + exchange_name, e + ) + ) + + return exchange_names def account(self): return None @@ -85,6 +165,30 @@ class CCXT(Exchange): def time_skew(self): return None + def get_candle_frequencies(self, data_frequency=None): + frequencies = [] + try: + for timeframe in self.api.timeframes: + freq = CCXT.get_frequency(timeframe, raise_error=False) + + # TODO: support all frequencies + if data_frequency == 'minute' and not freq.endswith('T'): + continue + + elif data_frequency == 'daily' and not freq.endswith('D'): + continue + + frequencies.append(freq) + + except Exception as e: + log.warn( + 'candle frequencies not available for exchange {}'.format( + self.name + ) + ) + + return frequencies + def get_market(self, symbol): """ The CCXT market. @@ -106,7 +210,7 @@ class CCXT(Exchange): ) return market - def get_symbol(self, asset_or_symbol): + def get_symbol(self, asset_or_symbol, source='catalyst'): """ The CCXT symbol. @@ -118,36 +222,109 @@ class CCXT(Exchange): ------- """ - symbol = asset_or_symbol if isinstance( - asset_or_symbol, string_types - ) else asset_or_symbol.symbol - parts = symbol.split('_') - return '{}/{}'.format(parts[0].upper(), parts[1].upper()) + if source == 'ccxt': + if isinstance(asset_or_symbol, string_types): + parts = asset_or_symbol.split('/') + return '{}_{}'.format(parts[0].lower(), parts[1].lower()) - def get_catalyst_symbol(self, market_or_symbol): + else: + return asset_or_symbol.symbol + + else: + symbol = asset_or_symbol if isinstance( + asset_or_symbol, string_types + ) else asset_or_symbol.symbol + + parts = symbol.split('_') + return '{}/{}'.format(parts[0].upper(), parts[1].upper()) + + @staticmethod + def map_frequency(value, source='ccxt', raise_error=True): """ - The Catalyst symbol. + Map a frequency value between CCXT and Catalyst Parameters ---------- - market_or_symbol + value: str + source: str + raise_error: bool Returns ------- + Notes + ----- + The Pandas offset aliases supported by Catalyst: + Alias Description + W weekly frequency + M month end frequency + D calendar day frequency + H hourly frequency + T, min minutely frequency + + The CCXT timeframes: + '1m': '1minute', + '1h': '1hour', + '1d': '1day', + '1w': '1week', + '1M': '1month', + '1y': '1year', """ - if isinstance(market_or_symbol, string_types): - parts = market_or_symbol.split('/') - return '{}_{}'.format(parts[0].lower(), parts[1].lower()) + match = re.match( + r'([0-9].*)?(m|M|d|D|h|H|T|w|W|min)', value, re.M | re.I + ) + if match: + candle_size = int(match.group(1)) \ + if match.group(1) else 1 + + unit = match.group(2) else: - return '{}_{}'.format( - market_or_symbol['base'].lower(), - market_or_symbol['quote'].lower(), - ) + raise ValueError('Unable to parse frequency or timeframe') - def get_timeframe(self, freq): + if source == 'ccxt': + if unit == 'd': + result = '{}D'.format(candle_size) + + elif unit == 'm': + result = '{}T'.format(candle_size) + + elif unit == 'h': + result = '{}H'.format(candle_size) + + elif unit == 'w': + result = '{}W'.format(candle_size) + + elif unit == 'M': + result = '{}M'.format(candle_size) + + elif raise_error: + raise InvalidHistoryTimeframeError(timeframe=value) + + else: + if unit == 'D': + result = '{}d'.format(candle_size) + + elif unit == 'min' or unit == 'T': + result = '{}m'.format(candle_size) + + elif unit == 'H': + result = '{}h'.format(candle_size) + + elif unit == 'W': + result = '{}w'.format(candle_size) + + elif unit == 'M': + result = '{}M'.format(candle_size) + + elif raise_error: + raise InvalidHistoryFrequencyError(frequency=value) + + return result + + @staticmethod + def get_timeframe(freq, raise_error=True): """ The CCXT timeframe from the Catalyst frequency. @@ -161,26 +338,29 @@ class CCXT(Exchange): str """ - freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I) - if freq_match: - candle_size = int(freq_match.group(1)) \ - if freq_match.group(1) else 1 + return CCXT.map_frequency( + freq, source='catalyst', raise_error=raise_error + ) - unit = freq_match.group(2) + @staticmethod + def get_frequency(timeframe, raise_error=True): + """ + Test Catalyst frequency from the CCXT timeframe - else: - raise InvalidHistoryFrequencyError(frequency=freq) + Catalyst uses the Pandas offset alias convention: + http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases - if unit.lower() == 'd': - timeframe = '{}d'.format(candle_size) + Parameters + ---------- + timeframe - elif unit.lower() == 'm' or unit == 'T': - timeframe = '{}m'.format(candle_size) + Returns + ------- - elif unit.lower() == 'h' or unit == 'T': - timeframe = '{}h'.format(candle_size) - - return timeframe + """ + return CCXT.map_frequency( + timeframe, source='ccxt', raise_error=raise_error + ) def get_candles(self, freq, assets, bar_count=None, start_dt=None, end_dt=None): @@ -189,7 +369,7 @@ class CCXT(Exchange): assets = [assets] symbols = self.get_symbols(assets) - timeframe = self.get_timeframe(freq) + timeframe = CCXT.get_timeframe(freq) ms = None if start_dt is not None: @@ -198,30 +378,26 @@ class CCXT(Exchange): candles = dict() for asset in assets: - try: - ohlcvs = self.api.fetch_ohlcv( - symbol=symbols[0], - timeframe=timeframe, - since=ms, - limit=bar_count, - params={} - ) + ohlcvs = self.api.fetch_ohlcv( + symbol=symbols[0], + timeframe=timeframe, + since=ms, + limit=bar_count, + params={} + ) - candles[asset] = [] - for ohlcv in ohlcvs: - candles[asset].append(dict( - last_traded=pd.to_datetime( - ohlcv[0], unit='ms', utc=True - ), - open=ohlcv[1], - high=ohlcv[2], - low=ohlcv[3], - close=ohlcv[4], - volume=ohlcv[5] - )) - - except Exception as e: - raise ExchangeRequestError(error=e) + candles[asset] = [] + for ohlcv in ohlcvs: + candles[asset].append(dict( + last_traded=pd.to_datetime( + ohlcv[0], unit='ms', utc=True + ), + open=ohlcv[1], + high=ohlcv[2], + low=ohlcv[3], + close=ohlcv[4], + volume=ohlcv[5] + )) if is_single: return six.next(six.itervalues(candles)) @@ -339,16 +515,21 @@ class CCXT(Exchange): and asset_def['end_minute'] != 'N/A' else None else: - params['symbol'] = self.get_catalyst_symbol(market) + params['symbol'] = get_catalyst_symbol(market) # TODO: add as an optional column params['leverage'] = 1.0 return TradingPair(**params) def load_assets(self): + log.debug('loading assets for {}'.format(self.name)) self.assets = [] for market in self.markets: + if 'id' not in market: + log.warn('invalid market: {}'.format(market)) + continue + asset_defs = self.get_asset_defs(market) asset = None @@ -399,21 +580,61 @@ class CCXT(Exchange): The Catalyst order object """ - if order_status['status'] == 'canceled': + order_id = order_status['id'] + symbol = self.get_symbol(order_status['symbol'], source='ccxt') + asset = self.get_asset(symbol) + + s = order_status['status'] + amount = order_status['amount'] + filled = order_status['filled'] + + if s == 'canceled' or (s == 'closed' and filled == 0): status = ORDER_STATUS.CANCELLED - elif order_status['status'] == 'closed' and order_status['filled'] > 0: - log.debug('found executed order {}'.format(order_status)) + elif s == 'closed' and filled > 0: + if filled < amount: + log.warn( + 'order {id} is executed but only partially filled:' + ' {filled} {symbol} out of {amount}'.format( + id=order_status['status'], + filled=order_status['filled'], + symbol=asset.symbol, + amount=order_status['amount'], + ) + ) + else: + log.info( + 'order {id} executed in full: {filled} {symbol}'.format( + id=order_id, + filled=filled, + symbol=asset.symbol, + ) + ) + status = ORDER_STATUS.FILLED - elif order_status['status'] == 'open': + elif s == 'open': + status = ORDER_STATUS.OPEN + + elif filled > 0: + log.info( + 'order {id} partially filled: {filled} {symbol} out of ' + '{amount}, waiting for complete execution'.format( + id=order_id, + filled=filled, + symbol=asset.symbol, + amount=amount, + ) + ) status = ORDER_STATUS.OPEN else: - raise ValueError('invalid state for order') - - amount = order_status['amount'] - filled = order_status['filled'] + log.warn( + 'invalid state {} for order {}'.format( + s, order_id + ) + ) + status = ORDER_STATUS.OPEN if order_status['side'] == 'sell': amount = -amount @@ -423,25 +644,16 @@ class CCXT(Exchange): order_type = order_status['type'] limit_price = price if order_type == 'limit' else None - stop_price = None # TODO: add support executed_price = order_status['cost'] / order_status['amount'] commission = order_status['fee'] date = from_ms_timestamp(order_status['timestamp']) - # order_id = str(order_status['info']['clientOrderId']) - order_id = order_status['id'] - - # TODO: this won't work, redo the packages with a different key. - symbol = order_status['info']['symbol'] \ - if 'symbol' in order_status['info'] \ - else order_status['info']['Exchange'] - order = Order( dt=date, - asset=self.get_asset(symbol, is_exchange_symbol=True), + asset=asset, amount=amount, - stop=stop_price, + stop=None, limit=limit_price, filled=filled, id=order_id, @@ -469,7 +681,6 @@ class CCXT(Exchange): ) side = 'buy' if amount > 0 else 'sell' - if hasattr(self.api, 'amount_to_lots'): adj_amount = self.api.amount_to_lots( symbol=symbol, @@ -583,10 +794,19 @@ class CCXT(Exchange): """ tickers = dict() - for asset in assets: - try: - ccxt_symbol = self.get_symbol(asset) - ticker = self.api.fetch_ticker(ccxt_symbol) + try: + for asset in assets: + symbol = self.get_symbol(asset) + # TODO: use fetch_tickers() for efficiency + # I tried using fetch_tickers() but noticed some + # inconsistencies, see issue: + # https://github.com/ccxt/ccxt/issues/870 + ticker = self.api.fetch_ticker(symbol=symbol) + if not ticker: + log.warn('ticker not found for {} {}'.format( + self.name, symbol + )) + continue ticker['last_traded'] = from_ms_timestamp(ticker['timestamp']) @@ -594,19 +814,27 @@ class CCXT(Exchange): # TODO: any more exceptions? ticker['last_price'] = ticker['last'] - # Using the volume represented in the base currency - ticker['volume'] = ticker['baseVolume'] \ - if 'baseVolume' in ticker else 0 + if 'baseVolume' in ticker and ticker['baseVolume'] is not None: + # Using the volume represented in the base currency + ticker['volume'] = ticker['baseVolume'] + + elif 'info' in ticker and 'bidQty' in ticker['info'] \ + and 'askQty' in ticker['info']: + ticker['volume'] = float(ticker['info']['bidQty']) + \ + float(ticker['info']['askQty']) + + else: + ticker['volume'] = 0 tickers[asset] = ticker - except ExchangeNotAvailable as e: - log.warn( - 'unable to fetch ticker: {} {}'.format( - self.name, asset.symbol - ) + except ExchangeNotAvailable as e: + log.warn( + 'unable to fetch ticker: {} {}'.format( + self.name, asset.symbol ) - raise ExchangeRequestError(error=e) + ) + raise ExchangeRequestError(error=e) return tickers diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index cb76b885..b6b24d20 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -9,15 +9,16 @@ from logbook import Logger from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS -from catalyst.exchange.bundle_utils import get_start_dt, \ - get_delta, get_periods, get_periods_range from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ - BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \ + SymbolNotFoundOnExchange, \ PricingDataNotLoadedError, \ - NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError -from catalyst.exchange.exchange_utils import get_exchange_symbols, \ - get_frequency, resample_history_df + NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \ + TickerNotFoundError, NotEnoughCashError +from catalyst.exchange.utils.bundle_utils import get_start_dt, \ + get_delta, get_periods, get_periods_range +from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ + get_frequency, resample_history_df, has_bundle log = Logger('Exchange', level=LOG_LEVEL) @@ -38,6 +39,8 @@ class Exchange: self.request_cpt = None self.bundle = ExchangeBundle(self.name) + self.low_balance_threshold = None + @abstractproperty def account(self): pass @@ -46,6 +49,9 @@ class Exchange: def time_skew(self): pass + def has_bundle(self, data_frequency): + return has_bundle(self.name, data_frequency) + def is_open(self, dt): """ Is the exchange open @@ -148,7 +154,7 @@ class Exchange: def get_assets(self, symbols=None, data_frequency=None, is_exchange_symbol=False, - is_local=None): + is_local=None, quote_currency=None): """ The list of markets for the specified symbols. @@ -172,6 +178,14 @@ class Exchange: if symbols is None: # Make a distinct list of all symbols symbols = list(set([asset.symbol for asset in self.assets])) + + if quote_currency is not None: + for symbol in symbols[:]: + suffix = '_{}'.format(quote_currency.lower()) + + if not symbol.endswith(suffix): + symbols.remove(symbol) + is_exchange_symbol = False assets = [] @@ -235,10 +249,10 @@ class Exchange: elif data_frequency is not None: applies = ( - ( - data_frequency == 'minute' and a.end_minute is not None) - or ( - data_frequency == 'daily' and a.end_daily is not None) + ( + data_frequency == 'minute' and a.end_minute is not None) + or ( + data_frequency == 'daily' and a.end_daily is not None) ) else: @@ -247,8 +261,16 @@ class Exchange: # The symbol provided may use the Catalyst or the exchange # convention key = a.exchange_symbol if is_exchange_symbol else a.symbol - if not asset and key.lower() == symbol.lower() and applies: - asset = a + if not asset and key.lower() == symbol.lower(): + if applies: + asset = a + + else: + raise NoDataAvailableOnExchange( + symbol=key, + exchange=self.name, + data_frequency=data_frequency, + ) if asset is None: supported_symbols = sorted([a.symbol for a in self.assets]) @@ -272,6 +294,16 @@ class Exchange: self._symbol_maps[index] = symbol_map return symbol_map + @abstractmethod + def init(self): + """ + Load the asset list from the network. + + Returns + ------- + + """ + @abstractmethod def load_assets(self, is_local=False): """ @@ -377,6 +409,7 @@ class Exchange: return value + # TODO: replace with catalyst.exchange.exchange_utils.get_candles_df def get_series_from_candles(self, candles, start_dt, end_dt, data_frequency, field, previous_value=None): """ @@ -619,46 +652,98 @@ class Exchange: return df - def calculate_totals(self, check_cash=False, positions=None): + def _check_low_balance(self, currency, balances, amount): + free = balances[currency]['free'] if currency in balances else 0.0 + + if free < amount: + return free, True + + else: + return free, False + + def sync_positions(self, positions, cash=None, check_balances=False): """ Update the portfolio cash and position balances based on the latest ticker prices. + Parameters + ---------- + positions: + The positions to synchronize. + + check_balances: + Check balances amounts against the exchange. + """ - log.debug('synchronizing portfolio with exchange {}'.format(self.name)) - - cash = None - if check_cash: + free_cash = 0.0 + if check_balances: + log.debug('fetching {} balances'.format(self.name)) balances = self.get_balances() - - cash = balances[self.base_currency]['free'] \ - if self.base_currency in balances else None - - if cash is None: - raise BaseCurrencyNotFoundError( - base_currency=self.base_currency, - exchange=self.name + log.debug( + 'got free balances for {} currencies'.format( + len(balances) ) - log.debug('found base currency balance: {}'.format(cash)) + ) + if cash is not None: + free_cash, is_lower = self._check_low_balance( + currency=self.base_currency, + balances=balances, + amount=cash, + ) + if is_lower: + raise NotEnoughCashError( + currency=self.base_currency, + exchange=self.name, + free=free_cash, + cash=cash, + ) positions_value = 0.0 - if positions: + if positions is not None: assets = set([position.asset for position in positions]) tickers = self.tickers(assets) - log.debug('got tickers for positions: {}'.format(tickers)) - for asset in tickers: + for position in positions: + asset = position.asset + if asset not in tickers: + raise TickerNotFoundError( + symbol=asset.symbol, + exchange=self.name, + ) + ticker = tickers[asset] - positions = [p for p in positions if p.asset == asset] + log.debug( + 'updating {symbol} position, last traded on {dt} for ' + '{price}{currency}'.format( + symbol=asset.symbol, + dt=ticker['last_traded'], + price=ticker['last_price'], + currency=asset.quote_currency, + ) + ) + position.last_sale_price = ticker['last_price'] + position.last_sale_date = ticker['last_traded'] - for position in positions: - position.last_sale_price = ticker['last_price'] - position.last_sale_date = ticker['last_traded'] + positions_value += \ + position.amount * position.last_sale_price - positions_value += \ - position.amount * position.last_sale_price + if check_balances: + free, is_lower = self._check_low_balance( + currency=asset.base_currency, + balances=balances, + amount=position.amount, + ) - return cash, positions_value + if is_lower: + log.warn( + 'detected lower balance for {} on {}: {} < {}, ' + 'updating position amount'.format( + asset.symbol, self.name, free, position.amount + ) + ) + position.amount = free + + return free_cash, positions_value def order(self, asset, amount, style): """Place an order. diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index cc82b8aa..491a9ff1 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -10,16 +10,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy import pickle import signal import sys from datetime import timedelta from os import listdir from os.path import isfile, join -from time import sleep import logbook import pandas as pd +from redo import retry import catalyst.protocol as zp from catalyst.algorithm import TradingAlgorithm @@ -27,21 +28,21 @@ from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange_blotter import ExchangeBlotter from catalyst.exchange.exchange_errors import ( ExchangeRequestError, - ExchangePortfolioDataError, - OrderTypeNotSupported, ) + OrderTypeNotSupported) from catalyst.exchange.exchange_execution import ExchangeLimitOrder -from catalyst.exchange.exchange_utils import ( +from catalyst.exchange.live_graph_clock import LiveGraphClock +from catalyst.exchange.simple_clock import SimpleClock +from catalyst.exchange.utils.exchange_utils import ( save_algo_object, get_algo_object, get_algo_folder, get_algo_df, save_algo_df, group_assets_by_exchange, ) -from catalyst.exchange.live_graph_clock import LiveGraphClock -from catalyst.exchange.simple_clock import SimpleClock -from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \ +from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \ stats_to_algo_folder from catalyst.finance.execution import MarketOrder +from catalyst.finance.performance import PerformanceTracker from catalyst.finance.performance.period import calc_period_stats from catalyst.gens.tradesimulation import AlgorithmSimulator from catalyst.utils.api_support import api_method @@ -70,12 +71,26 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): and self.sim_params.arena == 'backtest': self.simulate_orders = True + # Operations with retry features + self.attempts = dict( + get_transactions_attempts=5, + order_attempts=5, + synchronize_portfolio_attempts=5, + get_order_attempts=5, + get_open_orders_attempts=5, + cancel_order_attempts=5, + get_spot_value_attempts=5, + get_history_window_attempts=5, + retry_sleeptime=5, + ) + self.blotter = ExchangeBlotter( data_frequency=self.data_frequency, # Default to NeverCancel in catalyst cancel_policy=self.cancel_policy, simulate_orders=self.simulate_orders, - exchanges=self.exchanges + exchanges=self.exchanges, + attempts=self.attempts, ) @staticmethod @@ -218,28 +233,28 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): """ tracker = self.perf_tracker - period = tracker.todays_performance + cum = tracker.cumulative_performance - pos_stats = period.position_tracker.stats() - period_stats = calc_period_stats(pos_stats, period.ending_cash) + pos_stats = cum.position_tracker.stats() + period_stats = calc_period_stats(pos_stats, cum.ending_cash) stats = dict( period_start=tracker.period_start, period_end=tracker.period_end, capital_base=tracker.capital_base, progress=tracker.progress, - ending_value=period.ending_value, - ending_exposure=period.ending_exposure, - capital_used=period.cash_flow, - starting_value=period.starting_value, - starting_exposure=period.starting_exposure, - starting_cash=period.starting_cash, - ending_cash=period.ending_cash, - portfolio_value=period.ending_cash + period.ending_value, - pnl=period.pnl, - returns=period.returns, - period_open=period.period_open, - period_close=period.period_close, + ending_value=cum.ending_value, + ending_exposure=cum.ending_exposure, + capital_used=cum.cash_flow, + starting_value=cum.starting_value, + starting_exposure=cum.starting_exposure, + starting_cash=cum.starting_cash, + ending_cash=cum.ending_cash, + portfolio_value=cum.ending_cash + cum.ending_value, + pnl=cum.pnl, + returns=cum.returns, + period_open=start_dt, + period_close=end_dt, gross_leverage=period_stats.gross_leverage, net_leverage=period_stats.net_leverage, short_exposure=pos_stats.short_exposure, @@ -256,8 +271,9 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): # Merging latest recorded variables stats.update(self.recorded_vars) - stats['positions'] = period.position_tracker.get_positions_list() + stats['positions'] = cum.position_tracker.get_positions_list() + period = tracker.todays_performance # we want the key to be absent, not just empty # Only include transactions for given dt stats['transactions'] = [] @@ -276,6 +292,12 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): return stats + def run(self, data=None, overwrite_sim_params=True): + data.attempts = self.attempts + return super(ExchangeTradingAlgorithmBase, self).run( + data, overwrite_sim_params + ) + class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase): def __init__(self, *args, **kwargs): @@ -328,6 +350,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.algo_namespace = kwargs.pop('algo_namespace', None) self.live_graph = kwargs.pop('live_graph', None) self.stats_output = kwargs.pop('stats_output', None) + self._analyze_live = kwargs.pop('analyze_live', None) self._clock = None self.frame_stats = list() @@ -342,42 +365,30 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.is_running = True - self.retry_check_open_orders = 5 - self.retry_synchronize_portfolio = 5 - self.retry_get_open_orders = 5 - self.retry_order = 2 - self.retry_delay = 5 + self.stats_minutes = 1 - self.stats_minutes = 10 + self._last_orders = [] + self.trading_client = None super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs) - signal.signal(signal.SIGINT, self.signal_handler) + try: + signal.signal(signal.SIGINT, self.signal_handler) + except ValueError: + log.warn("Can't initialize signal handler inside another thread." + "Exit should be handled by the user.") log.info('initialized trading algorithm in live mode') - def signal_handler(self, signal, frame): - """ - Handles the keyboard interruption signal. - - Parameters - ---------- - signal - frame - - Returns - ------- - - """ + def interrupt_algorithm(self): self.is_running = False if self._analyze is None: - log.info('Interruption signal detected {}, exiting the ' - 'algorithm'.format(signal)) + log.info('Exiting the algorithm.') else: - log.info('Interruption signal detected {}, calling `analyze()` ' - 'before exiting the algorithm'.format(signal)) + log.info('Exiting the algorithm. Calling `analyze()` ' + 'before exiting the algorithm.') algo_folder = get_algo_folder(self.algo_namespace) folder = join(algo_folder, 'daily_perf') @@ -395,6 +406,23 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): sys.exit(0) + def signal_handler(self, signal, frame): + """ + Handles the keyboard interruption signal. + + Parameters + ---------- + signal + frame + + Returns + ------- + + """ + log.info('Interruption signal detected {}, exiting the ' + 'algorithm'.format(signal)) + self.interrupt_algorithm() + @property def clock(self): if self._clock is None: @@ -419,10 +447,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # TODO: should we apply time skew? not sure to understand the utility. log.debug('creating clock') - if self.live_graph: + if self.live_graph or self._analyze_live is not None: self._clock = LiveGraphClock( self.sim_params.sessions, - context=self + context=self, + callback=self._analyze_live, ) else: self._clock = SimpleClock( @@ -431,26 +460,52 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): return self._clock - def _create_generator(self, sim_params): + def get_generator(self): + if self.trading_client is not None: + return self.trading_client.transform() + + perf = None if self.perf_tracker is None: - self.perf_tracker = get_algo_object( - algo_name=self.algo_namespace, - key='perf_tracker' + tracker = self.perf_tracker = PerformanceTracker( + sim_params=self.sim_params, + trading_calendar=self.trading_calendar, + env=self.trading_environment, ) + # Set the dt initially to the period start by forcing it to change. + self.on_dt_changed(self.sim_params.start_session) + + # Unpacking the perf_tracker and positions if available + perf = get_algo_object( + algo_name=self.algo_namespace, + key='cumulative_performance', + ) + + if not self.initialized: + self.initialize(*self.initialize_args, **self.initialize_kwargs) + self.initialized = True + # Call the simulation trading algorithm for side-effects: # it creates the perf tracker - TradingAlgorithm._create_generator(self, sim_params) - self.trading_client = ExchangeAlgorithmExecutor( - self, - sim_params, - self.data_portal, - self.clock, - self._create_benchmark_source(), - self.restrictions, - universe_func=self._calculate_universe - ) + # TradingAlgorithm._create_generator(self, self.sim_params) + if perf is not None: + tracker.cumulative_performance = perf + period = self.perf_tracker.todays_performance + period.starting_cash = perf.ending_cash + period.starting_exposure = perf.ending_exposure + period.starting_value = perf.ending_value + period.position_tracker = perf.position_tracker + + self.trading_client = ExchangeAlgorithmExecutor( + algo=self, + sim_params=self.sim_params, + data_portal=self.data_portal, + clock=self.clock, + benchmark_source=self._create_benchmark_source(), + restrictions=self.restrictions, + universe_func=self._calculate_universe, + ) return self.trading_client.transform() def updated_portfolio(self): @@ -459,7 +514,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): def updated_account(self): return self.perf_tracker.get_account(False) - def synchronize_portfolio(self, attempt_index=0): + def synchronize_portfolio(self): """ Synchronizes the portfolio tracked by the algorithm to refresh its current value. @@ -481,63 +536,51 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): The total value of all tracked positions. """ + check_balances = (not self.simulate_orders) + base_currency = None tracker = self.perf_tracker.position_tracker total_cash = 0.0 total_positions_value = 0.0 - try: - # Position keys correspond to assets - positions = self.portfolio.positions - assets = list(positions) - exchange_assets = group_assets_by_exchange(assets) - for exchange_name in self.exchanges: - assets = exchange_assets[exchange_name] \ - if exchange_name in exchange_assets else [] + # Position keys correspond to assets + positions = self.portfolio.positions + assets = list(positions) + exchange_assets = group_assets_by_exchange(assets) + for exchange_name in self.exchanges: + assets = exchange_assets[exchange_name] \ + if exchange_name in exchange_assets else [] - exchange_positions = \ - [positions[asset] for asset in assets] - - check_cash = (not self.simulate_orders) - - exchange = self.exchanges[exchange_name] # Type: Exchange - cash, positions_value = exchange.calculate_totals( - positions=exchange_positions, - check_cash=check_cash, - ) - total_positions_value += positions_value - - if cash is not None: - total_cash += cash - - for position in exchange_positions: - tracker.update_position( - asset=position.asset, - last_sale_date=position.last_sale_date, - last_sale_price=position.last_sale_price - ) - - if cash is None: - total_cash = self.portfolio.cash - - elif total_cash < self.portfolio.cash: - raise ValueError('Cash on exchanges is lower than the algo.') - - return total_cash, total_positions_value - - except ExchangeRequestError as e: - log.warn( - 'update portfolio attempt {}: {}'.format(attempt_index, e) + exchange_positions = copy.deepcopy( + [positions[asset] for asset in assets if asset in positions] ) - if attempt_index < self.retry_synchronize_portfolio: - sleep(self.retry_delay) - return self.synchronize_portfolio(attempt_index + 1) - else: - raise ExchangePortfolioDataError( - data_type='update-portfolio', - attempts=attempt_index, - error=e + + exchange = self.exchanges[exchange_name] # Type: Exchange + + if base_currency is None: + base_currency = exchange.base_currency + + cash, positions_value = exchange.sync_positions( + positions=exchange_positions, + check_balances=check_balances, + cash=self.portfolio.cash, + ) + total_cash += cash + total_positions_value += positions_value + + # Applying modifications to the original positions + for position in exchange_positions: + tracker.update_position( + asset=position.asset, + amount=position.amount, + last_sale_date=position.last_sale_date, + last_sale_price=position.last_sale_price, ) + if not check_balances: + total_cash = self.portfolio.cash + + return total_cash, total_positions_value + def add_pnl_stats(self, period_stats): """ Save p&l stats. @@ -632,13 +675,27 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if self.current_day is not None and today > self.current_day: self.frame_stats = list() - new_transactions, new_commissions, closed_orders = \ - self.blotter.get_transactions(data) + self.performance_needs_update = False + new_orders = self.perf_tracker.todays_performance.orders_by_id.keys() + if new_orders != self._last_orders: + self.performance_needs_update = True - if len(new_transactions) > 0: + self._last_orders = new_orders + + if self.performance_needs_update: self.perf_tracker.update_performance() + self.performance_needs_update = False + + if self.portfolio_needs_update: + cash, positions_value = retry( + action=self.synchronize_portfolio, + attempts=self.attempts['synchronize_portfolio_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn('Ordering again.') + ) + self.portfolio_needs_update = False - cash, positions_value = self.synchronize_portfolio() log.info( 'got totals from exchanges, cash: {} positions: {}'.format( cash, positions_value @@ -657,15 +714,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): except Exception as e: log.warn('unable to calculate performance: {}'.format(e)) - # TODO: pickle does not seem to work in python 3 - try: - save_algo_object( - algo_name=self.algo_namespace, - key='perf_tracker', - obj=self.perf_tracker - ) - except Exception as e: - log.warn('unable to save minute perfs to disk: {}'.format(e)) + save_algo_object( + algo_name=self.algo_namespace, + key='cumulative_performance', + obj=self.perf_tracker.cumulative_performance, + ) self.current_day = data.current_dt.floor('1D') @@ -677,7 +730,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.perf_tracker.update_performance() frame_stats = self.prepare_period_stats( - data.current_dt, data.current_dt + timedelta(minutes=1)) + data.current_dt, data.current_dt + timedelta(minutes=1) + ) # Saving the last hour in memory self.frame_stats.append(frame_stats) @@ -699,7 +753,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): stats=get_pretty_stats( stats=self.frame_stats, recorded_cols=recorded_cols, - num_rows=self.stats_minutes + num_rows=self.stats_minutes, ) )) @@ -751,33 +805,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): def batch_market_order(self, share_counts): raise NotImplementedError() - def _get_open_orders(self, asset=None, attempt_index=0): - try: - if asset: - exchange = self.exchanges[asset.exchange] - return exchange.get_open_orders(asset) + def _get_open_orders(self, asset=None): + if asset: + exchange = self.exchanges[asset.exchange] + return exchange.get_open_orders(asset) - else: - open_orders = [] - for exchange_name in self.exchanges: - exchange = self.exchanges[exchange_name] - exchange_orders = exchange.get_open_orders() - open_orders.append(exchange_orders) + else: + open_orders = [] + for exchange_name in self.exchanges: + exchange = self.exchanges[exchange_name] + exchange_orders = exchange.get_open_orders() + open_orders.append(exchange_orders) - return open_orders - except ExchangeRequestError as e: - log.warn( - 'open orders attempt {}: {}'.format(attempt_index, e) - ) - if attempt_index < self.retry_get_open_orders: - sleep(self.retry_delay) - return self._get_open_orders(asset, attempt_index + 1) - else: - raise ExchangePortfolioDataError( - data_type='open-orders', - attempts=attempt_index, - error=e - ) + return open_orders @error_keywords(sid='Keyword argument `sid` is no longer supported for ' 'get_open_orders. Use `asset` instead.') @@ -799,7 +839,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): If an asset is passed then this will return a list of the open orders for this asset. """ - return self._get_open_orders(asset) + return retry( + action=self._get_open_orders, + attempts=self.attempts['get_open_orders_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn('Fetching open orders again.'), + args=(asset,)) @api_method def get_order(self, order_id, exchange_name): @@ -819,7 +865,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): The execution price per share of the order """ exchange = self.exchanges[exchange_name] - return exchange.get_order(order_id) + return retry( + action=exchange.get_order, + attempts=self.attempts['get_order_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn('Fetching orders again.'), + args=(order_id,)) @api_method def cancel_order(self, order_param, exchange_name): @@ -836,4 +888,10 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if isinstance(order_param, zp.Order): order_id = order_param.id - exchange.cancel_order(order_id) + retry( + action=exchange.cancel_order, + attempts=self.attempts['cancel_order_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn('cancelling order again.'), + args=(order_id,)) diff --git a/catalyst/exchange/exchange_asset_finder.py b/catalyst/exchange/exchange_asset_finder.py new file mode 100644 index 00000000..0a887986 --- /dev/null +++ b/catalyst/exchange/exchange_asset_finder.py @@ -0,0 +1,180 @@ +import pandas as pd +from logbook import Logger + +from catalyst.constants import LOG_LEVEL +from catalyst.exchange.utils.factory import find_exchanges + +log = Logger('ExchangeAssetFinder', level=LOG_LEVEL) + + +class ExchangeAssetFinder(object): + def __init__(self, exchanges): + self.exchanges = exchanges + + @property + def sids(self): + """ + This seems to be used to pre-fetch assets. + I don't think that we need this for live-trading. + Leaving the list empty. + """ + all_sids = [] + for exchange_name in self.exchanges: + # This is what initializes each exchanges at the beginning + # of an algo + exchange = self.exchanges[exchange_name] + exchange.init() + + all_sids += [asset.sid for asset in exchange.assets] + + sids = list(set(all_sids)) + return sids + + def retrieve_asset(self, sid, default_none=False): + """ + Retrieve the first Asset found for a given sid. + """ + asset = None + for exchange_name in self.exchanges: + if asset is not None: + break + + exchange = self.exchanges[exchange_name] + assets = [asset for asset in exchange.assets if asset.sid == sid] + if assets: + asset = assets[0] + + return asset + + def retrieve_all(self, sids, default_none=False): + """ + Retrieve all assets in `sids`. + + Parameters + ---------- + sids : iterable of int + Assets to retrieve. + default_none : bool + If True, return None for failed lookups. + If False, raise `SidsNotFound`. + + Returns + ------- + assets : list[Asset or None] + A list of the same length as `sids` containing Assets (or Nones) + corresponding to the requested sids. + + Raises + ------ + SidsNotFound + When a requested sid is not found and default_none=False. + """ + assets = [] + for exchange_name in self.exchanges: + exchange = self.exchanges[exchange_name] + xas = [asset for asset in exchange.assets if asset.sid in sids] + assets += xas + + return assets + + def lookup_symbol(self, symbol, exchange, data_frequency=None, + as_of_date=None, fuzzy=False): + """Lookup an asset by symbol. + + Parameters + ---------- + symbol : str + The ticker symbol to resolve. + as_of_date : datetime or None + Look up the last owner of this symbol as of this datetime. + If ``as_of_date`` is None, then this can only resolve the equity + if exactly one equity has ever owned the ticker. + fuzzy : bool, optional + Should fuzzy symbol matching be used? Fuzzy symbol matching + attempts to resolve differences in representations for + shareclasses. For example, some people may represent the ``A`` + shareclass of ``BRK`` as ``BRK.A``, where others could write + ``BRK_A``. + + Returns + ------- + equity : Asset + The equity that held ``symbol`` on the given ``as_of_date``, or the + only equity to hold ``symbol`` if ``as_of_date`` is None. + + Raises + ------ + SymbolNotFound + Raised when no equity has ever held the given symbol. + MultipleSymbolsFound + Raised when no ``as_of_date`` is given and more than one equity + has held ``symbol``. This is also raised when ``fuzzy=True`` and + there are multiple candidates for the given ``symbol`` on the + ``as_of_date``. + """ + log.debug('looking up symbol: {} {}'.format(symbol, exchange.name)) + + return exchange.get_asset(symbol, data_frequency) + + def lifetimes(self, dates, include_start_date): + """ + Compute a DataFrame representing asset lifetimes for the specified date + range. + + Parameters + ---------- + dates : pd.DatetimeIndex + The dates for which to compute lifetimes. + include_start_date : bool + Whether or not to count the asset as alive on its start_date. + + This is useful in a backtesting context where `lifetimes` is being + used to signify "do I have data for this asset as of the morning of + this date?" For many financial metrics, (e.g. daily close), data + isn't available for an asset until the end of the asset's first + day. + + Returns + ------- + lifetimes : pd.DataFrame + A frame of dtype bool with `dates` as index and an Int64Index of + assets as columns. The value at `lifetimes.loc[date, asset]` will + be True iff `asset` existed on `date`. If `include_start_date` is + False, then lifetimes.loc[date, asset] will be false when date == + asset.start_date. + + See Also + -------- + numpy.putmask + catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask + """ + exchanges = find_exchanges(features=['minuteBundle']) + if not exchanges: + raise ValueError('exchange with minute bundles not found') + + # TODO: find a way to support multiple exchanges + exchange = exchanges[0] + # Using a single exchange for now because are not unique for the + # same asset in different exchanges. I'd like to avoid binding + # pipeline to a single exchange. + exchange.init() + + data = [] + for dt in dates: + exists = [] + + for asset in exchange.assets: + if include_start_date: + condition = (asset.start_date <= dt < asset.end_minute) + + else: + condition = (asset.start_date < dt < asset.end_minute) + + exists.append(condition) + + data.append(exists) + + sids = [asset.sid for asset in exchange.assets] + df = pd.DataFrame(data, index=dates, columns=exchange.assets) + + return df diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index ab8056d2..41a673e1 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -1,15 +1,13 @@ -from time import sleep - import pandas as pd from catalyst.assets._assets import TradingPair from logbook import Logger +from redo import retry from catalyst.constants import LOG_LEVEL -from catalyst.exchange.exchange_errors import ExchangeRequestError, \ - ExchangePortfolioDataError, ExchangeTransactionError +from catalyst.exchange.exchange_errors import ExchangeRequestError from catalyst.finance.blotter import Blotter from catalyst.finance.commission import CommissionModel -from catalyst.finance.order import ORDER_STATUS, Order +from catalyst.finance.order import ORDER_STATUS from catalyst.finance.slippage import SlippageModel from catalyst.finance.transaction import create_transaction, Transaction from catalyst.utils.input_validation import expect_types @@ -60,12 +58,13 @@ class TradingPairFeeSchedule(CommissionModel): maker = self.maker if self.maker is not None else asset.maker taker = self.taker if self.taker is not None else asset.taker - multiplier = maker \ - if ((order.amount > 0 and order.limit < transaction.price) - or (order.amount < 0 and order.limit > transaction.price)) \ - and order.limit_reached else taker + multiplier = taker + if order.limit is not None: + multiplier = maker \ + if ((order.amount > 0 and order.limit < transaction.price) + or (order.amount < 0 and order.limit > transaction.price)) \ + and order.limit_reached else taker - # Assuming just the taker fee for now fee = cost * multiplier return fee @@ -132,6 +131,7 @@ class TradingPairFixedSlippage(SlippageModel): class ExchangeBlotter(Blotter): def __init__(self, *args, **kwargs): self.simulate_orders = kwargs.pop('simulate_orders', False) + self.attempts = kwargs.pop('attempts', False) self.exchanges = kwargs.pop('exchanges', None) if not self.exchanges: @@ -151,31 +151,11 @@ class ExchangeBlotter(Blotter): TradingPair: TradingPairFeeSchedule() } - self.retry_delay = 5 - self.retry_check_open_orders = 5 - - def exchange_order(self, asset, amount, style=None, attempt_index=0): - try: - exchange = self.exchanges[asset.exchange] - return exchange.order( - asset, amount, style - ) - except ExchangeRequestError as e: - log.warn( - 'order attempt {}: {}'.format(attempt_index, e) - ) - if attempt_index < self.retry_order: - sleep(self.retry_delay) - - return self.exchange_order( - asset, amount, style, attempt_index + 1 - ) - else: - raise ExchangeTransactionError( - transaction_type='order', - attempts=attempt_index, - error=e - ) + def exchange_order(self, asset, amount, style=None): + exchange = self.exchanges[asset.exchange] + return exchange.order( + asset, amount, style + ) @expect_types(asset=TradingPair) def order(self, asset, amount, style, order_id=None): @@ -190,8 +170,13 @@ class ExchangeBlotter(Blotter): ) else: - order = self.exchange_order( - asset, amount, style + order = retry( + action=self.exchange_order, + attempts=self.attempts['order_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn('Ordering again.'), + args=(asset, amount, style), ) self.open_orders[order.asset].append(order) @@ -258,40 +243,32 @@ class ExchangeBlotter(Blotter): ) ) - def get_exchange_transactions(self, attempt_index=0): + def get_exchange_transactions(self): closed_orders = [] transactions = [] commissions = [] - try: - for order, txn in self.check_open_orders(): - order.dt = txn.dt + for order, txn in self.check_open_orders(): + order.dt = txn.dt - transactions.append(txn) + transactions.append(txn) - if not order.open: - closed_orders.append(order) + if not order.open: + closed_orders.append(order) - return transactions, commissions, closed_orders - - except ExchangeRequestError as e: - log.warn( - 'check open orders attempt {}: {}'.format(attempt_index, e) - ) - if attempt_index < self.retry_check_open_orders: - sleep(self.retry_delay) - return self.get_exchange_transactions(attempt_index + 1) - - else: - raise ExchangePortfolioDataError( - data_type='order-status', - attempts=attempt_index, - error=e - ) + return transactions, commissions, closed_orders def get_transactions(self, bar_data): if self.simulate_orders: return super(ExchangeBlotter, self).get_transactions(bar_data) else: - return self.get_exchange_transactions() + return retry( + action=self.get_exchange_transactions, + attempts=self.attempts['get_transactions_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn( + 'Fetching exchange transactions again.' + ) + ) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 46c0c9e7..c3d032d2 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -18,18 +18,18 @@ from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST from catalyst.constants import LOG_LEVEL from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ BcolzMinuteBarMetadata -from catalyst.exchange.bundle_utils import range_in_bundle, \ - get_bcolz_chunk, get_month_start_end, \ - get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \ - get_delta, get_assets from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \ BcolzExchangeBarWriter from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \ TempBundleNotFoundError, \ NoDataAvailableOnExchange, \ PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError -from catalyst.exchange.exchange_utils import get_exchange_folder, \ - save_exchange_symbols, mixin_market_params +from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ + get_bcolz_chunk, get_month_start_end, \ + get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \ + get_delta, get_assets +from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ + save_exchange_symbols, mixin_market_params, get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory @@ -462,7 +462,7 @@ class ExchangeBundle: (earliest_trade is not None and earliest_trade > start): start = earliest_trade - if end is None or (last_entry is not None and end > last_entry): + if last_entry is not None and (end is None or end > last_entry): end = last_entry.replace(minute=59, hour=23) \ if data_frequency == 'minute' else last_entry @@ -668,7 +668,7 @@ class ExchangeBundle: if self.exchange is None: # Avoid circular dependencies - from catalyst.exchange.factory import get_exchange + from catalyst.exchange.utils.factory import get_exchange self.exchange = get_exchange(self.exchange_name) problems = [] @@ -681,6 +681,7 @@ class ExchangeBundle: last_traded=np.object_, open=np.float64, high=np.float64, + low=np.float64, close=np.float64, volume=np.float64 ), @@ -730,7 +731,7 @@ class ExchangeBundle: if data_frequency == 'minute' else asset_def['end_minute'] else: - params['symbol'] = self.exchange.get_catalyst_symbol(market) + params['symbol'] = get_catalyst_symbol(market) params['end_daily'] = end_dt \ if data_frequency == 'daily' else 'N/A' @@ -755,9 +756,10 @@ class ExchangeBundle: ) for symbol in assets: + # here the symbol is the market['id'] asset = assets[symbol] ohlcv_df = df.loc[ - (df.index.get_level_values(0) == symbol) + (df.index.get_level_values(0) == asset.symbol) ] # type: pd.DataFrame ohlcv_df.index = ohlcv_df.index.droplevel(0) @@ -805,7 +807,7 @@ class ExchangeBundle: else: if self.exchange is None: # Avoid circular dependencies - from catalyst.exchange.factory import get_exchange + from catalyst.exchange.utils.factory import get_exchange self.exchange = get_exchange(self.exchange_name) assets = get_assets( diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index 02d88ca0..a37d1025 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -1,19 +1,18 @@ import abc -from time import sleep import numpy as np import pandas as pd from catalyst.assets._assets import TradingPair from logbook import Logger +from redo import retry from catalyst.constants import LOG_LEVEL, AUTO_INGEST from catalyst.data.data_portal import DataPortal from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import ( ExchangeRequestError, - ExchangeBarDataError, PricingDataNotLoadedError) -from catalyst.exchange.exchange_utils import get_frequency, \ +from catalyst.exchange.utils.exchange_utils import get_frequency, \ resample_history_df, group_assets_by_exchange log = Logger('DataPortalExchange', level=LOG_LEVEL) @@ -21,11 +20,11 @@ log = Logger('DataPortalExchange', level=LOG_LEVEL) class DataPortalExchangeBase(DataPortal): def __init__(self, *args, **kwargs): - - # TODO: put somewhere accessible by each algo - self.retry_get_history_window = 5 - self.retry_get_spot_value = 5 - self.retry_delay = 5 + self.attempts = dict( + get_spot_value_attempts=5, + get_history_window_attempts=5, + retry_sleeptime=5, + ) super(DataPortalExchangeBase, self).__init__(*args, **kwargs) @@ -36,33 +35,14 @@ class DataPortalExchangeBase(DataPortal): frequency, field, data_frequency, - ffill=True, - attempt_index=0): - try: - exchange_assets = group_assets_by_exchange(assets) - if len(exchange_assets) > 1: - df_list = [] - for exchange_name in exchange_assets: - assets = exchange_assets[exchange_name] + ffill=True): + exchange_assets = group_assets_by_exchange(assets) + if len(exchange_assets) > 1: + df_list = [] + for exchange_name in exchange_assets: + assets = exchange_assets[exchange_name] - df_exchange = self.get_exchange_history_window( - exchange_name, - assets, - end_dt, - bar_count, - frequency, - field, - data_frequency, - ffill) - - df_list.append(df_exchange) - - # Merging the values values of each exchange - return pd.concat(df_list) - - else: - exchange_name = list(exchange_assets.keys())[0] - return self.get_exchange_history_window( + df_exchange = self.get_exchange_history_window( exchange_name, assets, end_dt, @@ -72,26 +52,22 @@ class DataPortalExchangeBase(DataPortal): data_frequency, ffill) - except ExchangeRequestError as e: - log.warn( - 'get history attempt {}: {}'.format(attempt_index, e) - ) - if attempt_index < self.retry_get_history_window: - sleep(self.retry_delay) - return self._get_history_window(assets, - end_dt, - bar_count, - frequency, - field, - data_frequency, - ffill, - attempt_index + 1) - else: - raise ExchangeBarDataError( - data_type='history', - attempts=attempt_index, - error=e - ) + df_list.append(df_exchange) + + # Merging the values values of each exchange + return pd.concat(df_list) + + else: + exchange_name = list(exchange_assets.keys())[0] + return self.get_exchange_history_window( + exchange_name, + assets, + end_dt, + bar_count, + frequency, + field, + data_frequency, + ffill) def get_history_window(self, assets, @@ -105,13 +81,19 @@ class DataPortalExchangeBase(DataPortal): if field == 'price': field = 'close' - return self._get_history_window(assets, - end_dt, - bar_count, - frequency, - field, - data_frequency, - ffill) + return retry( + action=self._get_history_window, + attempts=self.attempts['get_history_window_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn('fetching history again.'), + args=(assets, + end_dt, + bar_count, + frequency, + field, + data_frequency, + ffill)) @abc.abstractmethod def get_exchange_history_window(self, @@ -125,69 +107,58 @@ class DataPortalExchangeBase(DataPortal): ffill=True): pass - def _get_spot_value(self, assets, field, dt, data_frequency, - attempt_index=0): - try: - if isinstance(assets, TradingPair): - spot_values = self.get_exchange_spot_value( - assets.exchange, [assets], field, dt, data_frequency) + def _get_spot_value(self, assets, field, dt, data_frequency): + if isinstance(assets, TradingPair): + spot_values = self.get_exchange_spot_value( + assets.exchange, [assets], field, dt, data_frequency) - if not spot_values: - return np.nan + if not spot_values: + return np.nan - return spot_values[0] + return spot_values[0] + + else: + exchange_assets = dict() + for asset in assets: + if asset.exchange not in exchange_assets: + exchange_assets[asset.exchange] = list() + + exchange_assets[asset.exchange].append(asset) + + if len(list(exchange_assets.keys())) == 1: + exchange_name = list(exchange_assets.keys())[0] + return self.get_exchange_spot_value( + exchange_name, assets, field, dt, data_frequency) else: - exchange_assets = dict() - for asset in assets: - if asset.exchange not in exchange_assets: - exchange_assets[asset.exchange] = list() + spot_values = [] + for exchange_name in exchange_assets: + assets = exchange_assets[exchange_name] + exchange_spot_values = self.get_exchange_spot_value( + exchange_name, + assets, + field, + dt, + data_frequency + ) + if len(assets) == 1: + spot_values.append(exchange_spot_values) + else: + spot_values += exchange_spot_values - exchange_assets[asset.exchange].append(asset) - - if len(list(exchange_assets.keys())) == 1: - exchange_name = list(exchange_assets.keys())[0] - return self.get_exchange_spot_value( - exchange_name, assets, field, dt, data_frequency) - - else: - spot_values = [] - for exchange_name in exchange_assets: - assets = exchange_assets[exchange_name] - exchange_spot_values = self.get_exchange_spot_value( - exchange_name, - assets, - field, - dt, - data_frequency - ) - if len(assets) == 1: - spot_values.append(exchange_spot_values) - else: - spot_values += exchange_spot_values - - return spot_values - - except ExchangeRequestError as e: - log.warn( - 'get spot value attempt {}: {}'.format(attempt_index, e) - ) - if attempt_index < self.retry_get_spot_value: - sleep(self.retry_delay) - return self._get_spot_value(assets, field, dt, data_frequency, - attempt_index + 1) - else: - raise ExchangeBarDataError( - data_type='spot', - attempts=attempt_index, - error=e - ) + return spot_values def get_spot_value(self, assets, field, dt, data_frequency): if field == 'price': field = 'close' - return self._get_spot_value(assets, field, dt, data_frequency) + return retry( + action=self._get_spot_value, + attempts=self.attempts['get_spot_value_attempts'], + sleeptime=self.attempts['retry_sleeptime'], + retry_exceptions=(ExchangeRequestError,), + cleanup=lambda: log.warn('fetching spot value again.'), + args=(assets, field, dt, data_frequency)) @abc.abstractmethod def get_exchange_spot_value(self, exchange_name, assets, field, dt, @@ -339,7 +310,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): field=field, data_frequency=adj_data_frequency, algo_end_dt=self._last_available_session, - trailing_bar_count=trailing_bar_count + trailing_bar_count=trailing_bar_count, ) df = resample_history_df(pd.DataFrame(series), freq, field) diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index bb393721..0e22868f 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -100,6 +100,12 @@ class InvalidHistoryFrequencyError(ZiplineError): ).strip() +class InvalidHistoryTimeframeError(ZiplineError): + msg = ( + 'CCXT timeframe {timeframe} not supported by the exchange.' + ).strip() + + class MismatchingFrequencyError(ZiplineError): msg = ( 'Bar aggregate frequency {frequency} not compatible with ' @@ -162,8 +168,8 @@ class SidHashError(ZiplineError): class BaseCurrencyNotFoundError(ZiplineError): msg = ( - 'Algorithm base currency {base_currency} not found in exchange ' - '{exchange}.' + 'Algorithm base currency {base_currency} not found in account ' + 'balances on {exchange}: {balances}' ).strip() @@ -226,16 +232,20 @@ class PricingDataValueError(ZiplineError): class DataCorruptionError(ZiplineError): - msg = ('Unable to validate data for {exchange} {symbols} in date range ' - '[{start_dt} - {end_dt}]. The data is either corrupted or ' - 'unavailable. Please try deleting this bundle:' - '\n`catalyst clean-exchange -x {exchange}\n' - 'Then, ingest the data again. Please contact the Catalyst team if ' - 'the issue persists.').strip() + msg = ( + 'Unable to validate data for {exchange} {symbols} in date range ' + '[{start_dt} - {end_dt}]. The data is either corrupted or ' + 'unavailable. Please try deleting this bundle:' + '\n`catalyst clean-exchange -x {exchange}\n' + 'Then, ingest the data again. Please contact the Catalyst team if ' + 'the issue persists.' + ).strip() class ApiCandlesError(ZiplineError): - msg = ('Unable to fetch candles from the remote API: {error}.').strip() + msg = ( + 'Unable to fetch candles from the remote API: {error}.' + ).strip() class NoDataAvailableOnExchange(ZiplineError): @@ -248,13 +258,16 @@ class NoDataAvailableOnExchange(ZiplineError): class NoValueForField(ZiplineError): - msg = ('Value not found for field: {field}.').strip() + msg = ( + 'Value not found for field: {field}.' + ).strip() class OrderTypeNotSupported(ZiplineError): msg = ( - 'Order type `{order_type}` not currencly supported by Catalyst. ' - 'Please use `limit` or `market` orders only.').strip() + 'Order type `{order_type}` not currency supported by Catalyst. ' + 'Please use `limit` or `market` orders only.' + ).strip() class NotEnoughCapitalError(ZiplineError): @@ -262,10 +275,43 @@ class NotEnoughCapitalError(ZiplineError): 'Not enough capital on exchange {exchange} for trading. Each ' 'exchange should contain at least as much {base_currency} ' 'as the specified `capital_base`. The current balance {balance} is ' - 'lower than the `capital_base`: {capital_base}').strip() + 'lower than the `capital_base`: {capital_base}' + ).strip() + + +class NotEnoughCashError(ZiplineError): + msg = ( + 'Total {currency} amount on {exchange} is lower than the cash ' + 'reserved for this algo: {free} < {cash}. While trades can be made on ' + 'the exchange accounts outside of the algo, exchange must have enough ' + 'free {currency} to cover the algo cash.' + ).strip() + class LastCandleTooEarlyError(ZiplineError): msg = ( 'The trade date of the last candle {last_traded} is before the ' 'specified end date minus one candle {end_dt}. Please verify how ' - '{exchange} calculates the start date of OHLCV candles.').strip() + '{exchange} calculates the start date of OHLCV candles.' + ).strip() + + +class TickerNotFoundError(ZiplineError): + msg = ( + 'Unable to fetch ticker for {symbol} on {exchange}.' + ).strip() + + +class BalanceNotFoundError(ZiplineError): + msg = ( + '{currency} not found in account balance on {exchange}: {balances}.' + ).strip() + + +class BalanceTooLowError(ZiplineError): + msg = ( + 'Balance for {currency} on {exchange} too low: {free} < {amount}. ' + 'Positions have likely been sold outside of this algorithm. Please ' + 'add positions to hold a free amount greater than {amount}, or clean ' + 'the state of this algo and restart.' + ).strip() diff --git a/catalyst/exchange/exchange_pricing_loader.py b/catalyst/exchange/exchange_pricing_loader.py new file mode 100644 index 00000000..38663962 --- /dev/null +++ b/catalyst/exchange/exchange_pricing_loader.py @@ -0,0 +1,178 @@ +# Copyright 2015 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from logbook import Logger +from numpy import ( + iinfo, + uint32, +) + +from catalyst.constants import LOG_LEVEL +from catalyst.data.us_equity_pricing import BcolzDailyBarReader +from catalyst.errors import NoFurtherDataError +from catalyst.exchange.utils.factory import get_exchange +from catalyst.lib.adjusted_array import AdjustedArray +from catalyst.pipeline.data import DataSet, Column +from catalyst.pipeline.loaders.base import PipelineLoader +from catalyst.utils.calendars import get_calendar +from catalyst.utils.numpy_utils import float64_dtype + +UINT32_MAX = iinfo(uint32).max + +log = Logger('ExchangePriceLoader', level=LOG_LEVEL) + + +class TradingPairPricing(DataSet): + """ + Dataset representing daily trading prices and volumes. + """ + open = Column(float64_dtype) + high = Column(float64_dtype) + low = Column(float64_dtype) + close = Column(float64_dtype) + volume = Column(float64_dtype) + + +class ExchangePricingLoader(PipelineLoader): + """ + PipelineLoader for Crypto Pricing data + + Delegates loading of baselines and adjustments. + """ + + def __init__(self, data_frequency): + + cal = get_calendar('OPEN') + + if data_frequency == 'daily': + reader = None + all_sessions = cal.all_sessions + + elif data_frequency == 'minute': + reader = None + all_sessions = cal.all_minutes + + else: + raise ValueError( + 'Invalid data frequency: {}'.format(data_frequency) + ) + + self.data_frequency = data_frequency + self.raw_price_loader = reader + self._columns = TradingPairPricing.columns + self._all_sessions = all_sessions + + @classmethod + def from_files(cls, pricing_path): + """ + Create a loader from a bcolz equity pricing dir and a SQLite + adjustments path. + + Parameters + ---------- + pricing_path : str + Path to a bcolz directory written by a BcolzDailyBarWriter. + """ + return cls( + BcolzDailyBarReader(pricing_path), + ) + + def load_adjusted_array(self, columns, dates, assets, mask): + # load_adjusted_array is called with dates on which the user's algo + # will be shown data, which means we need to return the data that would + # be known at the start of each date. We assume that the latest data + # known on day N is the data from day (N - 1), so we shift all query + # dates back by a day. + start_date, end_date = _shift_dates( + self._all_sessions, dates[0], dates[-1], shift=1, + ) + colnames = [c.name for c in columns] + + if len(assets) == 0: + raise ValueError( + 'Pipeline cannot load data with eligible assets.' + ) + + exchange_names = [] + for asset in assets: + if asset.exchange not in exchange_names: + exchange_names.append(asset.exchange) + + exchange = get_exchange(exchange_names[0]) + reader = exchange.bundle.get_reader(self.data_frequency) + + raw_arrays = reader.load_raw_arrays( + colnames, + start_date, + end_date, + assets, + ) + + out = {} + for c, c_raw in zip(columns, raw_arrays): + out[c] = AdjustedArray( + c_raw.astype(c.dtype), + mask, + {}, + c.missing_value, + ) + return out + + @property + def columns(self): + return self._columns + + +def _shift_dates(dates, start_date, end_date, shift): + try: + start = dates.get_loc(start_date) + except KeyError: + if start_date < dates[0]: + raise NoFurtherDataError( + msg=( + "Pipeline Query requested data starting on {query_start}, " + "but first known date is {calendar_start}" + ).format( + query_start=str(start_date), + calendar_start=str(dates[0]), + ) + ) + else: + raise ValueError("Query start %s not in calendar" % start_date) + + # Make sure that shifting doesn't push us out of the calendar. + if start < shift: + raise NoFurtherDataError( + msg=( + "Pipeline Query requested data from {shift}" + " days before {query_start}, but first known date is only " + "{start} days earlier." + ).format(shift=shift, query_start=start_date, start=start), + ) + + try: + end = dates.get_loc(end_date) + except KeyError: + if end_date > dates[-1]: + raise NoFurtherDataError( + msg=( + "Pipeline Query requesting data up to {query_end}, " + "but last known date is {calendar_end}" + ).format( + query_end=end_date, + calendar_end=dates[-1], + ) + ) + else: + raise ValueError("Query end %s not in calendar" % end_date) + return dates[start - shift], dates[end - shift] diff --git a/catalyst/exchange/factory.py b/catalyst/exchange/factory.py deleted file mode 100644 index b0002f32..00000000 --- a/catalyst/exchange/factory.py +++ /dev/null @@ -1,34 +0,0 @@ -import os - -from catalyst.exchange.ccxt.ccxt_exchange import CCXT -from catalyst.exchange.exchange_errors import ExchangeAuthEmpty -from catalyst.exchange.exchange_utils import get_exchange_auth, \ - get_exchange_folder - - -def get_exchange(exchange_name, base_currency=None, must_authenticate=False): - exchange_auth = get_exchange_auth(exchange_name) - - has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '') - if must_authenticate and not has_auth: - raise ExchangeAuthEmpty( - exchange=exchange_name.title(), - filename=os.path.join( - get_exchange_folder(exchange_name), 'auth.json' - ) - ) - - return CCXT( - exchange_name=exchange_name, - key=exchange_auth['key'], - secret=exchange_auth['secret'], - base_currency=base_currency, - ) - - -def get_exchanges(exchange_names): - exchanges = dict() - for exchange_name in exchange_names: - exchanges[exchange_name] = get_exchange(exchange_name) - - return exchanges diff --git a/catalyst/exchange/live_graph_clock.py b/catalyst/exchange/live_graph_clock.py index 6f674455..40ad9c24 100644 --- a/catalyst/exchange/live_graph_clock.py +++ b/catalyst/exchange/live_graph_clock.py @@ -6,8 +6,7 @@ from catalyst.gens.sim_engine import ( from logbook import Logger from catalyst.constants import LOG_LEVEL -from catalyst.exchange.exchange_errors import \ - MismatchingBaseCurrenciesExchanges +from catalyst.exchange.utils.stats_utils import prepare_stats log = Logger('LiveGraphClock', level=LOG_LEVEL) @@ -38,177 +37,23 @@ class LiveGraphClock(object): the exchange and the live trading machine's clock. It's not used currently. """ - def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')): - - global mdates, plt # TODO: Could be cleaner - import matplotlib.dates as mdates - from matplotlib import pyplot as plt - from matplotlib import style + def __init__(self, sessions, context, callback=None, + time_skew=pd.Timedelta('0s')): self.sessions = sessions self.time_skew = time_skew self._last_emit = None self._before_trading_start_bar_yielded = True self.context = context - self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M') - - style.use('dark_background') - - fig = plt.figure() - fig.canvas.set_window_title('Enigma Catalyst: {}'.format( - self.context.algo_namespace)) - - self.ax_pnl = fig.add_subplot(311) - - self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl) - - self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl) - - if len(context.minute_stats) > 0: - self.draw_pnl() - self.draw_custom_signals() - self.draw_exposure() - - # rotates and right aligns the x labels, and moves the bottom of the - # axes up to make room for them - fig.autofmt_xdate() - fig.subplots_adjust(hspace=0.5) - - plt.tight_layout() - plt.ion() - plt.show() - - def format_ax(self, ax): - """ - Trying to assign reasonable parameters to the time axis. - - Parameters - ---------- - ax: - - """ - # TODO: room for improvement - ax.xaxis.set_major_locator(mdates.DayLocator(interval=1)) - ax.xaxis.set_major_formatter(self.fmt) - - locator = mdates.HourLocator(interval=4) - locator.MAXTICKS = 5000 - ax.xaxis.set_minor_locator(locator) - - datemin = pd.Timestamp.utcnow() - ax.set_xlim(datemin) - - ax.grid(True) - - def set_legend(self, ax): - """ - Set legend on the chart. - - Parameters - ---------- - ax - - """ - ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1) - - def draw_pnl(self): - """ - Draw p&l line on the chart. - - """ - ax = self.ax_pnl - df = self.context.pnl_stats - - ax.clear() - ax.set_title('Performance') - ax.plot(df.index, df['performance'], '-', - color='green', - linewidth=1.0, - label='Performance' - ) - - def perc(val): - return '{:2f}'.format(val) - - ax.format_ydata = perc - - self.set_legend(ax) - self.format_ax(ax) - - def draw_custom_signals(self): - """ - Draw custom signals on the chart. - - """ - ax = self.ax_custom_signals - df = self.context.custom_signals_stats - - colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink'] - - ax.clear() - ax.set_title('Custom Signals') - for index, column in enumerate(df.columns.values.tolist()): - ax.plot(df.index, df[column], '-', - color=colors[index], - linewidth=1.0, - label=column - ) - - self.set_legend(ax) - self.format_ax(ax) - - def draw_exposure(self): - """ - Draw exposure line on the chart. - - """ - ax = self.ax_exposure - context = self.context - df = context.exposure_stats - - # TODO: list exchanges in graph - base_currency = None - positions = [] - for exchange_name in context.exchanges: - exchange = context.exchanges[exchange_name] - - if not base_currency: - base_currency = exchange.base_currency - elif base_currency != exchange.base_currency: - raise MismatchingBaseCurrenciesExchanges( - base_currency=base_currency, - exchange_name=exchange.name, - exchange_currency=exchange.base_currency - ) - - positions += exchange.portfolio.positions - - ax.clear() - ax.set_title('Exposure') - ax.plot(df.index, df['base_currency'], '-', - color='green', - linewidth=1.0, - label='Base Currency: {}'.format(base_currency.upper()) - ) - - symbols = [] - for position in positions: - symbols.append(position.symbol) - - ax.plot(df.index, df['long_exposure'], '-', - color='blue', - linewidth=1.0, - label='Long Exposure: {}'.format(', '.join(symbols).upper())) - - self.set_legend(ax) - self.format_ax(ax) + self.callback = callback def __iter__(self): + from matplotlib import pyplot as plt yield pd.Timestamp.utcnow(), SESSION_START while True: current_time = pd.Timestamp.utcnow() - current_minute = current_time.floor('1 min') + current_minute = current_time.floor('1T') if self._last_emit is None or current_minute > self._last_emit: log.debug('emitting minutely bar: {}'.format(current_minute)) @@ -216,14 +61,11 @@ class LiveGraphClock(object): self._last_emit = current_minute yield current_minute, BAR - try: - self.draw_pnl() - self.draw_custom_signals() - self.draw_exposure() - - plt.draw() - except Exception as e: - log.warn('Unable to update the graph: {}'.format(e)) + recorded_cols = list(self.context.recorded_vars.keys()) + df, _ = prepare_stats( + self.context.frame_stats, recorded_cols=recorded_cols + ) + self.callback(self.context, df) else: # I can't use the "animate" reactive approach here because diff --git a/catalyst/exchange/poloniex/__init__.py b/catalyst/exchange/poloniex/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/catalyst/exchange/poloniex/poloniex.py b/catalyst/exchange/poloniex/poloniex.py deleted file mode 100644 index 49ccbd40..00000000 --- a/catalyst/exchange/poloniex/poloniex.py +++ /dev/null @@ -1,661 +0,0 @@ -import json -import time -from collections import defaultdict - -import numpy as np -import pandas as pd -import pytz -from catalyst.assets._assets import TradingPair -from logbook import Logger -# import six -from six import iteritems - -from catalyst.constants import LOG_LEVEL -# from websocket import create_connection -from catalyst.exchange.exchange import Exchange -from catalyst.exchange.exchange_bundle import ExchangeBundle -from catalyst.exchange.exchange_errors import ( - ExchangeRequestError, - InvalidHistoryFrequencyError, - InvalidOrderStyle, - OrphanOrderError, - OrphanOrderReverseError) -from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ - ExchangeStopLimitOrder -from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ - download_exchange_symbols, get_symbols_string -from catalyst.exchange.poloniex.poloniex_api import Poloniex_api -from catalyst.finance.order import Order, ORDER_STATUS -from catalyst.finance.transaction import Transaction -from catalyst.protocol import Account -from catalyst.utils.deprecate import deprecated - -log = Logger('Poloniex', level=LOG_LEVEL) - - -@deprecated -class Poloniex(Exchange): - def __init__(self, key, secret, base_currency, portfolio=None): - self.api = Poloniex_api(key=key, secret=secret) - self.name = 'poloniex' - - self.assets = dict() - self.load_assets() - - self.local_assets = dict() - self.load_assets(is_local=True) - - self.base_currency = base_currency - self._portfolio = portfolio - self.minute_writer = None - self.minute_reader = None - self.transactions = defaultdict(list) - - self.num_candles_limit = 2000 - self.max_requests_per_minute = 60 - self.request_cpt = dict() - - self.bundle = ExchangeBundle(self.name) - - def sanitize_curency_symbol(self, exchange_symbol): - """ - Helper method used to build the universal pair. - Include any symbol mapping here if appropriate. - - :param exchange_symbol: - :return universal_symbol: - """ - return exchange_symbol.lower() - - def _create_order(self, order_status): - """ - Create a Catalyst order object from the Exchange order dictionary - :param order_status: - :return: Order - """ - # if order_status['is_cancelled']: - # status = ORDER_STATUS.CANCELLED - # elif not order_status['is_live']: - # log.info('found executed order {}'.format(order_status)) - # status = ORDER_STATUS.FILLED - # else: - status = ORDER_STATUS.OPEN - - amount = float(order_status['amount']) - # filled = float(order_status['executed_amount']) - filled = None - - if order_status['type'] == 'sell': - amount = -amount - # filled = -filled - - price = float(order_status['rate']) - - stop_price = None - limit_price = None - - # TODO: is this comprehensive enough? - # if order_type.endswith('limit'): - # limit_price = price - # elif order_type.endswith('stop'): - # stop_price = price - - # executed_price = float(order_status['avg_execution_price']) - executed_price = price - - # TODO: Set Poloniex comission - commission = None - - # date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) - # date=pytz.utc.localize(date) - date = None - - order = Order( - dt=date, - asset=self.assets[order_status['symbol']], - # No such field in Poloniex - amount=amount, - stop=stop_price, - limit=limit_price, - filled=filled, - id=str(order_status['orderNumber']), - commission=commission - ) - order.status = status - - return order, executed_price - - def get_balances(self): - balances = self.api.returnbalances() - try: - log.debug('retrieving wallets balances') - except Exception as e: - log.debug(e) - raise ExchangeRequestError(error=e) - - if 'error' in balances: - raise ExchangeRequestError( - error='unable to fetch balance {}'.format(balances['error']) - ) - - std_balances = dict() - for (key, value) in iteritems(balances): - currency = key.lower() - std_balances[currency] = float(value) - - return std_balances - - @property - def account(self): - account = Account() - - account.settled_cash = None - account.accrued_interest = None - account.buying_power = None - account.equity_with_loan = None - account.total_positions_value = None - account.total_positions_exposure = None - account.regt_equity = None - account.regt_margin = None - account.initial_margin_requirement = None - account.maintenance_margin_requirement = None - account.available_funds = None - account.excess_liquidity = None - account.cushion = None - account.day_trades_remaining = None - account.leverage = None - account.net_leverage = None - account.net_liquidation = None - - return account - - @property - def time_skew(self): - # TODO: research the time skew conditions - return pd.Timedelta('0s') - - def get_account(self): - # TODO: fetch account data and keep in cache - return None - - def get_candles(self, freq, assets, bar_count=None, - start_dt=None, end_dt=None): - """ - Retrieve OHLVC candles from Poloniex - - :param freq: - :param assets: - :param bar_count: - :return: - - Available Frequencies - --------------------- - '5m', '15m', '30m', '2h', '4h', '1D' - """ - - if end_dt is None: - end_dt = pd.Timestamp.utcnow() - - log.debug( - 'retrieving {bars} {freq} candles on {exchange} from ' - '{end_dt} for markets {symbols}, '.format( - bars=bar_count, - freq=freq, - exchange=self.name, - end_dt=end_dt, - symbols=get_symbols_string(assets) - ) - ) - - if freq == '1T' and (bar_count == 1 or bar_count is None): - # TODO: use the order book instead - # We use the 5m to fetch the last bar - frequency = 300 - elif freq == '5T': - frequency = 300 - elif freq == '15T': - frequency = 900 - elif freq == '30T': - frequency = 1800 - elif freq == '120T': - frequency = 7200 - elif freq == '240T': - frequency = 14400 - elif freq == '1D': - frequency = 86400 - else: - # Poloniex does not offer 1m data candles - # It is likely to error out there frequently - raise InvalidHistoryFrequencyError(frequency=freq) - - # Making sure that assets are iterable - asset_list = [assets] if isinstance(assets, TradingPair) else assets - ohlc_map = dict() - - for asset in asset_list: - delta = end_dt - pd.to_datetime('1970-1-1', utc=True) - end = int(delta.total_seconds()) - - if bar_count is None: - start = end - 2 * frequency - else: - start = end - bar_count * frequency - - try: - response = self.api.returnchartdata( - self.get_symbol(asset), frequency, start, end - ) - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'error' in response: - raise ExchangeRequestError( - error='Unable to retrieve candles: {}'.format( - response.content) - ) - - def ohlc_from_candle(candle): - last_traded = pd.Timestamp.utcfromtimestamp(candle['date']) - last_traded = last_traded.replace(tzinfo=pytz.UTC) - - ohlc = dict( - open=np.float64(candle['open']), - high=np.float64(candle['high']), - low=np.float64(candle['low']), - close=np.float64(candle['close']), - volume=np.float64(candle['volume']), - price=np.float64(candle['close']), - last_traded=last_traded - ) - - return ohlc - - if bar_count is None: - ohlc_map[asset] = ohlc_from_candle(response[0]) - else: - ohlc_bars = [] - for candle in response: - ohlc = ohlc_from_candle(candle) - ohlc_bars.append(ohlc) - ohlc_map[asset] = ohlc_bars - - return ohlc_map[assets] \ - if isinstance(assets, TradingPair) else ohlc_map - - def create_order(self, asset, amount, is_buy, style): - """ - Creating order on the exchange. - - :param asset: - :param amount: - :param is_buy: - :param style: - :return: - """ - exchange_symbol = self.get_symbol(asset) - - if (isinstance(style, ExchangeLimitOrder) - or isinstance(style, ExchangeStopLimitOrder)): - if isinstance(style, ExchangeStopLimitOrder): - log.warn('{} will ignore the stop price'.format(self.name)) - - price = style.get_limit_price(is_buy) - - try: - if (is_buy): - response = self.api.buy(exchange_symbol, amount, price) - else: - response = self.api.sell(exchange_symbol, -amount, price) - except Exception as e: - raise ExchangeRequestError(error=e) - - date = pd.Timestamp.utcnow() - - if ('orderNumber' in response): - order_id = str(response['orderNumber']) - order = Order( - dt=date, - asset=asset, - amount=amount, - stop=style.get_stop_price(is_buy), - limit=style.get_limit_price(is_buy), - id=order_id - ) - return order - else: - log.warn( - '{} order failed: {}'.format('buy' if is_buy else 'sell', - response['error'])) - return None - else: - raise InvalidOrderStyle(exchange=self.name, - style=style.__class__.__name__) - - def get_open_orders(self, asset='all'): - """Retrieve all of the current open orders. - - Parameters - ---------- - asset : Asset - If passed and not 'all', return only the open orders for the given - asset instead of all open orders. - - Returns - ------- - open_orders : dict[list[Order]] or list[Order] - If 'all' is passed this will return a dict mapping Assets - to a list containing all the open orders for the asset. - If an asset is passed then this will return a list of the open - orders for this asset. - """ - - return self.portfolio.open_orders - - """ - TODO: Why going to the exchange if we already have this info locally? - And why creating all these Orders if we later discard them? - """ - - try: - if (asset == 'all'): - response = self.api.returnopenorders('all') - else: - response = self.api.returnopenorders(self.get_symbol(asset)) - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'error' in response: - raise ExchangeRequestError( - error='Unable to retrieve open orders: {}'.format( - response['message']) - ) - - print(self.portfolio.open_orders) - - # TODO: Need to handle openOrders for 'all' - orders = list() - for order_status in response: - # will Throw error b/c Polo doesn't track order['symbol'] - order, executed_price = self._create_order(order_status) - if asset is None or asset == order.sid: - orders.append(order) - - return orders - - def get_order(self, order_id): - """Lookup an order based on the order id returned from one of the - order functions. - - Parameters - ---------- - order_id : str - The unique identifier for the order. - - Returns - ------- - order : Order - The order object. - """ - - try: - order = self._portfolio.open_orders[order_id] - except Exception as e: - raise OrphanOrderError(order_id=order_id, exchange=self.name) - - return order - - # TODO: Need to decide whether we fetch orders locally or from exchnage - # The code below is ignored - - try: - response = self.api.returnopenorders(self.get_symbol(order.sid)) - except Exception as e: - raise ExchangeRequestError(error=e) - - for o in response: - if (int(o['orderNumber']) == int(order_id)): - return order - - return None - - def cancel_order(self, order_param): - """Cancel an open order. - - Parameters - ---------- - order_param : str or Order - The order_id or order object to cancel. - """ - - if (isinstance(order_param, Order)): - order = order_param - else: - order = self._portfolio.open_orders[order_param] - - try: - response = self.api.cancelorder(order.id) - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'error' in response: - log.info( - 'Unable to cancel order {order_id} on exchange {exchange} ' - '{error}.'.format( - order_id=order.id, - exchange=self.name, - error=response['error'] - )) - - # raise OrderCancelError( - # order_id=order.id, - # exchange=self.name, - # error=response['error'] - # ) - - self.portfolio.remove_order(order) - - def tickers(self, assets): - """ - Fetch ticket data for assets - https://docs.bitfinex.com/v2/reference#rest-public-tickers - - :param assets: - :return: - """ - symbols = self.get_symbols(assets) - - log.debug('fetching tickers {}'.format(symbols)) - - try: - response = self.api.returnticker() - except Exception as e: - raise ExchangeRequestError(error=e) - - if 'error' in response: - raise ExchangeRequestError( - error='Unable to retrieve tickers: {}'.format( - response['error']) - ) - - ticks = dict() - - for index, symbol in enumerate(symbols): - ticks[assets[index]] = dict( - timestamp=pd.Timestamp.utcnow(), - bid=float(response[symbol]['highestBid']), - ask=float(response[symbol]['lowestAsk']), - last_price=float(response[symbol]['last']), - low=float(response[symbol]['lowestAsk']), - # TODO: Polo does not provide low - high=float(response[symbol]['highestBid']), - # TODO: Polo does not provide high - volume=float(response[symbol]['baseVolume']), - ) - - log.debug('got tickers {}'.format(ticks)) - return ticks - - def generate_symbols_json(self, filename=None, source_dates=False): - symbol_map = {} - - if not source_dates: - fn, r = download_exchange_symbols(self.name) - with open(fn) as data_file: - cached_symbols = json.load(data_file) - - response = self.api.returnticker() - - for exchange_symbol in response: - base, market = self.sanitize_curency_symbol(exchange_symbol).split( - '_') - symbol = '{market}_{base}'.format(market=market, base=base) - - if (source_dates): - start_date = self.get_symbol_start_date(exchange_symbol) - else: - try: - start_date = cached_symbols[exchange_symbol]['start_date'] - except KeyError: - start_date = time.strftime('%Y-%m-%d') - - try: - end_daily = cached_symbols[exchange_symbol]['end_daily'] - except KeyError: - end_daily = 'N/A' - - try: - end_minute = cached_symbols[exchange_symbol]['end_minute'] - except KeyError: - end_minute = 'N/A' - - symbol_map[exchange_symbol] = dict( - symbol=symbol, - start_date=start_date, - end_daily=end_daily, - end_minute=end_minute, - ) - - if (filename is None): - filename = get_exchange_symbols_filename(self.name) - - with open(filename, 'w') as f: - json.dump(symbol_map, f, sort_keys=True, indent=2, - separators=(',', ':')) - - def get_symbol_start_date(self, symbol): - try: - r = self.api.returnchartdata(symbol, 86400, pd.to_datetime( - '2010-1-1').value // 10 ** 9) - except Exception as e: - raise ExchangeRequestError(error=e) - - return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date']))) - - def check_open_orders(self): - """ - Need to override this function for Poloniex: - - Loop through the list of open orders in the Portfolio object. - Check if any transactions have been executed: - If so, create a transaction and apply to the Portfolio. - Check if the order is still open: - If not, remove it from open orders - - :return: - transactions: Transaction[] - """ - transactions = list() - if self.portfolio.open_orders: - for order_id in list(self.portfolio.open_orders): - - order = self._portfolio.open_orders[order_id] - log.debug('found open order: {}'.format(order_id)) - - try: - order_open = self.get_order(order_id) - except Exception as e: - raise ExchangeRequestError(error=e) - - if (order_open): - delta = pd.Timestamp.utcnow() - order.dt - log.info( - 'order {order_id} still open after {delta}'.format( - order_id=order_id, - delta=delta) - ) - - try: - response = self.api.returnordertrades(order_id) - except Exception as e: - raise ExchangeRequestError(error=e) - - if ('error' in response): - if (not order_open): - raise OrphanOrderReverseError(order_id=order_id, - exchange=self.name) - else: - for tx in response: - """ - We maintain a list of dictionaries of transactions that - correspond to partially filled orders, indexed by - order_id. Every time we query executed transactions - from the exchange, we check if we had that transaction - for that order already. If not, we process it. - - When an order if fully filled, we flush the dict of - transactions associated with that order. - """ - if (not filter( - lambda item: item['order_id'] == tx['tradeID'], - self.transactions[order_id])): - log.debug( - 'Got new transaction for order {}: amount {}, ' - 'price {}'.format( - order_id, tx['amount'], tx['rate'])) - tx['amount'] = float(tx['amount']) - if (tx['type'] == 'sell'): - tx['amount'] = -tx['amount'] - transaction = Transaction( - asset=order.asset, - amount=tx['amount'], - dt=pd.to_datetime(tx['date'], utc=True), - price=float(tx['rate']), - order_id=tx['tradeID'], - # it's a misnomer, but keep for compatibility - commission=float(tx['fee']) - ) - self.transactions[order_id].append(transaction) - self.portfolio.execute_transaction(transaction) - transactions.append(transaction) - - if (not order_open): - """ - Since transactions have been executed individually - the only thing left to do is remove them from list - of open_orders - """ - del self.portfolio.open_orders[order_id] - del self.transactions[order_id] - - return transactions - - def get_orderbook(self, asset, order_type='all'): - exchange_symbol = asset.exchange_symbol - data = self.api.returnOrderBook(market=exchange_symbol) - - result = dict() - for order_type in data: - # TODO: filter by type - if order_type != 'asks' and order_type != 'bids': - continue - - result[order_type] = [] - for entry in data[order_type]: - if len(entry) == 2: - result[order_type].append( - dict( - rate=float(entry[0]), - quantity=float(entry[1]) - ) - ) - return result diff --git a/catalyst/exchange/poloniex/poloniex_api.py b/catalyst/exchange/poloniex/poloniex_api.py deleted file mode 100644 index 6f339192..00000000 --- a/catalyst/exchange/poloniex/poloniex_api.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python -import json -import time -import hmac -import hashlib -import ssl - -from six.moves import urllib - -# Workaround for backwards compatibility -# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7 -urlopen = urllib.request.urlopen - - -class Poloniex_api(object): - def __init__(self, key, secret): - self.key = key - self.secret = secret - - self.max_requests_per_second = 6 - self.request_cpt = dict() - - self.public = ['returnTicker', 'return24Volume', 'returnOrderBook', - 'returnTradeHistory', 'returnChartData', - 'returnCurrencies', 'returnLoanOrders'] - self.trading = ['returnBalances', 'returnCompleteBalances', - 'returnDepositAddresses', - 'generateNewAddress', 'returnDepositsWithdrawals', - 'returnOpenOrders', - 'returnTradeHistory', 'returnOrderTrades', - 'buy', 'sell', 'cancelOrder', 'moveOrder', - 'withdraw', 'returnFeeInfo', - 'returnAvailableAccountBalances', - 'returnTradableBalances', 'transferBalance', - 'returnMarginAccountSummary', 'marginBuy', - 'marginSell', - 'getMarginPosition', 'closeMarginPosition', - 'createLoanOffer', - 'cancelLoanOffer', 'returnOpenLoanOffers', - 'returnActiveLoans', - 'returnLendingHistory', 'toggleAutoRenew'] - - def ask_request(self): - """ - Asks permission to issue a request to the exchange. - The primary purpose is to avoid hitting rate limits. - - The application will pause if the maximum requests per minute - permitted by the exchange is exceeded. - - :return boolean: - - """ - now = time.time() - if not self.request_cpt: - self.request_cpt = dict() - self.request_cpt[now] = 0 - return True - - cpt_date = list(self.request_cpt.keys())[0] - cpt = self.request_cpt[cpt_date] - - if now > cpt_date + 1: - self.request_cpt = dict() - self.request_cpt[now] = 0 - return True - - if cpt >= self.max_requests_per_second: - - time.sleep(1) - - now = time.time() - self.request_cpt = dict() - self.request_cpt[now] = 0 - return True - else: - self.request_cpt[cpt_date] += 1 - - def query(self, method, req={}): - - if method in self.public: - url = 'https://poloniex.com/public?command=' + method + '&' + \ - urllib.parse.urlencode(req) - headers = {} - post_data = None - elif method in self.trading: - url = 'https://poloniex.com/tradingApi' - req['command'] = method - req['nonce'] = int(time.time() * 1000) - post_data = urllib.parse.urlencode(req) - - signature = hmac.new(self.secret.encode('utf-8'), - post_data.encode('utf-8'), - hashlib.sha512).hexdigest() - headers = {'Sign': signature, 'Key': self.key} - - post_data = post_data.encode('utf-8') - else: - raise ValueError( - 'Method "' + method + '" not found in neither the Public API ' - 'or Trading API endpoints' - ) - - self.ask_request() - req = urllib.request.Request( - url, - data=post_data, - headers=headers, - ) - resource = urlopen(req, context=ssl._create_unverified_context()) - content = resource.read().decode('utf-8') - return json.loads(content) - - def returnticker(self): - return self.query('returnTicker', {}) - - def return24volume(self): - return self.query('return24Volume', {}) - - def returnOrderBook(self, market='all'): - return self.query('returnOrderBook', {'currencyPair': market}) - - def returntradehistory(self, market, start=None, end=None): - if (start is not None and end is not None): - return self.query('returntradehistory', - {'currencyPair': market, 'start': start, - 'end': end}) - else: - return self.query('returntradehistory', {'currencyPair': market}) - - def returnchartdata(self, market, period, start, end=9999999999): - return self.query('returnChartData', - {'currencyPair': market, 'period': period, - 'start': start, 'end': end}) - - def returncurrencies(self): - return self.query('returnCurrencies', {}) - - def returnloadorders(self, market): - return self.query('returnLoanOrders', {'currency': market}) - - def returnbalances(self): - return self.query('returnBalances') - - def returncompletebalances(self, account): - if (account): - return self.query('returnCompleteBalances', {'account': account}) - else: - return self.query('returnCompleteBalances') - - def returndepositaddresses(self): - return self.query('returnDepositAddresses') - - def generatenewaddress(self, currency): - return self.query('generateNewAddress', {'currency': currency}) - - def returnDepositsWithdrawals(self, start, end): - return self.query('returnDepositsWithdrawals', - {'start': start, 'end': end}) - - def returnopenorders(self, market): - return self.query('returnOpenOrders', {'currencyPair': market}) - - def returnordertrades(self, ordernumber): - return self.query('returnOrderTrades', {'orderNumber': ordernumber}) - - def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0, - postonly=0): - if (fillorkill): - return self.query('buy', {'currencyPair': market, 'rate': rate, - 'amount': amount, - 'fillOrKill': fillorkill, }) - elif (immediateorcancel): - return self.query('buy', {'currencyPair': market, 'rate': rate, - 'amount': amount, - 'immediateOrCancel': immediateorcancel}) - elif (postonly): - return self.query('buy', {'currencyPair': market, 'rate': rate, - 'amount': amount, - 'postOnly': postonly, }) - else: - return self.query('buy', {'currencyPair': market, 'rate': rate, - 'amount': amount, }) - - def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0, - postonly=0): - if (fillorkill): - return self.query('sell', {'currencyPair': market, 'rate': rate, - 'amount': amount, - 'fillOrKill': fillorkill, }) - elif (immediateorcancel): - return self.query('sell', {'currencyPair': market, 'rate': rate, - 'amount': amount, - 'immediateOrCancel': immediateorcancel}) - elif (postonly): - return self.query('sell', {'currencyPair': market, 'rate': rate, - 'amount': amount, - 'postOnly': postonly, }) - else: - return self.query('sell', {'currencyPair': market, 'rate': rate, - 'amount': amount, }) - - def cancelorder(self, ordernumber): - return self.query('cancelOrder', {'orderNumber': ordernumber}) - - def withdraw(self, currency, quantity, address): - return self.query('withdraw', - {'currency': currency, 'amount': quantity, - 'address': address}) - - def returnfeeinfo(self): - return self.query('returnFeeInfo') diff --git a/catalyst/exchange/bitfinex/__init__.py b/catalyst/exchange/utils/__init__.py similarity index 100% rename from catalyst/exchange/bitfinex/__init__.py rename to catalyst/exchange/utils/__init__.py diff --git a/catalyst/exchange/bundle_utils.py b/catalyst/exchange/utils/bundle_utils.py similarity index 98% rename from catalyst/exchange/bundle_utils.py rename to catalyst/exchange/utils/bundle_utils.py index 0d758c7d..6107bf79 100644 --- a/catalyst/exchange/bundle_utils.py +++ b/catalyst/exchange/utils/bundle_utils.py @@ -8,7 +8,7 @@ import pandas as pd import pytz from catalyst.data.bundles.core import download_without_progress -from catalyst.exchange.exchange_utils import get_exchange_bundles_folder +from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex'] API_URL = 'http://data.enigma.co/api/v1' diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py similarity index 78% rename from catalyst/exchange/exchange_utils.py rename to catalyst/exchange/utils/exchange_utils.py index 19938d3e..ab16f71d 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -14,6 +14,8 @@ from six.moves.urllib import request from catalyst.constants import DATE_FORMAT, SYMBOLS_URL from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \ InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias +from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ + ExchangeJSONDecoder from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time @@ -62,6 +64,13 @@ def get_exchange_folder(exchange_name, environ=None): return exchange_folder +def is_blacklist(exchange_name, environ=None): + exchange_folder = get_exchange_folder(exchange_name, environ) + filename = os.path.join(exchange_folder, 'blacklist.txt') + + return os.path.exists(filename) + + def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): """ The absolute path of the exchange's symbol.json file. @@ -101,20 +110,6 @@ def download_exchange_symbols(exchange_name, environ=None): return response -def symbols_parser(asset_def): - for key, value in asset_def.items(): - match = isinstance(value, string_types) \ - and re.search(r'(\d{4}-\d{2}-\d{2})', value) - - if match: - try: - asset_def[key] = pd.to_datetime(value, utc=True) - except ValueError: - pass - - return asset_def - - def get_exchange_symbols(exchange_name, is_local=False, environ=None): """ The de-serialized content of the exchange's symbols.json. @@ -134,13 +129,13 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): if not is_local and (not os.path.isfile(filename) or pd.Timedelta( pd.Timestamp('now', tz='UTC') - last_modified_time( - filename)).days > 1): + filename)).days > 1): download_exchange_symbols(exchange_name, environ) if os.path.isfile(filename): with open(filename) as data_file: try: - data = json.load(data_file, object_hook=symbols_parser) + data = json.load(data_file, cls=ExchangeJSONDecoder) return data except ValueError: @@ -266,7 +261,7 @@ def get_algo_folder(algo_name, environ=None): return algo_folder -def get_algo_object(algo_name, key, environ=None, rel_path=None): +def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'): """ The de-serialized object of the algo name and key. @@ -290,19 +285,25 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None): if rel_path is not None: folder = os.path.join(folder, rel_path) - filename = os.path.join(folder, key + '.p') + name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key) + filename = os.path.join(folder, name) if os.path.isfile(filename): - try: + if how == 'pickle': with open(filename, 'rb') as handle: return pickle.load(handle) - except Exception: - return None + + else: + with open(filename) as data_file: + data = json.load(data_file, cls=ExchangeJSONDecoder) + return data + else: return None -def save_algo_object(algo_name, key, obj, environ=None, rel_path=None): +def save_algo_object(algo_name, key, obj, environ=None, rel_path=None, + how='pickle'): """ Serialize and save an object by algo name and key. @@ -321,10 +322,15 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None): folder = os.path.join(folder, rel_path) ensure_directory(folder) - filename = os.path.join(folder, key + '.p') + if how == 'json': + filename = os.path.join(folder, '{}.json'.format(key)) + with open(filename, 'wt') as handle: + json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder) - with open(filename, 'wb') as handle: - pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL) + else: + filename = os.path.join(folder, '{}.p'.format(key)) + with open(filename, 'wb') as handle: + pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL) def get_algo_df(algo_name, key, environ=None, rel_path=None): @@ -428,6 +434,15 @@ def get_exchange_bundles_folder(exchange_name, environ=None): return temp_bundles +def has_bundle(exchange_name, data_frequency, environ=None): + exchange_folder = get_exchange_folder(exchange_name, environ) + + folder_name = '{}_bundle'.format(data_frequency.lower()) + folder = os.path.join(exchange_folder, folder_name) + + return os.path.isdir(folder) + + def symbols_serial(obj): """ JSON serializer for objects not serializable by default json code @@ -531,6 +546,11 @@ def get_frequency(freq, data_frequency): else: raise InvalidHistoryFrequencyError(frequency=freq) + # TODO: some exchanges support H and W frequencies but not bundles + # Find a way to pass-through these parameters to exchanges + # but resample from minute or daily in backtest mode + # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between + # Pandas offet aliases (used by Catalyst) and the CCXT timeframes if unit.lower() == 'd': alias = '{}D'.format(candle_size) @@ -646,3 +666,70 @@ def group_assets_by_exchange(assets): exchange_assets[asset.exchange].append(asset) return exchange_assets + + +def get_catalyst_symbol(market_or_symbol): + """ + The Catalyst symbol. + + Parameters + ---------- + market_or_symbol + + Returns + ------- + + """ + if isinstance(market_or_symbol, string_types): + parts = market_or_symbol.split('/') + return '{}_{}'.format(parts[0].lower(), parts[1].lower()) + + else: + return '{}_{}'.format( + market_or_symbol['base'].lower(), + market_or_symbol['quote'].lower(), + ) + + +def save_asset_data(folder, df, decimals=8): + symbols = df.index.get_level_values('symbol') + for symbol in symbols: + symbol_df = df.loc[(symbols == symbol)] # Type: pd.DataFrame + + filename = os.path.join(folder, '{}.csv'.format(symbol)) + if os.path.exists(filename): + print_headers = False + + else: + print_headers = True + + with open(filename, 'a') as f: + symbol_df.to_csv( + path_or_buf=f, + header=print_headers, + float_format='%.{}f'.format(decimals), + ) + + +def get_candles_df(candles, field, freq, bar_count, end_dt, + previous_value=None): + all_series = dict() + for asset in candles: + periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq) + + dates = [candle['last_traded'] for candle in candles[asset]] + values = [candle[field] for candle in candles[asset]] + series = pd.Series(values, index=dates) + + series = series.reindex( + periods, + method='ffill', + fill_value=previous_value, + ) + series.sort_index(inplace=True) + all_series[asset] = series + + df = pd.DataFrame(all_series) + df.dropna(inplace=True) + + return df diff --git a/catalyst/exchange/utils/factory.py b/catalyst/exchange/utils/factory.py new file mode 100644 index 00000000..17499442 --- /dev/null +++ b/catalyst/exchange/utils/factory.py @@ -0,0 +1,98 @@ +import os + +from logbook import Logger + +from catalyst.constants import LOG_LEVEL +from catalyst.exchange.ccxt.ccxt_exchange import CCXT +from catalyst.exchange.exchange import Exchange +from catalyst.exchange.exchange_errors import ExchangeAuthEmpty +from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \ + get_exchange_folder, is_blacklist + +log = Logger('factory', level=LOG_LEVEL) +exchange_cache = dict() + + +def get_exchange(exchange_name, base_currency=None, must_authenticate=False, + skip_init=False): + key = (exchange_name, base_currency) + if key in exchange_cache: + return exchange_cache[key] + + exchange_auth = get_exchange_auth(exchange_name) + + has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '') + if must_authenticate and not has_auth: + raise ExchangeAuthEmpty( + exchange=exchange_name.title(), + filename=os.path.join( + get_exchange_folder(exchange_name), 'auth.json' + ) + ) + + exchange = CCXT( + exchange_name=exchange_name, + key=exchange_auth['key'], + secret=exchange_auth['secret'], + base_currency=base_currency, + ) + exchange_cache[key] = exchange + + if not skip_init: + exchange.init() + + return exchange + + +def get_exchanges(exchange_names): + exchanges = dict() + for exchange_name in exchange_names: + exchanges[exchange_name] = get_exchange(exchange_name) + + return exchanges + + +def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False, + base_currency=None): + """ + Find exchanges filtered by a list of feature. + + Parameters + ---------- + features: str + The list of features. + + skip_blacklist: bool + is_authenticated: bool + base_currency: bool + + Returns + ------- + list[Exchange] + + """ + exchange_names = CCXT.find_exchanges(features, is_authenticated) + + exchanges = [] + for exchange_name in exchange_names: + if skip_blacklist and is_blacklist(exchange_name): + continue + + exchange = get_exchange( + exchange_name=exchange_name, + skip_init=True, + base_currency=base_currency, + ) + + if features is not None: + if 'dailyBundle' in features \ + and not exchange.has_bundle('daily'): + continue + + elif 'minuteBundle' in features \ + and not exchange.has_bundle('minute'): + continue + + exchanges.append(exchange) + + return exchanges diff --git a/catalyst/exchange/utils/live_chart_utils.py b/catalyst/exchange/utils/live_chart_utils.py new file mode 100644 index 00000000..11988cbc --- /dev/null +++ b/catalyst/exchange/utils/live_chart_utils.py @@ -0,0 +1,131 @@ +import matplotlib.dates as mdates +import pandas as pd + +from catalyst.exchange.exchange_errors import \ + MismatchingBaseCurrenciesExchanges + +fmt = mdates.DateFormatter('%Y-%m-%d %H:%M') + + +def format_ax(ax): + """ + Trying to assign reasonable parameters to the time axis. + + Parameters + ---------- + ax: + + """ + # TODO: room for improvement + ax.xaxis.set_major_locator(mdates.DayLocator(interval=1)) + ax.xaxis.set_major_formatter(fmt) + + locator = mdates.HourLocator(interval=4) + locator.MAXTICKS = 5000 + ax.xaxis.set_minor_locator(locator) + + datemin = pd.Timestamp.utcnow() + ax.set_xlim(datemin) + + ax.grid(True) + + +def set_legend(ax): + """ + Set legend on the chart. + + Parameters + ---------- + ax + + """ + ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1) + + +def draw_pnl(ax, df): + """ + Draw p&l line on the chart. + + """ + ax.clear() + ax.set_title('Performance') + index = df.index.unique() + dt = index.get_level_values(level=0) + pnl = index.get_level_values(level=4) + ax.plot( + dt, pnl, '-', + color='green', + linewidth=1.0, + label='Performance' + ) + + def perc(val): + return '{:2f}'.format(val) + + ax.format_ydata = perc + + set_legend(ax) + format_ax(ax) + + +def draw_custom_signals(ax, df): + """ + Draw custom signals on the chart. + + """ + colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink'] + + ax.clear() + ax.set_title('Custom Signals') + for index, column in enumerate(df.columns.values.tolist()): + ax.plot(df.index, df[column], '-', + color=colors[index], + linewidth=1.0, + label=column + ) + + set_legend(ax) + format_ax(ax) + + +def draw_exposure(ax, df, context): + """ + Draw exposure line on the chart. + + """ + # TODO: list exchanges in graph + base_currency = None + positions = [] + for exchange_name in context.exchanges: + exchange = context.exchanges[exchange_name] + + if not base_currency: + base_currency = exchange.base_currency + elif base_currency != exchange.base_currency: + raise MismatchingBaseCurrenciesExchanges( + base_currency=base_currency, + exchange_name=exchange.name, + exchange_currency=exchange.base_currency + ) + + positions += exchange.portfolio.positions + + ax.clear() + ax.set_title('Exposure') + ax.plot(df.index, df['base_currency'], '-', + color='green', + linewidth=1.0, + label='Base Currency: {}'.format(base_currency.upper()) + ) + + symbols = [] + for position in positions: + symbols.append(position.symbol) + + ax.plot(df.index, df['long_exposure'], '-', + color='blue', + linewidth=1.0, + label='Long Exposure: {}'.format(', '.join(symbols).upper())) + + set_legend(ax) + format_ax(ax) diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py new file mode 100644 index 00000000..4b098a02 --- /dev/null +++ b/catalyst/exchange/utils/serialization_utils.py @@ -0,0 +1,70 @@ +import json +import re +from json import JSONEncoder + +import pandas as pd +from six import string_types + +from catalyst.constants import DATE_TIME_FORMAT + + +class ExchangeJSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, pd.Timestamp): + return obj.strftime(DATE_TIME_FORMAT) + + # Let the base class default method raise the TypeError + return JSONEncoder.default(self, obj) + + +class ExchangeJSONDecoder(json.JSONDecoder): + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__( + self, object_hook=self.object_hook, *args, **kwargs + ) + + def recursive_iter(self, obj): + if isinstance(obj, dict): + for key, value in obj.items(): + match = isinstance(value, string_types) and re.search( + r'(\d{4}-\d{2}-\d{2}).*', value + ) + if match: + try: + obj[key] = pd.to_datetime(value, utc=True) + except ValueError: + pass + + elif any(isinstance(obj, t) for t in (list, tuple)): + for item in obj: + self.recursive_iter(item) + + def object_hook(self, obj): + self.recursive_iter(obj) + return obj + + +def portfolio_to_dict(portfolio): + positions = [] + for asset in portfolio.positions: + p = portfolio.positions[asset] # Type: Position + + position = dict( + symbol=asset.symbol, + exchange=asset.exchange, + amount=p.amount, + cost_basis=p.cost_basis, + last_sale_price=p.last_sale_price, + last_sale_date=p.last_sale_date, + ) + positions.append(position) + + portfolio_dict = vars(portfolio) + portfolio_dict['positions'] = positions + + return portfolio_dict + + +def portfolio_from_dict(self, portfolio_data): + from catalyst.protocol import Portfolio + return Portfolio() diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/utils/stats_utils.py similarity index 87% rename from catalyst/exchange/stats_utils.py rename to catalyst/exchange/utils/stats_utils.py index faa378ba..dd2d4899 100644 --- a/catalyst/exchange/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -1,18 +1,19 @@ -import csv -import numbers - import copy -import numpy as np +import csv +import json +import numbers import os -import pandas as pd -import boto3 import time +import numpy as np +import pandas as pd from catalyst.assets._assets import TradingPair -from catalyst.exchange.exchange_utils import get_algo_folder +from catalyst.exchange.utils.exchange_utils import get_algo_folder +from catalyst.utils.paths import data_root, ensure_directory -s3 = boto3.resource('s3') +s3_conn = [] +mailgun = [] def trend_direction(series): @@ -195,6 +196,9 @@ def prepare_stats(stats, recorded_cols=list()): if recorded_cols is not None: for column in recorded_cols[:]: value = row_data[column] + if isinstance(value, pd.Series): + value = value.to_dict() + if type(value) is dict: for asset in value: if not isinstance(asset, TradingPair): @@ -278,21 +282,17 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10): if isinstance(stats, pd.DataFrame): stats = stats.T.to_dict().values() - df, columns = prepare_stats(stats, recorded_cols=recorded_cols) + display_stats = stats[-num_rows:] if len(stats) > num_rows else stats + df, columns = prepare_stats( + display_stats, recorded_cols=recorded_cols + ) pd.set_option('display.expand_frame_repr', False) pd.set_option('precision', 8) pd.set_option('display.width', 1000) pd.set_option('display.max_colwidth', 1000) - formatters = { - 'returns': lambda returns: "{0:.4f}".format(returns), - } - - return df.tail(num_rows).to_string( - columns=columns, - formatters=formatters - ) + return df.to_string(columns=columns) def get_csv_stats(stats, recorded_cols=None): @@ -338,6 +338,12 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None, ------- """ + if not s3_conn: + import boto3 + s3_conn.append(boto3.resource('s3')) + + s3 = s3_conn[0] + if bytes_to_write is None: bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols) @@ -352,6 +358,35 @@ def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None, obj.put(Body=bytes_to_write) +def email_error(algo_name, dt, e, environ=None): + import requests + import traceback + + if not mailgun: + root = data_root(environ) + filename = os.path.join(root, 'mailgun.json') + if not os.path.exists(filename): + raise ValueError( + 'mailgun.json not found in the catalyst data folder' + ) + + with open(filename) as data_file: + mailgun.append(json.load(data_file)) + + mg = mailgun[0] + + return requests.post( + mg['url'], + auth=("api", mg['api']), + data={ + "from": mg['from'], + "to": mg['to'], + "subject": 'Error: {}'.format(algo_name), + "text": '{}\n\n{}\n{}'.format( + dt, e, traceback.format_exc() + )}) + + def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): """ Saves the performance stats to the algo local folder. @@ -372,7 +407,10 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): timestr = time.strftime('%Y%m%d') folder = get_algo_folder(algo_namespace) - filename = os.path.join(folder, '{}-{}.csv'.format(timestr, 'frames')) + stats_folder = os.path.join(folder, 'stats') + ensure_directory(stats_folder) + + filename = os.path.join(stats_folder, '{}.csv'.format(timestr)) with open(filename, 'wb') as handle: handle.write(bytes_to_write) diff --git a/catalyst/exchange/utils/test_utils.py b/catalyst/exchange/utils/test_utils.py new file mode 100644 index 00000000..caae1e23 --- /dev/null +++ b/catalyst/exchange/utils/test_utils.py @@ -0,0 +1,83 @@ +import os +import random +import tempfile + +from catalyst.assets._assets import TradingPair + +from catalyst.exchange.utils.exchange_utils import get_exchange_folder +from catalyst.exchange.utils.factory import find_exchanges +from catalyst.utils.paths import ensure_directory + + +def handle_exchange_error(exchange, e): + try: + message = '{}: {}'.format( + e.__class__, e.message.decode('ascii', 'ignore') + ) + except Exception: + message = 'unexpected error' + + folder = get_exchange_folder(exchange.name) + filename = os.path.join(folder, 'blacklist.txt') + with open(filename, 'wt') as handle: + handle.write(message) + + +def select_random_exchanges(population=3, features=None, + is_authenticated=False, base_currency=None): + all_exchanges = find_exchanges( + features=features, + is_authenticated=is_authenticated, + base_currency=base_currency, + ) + + if population is not None: + if len(all_exchanges) < population: + population = len(all_exchanges) + + exchanges = random.sample(all_exchanges, population) + + else: + exchanges = all_exchanges + + return exchanges + + +def select_random_assets(all_assets, population=3): + assets = random.sample(all_assets, population) + return assets + + +def output_df(df, assets, name=None): + """ + Outputs a price DataFrame to a temp folder. + + Parameters + ---------- + df: pd.DataFrame + assets + name + + Returns + ------- + + """ + if isinstance(assets, TradingPair): + exchange_folder = assets.exchange + asset_folder = assets.symbol + else: + exchange_folder = ','.join([asset.exchange for asset in assets]) + asset_folder = ','.join([asset.symbol for asset in assets]) + + folder = os.path.join( + tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder + ) + ensure_directory(folder) + + if name is None: + name = 'output' + + path = os.path.join(folder, '{}.csv'.format(name)) + df.to_csv(path) + + return path diff --git a/catalyst/exchange/validator.py b/catalyst/exchange/validator.py deleted file mode 100644 index 5cd0b1e2..00000000 --- a/catalyst/exchange/validator.py +++ /dev/null @@ -1,142 +0,0 @@ -import os -import tempfile - -import pandas as pd -import six -from catalyst.assets._assets import TradingPair, get_calendar -from logbook import Logger -from pandas.util.testing import assert_frame_equal - -from catalyst.constants import LOG_LEVEL -from catalyst.exchange.asset_finder_exchange import AssetFinderExchange -from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest -from catalyst.exchange.factory import get_exchanges -from catalyst.utils.paths import ensure_directory - -log = Logger('Validator', level=LOG_LEVEL) - - -def output_df(df, assets, name=None): - """ - Outputs a price DataFrame to a temp folder. - - Parameters - ---------- - df: pd.DataFrame - assets - name - - Returns - ------- - - """ - if isinstance(assets, TradingPair): - exchange_folder = assets.exchange - asset_folder = assets.symbol - else: - exchange_folder = ','.join([asset.exchange for asset in assets]) - asset_folder = ','.join([asset.symbol for asset in assets]) - - folder = os.path.join( - tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder - ) - ensure_directory(folder) - - if name is None: - name = 'output' - - path = os.path.join(folder, '{}.csv'.format(name)) - df.to_csv(path) - - return path - - -class Validator(object): - def __init__(self, data_portal): - self.data_portal = data_portal - - def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count, - sample_minutes): - """ - Creates DataFrames from the bundle and exchange for the specified - data set. - - Parameters - ---------- - exchange: Exchange - assets - end_dt - bar_count - sample_minutes - - Returns - ------- - - """ - freq = '{}T'.format(sample_minutes) - - log.info('creating data sample from bundle') - df1 = self.data_portal.get_history_window( - assets=assets, - end_dt=end_dt, - bar_count=bar_count, - frequency=freq, - field='close', - data_frequency='minute' - ) - path = output_df(df1, assets, '{}_resampled'.format(freq)) - log.info('saved resampled bundle candles: {}\n{}'.format( - path, df1.tail(10)) - ) - - log.info('creating data sample from exchange api') - candles = exchange.get_candles( - end_dt=end_dt, - freq='{}T'.format(sample_minutes), - assets=assets, - bar_count=bar_count - ) - - series = dict() - for asset in assets: - series[asset] = pd.Series( - data=[candle['close'] for candle in candles[asset]], - index=[candle['last_traded'] for candle in candles[asset]] - ) - - df2 = pd.DataFrame(series) - path = output_df(df2, assets, '{}_api'.format(freq)) - log.info('saved exchange api candles: {}\n{}'.format( - path, df2.tail(10)) - ) - - try: - assert_frame_equal(df1, df2) - return True - except: - log.warn('differences found in dataframes') - return False - - -if __name__ == '__main__': - exchanges = get_exchanges(['poloniex']) - exchange = six.next(six.itervalues(exchanges)) - assets = exchange.get_assets(symbols=['eth_btc']) - - open_calendar = get_calendar('OPEN') - asset_finder = AssetFinderExchange() - data_portal = DataPortalExchangeBacktest( - exchanges=exchanges, - asset_finder=asset_finder, - trading_calendar=open_calendar, - first_trading_day=None # will set dynamically based on assets - ) - validator = Validator(data_portal=data_portal) - - validator.compare_bundle_with_exchange( - exchange=exchange, - assets=assets, - end_dt=pd.to_datetime('2017-11-10 1:00', utc=True), - bar_count=200, - sample_minutes=30 - ) diff --git a/catalyst/finance/risk/cumulative.py b/catalyst/finance/risk/cumulative.py index 37bd349b..b04b67a3 100644 --- a/catalyst/finance/risk/cumulative.py +++ b/catalyst/finance/risk/cumulative.py @@ -27,15 +27,15 @@ from .risk import ( choose_treasury ) -from empyrical import ( +from catalyst.patches.stats import ( alpha_beta_aligned, annual_volatility, - cum_returns, downside_risk, information_ratio, max_drawdown, sharpe_ratio, sortino_ratio, + cum_returns, ) import warnings from catalyst.constants import LOG_LEVEL @@ -161,9 +161,13 @@ class RiskMetricsCumulative(object): if len(self.algorithm_returns) == 1: self.algorithm_returns = np.append(0.0, self.algorithm_returns) - self.algorithm_cumulative_returns[dt_loc] = cum_returns( - self.algorithm_returns - )[-1] + try: + self.algorithm_cumulative_returns[dt_loc] = cum_returns( + self.algorithm_returns + )[-1] + except Exception as e: + log.debug('unable to calculate cum returns: {}'.format(e)) + self.algorithm_cumulative_returns[dt_loc] = np.nan algo_cumulative_returns_to_date = \ self.algorithm_cumulative_returns[:dt_loc + 1] @@ -196,8 +200,11 @@ class RiskMetricsCumulative(object): self.benchmark_cumulative_returns[dt_loc] = cum_returns( self.benchmark_returns )[-1] - except Exception: - self.benchmark_cumulative_returns[dt_loc] = 0 + except Exception as e: + log.debug( + 'unable to calculate benchmark cum returns: {}'.format(e) + ) + self.benchmark_cumulative_returns[dt_loc] = np.nan benchmark_cumulative_returns_to_date = \ self.benchmark_cumulative_returns[:dt_loc + 1] @@ -269,9 +276,16 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}" self.sharpe[dt_loc] = sharpe_ratio( self.algorithm_returns, ) - self.downside_risk[dt_loc] = downside_risk( - self.algorithm_returns - ) + + try: + self.downside_risk[dt_loc] = downside_risk( + self.algorithm_returns + ) + except Exception as e: + log.debug( + 'unable to calculate downside risk returns: {}'.format(e) + ) + self.downside_risk[dt_loc] = np.nan try: risk = self.downside_risk[dt_loc] @@ -279,17 +293,26 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}" self.algorithm_returns, _downside_risk=risk ) - except Exception: - # TODO: what causes it to error out? - self.sortino[dt_loc] = 0 + except Exception as e: + log.debug( + 'unable to calculate benchmark cum returns: {}'.format(e) + ) + self.sortino[dt_loc] = np.nan self.information[dt_loc] = information_ratio( self.algorithm_returns, self.benchmark_returns, ) - self.max_drawdown = max_drawdown( - self.algorithm_returns - ) + try: + self.max_drawdown = max_drawdown( + self.algorithm_returns + ) + except Exception as e: + log.debug( + 'unable to calculate max drawdown: {}'.format(e) + ) + self.max_drawdown = np.nan + self.max_drawdowns[dt_loc] = self.max_drawdown self.max_leverage = self.calculate_max_leverage() self.max_leverages[dt_loc] = self.max_leverage diff --git a/catalyst/exchange/bittrex/__init__.py b/catalyst/patches/__init__.py similarity index 100% rename from catalyst/exchange/bittrex/__init__.py rename to catalyst/patches/__init__.py diff --git a/catalyst/patches/stats.py b/catalyst/patches/stats.py new file mode 100644 index 00000000..ac8b8688 --- /dev/null +++ b/catalyst/patches/stats.py @@ -0,0 +1,1112 @@ +# +# Copyright 2016 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import division + +from functools import wraps + +import pandas as pd +import numpy as np +from scipy import stats +from six import iteritems + +from empyrical.utils import nanmean, nanstd, nanmin + +APPROX_BDAYS_PER_MONTH = 21 +APPROX_BDAYS_PER_YEAR = 252 + +MONTHS_PER_YEAR = 12 +WEEKS_PER_YEAR = 52 + +DAILY = 'daily' +WEEKLY = 'weekly' +MONTHLY = 'monthly' +YEARLY = 'yearly' + +ANNUALIZATION_FACTORS = { + DAILY: APPROX_BDAYS_PER_YEAR, + WEEKLY: WEEKS_PER_YEAR, + MONTHLY: MONTHS_PER_YEAR, + YEARLY: 1 +} + + +def _adjust_returns(returns, adjustment_factor): + """ + Returns the returns series adjusted by adjustment_factor. Optimizes for the + case of adjustment_factor being 0 by returning returns itself, not a copy! + + Parameters + ---------- + returns : pd.Series or np.ndarray + adjustment_factor : pd.Series or np.ndarray or float or int + + Returns + ------- + pd.Series or np.ndarray + """ + if isinstance(adjustment_factor, (float, int)) and adjustment_factor == 0: + return returns + return returns - adjustment_factor + + +def annualization_factor(period, annualization): + """ + Return annualization factor from period entered or if a custom + value is passed in. + + Parameters + ---------- + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + Returns + ------- + float + Annualization factor. + """ + if annualization is None: + try: + factor = ANNUALIZATION_FACTORS[period] + except KeyError: + raise ValueError( + "Period cannot be '{}'. " + "Can be '{}'.".format( + period, "', '".join(ANNUALIZATION_FACTORS.keys()) + ) + ) + else: + factor = annualization + return factor + + +def cum_returns(returns, starting_value=0): + """ + Compute cumulative returns from simple returns. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Returns of the strategy as a percentage, noncumulative. + - Time series with decimal returns. + - Example: + 2015-07-16 -0.012143 + 2015-07-17 0.045350 + 2015-07-20 0.030957 + 2015-07-21 0.004902. + starting_value : float, optional + The starting returns. + + Returns + ------- + pd.Series or np.ndarray + Series of cumulative returns. + + Notes + ----- + For increased numerical accuracy, convert input to log returns + where it is possible to sum instead of multiplying. + PI((1+r_i)) - 1 = exp(ln(PI(1+r_i))) # x = exp(ln(x)) + = exp(SIGMA(ln(1+r_i)) # ln(a*b) = ln(a) + ln(b) + """ + # df_price.pct_change() adds a nan in first position, we can use + # that to have cum_logarithmic_returns start at the origin so that + # df_cum.iloc[0] == starting_value + # Note that we can't add that ourselves as we don't know which dt + # to use. + + if len(returns) < 1: + return type(returns)([]) + + if np.any(np.isnan(returns)): + returns = returns.copy() + returns[np.isnan(returns)] = 0. + + df_cum = (returns + 1).cumprod(axis=0) + + if starting_value == 0: + return df_cum - 1 + else: + return df_cum * starting_value + + +def cum_returns_final(returns, starting_value=0): + """ + Compute total returns from simple returns. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Returns of the strategy as a percentage, noncumulative. + - Time series with decimal returns. + - Example: + 2015-07-16 -0.012143 + 2015-07-17 0.045350 + 2015-07-20 0.030957 + 2015-07-21 0.004902. + starting_value : float, optional + The starting returns. + + Returns + ------- + float + + """ + + if len(returns) == 0: + return np.nan + + return cum_returns(np.asanyarray(returns), + starting_value=starting_value)[-1] + + +def array_wrap(arg_name, _not_specified=object()): + """ + Decorator for functions working on array_likes that ensures the type of + output matches that of the input, delegating to the input's __array_wrap__. + + Parameters + ---------- + arg_name : str + + The name of the array_like arg to the wrapped function. Should be the + first positional parameter to the wrapped function. + + """ + + def dec(f): + @wraps(f) + def _wrapit(*args, **kwds): + obj = kwds.get(arg_name, _not_specified) + if obj is _not_specified: + obj = args[0] + + try: + wrap = obj.__array_wrap__ + except AttributeError: + wrap = None + result = f(*args, **kwds) + if wrap: + if not isinstance(result, np.ndarray): + result = np.asarray(result) + result = wrap(result) + return result + + return _wrapit + + return dec + + +@array_wrap('a') +def nancumsum(a, axis=None, dtype=None): + """ + Return the cumulative sum of array elements over a given axis treating Not + a Numbers (NaNs) as zero. The cumulative sum does not change when NaNs are + encountered and leading NaNs are replaced by zeros. + + Handles a subset of the edge cases handled by the nancumsum added in numpy + 1.12.0. + + Parameters + ---------- + a : np.ndarray or pd.Series + + Input array. + + axis : int, optional + + Axis along which the cumulative sum is computed. The default + (None) is to compute the cumsum over the flattened array. + + dtype : np.dtype, optional + + Type of the returned array and of the accumulator in which the + elements are summed. If `dtype` is not specified, it defaults + to the dtype of `a`, unless `a` has an integer dtype with a + precision less than that of the default platform integer. In + that case, the default platform integer is used. + + Returns + ------- + nancumsum : np.ndarray or pd.Series + + A new array that has the same size as a, and the same shape as a. + + See Also + -------- + numpy.cumsum : Cumulative sum across array propagating NaNs. + + """ + y = np.array(a, subok=True) + mask = np.isnan(a) + np.putmask(y, mask, 0.) + result = np.cumsum(y, axis=axis, dtype=dtype) + np.putmask(result, mask, np.nan) + return result + + +def aggregate_returns(returns, convert_to): + """ + Aggregates returns by week, month, or year. + + Parameters + ---------- + returns : pd.Series + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + convert_to : str + Can be 'weekly', 'monthly', or 'yearly'. + + Returns + ------- + pd.Series + Aggregated returns. + """ + + def cumulate_returns(x): + return cum_returns(x).iloc[-1] + + if convert_to == WEEKLY: + grouping = [lambda x: x.year, lambda x: x.isocalendar()[1]] + elif convert_to == MONTHLY: + grouping = [lambda x: x.year, lambda x: x.month] + elif convert_to == YEARLY: + grouping = [lambda x: x.year] + else: + raise ValueError( + 'convert_to must be {}, {} or {}'.format(WEEKLY, MONTHLY, YEARLY) + ) + + return returns.groupby(grouping).apply(cumulate_returns) + + +def max_drawdown(returns): + """ + Determines the maximum drawdown of a strategy. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + + Returns + ------- + float + Maximum drawdown. + + Note + ----- + See https://en.wikipedia.org/wiki/Drawdown_(economics) for more details. + """ + + if len(returns) < 1: + return np.nan + + cumulative = cum_returns(returns, starting_value=100) + max_return = np.fmax.accumulate(cumulative) + return nanmin((cumulative - max_return) / max_return) + + +def annual_return(returns, period=DAILY, annualization=None): + """Determines the mean annual growth rate of returns. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Periodic returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + Returns + ------- + float + Annual Return as CAGR (Compounded Annual Growth Rate). + + """ + + if len(returns) < 1: + return np.nan + + ann_factor = annualization_factor(period, annualization) + + num_years = float(len(returns)) / ann_factor + start_value = 100 + # Pass array to ensure index -1 looks up successfully. + end_value = cum_returns(np.asanyarray(returns), + starting_value=start_value)[-1] + cum_returns_final = (end_value - start_value) / start_value + annual_return = (1. + cum_returns_final) ** (1. / num_years) - 1 + + return annual_return + + +def annual_volatility(returns, period=DAILY, alpha=2.0, + annualization=None): + """ + Determines the annual volatility of a strategy. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Periodic returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + alpha : float, optional + Scaling relation (Levy stability exponent). + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + Returns + ------- + float + Annual volatility. + """ + + if len(returns) < 2: + return np.nan + + ann_factor = annualization_factor(period, annualization) + + volatility = nanstd(returns, ddof=1) * (ann_factor ** (1.0 / alpha)) + + return volatility + + +def calmar_ratio(returns, period=DAILY, annualization=None): + """ + Determines the Calmar ratio, or drawdown ratio, of a strategy. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + + Returns + ------- + float + Calmar ratio (drawdown ratio) as float. Returns np.nan if there is no + calmar ratio. + + Note + ----- + See https://en.wikipedia.org/wiki/Calmar_ratio for more details. + """ + + max_dd = max_drawdown(returns=returns) + if max_dd < 0: + temp = annual_return( + returns=returns, + period=period, + annualization=annualization + ) / abs(max_dd) + else: + return np.nan + + if np.isinf(temp): + return np.nan + + return temp + + +def omega_ratio(returns, risk_free=0.0, required_return=0.0, + annualization=APPROX_BDAYS_PER_YEAR): + """Determines the Omega ratio of a strategy. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + risk_free : int, float + Constant risk-free return throughout the period + required_return : float, optional + Minimum acceptance return of the investor. Threshold over which to + consider positive vs negative returns. It will be converted to a + value appropriate for the period of the returns. E.g. An annual minimum + acceptable return of 100 will translate to a minimum acceptable + return of 0.018. + annualization : int, optional + Factor used to convert the required_return into a daily + value. Enter 1 if no time period conversion is necessary. + + Returns + ------- + float + Omega ratio. + + Note + ----- + See https://en.wikipedia.org/wiki/Omega_ratio for more details. + + """ + + if len(returns) < 2: + return np.nan + + if annualization == 1: + return_threshold = required_return + elif required_return <= -1: + return np.nan + else: + return_threshold = (1 + required_return) ** \ + (1. / annualization) - 1 + + returns_less_thresh = returns - risk_free - return_threshold + + numer = sum(returns_less_thresh[returns_less_thresh > 0.0]) + denom = -1.0 * sum(returns_less_thresh[returns_less_thresh < 0.0]) + + if denom > 0.0: + return numer / denom + else: + return np.nan + + +def sharpe_ratio(returns, risk_free=0, period=DAILY, annualization=None): + """ + Determines the Sharpe ratio of a strategy. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + risk_free : int, float + Constant risk-free return throughout the period. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + Returns + ------- + float + Sharpe ratio. + + np.nan + If insufficient length of returns or if if adjusted returns are 0. + + Note + ----- + See https://en.wikipedia.org/wiki/Sharpe_ratio for more details. + + """ + + if len(returns) < 2: + return np.nan + + ann_factor = annualization_factor(period, annualization) + + returns_risk_adj = np.asanyarray(_adjust_returns(returns, risk_free)) + returns_risk_adj = returns_risk_adj[~np.isnan(returns_risk_adj)] + + if np.std(returns_risk_adj, ddof=1) == 0: + return np.nan + + return np.mean(returns_risk_adj) / np.std(returns_risk_adj, ddof=1) * \ + np.sqrt(ann_factor) + + +def sortino_ratio(returns, required_return=0, period=DAILY, + annualization=None, _downside_risk=None): + """ + Determines the Sortino ratio of a strategy. + + Parameters + ---------- + returns : pd.Series or np.ndarray or pd.DataFrame + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + required_return: float / series + minimum acceptable return + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + _downside_risk : float, optional + The downside risk of the given inputs, if known. Will be calculated if + not provided. + + Returns + ------- + float, pd.Series + + depends on input type + series ==> float + DataFrame ==> pd.Series + + Annualized Sortino ratio. + + """ + + if len(returns) < 2: + return np.nan + + ann_factor = annualization_factor(period, annualization) + + adj_returns = _adjust_returns(returns, required_return) + mu = nanmean(adj_returns, axis=0) + dsr = (_downside_risk if _downside_risk is not None + else downside_risk(returns, required_return)) + sortino = mu / dsr + return sortino * ann_factor + + +def downside_risk(returns, required_return=0, period=DAILY, + annualization=None): + """ + Determines the downside deviation below a threshold + + Parameters + ---------- + returns : pd.Series or np.ndarray or pd.DataFrame + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + required_return: float / series + minimum acceptable return + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + Returns + ------- + float, pd.Series + depends on input type + series ==> float + DataFrame ==> pd.Series + + Annualized downside deviation + + """ + + if len(returns) < 1: + return np.nan + + ann_factor = annualization_factor(period, annualization) + + downside_diff = _adjust_returns(returns, required_return).copy() + mask = downside_diff > 0 + downside_diff[mask] = 0.0 + squares = np.square(downside_diff) + mean_squares = nanmean(squares, axis=0) + dside_risk = np.sqrt(mean_squares) * np.sqrt(ann_factor) + + if len(returns.shape) == 2 and isinstance(returns, pd.DataFrame): + dside_risk = pd.Series(dside_risk, index=returns.columns) + return dside_risk + + +def information_ratio(returns, factor_returns): + """ + Determines the Information ratio of a strategy. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + factor_returns: float / series + Benchmark return to compare returns against. + + Returns + ------- + float + The information ratio. + + Note + ----- + See https://en.wikipedia.org/wiki/information_ratio for more details. + + """ + if len(returns) < 2: + return np.nan + + active_return = _adjust_returns(returns, factor_returns) + tracking_error = nanstd(active_return, ddof=1) + if np.isnan(tracking_error): + return 0.0 + if tracking_error == 0: + return np.nan + return nanmean(active_return) / tracking_error + + +def _aligned_series(*many_series): + """ + Return a new list of series containing the data in the input series, but + with their indices aligned. NaNs will be filled in for missing values. + + Parameters + ---------- + many_series : list[pd.Series] + + Returns + ------- + aligned_series : list[pd.Series] + + A new list of series containing the data in the input series, but + with their indices aligned. NaNs will be filled in for missing values. + + """ + return [series + for col, series in iteritems(pd.concat(many_series, axis=1))] + + +def alpha_beta(returns, factor_returns, risk_free=0.0, period=DAILY, + annualization=None): + """Calculates annualized alpha and beta. + + Parameters + ---------- + returns : pd.Series + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + factor_returns : pd.Series + Daily noncumulative returns of the factor to which beta is + computed. Usually a benchmark such as the market. + - This is in the same style as returns. + risk_free : int, float, optional + Constant risk-free return throughout the period. For example, the + interest rate on a three month us treasury bill. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + Returns + ------- + float + Alpha. + float + Beta. + + """ + if len(returns) < 2 or len(factor_returns) < 2: + return np.nan, np.nan + + return alpha_beta_aligned(*_aligned_series(returns, factor_returns), + risk_free=risk_free, period=period, + annualization=annualization) + + +def alpha_beta_aligned(returns, factor_returns, risk_free=0.0, period=DAILY, + annualization=None): + """Calculates annualized alpha and beta. + + If they are pd.Series, expects returns and factor_returns have already + been aligned on their labels. If np.ndarray, these arguments should have + the same shape. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + factor_returns : pd.Series or np.ndarray + Daily noncumulative returns of the factor to which beta is + computed. Usually a benchmark such as the market. + - This is in the same style as returns. + risk_free : int, float, optional + Constant risk-free return throughout the period. For example, the + interest rate on a three month us treasury bill. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + + Returns + ------- + float + Alpha. + float + Beta. + + """ + b = beta_aligned(returns, factor_returns, risk_free) + a = alpha_aligned(returns, factor_returns, risk_free, period, + annualization, _beta=b) + return a, b + + +def alpha(returns, factor_returns, risk_free=0.0, period=DAILY, + annualization=None, _beta=None): + """Calculates annualized alpha. + + Parameters + ---------- + returns : pd.Series + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + factor_returns : pd.Series + Daily noncumulative returns of the factor to which beta is + computed. Usually a benchmark such as the market. + - This is in the same style as returns. + risk_free : int, float, optional + Constant risk-free return throughout the period. For example, the + interest rate on a three month us treasury bill. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + - See full explanation in :func:`~empyrical.stats.annual_return`. + _beta : float, optional + The beta for the given inputs, if already known. Will be calculated + internally if not provided. + + Returns + ------- + float + Alpha. + """ + if len(returns) < 2 or len(factor_returns) < 2: + return np.nan + + return alpha_aligned(*_aligned_series(returns, factor_returns), + risk_free=risk_free, period=period, + annualization=annualization, _beta=_beta) + + +def alpha_aligned(returns, factor_returns, risk_free=0.0, period=DAILY, + annualization=None, _beta=None): + """Calculates annualized alpha. + + If they are pd.Series, expects returns and factor_returns have already + been aligned on their labels. If np.ndarray, these arguments should have + the same shape. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + factor_returns : pd.Series or np.ndarray + Daily noncumulative returns of the factor to which beta is + computed. Usually a benchmark such as the market. + - This is in the same style as returns. + risk_free : int, float, optional + Constant risk-free return throughout the period. For example, the + interest rate on a three month us treasury bill. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + - See full explanation in :func:`~empyrical.stats.annual_return`. + _beta : float, optional + The beta for the given inputs, if already known. Will be calculated + internally if not provided. + + Returns + ------- + float + Alpha. + """ + if len(returns) < 2: + return np.nan + + ann_factor = annualization_factor(period, annualization) + + if _beta is None: + _beta = beta_aligned(returns, factor_returns, risk_free) + + adj_returns = _adjust_returns(returns, risk_free) + adj_factor_returns = _adjust_returns(factor_returns, risk_free) + alpha_series = adj_returns - (_beta * adj_factor_returns) + + return nanmean(alpha_series) * ann_factor + + +def beta(returns, factor_returns, risk_free=0.0): + """Calculates beta. + + Parameters + ---------- + returns : pd.Series + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + factor_returns : pd.Series + Daily noncumulative returns of the factor to which beta is + computed. Usually a benchmark such as the market. + - This is in the same style as returns. + risk_free : int, float, optional + Constant risk-free return throughout the period. For example, the + interest rate on a three month us treasury bill. + + Returns + ------- + float + Beta. + """ + if len(returns) < 2 or len(factor_returns) < 2: + return np.nan + + return beta_aligned(*_aligned_series(returns, factor_returns), + risk_free=risk_free) + + +def beta_aligned(returns, factor_returns, risk_free=0.0): + """Calculates beta. + + If they are pd.Series, expects returns and factor_returns have already + been aligned on their labels. If np.ndarray, these arguments should have + the same shape. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + factor_returns : pd.Series or np.ndarray + Daily noncumulative returns of the factor to which beta is + computed. Usually a benchmark such as the market. + - This is in the same style as returns. + risk_free : int, float, optional + Constant risk-free return throughout the period. For example, the + interest rate on a three month us treasury bill. + + Returns + ------- + float + Beta. + """ + + if len(returns) < 2 or len(factor_returns) < 2: + return np.nan + # Filter out dates with np.nan as a return value + joint = np.vstack([_adjust_returns(returns, risk_free), + factor_returns]) + joint = joint[:, ~np.isnan(joint).any(axis=0)] + if joint.shape[1] < 2: + return np.nan + + cov = np.cov(joint, ddof=0) + + if np.absolute(cov[1, 1]) < 1.0e-30: + return np.nan + + return cov[0, 1] / cov[1, 1] + + +def stability_of_timeseries(returns): + """Determines R-squared of a linear fit to the cumulative + log returns. Computes an ordinary least squares linear fit, + and returns R-squared. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + + Returns + ------- + float + R-squared. + + """ + if len(returns) < 2: + return np.nan + + returns = np.asanyarray(returns) + returns = returns[~np.isnan(returns)] + + cum_log_returns = np.log1p(returns).cumsum() + rhat = stats.linregress(np.arange(len(cum_log_returns)), + cum_log_returns)[2] + + return rhat ** 2 + + +def tail_ratio(returns): + """Determines the ratio between the right (95%) and left tail (5%). + + For example, a ratio of 0.25 means that losses are four times + as bad as profits. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + + Returns + ------- + float + tail ratio + + """ + + if len(returns) < 1: + return np.nan + + returns = np.asanyarray(returns) + # Be tolerant of nan's + returns = returns[~np.isnan(returns)] + if len(returns) < 1: + return np.nan + + return np.abs(np.percentile(returns, 95)) / \ + np.abs(np.percentile(returns, 5)) + + +def cagr(returns, period=DAILY, annualization=None): + """ + Compute compound annual growth rate. + + Parameters + ---------- + returns : pd.Series or np.ndarray + Daily returns of the strategy, noncumulative. + - See full explanation in :func:`~empyrical.stats.cum_returns`. + period : str, optional + Defines the periodicity of the 'returns' data for purposes of + annualizing. Value ignored if `annualization` parameter is specified. + Defaults are: + 'monthly':12 + 'weekly': 52 + 'daily': 252 + annualization : int, optional + Used to suppress default values available in `period` to convert + returns into annual returns. Value should be the annual frequency of + `returns`. + - See full explanation in :func:`~empyrical.stats.annual_return`. + + Returns + ------- + float, np.nan + The CAGR value. + + """ + if len(returns) < 1: + return np.nan + + ann_factor = annualization_factor(period, annualization) + no_years = len(returns) / float(ann_factor) + # Pass array to ensure index -1 looks up successfully. + ending_value = cum_returns(np.asanyarray(returns), starting_value=1)[-1] + + return ending_value ** (1. / no_years) - 1 + + +SIMPLE_STAT_FUNCS = [ + cum_returns_final, + annual_return, + annual_volatility, + sharpe_ratio, + calmar_ratio, + stability_of_timeseries, + max_drawdown, + omega_ratio, + sortino_ratio, + stats.skew, + stats.kurtosis, + tail_ratio, + cagr +] + +FACTOR_STAT_FUNCS = [ + information_ratio, + alpha, + beta, +] diff --git a/catalyst/pipeline/engine.py b/catalyst/pipeline/engine.py index 8b8eb4b6..1d2098c5 100644 --- a/catalyst/pipeline/engine.py +++ b/catalyst/pipeline/engine.py @@ -7,6 +7,7 @@ from abc import ( ) from uuid import uuid4 +import six from six import ( iteritems, with_metaclass, @@ -33,7 +34,6 @@ from catalyst.utils.sharedoc import copydoc class PipelineEngine(with_metaclass(ABCMeta)): - @abstractmethod def run_pipeline(self, pipeline, start_date, end_date): """ @@ -118,6 +118,7 @@ class ExplodingPipelineEngine(PipelineEngine): """ A PipelineEngine that doesn't do anything. """ + def run_pipeline(self, pipeline, start_date, end_date): raise NoEngineRegistered( "Attempted to run a pipeline but no pipeline " @@ -484,8 +485,10 @@ class SimplePipelineEngine(PipelineEngine): ) if isinstance(term, LoadableTerm): + term_key = loader_group_key(term) + # TODO: temp workaround to_load = sorted( - loader_groups[loader_group_key(term)], + six.next(six.itervalues(loader_groups)), key=lambda t: t.dataset ) loader = get_loader(term) @@ -565,9 +568,10 @@ class SimplePipelineEngine(PipelineEngine): index=MultiIndex.from_arrays([empty_dates, empty_assets]), ) - resolved_assets = array(self._finder.retrieve_all(assets)) + # TODO: not sure what's wrong with the resolved_assets + # resolved_assets = array(self._finder.retrieve_all(assets)) dates_kept = repeat_last_axis(dates.values, len(assets))[mask] - assets_kept = repeat_first_axis(resolved_assets, len(dates))[mask] + assets_kept = repeat_first_axis(assets, len(dates))[mask] final_columns = {} for name in data: diff --git a/catalyst/support/issue_126.py b/catalyst/support/issue_126.py new file mode 100644 index 00000000..bb6435d1 --- /dev/null +++ b/catalyst/support/issue_126.py @@ -0,0 +1,52 @@ +from catalyst import run_algorithm +from catalyst.api import order, record, symbol +import pandas as pd + +from catalyst.exchange.utils.stats_utils import get_pretty_stats + + +def initialize(context): + context.assets = [symbol('eth_btc'), symbol('eth_usdt')] + + +def handle_data(context, data): + order(context.assets[0], 1) + + prices = data.current(context.assets, 'price') + record(price=prices) + pass + + +def analyze(context, perf): + stats = get_pretty_stats(perf) + print(stats) + pass + + +if __name__ == '__main__': + live = True + if live: + run_algorithm( + capital_base=0.01, + initialize=initialize, + handle_data=handle_data, + exchange_name='poloniex', + algo_namespace='buy_btc_polo_jh', + base_currency='btc', + analyze=analyze, + live=True, + simulate_orders=True, + ) + else: + run_algorithm( + capital_base=1000, + data_frequency='daily', + initialize=initialize, + handle_data=handle_data, + exchange_name='poloniex', + algo_namespace='buy_btc_polo_jh', + base_currency='usd', + analyze=analyze, + start=pd.to_datetime('2017-01-01', utc=True), + end=pd.to_datetime('2017-12-25', utc=True), + ) diff --git a/catalyst/support/issue_30.py b/catalyst/support/issue_30.py new file mode 100644 index 00000000..89ceb92a --- /dev/null +++ b/catalyst/support/issue_30.py @@ -0,0 +1,28 @@ +from catalyst.api import symbol +from catalyst.utils.run_algo import run_algorithm + + +def initialize(context): + context.asset = symbol('bcc_usdt') + + +def handle_data(context, data): + data.history(context.asset, ['close'], bar_count=100, frequency='5T') + + +def analyze(context=None, results=None): + pass + + +if __name__ == '__main__': + run_algorithm( + capital_base=100, + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bittrex', + algo_namespace="bittrex_is_broken", + base_currency='usdt', + data_frequency='minute', + simulate_orders=True, + live=True) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 153e3af6..d0129f28 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -12,7 +12,9 @@ from logbook import Logger from catalyst.data.bundles import load from catalyst.data.data_portal import DataPortal -from catalyst.exchange.factory import get_exchange +from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \ + TradingPairPricing +from catalyst.exchange.utils.factory import get_exchange try: from pygments import highlight @@ -37,7 +39,7 @@ from catalyst.exchange.exchange_algorithm import ( ) from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \ DataPortalExchangeBacktest -from catalyst.exchange.asset_finder_exchange import AssetFinderExchange +from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder from catalyst.exchange.exchange_errors import ( ExchangeRequestError, ExchangeRequestErrorTooManyAttempts, BaseCurrencyNotFoundError, NotEnoughCapitalError) @@ -68,35 +70,7 @@ class _RunAlgoError(click.ClickException, ValueError): return self.pyfunc_msg -def _run(handle_data, - initialize, - before_trading_start, - analyze, - algofile, - algotext, - defines, - data_frequency, - capital_base, - data, - bundle, - bundle_timestamp, - start, - end, - output, - print_algo, - local_namespace, - environ, - live, - exchange, - algo_namespace, - base_currency, - live_graph, - simulate_orders, - stats_output): - """Run a backtest for the given algorithm. - - This is shared between the cli and :func:`catalyst.run_algo`. - """ +def _build_namespace(algotext, local_namespace, defines): if algotext is not None: if local_namespace: ip = get_ipython() # noqa @@ -110,164 +84,197 @@ def _run(handle_data, except ValueError: raise ValueError( 'invalid define %r, should be of the form name=value' % - assign, - ) + assign) try: # evaluate in the same namespace so names may refer to # eachother namespace[name] = eval(value, namespace) except Exception as e: raise ValueError( - 'failed to execute definition for name %r: %s' % (name, e), - ) + 'failed to execute definition for name %r: %s' % (name, e)) elif defines: raise _RunAlgoError( 'cannot pass define without `algotext`', - "cannot pass '-D' / '--define' without '-t' / '--algotext'", - ) + "cannot pass '-D' / '--define' without '-t' / '--algotext'") else: namespace = {} - if algofile is not None: - algotext = algofile.read() - if print_algo: - if PYGMENTS: - highlight( - algotext, - PythonLexer(), - TerminalFormatter(), - outfile=sys.stdout, - ) - else: - click.echo(algotext) + return namespace - mode = 'paper-trading' if simulate_orders else 'live-trading' \ - if live else 'backtest' - log.info('running algo in {mode} mode'.format(mode=mode)) +def _mode(simulate_orders, live): + if not live: + return 'backtest' + elif simulate_orders: + return 'paper-trading' + else: + return 'live-trading' + + +def _build_exchanges_dict(exchange, live, simulate_orders, base_currency): exchange_name = exchange if exchange_name is None: raise ValueError('Please specify at least one exchange.') exchange_list = [x.strip().lower() for x in exchange.split(',')] - exchanges = dict() - for exchange_name in exchange_list: - exchanges[exchange_name] = get_exchange( - exchange_name=exchange_name, + exchanges = {exchange_name: get_exchange( + exchange_name=exchange_name, + base_currency=base_currency, + must_authenticate=(live and not simulate_orders)) + for exchange_name in exchange_list} + + return exchanges + + +def _pretty_print_code(algotext): + if PYGMENTS: + highlight( + algotext, + PythonLexer(), + TerminalFormatter(), + outfile=sys.stdout) + else: + click.echo(algotext) + + +def _choose_loader(data_frequency, column): + bound_cols = TradingPairPricing.columns + if column in bound_cols: + return ExchangePricingLoader(data_frequency) + raise ValueError( + "No PipelineLoader registered for column %s." % column) + + +def _get_live_time_range(): + start = pd.Timestamp.utcnow() + # TODO: fix the end data. + end = start + timedelta(hours=8760) + return start, end + + +def _data_for_live_trading(sim_params, exchanges, env, open_calendar): + data = DataPortalExchangeLive( + exchanges=exchanges, + asset_finder=env.asset_finder, + trading_calendar=open_calendar, + first_trading_day=pd.to_datetime('today', utc=True)) + + return data + + +# TODO use proper retry here +def _fetch_capital_base(base_currency, exchange_name, exchange, + attempt_index=0): + """ + Fetch the base currency amount required to bootstrap + the algorithm against the exchange. + + The algorithm cannot continue without this value. + + :param exchange: the targeted exchange + :param attempt_index: + :return capital_base: the amount of base currency available for + trading + """ + try: + log.debug('retrieving capital base in {} to bootstrap ' + 'exchange {}'.format(base_currency, exchange_name)) + balances = exchange.get_balances() + except ExchangeRequestError as e: + if attempt_index < 20: + log.warn( + 'could not retrieve balances on {}: {}'.format( + exchange.name, e)) + sleep(5) + return _fetch_capital_base(base_currency, exchange_name, exchange, + attempt_index + 1) + + else: + raise ExchangeRequestErrorTooManyAttempts( + attempts=attempt_index, + error=e) + + if base_currency in balances: + base_currency_available = balances[base_currency]['free'] + log.info( + 'base currency available in the account: {} {}'.format( + base_currency_available, base_currency)) + + return base_currency_available + else: + raise BaseCurrencyNotFoundError( base_currency=base_currency, - must_authenticate=(live and not simulate_orders), - ) + exchange=exchange_name) - open_calendar = get_calendar('OPEN') - env = TradingEnvironment( - load=partial( - load_crypto_market_data, - environ=environ, - start_dt=start, - end_dt=end - ), - environ=environ, - exchange_tz='UTC', - asset_db_path=None # We don't need an asset db, we have exchanges - ) - env.asset_finder = AssetFinderExchange() - choose_loader = None # TODO: use the DataPortal in the algo class for this +def _algorithm_class_for_live(algo_namespace, live_graph, stats_output, + analyze_live, base_currency, simulate_orders, + exchanges, capital_base): + if not simulate_orders: + for exchange_name in exchanges: + exchange = exchanges[exchange_name] + balance = _fetch_capital_base(base_currency, exchange_name, + exchange) - if live: - start = pd.Timestamp.utcnow() - - # TODO: fix the end data. - end = start + timedelta(hours=8760) - - data = DataPortalExchangeLive( - exchanges=exchanges, - asset_finder=env.asset_finder, - trading_calendar=open_calendar, - first_trading_day=pd.to_datetime('today', utc=True) - ) - - def fetch_capital_base(exchange, attempt_index=0): - """ - Fetch the base currency amount required to bootstrap - the algorithm against the exchange. - - The algorithm cannot continue without this value. - - :param exchange: the targeted exchange - :param attempt_index: - :return capital_base: the amount of base currency available for - trading - """ - try: - log.debug('retrieving capital base in {} to bootstrap ' - 'exchange {}'.format(base_currency, exchange_name)) - balances = exchange.get_balances() - except ExchangeRequestError as e: - if attempt_index < 20: - log.warn( - 'could not retrieve balances on {}: {}'.format( - exchange.name, e - ) - ) - sleep(5) - return fetch_capital_base(exchange, attempt_index + 1) - - else: - raise ExchangeRequestErrorTooManyAttempts( - attempts=attempt_index, - error=e - ) - - if base_currency in balances: - base_currency_available = balances[base_currency]['free'] - log.info( - 'base currency available in the account: {} {}'.format( - base_currency_available, base_currency - ) - ) - - return base_currency_available - else: - raise BaseCurrencyNotFoundError( + if balance < capital_base: + raise NotEnoughCapitalError( + exchange=exchange_name, base_currency=base_currency, - exchange=exchange_name - ) + balance=balance, + capital_base=capital_base) - if not simulate_orders: - for exchange_name in exchanges: - exchange = exchanges[exchange_name] - balance = fetch_capital_base(exchange) + algorithm_class = partial( + ExchangeTradingAlgorithmLive, + exchanges=exchanges, + algo_namespace=algo_namespace, + live_graph=live_graph, + simulate_orders=simulate_orders, + stats_output=stats_output, + analyze_live=analyze_live,) - if balance < capital_base: - raise NotEnoughCapitalError( - exchange=exchange_name, - base_currency=base_currency, - balance=balance, - capital_base=capital_base, - ) + return algorithm_class - sim_params = create_simulation_parameters( - start=start, - end=end, - capital_base=capital_base, - emission_rate='minute', - data_frequency='minute' - ) - # TODO: use the constructor instead - sim_params._arena = 'live' +def _bundle_trading_environment(bundle_data, environ): + prefix, connstr = re.split( + r'sqlite:///', + str(bundle_data.asset_finder.engine.url), + maxsplit=1) + if prefix: + raise ValueError( + "invalid url %r, must begin with 'sqlite:///'" % + str(bundle_data.asset_finder.engine.url)) - algorithm_class = partial( - ExchangeTradingAlgorithmLive, - exchanges=exchanges, - algo_namespace=algo_namespace, - live_graph=live_graph, - simulate_orders=simulate_orders, - stats_output=stats_output, - ) - elif exchanges: + return TradingEnvironment(asset_db_path=connstr, environ=environ) + + +def _build_live_algo_and_data(sim_params, exchanges, env, open_calendar, + simulate_orders, algo_namespace, capital_base, + live_graph, stats_output, analyze_live, + base_currency, namespace, choose_loader, + algorithm_class_kwargs): + sim_params._arena = 'live' # TODO: use the constructor instead + + data = _data_for_live_trading(sim_params, exchanges, env, open_calendar) + + algorithm_class = _algorithm_class_for_live( + algo_namespace, live_graph, stats_output, analyze_live, + base_currency, simulate_orders, exchanges, capital_base) + + return data, algorithm_class( + namespace=namespace, + env=env, + get_pipeline_loader=choose_loader, + sim_params=sim_params, + **algorithm_class_kwargs) + + +def _build_backtest_algo_and_data( + exchanges, bundle, env, environ, bundle_timestamp, open_calendar, + start, end, namespace, choose_loader, sim_params, + algorithm_class_kwargs): + if exchanges: # Removed the existing Poloniex fork to keep things simple # We can add back the complexity if required. @@ -281,41 +288,19 @@ def _run(handle_data, asset_finder=None, trading_calendar=open_calendar, first_trading_day=start, - last_available_session=end - ) - - sim_params = create_simulation_parameters( - start=start, - end=end, - capital_base=capital_base, - data_frequency=data_frequency, - emission_rate=data_frequency, - ) + last_available_session=end) algorithm_class = partial( ExchangeTradingAlgorithmBacktest, - exchanges=exchanges - ) - + exchanges=exchanges) elif bundle is not None: - bundle_data = load( - bundle, - environ, - bundle_timestamp, - ) + # TODO This branch should probably be removed or fixed: it doesn't even + # build `algorithm_class`, so it will break when trying to instantiate + # it. + bundle_data = load(bundle, environ, bundle_timestamp) - prefix, connstr = re.split( - r'sqlite:///', - str(bundle_data.asset_finder.engine.url), - maxsplit=1, - ) - if prefix: - raise ValueError( - "invalid url %r, must begin with 'sqlite:///'" % - str(bundle_data.asset_finder.engine.url), - ) + env = _bundle_trading_environment(bundle_data, environ) - env = TradingEnvironment(asset_db_path=connstr, environ=environ) first_trading_day = \ bundle_data.equity_minute_bar_reader.first_trading_day @@ -324,27 +309,103 @@ def _run(handle_data, first_trading_day=first_trading_day, equity_minute_reader=bundle_data.equity_minute_bar_reader, equity_daily_reader=bundle_data.equity_daily_bar_reader, - adjustment_reader=bundle_data.adjustment_reader, - ) + adjustment_reader=bundle_data.adjustment_reader) - perf = algorithm_class( + return data, algorithm_class( namespace=namespace, env=env, get_pipeline_loader=choose_loader, sim_params=sim_params, - **{ - 'initialize': initialize, - 'handle_data': handle_data, - 'before_trading_start': before_trading_start, - 'analyze': analyze, - } if algotext is None else { - 'algo_filename': getattr(algofile, 'name', ''), - 'script': algotext, - } - ).run( + **algorithm_class_kwargs) + + +def _build_algo_and_data(handle_data, initialize, before_trading_start, + analyze, algofile, algotext, defines, data_frequency, + capital_base, data, bundle, bundle_timestamp, start, + end, output, print_algo, local_namespace, environ, + live, exchange, algo_namespace, base_currency, + live_graph, analyze_live, simulate_orders, + stats_output): + namespace = _build_namespace(algotext, local_namespace, defines) + if algotext is not None: + algotext = algofile.read() + + if print_algo: + _pretty_print_code(algotext) + + mode = _mode(simulate_orders, live) + log.info('running algo in {mode} mode'.format(mode=mode)) + + exchanges = _build_exchanges_dict(exchange, live, simulate_orders, + base_currency) + + open_calendar = get_calendar('OPEN') + + env = TradingEnvironment( + load=partial(load_crypto_market_data, environ=environ, start_dt=start, + end_dt=end), + environ=environ, + exchange_tz='UTC', + asset_db_path=None) # We don't need an asset db, we have exchanges + + env.asset_finder = ExchangeAssetFinder(exchanges=exchanges) + + choose_loader = partial(_choose_loader, data_frequency) + + if live: + start, end = _get_live_time_range() + data_frequency = 'minute' # TODO double check if this is the desired behavior + + sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=capital_base, + emission_rate=data_frequency, + data_frequency=data_frequency) + + if algotext is None: + algorithm_class_kwargs = {'initialize': initialize, + 'handle_data': handle_data, + 'before_trading_start': before_trading_start, + 'analyze': analyze} + else: + algorithm_class_kwargs = {'algo_filename': getattr(algofile, 'name', + ''), + 'script': algotext} + + if live: + return _build_live_algo_and_data( + sim_params, exchanges, env, open_calendar, simulate_orders, + algo_namespace, capital_base, live_graph, stats_output, + analyze_live, base_currency, namespace, choose_loader, + algorithm_class_kwargs) + else: + return _build_backtest_algo_and_data( + exchanges, bundle, env, environ, bundle_timestamp, open_calendar, + start, end, namespace, choose_loader, sim_params, + algorithm_class_kwargs) + + +def _run(handle_data, initialize, before_trading_start, analyze, algofile, + algotext, defines, data_frequency, capital_base, data, bundle, + bundle_timestamp, start, end, output, print_algo, local_namespace, + environ, live, exchange, algo_namespace, base_currency, live_graph, + analyze_live, simulate_orders, stats_output): + """Run an algorithm in backtest, + paper-trading or live-trading mode. + + This is shared between the cli and :func:`catalyst.run_algo`. + """ + + data, algorithm = _build_algo_and_data( + handle_data, initialize, before_trading_start, analyze, algofile, + algotext, defines, data_frequency, capital_base, data, bundle, + bundle_timestamp, start, end, output, print_algo, local_namespace, + environ, live, exchange, algo_namespace, base_currency, live_graph, + analyze_live, simulate_orders, stats_output) + perf = algorithm.run( data, - overwrite_sim_params=False, - ) + overwrite_sim_params=False) if output == '-': click.echo(str(perf)) @@ -401,8 +462,7 @@ def load_extensions(default, extensions, strict, environ, reload=False): # without `strict` we should just log the failure warnings.warn( 'Failed to load extension: %r\n%s' % (ext, e), - stacklevel=2 - ) + stacklevel=2) else: _loaded_extensions.add(ext) @@ -427,6 +487,7 @@ def run_algorithm(initialize, base_currency=None, algo_namespace=None, live_graph=False, + analyze_live=None, simulate_orders=True, stats_output=None, output=os.devnull): @@ -500,8 +561,7 @@ def run_algorithm(initialize, catalyst.data.bundles.bundles : The available data bundles. """ load_extensions( - default_extension, extensions, strict_extensions, environ - ) + default_extension, extensions, strict_extensions, environ) if capital_base is None: raise ValueError( @@ -509,8 +569,7 @@ def run_algorithm(initialize, 'amount of base currency available for trading. For example, ' 'if the `capital_base` is 5ETH, the ' '`order_target_percent(asset, 1)` command will order 5ETH worth ' - 'of the specified asset.' - ) + 'of the specified asset.') # I'm not sure that we need this since the modified DataPortal # does not require extensions to be explicitly loaded. @@ -528,13 +587,11 @@ def run_algorithm(initialize, elif len(non_none_data) != 1: raise ValueError( 'must specify one of `data`, `data_portal`, or `bundle`,' - ' got: %r' % non_none_data, - ) + ' got: %r' % non_none_data) elif 'bundle' not in non_none_data and bundle_timestamp is not None: raise ValueError( - 'cannot specify `bundle_timestamp` without passing `bundle`', - ) + 'cannot specify `bundle_timestamp` without passing `bundle`') return _run( handle_data=handle_data, initialize=initialize, @@ -559,6 +616,6 @@ def run_algorithm(initialize, algo_namespace=algo_namespace, base_currency=base_currency, live_graph=live_graph, + analyze_live=analyze_live, simulate_orders=simulate_orders, - stats_output=stats_output - ) + stats_output=stats_output) diff --git a/docs/source/install.rst b/docs/source/install.rst index e95eea65..13f0b115 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -15,6 +15,9 @@ as an alternative installation method for MacOS and Linux, you can install Catalyst directly with ``pip`` (we recommend in combination with a virtual environemnt). See :ref:`Installing with pip `. +Alternatively you can install Catalyst using ``pipenv`` which is a mix of pip +and virtualenv. See :ref:`Installing with pipenv `. + Regardless of the method, each operating system (OS), has its own prerequisites, make sure to review the corresponding sections for your system: :ref:`Linux `, :ref:`MacOS ` and :ref:`Windows `. @@ -293,6 +296,39 @@ Troubleshooting ``pip`` Install sudo apt-get install python-dev +.. _pipenv: + +Installing with ``pipenv`` +------------------------- + +Installing Catalyst via ``pipenv`` is perhaps easier that installing it via +``pip`` itself but you need to install ``pipenv`` first via ``pip``. + +.. code-block:: bash + + $ pip install pipenv + +Once ``pipenv`` is installed you can proceed by creating a project folder and +installing Catalyst on that project automagically as follows: + +.. code-block:: bash + + $ mkdir project + $ cd project + $ pipenv --two + $ pipenv install enigma-catalyst matplotlib + +Until now the workflow compared to ``pip`` is almost identical, the difference +is that you don't need to load manually any virtualenv however you need to use +the `pipenv run` prefix to run the `catalyst` command as follows: + +.. code-block:: bash + + $ pipenv run catalyst --version + +If you want to know more about ``pipenv`` go to the `pipenv github repo`_ + +.. _`pipenv github repo`: https://github.com/pypa/pipenv .. _linux: diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 25ddf153..f9932aee 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,23 +2,31 @@ Release Notes ============= -Version 0.3.10 +Version 0.4.1 +^^^^^^^^^^^^^ +**Release Date**: 2017-01-03 + +Bug Fixes +~~~~~~~~~ +- Fixed cash synchronization issue (:issue:`133`) +- Fixed positions synchronization issue (:issue:`132`) +- Patched empyrical to resolve a np.log1p issue (:issue:`126`) +- Fixed a paper trading issue (:issue:`124`) +- Fixed a commission issue (:issue:`104`) +- Fixed a poloniex specific issue in live trading (:issue:`103`) + +Build +~~~~~ +- Caching CCXT market info to limit round-trips (:issue:`99`) +- Tentative support for Pipeline (:issue:`96`) + +Version 0.4.0 ^^^^^^^^^^^^^ **Release Date**: 2017-12-12 Bug Fixes ~~~~~~~~~ -- Fixed issue with fetching assets with daily frequency - -Version 0.3.10 -^^^^^^^^^^^^^ -**Release Date**: 2017-11-28 - -Bug Fixes -~~~~~~~~~ - -- Fixed issue with fetching assets with daily frequency - Changed Poloniex interface (should solve :issue:`95` and :issue:`94`) - Solved issue with overriding commission and slippage (:issue:`87`) - Fixed inefficiency with Bittrex current prices (:issue:`76`) @@ -30,6 +38,15 @@ Build - More granular commissions (:issue:`82`) - Added market orders in live mode (:issue:`81`) +Version 0.3.10 +^^^^^^^^^^^^^ +**Release Date**: 2017-11-28 + +Bug Fixes +~~~~~~~~~ + +- Fixed issue with fetching assets with daily frequency + Version 0.3.9 ^^^^^^^^^^^^^ **Release Date**: 2017-11-28 diff --git a/docs/source/unit-tests.rst b/docs/source/unit-tests.rst new file mode 100644 index 00000000..1da822f1 --- /dev/null +++ b/docs/source/unit-tests.rst @@ -0,0 +1,88 @@ +========== +Unit Tests +========== + +Exchanges +~~~~~~~~~ + +Markets +------- +Sample: + All markets in 3 random exchanges +Test: + Fetch all TradingPair instances +Assert: + No error + +Current Ticker +------------------ +Sample: + 3 random markets in each of the 3 random exchanges +Test: + Fetch current price and volume +Assert: + Not null and no error + +Historical Price Data +--------------------- +Sample: + - 3 random markets for each of the 3 random exchanges supporting historical data + - For each market, randomly select one supported frequency +Test: + Fetch historical data for each market using the selected frequency +Assert: + - No error and not blank + - Date of each candle is consistent with the Catalyst desired pattern, + - All candle start at fix intervals + - Last candle partial and forward looking from the end date + +Authentication and Orders +------------------------- +Sample: + 1 random market for each of 3 random authenticated exchanges +Test: + - Create one limit order randomly buying or selling at least 10% out from the current price + - Retrieve the open order from the exchange + - Cancel the open order +Assert: + No error + + +Bundles +~~~~~~~ + +Validate Bundle Data +-------------------- +Sample: + - 3 random market in bundles for exchanges supporting historical data + - For each market, randomly selected data range available in the exchange historical data +Test: + - Clean the target exchange bundle + - Ingest the selected market data for the selected data range + - Retrieve the bundle data into a dataframe + - Retrieve the equivalent OHLCV data from the exchange into a dataframe +Assert: + Matching data for the bundle and exchange + + +Algo Stats +---------- +Sample: + - 2 sample algorithms with built-in stats calculator + - 2 KPIs both calculated by each algo and by Catalyst +Test: + - Run each algorithm + - Compare the results of the two methods or calculating stats +Assert: + - Matching stats + +CSV Ingestion +------------- +Sample: + 3 random CSV files containing price data +Test: + - Ingest each CSV files + - Validate with the exchange like in the 'Validate Bundle Data' test +Assert: + Matching data between the bundle and the exchange + diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 3959b8df..b8b62360 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -20,7 +20,7 @@ dependencies: - bcolz==0.12.1 - bottleneck==1.2.1 - chardet==3.0.4 - - ccxt==1.10.319 + - ccxt==1.10.283 - click==6.7 - contextlib2==0.5.5 - cycler==0.10.0 @@ -45,6 +45,7 @@ dependencies: - python-dateutil==2.6.1 - python-editor==1.0.3 - pytz==2017.2 + - redo==1.6 - requests==2.18.4 - requests-file==1.4.2 - requests-ftp==0.3.1 diff --git a/etc/requirements.txt b/etc/requirements.txt index 5aa8ccb2..cc009908 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -83,3 +83,4 @@ tables==3.3.0 #Catalyst dependencies ccxt==1.10.283 boto3==1.4.8 +redo==1.6 diff --git a/tests/exchange/test_bcolz.py b/tests/exchange/test_bcolz.py index a842bee7..796a5da3 100644 --- a/tests/exchange/test_bcolz.py +++ b/tests/exchange/test_bcolz.py @@ -1,15 +1,14 @@ -import shutil import random +import shutil import tempfile -import pandas as pd -from catalyst.exchange.exchange_bundle import ExchangeBundle +import pandas as pd +from nose.tools import assert_equals + from catalyst.exchange.exchange_bcolz import BcolzExchangeBarWriter, \ BcolzExchangeBarReader - -from catalyst.exchange.bundle_utils import get_df_from_arrays - -from nose.tools import assert_equals +from catalyst.exchange.exchange_bundle import ExchangeBundle +from catalyst.exchange.utils.bundle_utils import get_df_from_arrays class TestBcolzWriter(object): diff --git a/tests/exchange/test_bitfinex.py b/tests/exchange/test_bitfinex.py deleted file mode 100644 index 4ac4e205..00000000 --- a/tests/exchange/test_bitfinex.py +++ /dev/null @@ -1,78 +0,0 @@ -from logbook import Logger - -from base import BaseExchangeTestCase -from catalyst.exchange.bitfinex.bitfinex import Bitfinex -from catalyst.exchange.exchange_utils import get_exchange_auth -from catalyst.finance.execution import (LimitOrder) -from catalyst.utils.deprecate import deprecated - -log = Logger('test_bitfinex') - - -@deprecated -class TestBitfinex(BaseExchangeTestCase): - @classmethod - def setup(self): - log.info('creating bitfinex object') - auth = get_exchange_auth('bitfinex') - self.exchange = Bitfinex( - key=auth['key'], - secret=auth['secret'], - base_currency='usd' - ) - - def test_order(self): - log.info('creating order') - asset = self.exchange.get_asset('eth_usd') - order_id = self.exchange.order( - asset=asset, - style=LimitOrder(limit_price=200), - limit_price=200, - amount=0.5, - stop_price=None - ) - log.info('order created {}'.format(order_id)) - pass - - def test_open_orders(self): - log.info('retrieving open orders') - # orders = self.exchange.get_open_orders() - pass - - def test_get_order(self): - log.info('retrieving order') - pass - - def test_cancel_order(self): - log.info('cancel order') - pass - - def test_get_candles(self): - log.info('retrieving candles') - # ohlcv_neo = self.exchange.get_candles( - # freq='1T', - # assets=self.exchange.get_asset('neo_btc')) - pass - - def test_tickers(self): - log.info('retrieving tickers') - # tickers = self.exchange.tickers([ - # self.exchange.get_asset('eth_btc'), - # self.exchange.get_asset('etc_btc') - # ]) - pass - - def test_get_account(self): - log.info('retrieving account data') - pass - - def test_get_balances(self): - log.info('testing exchange balances') - # balances = self.exchange.get_balances() - pass - - def test_orderbook(self): - log.info('testing order book for bitfinex') - # asset = self.exchange.get_asset('eth_btc') - # orderbook = self.exchange.get_orderbook(asset) - pass diff --git a/tests/exchange/test_bittrex.py b/tests/exchange/test_bittrex.py deleted file mode 100644 index d77c67b0..00000000 --- a/tests/exchange/test_bittrex.py +++ /dev/null @@ -1,95 +0,0 @@ -# import pandas as pd -from catalyst.exchange.bittrex.bittrex import Bittrex -from catalyst.finance.order import Order -from base import BaseExchangeTestCase -from logbook import Logger -from catalyst.exchange.exchange_utils import get_exchange_auth -from catalyst.utils.deprecate import deprecated - -log = Logger('test_bittrex') - - -@deprecated -class TestBittrex(BaseExchangeTestCase): - @classmethod - def setup(self): - auth = get_exchange_auth('bittrex') - self.exchange = Bittrex( - key=auth['key'], - secret=auth['secret'], - base_currency=None, - portfolio=None - ) - - def test_order(self): - log.info('creating order') - asset = self.exchange.get_asset('neo_btc') - order_id = self.exchange.order( - asset=asset, - limit_price=0.0005, - amount=1, - ) - log.info('order created {}'.format(order_id)) - assert order_id is not None - pass - - def test_open_orders(self): - log.info('retrieving open orders') - # asset = self.exchange.get_asset('neo_btc') - # orders = self.exchange.get_open_orders(asset) - pass - - def test_get_order(self): - log.info('retrieving order') - order = self.exchange.get_order( - u'2c584020-9caf-4af5-bde0-332c0bba17e2') - assert isinstance(order, Order) - pass - - def test_cancel_order(self, ): - log.info('cancel order') - self.exchange.cancel_order(u'dc7bcca2-5219-4145-8848-8a593d2a72f9') - pass - - def test_get_candles(self): - log.info('retrieving candles') - # ohlcv_neo = self.exchange.get_candles( - # freq='5T', - # assets=self.exchange.get_asset('neo_btc'), - # bar_count=20, - # end_dt=pd.to_datetime('2017-10-20', utc=True) - # ) - # ohlcv_neo_ubq = self.exchange.get_candles( - # freq='1D', - # assets=[ - # self.exchange.get_asset('neo_btc'), - # self.exchange.get_asset('ubq_btc') - # ], - # bar_count=14, - # end_dt=pd.to_datetime('2017-10-20', utc=True) - # ) - pass - - def test_tickers(self): - log.info('retrieving tickers') - tickers = self.exchange.tickers([ - self.exchange.get_asset('eth_btc'), - self.exchange.get_asset('etc_btc') - ]) - assert len(tickers) == 2 - pass - - def test_get_balances(self): - log.info('testing wallet balances') - # balances = self.exchange.get_balances() - pass - - def test_get_account(self): - log.info('testing account data') - pass - - def test_orderbook(self): - log.info('testing order book for bittrex') - # asset = self.exchange.get_asset('eth_btc') - # orderbook = self.exchange.get_orderbook(asset) - pass diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index a0d23319..3864d531 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -5,15 +5,15 @@ from logging import getLogger import pandas as pd -from catalyst.exchange.bundle_utils import get_bcolz_chunk, \ - get_start_dt, get_df_from_arrays from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \ BcolzExchangeBarWriter from catalyst.exchange.exchange_bundle import ExchangeBundle, \ BUNDLE_NAME_TEMPLATE -from catalyst.exchange.exchange_utils import get_exchange_folder -from catalyst.exchange.factory import get_exchange -from catalyst.exchange.stats_utils import df_to_string +from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \ + get_start_dt, get_df_from_arrays +from catalyst.exchange.utils.exchange_utils import get_exchange_folder +from catalyst.exchange.utils.factory import get_exchange +from catalyst.exchange.utils.stats_utils import df_to_string from catalyst.utils.paths import ensure_directory log = getLogger('test_exchange_bundle') diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 3ff44a8b..7be111d1 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -1,10 +1,11 @@ import pandas as pd from logbook import Logger -from base import BaseExchangeTestCase +from base import BaseExchangeTestCase from catalyst.exchange.ccxt.ccxt_exchange import CCXT +from catalyst.exchange.exchange_execution import ExchangeLimitOrder +from catalyst.exchange.utils.exchange_utils import get_exchange_auth from catalyst.finance.order import Order -from catalyst.exchange.exchange_utils import get_exchange_auth log = Logger('test_ccxt') @@ -12,22 +13,22 @@ log = Logger('test_ccxt') class TestCCXT(BaseExchangeTestCase): @classmethod def setup(self): - exchange_name = 'gdax' + exchange_name = 'binance' auth = get_exchange_auth(exchange_name) self.exchange = CCXT( exchange_name=exchange_name, key=auth['key'], secret=auth['secret'], base_currency='eth', - portfolio=None ) + self.exchange.init() def test_order(self): log.info('creating order') asset = self.exchange.get_asset('neo_eth') order_id = self.exchange.order( asset=asset, - limit_price=0.07, + style=ExchangeLimitOrder(limit_price=0.7), amount=1, ) log.info('order created {}'.format(order_id)) @@ -68,9 +69,10 @@ class TestCCXT(BaseExchangeTestCase): def test_tickers(self): log.info('retrieving tickers') - tickers = self.exchange.tickers([ - self.exchange.get_asset('eth_btc'), - ]) + assets = [ + self.exchange.get_asset('eng_eth'), + ] + tickers = self.exchange.tickers(assets) assert len(tickers) == 1 pass diff --git a/tests/exchange/test_data_portal.py b/tests/exchange/test_data_portal.py index 29ef4d46..4a9beba8 100644 --- a/tests/exchange/test_data_portal.py +++ b/tests/exchange/test_data_portal.py @@ -2,13 +2,13 @@ import pandas as pd from logbook import Logger from catalyst import get_calendar -from catalyst.exchange.asset_finder_exchange import AssetFinderExchange +from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder from catalyst.exchange.exchange_data_portal import ( DataPortalExchangeBacktest, DataPortalExchangeLive ) -from catalyst.exchange.exchange_utils import get_common_assets -from catalyst.exchange.factory import get_exchanges +from catalyst.exchange.utils.exchange_utils import get_common_assets +from catalyst.exchange.utils.factory import get_exchanges from test_utils import rnd_history_date_days, rnd_bar_count log = Logger('test_bitfinex') @@ -20,7 +20,7 @@ class TestExchangeDataPortal: log.info('creating bitfinex exchange') exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex']) open_calendar = get_calendar('OPEN') - asset_finder = AssetFinderExchange() + asset_finder = ExchangeAssetFinder() self.data_portal_live = DataPortalExchangeLive( exchanges=exchanges, diff --git a/tests/exchange/test_poloniex.py b/tests/exchange/test_poloniex.py deleted file mode 100644 index f263e9b0..00000000 --- a/tests/exchange/test_poloniex.py +++ /dev/null @@ -1,96 +0,0 @@ -from catalyst.exchange.poloniex.poloniex import Poloniex -from catalyst.finance.order import Order -from base import BaseExchangeTestCase -from logbook import Logger -from catalyst.exchange.exchange_utils import get_exchange_auth -import pandas as pd - -from catalyst.utils.deprecate import deprecated -from test_utils import output_df - -log = Logger('test_poloniex') - - -@deprecated -class TestPoloniex(BaseExchangeTestCase): - @classmethod - def setup(self): - print ('creating poloniex object') - auth = get_exchange_auth('poloniex') - self.exchange = Poloniex( - key=auth['key'], - secret=auth['secret'], - base_currency='btc' - ) - - def test_order(self): - log.info('creating order') - asset = self.exchange.get_asset('neos_btc') - order_id = self.exchange.order( - asset=asset, - limit_price=0.0005, - amount=1, - ) - log.info('order created {}'.format(order_id)) - assert order_id is not None - pass - - def test_open_orders(self): - log.info('retrieving open orders') - # asset = self.exchange.get_asset('neos_btc') - # orders = self.exchange.get_open_orders(asset) - pass - - def test_get_order(self): - log.info('retrieving order') - order = self.exchange.get_order( - u'2c584020-9caf-4af5-bde0-332c0bba17e2') - assert isinstance(order, Order) - pass - - def test_cancel_order(self, ): - log.info('cancel order') - self.exchange.cancel_order(u'dc7bcca2-5219-4145-8848-8a593d2a72f9') - pass - - def test_get_candles(self): - log.info('retrieving candles') - assets = self.exchange.get_asset('eth_btc') - ohlcv = self.exchange.get_candles( - # end_dt=pd.to_datetime('2017-11-01', utc=True), - end_dt=None, - freq='5T', - assets=assets, - bar_count=200 - ) - df = pd.DataFrame(ohlcv) - df.set_index('last_traded', drop=True, inplace=True) - log.info(df.tail(25)) - - path = output_df(df, assets, '5min_candles') - log.info('saved candles: {}'.format(path)) - pass - - def test_tickers(self): - log.info('retrieving tickers') - tickers = self.exchange.tickers([ - self.exchange.get_asset('eth_btc'), - self.exchange.get_asset('etc_btc') - ]) - assert len(tickers) == 2 - pass - - def test_get_balances(self): - log.info('testing wallet balances') - # balances = self.exchange.get_balances() - pass - - def test_get_account(self): - log.info('testing account data') - pass - - def test_orderbook(self): - log.info('testing order book for poloniex') - # asset = self.exchange.get_asset('eth_btc') - # orderbook = self.exchange.get_orderbook(asset) - pass diff --git a/tests/exchange/test_server_bundle.py b/tests/exchange/test_server_bundle.py index ea90c0f2..eb4f4703 100644 --- a/tests/exchange/test_server_bundle.py +++ b/tests/exchange/test_server_bundle.py @@ -1,17 +1,18 @@ -import os import importlib +import os -import pandas as pd import matplotlib import matplotlib.pyplot as plt -from matplotlib.finance import candlestick2_ohlc # from matplotlib.finance import volume_overlay import matplotlib.ticker as ticker +import pandas as pd +from matplotlib.finance import candlestick2_ohlc -from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader -from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk -from catalyst.exchange.factory import get_exchange +from catalyst.exchange.exchange_bundle import ExchangeBundle +from catalyst.exchange.utils.bundle_utils import get_df_from_arrays, \ + get_bcolz_chunk +from catalyst.exchange.utils.factory import get_exchange EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex'] exchanges = dict((e, getattr(importlib.import_module( diff --git a/tests/exchange/test_suite_bundle.py b/tests/exchange/test_suite_bundle.py new file mode 100644 index 00000000..94952b9c --- /dev/null +++ b/tests/exchange/test_suite_bundle.py @@ -0,0 +1,145 @@ +import random + +import pandas as pd +from logbook import Logger +from pandas.util.testing import assert_frame_equal + +from catalyst import get_calendar +from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder +from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest +from catalyst.exchange.utils.exchange_utils import get_candles_df +from catalyst.exchange.utils.factory import get_exchange +from catalyst.exchange.utils.test_utils import output_df, \ + select_random_assets + +log = Logger('TestSuiteExchange') + +pd.set_option('display.expand_frame_repr', False) +pd.set_option('precision', 8) +pd.set_option('display.width', 1000) +pd.set_option('display.max_colwidth', 1000) + + +class TestSuiteBundle: + @staticmethod + def get_data_portal(exchange_names): + open_calendar = get_calendar('OPEN') + asset_finder = ExchangeAssetFinder() + + data_portal = DataPortalExchangeBacktest( + exchange_names=exchange_names, + asset_finder=asset_finder, + trading_calendar=open_calendar, + first_trading_day=None # will set dynamically based on assets + ) + return data_portal + + def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count, + freq, data_frequency, data_portal): + """ + Creates DataFrames from the bundle and exchange for the specified + data set. + + Parameters + ---------- + exchange: Exchange + assets + end_dt + bar_count + sample_minutes + + Returns + ------- + + """ + data = dict() + + log.info('creating data sample from bundle') + data['bundle'] = data_portal.get_history_window( + assets=assets, + end_dt=end_dt, + bar_count=bar_count, + frequency=freq, + field='close', + data_frequency=data_frequency, + ) + log.info('bundle data:\n{}'.format( + data['bundle'].tail(10)) + ) + + log.info('creating data sample from exchange api') + candles = exchange.get_candles( + end_dt=end_dt, + freq=freq, + assets=assets, + bar_count=bar_count, + ) + data['exchange'] = get_candles_df( + candles=candles, + field='close', + freq=freq, + bar_count=bar_count, + end_dt=end_dt, + ) + log.info('exchange data:\n{}'.format( + data['exchange'].tail(10)) + ) + for source in data: + df = data[source] + path = output_df(df, assets, '{}_{}'.format(freq, source)) + log.info('saved {}:\n{}'.format(source, path)) + + assert_frame_equal( + right=data['bundle'], + left=data['exchange'], + check_less_precise=True, + ) + + def test_validate_bundles(self): + # exchange_population = 3 + asset_population = 3 + data_frequency = random.choice(['minute', 'daily']) + + # bundle = 'dailyBundle' if data_frequency + # == 'daily' else 'minuteBundle' + # exchanges = select_random_exchanges( + # population=exchange_population, + # features=[bundle], + # ) # Type: list[Exchange] + exchanges = [get_exchange('bitfinex', skip_init=True)] + + data_portal = TestSuiteBundle.get_data_portal( + [exchange.name for exchange in exchanges] + ) + for exchange in exchanges: + exchange.init() + + frequencies = exchange.get_candle_frequencies(data_frequency) + freq = random.sample(frequencies, 1)[0] + + bar_count = random.randint(1, 10) + + assets = select_random_assets( + exchange.assets, asset_population + ) + end_dt = None + for asset in assets: + attribute = 'end_{}'.format(data_frequency) + asset_end_dt = getattr(asset, attribute) + + if end_dt is None or asset_end_dt < end_dt: + end_dt = asset_end_dt + + dt_range = pd.date_range( + end=end_dt, periods=bar_count, freq=freq + ) + self.compare_bundle_with_exchange( + exchange=exchange, + assets=assets, + end_dt=dt_range[-1], + bar_count=bar_count, + freq=freq, + data_frequency=data_frequency, + data_portal=data_portal, + ) + pass diff --git a/tests/exchange/test_suite_exchange.py b/tests/exchange/test_suite_exchange.py new file mode 100644 index 00000000..cf92845e --- /dev/null +++ b/tests/exchange/test_suite_exchange.py @@ -0,0 +1,189 @@ +import json +import os +import random +from logging import Logger +from time import sleep + +import pandas as pd + +from catalyst.exchange.exchange_errors import ExchangeRequestError +from catalyst.exchange.exchange_execution import ExchangeLimitOrder +from catalyst.exchange.utils.exchange_utils import get_exchange_folder +from catalyst.exchange.utils.test_utils import select_random_exchanges, \ + handle_exchange_error, select_random_assets + +log = Logger('TestSuiteExchange') + + +class TestSuiteExchange: + def _test_markets_exchange(self, exchange, attempts=0): + assets = None + try: + exchange.init() + + # Verify that the assets and markets are populated + if not exchange.markets: + raise ValueError( + 'no markets found' + ) + if not exchange.assets: + raise ValueError( + 'no assets derived from markets' + ) + assets = exchange.assets + + except ExchangeRequestError as e: + sleep(5) + + if attempts > 5: + handle_exchange_error(exchange, e) + + else: + print( + 're-trying an exchange request {} {}'.format( + exchange.name, attempts + ) + ) + self._test_markets_exchange(exchange, attempts + 1) + + except Exception as e: + handle_exchange_error(exchange, e) + + return assets + + def test_markets(self): + population = 3 + results = dict() + + exchanges = select_random_exchanges(population) # Type: list[Exchange] + for exchange in exchanges: + assets = self._test_markets_exchange(exchange) + + if assets is not None: + results[exchange.name] = len(assets) + + folder = get_exchange_folder(exchange.name) + filename = os.path.join(folder, 'whitelist.json') + + symbols = [asset.symbol for asset in assets] + with open(filename, 'wt') as handle: + json.dump(symbols, handle, indent=4) + + series = pd.Series(results) + print('the tested markets\n{}'.format(series)) + + if population is not None: + assert (len(results) == population) + + pass + + def test_tickers(self): + exchange_population = 3 + asset_population = 3 + + exchanges = select_random_exchanges( + exchange_population, + features=['fetchTickers'], + ) # Type: list[Exchange] + for exchange in exchanges: + exchange.init() + + if exchange.assets and len(exchange.assets) >= asset_population: + assets = select_random_assets( + exchange.assets, asset_population + ) + tickers = exchange.tickers(assets) + + assert len(tickers) == asset_population + + else: + print( + 'skipping exchange without assets {}'.format(exchange.name) + ) + exchange_population -= 1 + pass + + def test_candles(self): + exchange_population = 3 + asset_population = 3 + + exchanges = select_random_exchanges( + population=exchange_population, + features=['fetchOHLCV'], + ) # Type: list[Exchange] + for exchange in exchanges: + exchange.init() + + if exchange.assets and len(exchange.assets) >= asset_population: + frequencies = exchange.get_candle_frequencies() + freq = random.sample(frequencies, 1)[0] + + bar_count = random.randint(1, 10) + end_dt = pd.Timestamp.utcnow().floor('1T') + dt_range = pd.date_range( + end=end_dt, periods=bar_count, freq=freq + ) + assets = select_random_assets( + exchange.assets, asset_population + ) + + candles = exchange.get_candles( + freq=freq, + assets=assets, + bar_count=bar_count, + start_dt=dt_range[0], + end_dt=dt_range[-1], + ) + + assert len(candles) == asset_population + + else: + print( + 'skipping exchange without assets {}'.format(exchange.name) + ) + exchange_population -= 1 + pass + + def test_orders(self): + population = 3 + quote_currency = 'eth' + order_amount = 0.1 + + exchanges = select_random_exchanges( + population=population, + features=['fetchOrder'], + is_authenticated=True, + base_currency=quote_currency, + ) # Type: list[Exchange] + + for exchange in exchanges: + exchange.init() + + assets = exchange.get_assets(quote_currency=quote_currency) + asset = select_random_assets(assets, 1)[0] + assert asset + + tickers = exchange.tickers([asset]) + price = tickers[asset]['last_price'] + + amount = order_amount / price + + limit_price = price * 0.8 + style = ExchangeLimitOrder(limit_price=limit_price) + + order = exchange.order( + asset=asset, + amount=amount, + style=style, + ) + sleep(1) + + open_order, _ = exchange.get_order(order.id, asset) + assert open_order.status == 0 + + exchange.cancel_order(open_order, asset) + sleep(1) + + canceled_order, _ = exchange.get_order(open_order.id, asset) + assert canceled_order.status == 2 + pass