From f2e4637f29815a203bbcb060e798a4b215512932 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 12 Dec 2017 13:34:18 -0500 Subject: [PATCH] BUG: trying to mitigate a date adjustment issue which occurs sometimes sometimes in live trading especially with Bitrrex at certain frequencies. --- catalyst/examples/mean_reversion_simple.py | 2 +- catalyst/examples/simple_loop.py | 43 +++++++------ catalyst/examples/simple_universe.py | 2 +- catalyst/exchange/ccxt/ccxt_exchange.py | 18 ++++-- catalyst/exchange/exchange.py | 72 ++++++++++++++++------ catalyst/exchange/exchange_algorithm.py | 18 ++++-- catalyst/exchange/exchange_data_portal.py | 1 + catalyst/exchange/exchange_errors.py | 6 ++ catalyst/utils/run_algo.py | 2 +- 9 files changed, 112 insertions(+), 52 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index b3fb7934..ca03baf0 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -245,7 +245,7 @@ def analyze(context=None, perf=None): if __name__ == '__main__': # The execution mode: backtest or live - MODE = 'backtest' + MODE = 'live' if MODE == 'backtest': folder = os.path.join( diff --git a/catalyst/examples/simple_loop.py b/catalyst/examples/simple_loop.py index 9aa3b9f1..636940d2 100644 --- a/catalyst/examples/simple_loop.py +++ b/catalyst/examples/simple_loop.py @@ -9,7 +9,7 @@ from catalyst.exchange.stats_utils import get_pretty_stats, \ def initialize(context): print('initializing') - context.asset = symbol('neo_eth') + context.asset = symbol('eth_btc') context.base_price = None @@ -23,8 +23,11 @@ def handle_data(context, data): context.asset, fields='price', bar_count=20, - frequency='15T' + frequency='5T' ) + last_traded = prices.index[-1] + print('last candle date: {}'.format(last_traded)) + rsi = talib.RSI(prices.values, timeperiod=14)[-1] print('got rsi: {}'.format(rsi)) @@ -107,25 +110,27 @@ def analyze(context, perf): pass -run_algorithm( - capital_base=250, - start=pd.to_datetime('2017-11-9 0:00', utc=True), - end=pd.to_datetime('2017-11-10 23:59', utc=True), - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace='simple_loop', - base_currency='usd' -) # run_algorithm( +# capital_base=250, +# start=pd.to_datetime('2017-11-9 0:00', utc=True), +# end=pd.to_datetime('2017-11-10 23:59', utc=True), +# data_frequency='minute', # initialize=initialize, # handle_data=handle_data, -# analyze=None, -# exchange_name='binance', -# live=True, +# analyze=analyze, +# exchange_name='bitfinex', # algo_namespace='simple_loop', -# base_currency='eth', -# live_graph=False, +# base_currency='usd' # ) +run_algorithm( + capital_base=1, + initialize=initialize, + handle_data=handle_data, + analyze=None, + exchange_name='binance', + live=True, + algo_namespace='simple_loop', + base_currency='eth', + live_graph=False, + simulate_orders=True +) diff --git a/catalyst/examples/simple_universe.py b/catalyst/examples/simple_universe.py index c27979b9..fec0c340 100644 --- a/catalyst/examples/simple_universe.py +++ b/catalyst/examples/simple_universe.py @@ -163,7 +163,7 @@ if __name__ == '__main__': initialize=initialize, handle_data=handle_data, analyze=analyze, - exchange_name='bitfinex', + exchange_name='poloniex', data_frequency='minute', base_currency='btc', live=False, diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 79203b5f..97b29cb1 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -4,12 +4,12 @@ from collections import defaultdict import ccxt import pandas as pd import six -from catalyst.assets._assets import TradingPair from ccxt import ExchangeNotAvailable, InvalidOrder from logbook import Logger from six import string_types from catalyst.algorithm import MarketOrder +from catalyst.assets._assets import TradingPair from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -58,8 +58,12 @@ class CCXT(Exchange): self._symbol_maps = [None, None] - markets_symbols = self.api.load_markets() - log.debug('the markets:\n{}'.format(markets_symbols)) + try: + markets_symbols = self.api.load_markets() + log.debug('the markets:\n{}'.format(markets_symbols)) + + except ExchangeNotAvailable as e: + raise ExchangeRequestError(error=e) self.name = exchange_name @@ -185,10 +189,12 @@ class CCXT(Exchange): assets = [assets] symbols = self.get_symbols(assets) - timeframe = self.get_timeframe(freq) - delta = start_dt - get_epoch() - ms = int(delta.total_seconds()) * 1000 + + ms = None + if start_dt is not None: + delta = start_dt - get_epoch() + ms = int(delta.total_seconds()) * 1000 candles = dict() for asset in assets: diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 3f2ac85a..36d006eb 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -15,7 +15,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \ PricingDataNotLoadedError, \ - NoDataAvailableOnExchange, NoValueForField + NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError from catalyst.exchange.exchange_utils import get_exchange_symbols, \ get_frequency, resample_history_df @@ -176,10 +176,18 @@ class Exchange: assets = [] for symbol in symbols: - asset = self.get_asset( - symbol, data_frequency, is_exchange_symbol, is_local - ) - assets.append(asset) + try: + asset = self.get_asset( + symbol, data_frequency, is_exchange_symbol, is_local + ) + assets.append(asset) + + except SymbolNotFoundOnExchange: + log.debug( + 'skipping non-existent market {} {}'.format( + self.name, symbol + ) + ) return assets def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False, @@ -227,8 +235,10 @@ class Exchange: elif data_frequency is not None: applies = ( - (data_frequency == 'minute' and a.end_minute is not None) - or (data_frequency == 'daily' and a.end_daily is not None) + ( + data_frequency == 'minute' and a.end_minute is not None) + or ( + data_frequency == 'daily' and a.end_daily is not None) ) else: @@ -441,6 +451,12 @@ class Exchange: Forward-fill missing values. Only has effect if field is 'price'. + Notes + ----- + Catalysts requires an end data with bar count both CCXT wants a + start data with bar count. Since we have to make calculations here, + we ensure that the last candle match the end_dt parameter. + Returns ------- DataFrame @@ -451,6 +467,7 @@ class Exchange: frequency, data_frequency ) adj_bar_count = candle_size * bar_count + start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) # The get_history method supports multiple asset @@ -459,11 +476,23 @@ class Exchange: assets=assets, bar_count=bar_count, start_dt=start_dt, - end_dt=end_dt + end_dt=end_dt, ) series = dict() for asset in candles: + if end_dt is not None and candles[asset]: + delta = get_delta(candle_size, data_frequency) + adj_end_dt = end_dt - delta + last_candle = candles[asset][-1] + + if last_candle['last_traded'] < adj_end_dt: + raise LastCandleTooEarlyError( + last_traded=last_candle['last_traded'], + end_dt=adj_end_dt, + exchange=self.name, + ) + asset_series = self.get_series_from_candles( candles=candles[asset], start_dt=start_dt, @@ -528,6 +557,7 @@ class Exchange: frequency, data_frequency ) adj_bar_count = candle_size * bar_count + try: series = self.bundle.get_history_window_series_and_load( assets=assets, @@ -537,6 +567,7 @@ class Exchange: data_frequency=data_frequency, force_auto_ingest=force_auto_ingest ) + except (PricingDataNotLoadedError, NoDataAvailableOnExchange): series = dict() @@ -548,7 +579,7 @@ class Exchange: start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ - if asset in series else start_dt + if asset in series else start_dt # The get_history method supports multiple asset # Use the original frequency to let each api optimize @@ -590,24 +621,27 @@ class Exchange: return df - def calculate_totals(self, positions=None): + def calculate_totals(self, check_cash=False, positions=None): """ Update the portfolio cash and position balances based on the latest ticker prices. """ log.debug('synchronizing portfolio with exchange {}'.format(self.name)) - balances = self.get_balances() - cash = balances[self.base_currency]['free'] \ - if self.base_currency in balances else None + cash = None + if check_cash: + balances = self.get_balances() - if cash is None: - raise BaseCurrencyNotFoundError( - base_currency=self.base_currency, - exchange=self.name - ) - log.debug('found base currency balance: {}'.format(cash)) + cash = balances[self.base_currency]['free'] \ + if self.base_currency in balances else None + + if cash is None: + raise BaseCurrencyNotFoundError( + base_currency=self.base_currency, + exchange=self.name + ) + log.debug('found base currency balance: {}'.format(cash)) positions_value = 0.0 if positions: diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 3505accf..b81a3132 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -498,13 +498,18 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): exchange_positions = \ [positions[asset] for asset in assets] - exchange = self.exchanges[exchange_name] # Type: Exchange - cash, positions_value = \ - exchange.calculate_totals(exchange_positions) + check_cash = (not self.simulate_orders) - total_cash += cash + exchange = self.exchanges[exchange_name] # Type: Exchange + cash, positions_value = exchange.calculate_totals( + positions=exchange_positions, + check_cash=check_cash, + ) total_positions_value += positions_value + if cash is not None: + total_cash += cash + for position in exchange_positions: tracker.update_position( asset=position.asset, @@ -512,7 +517,10 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): last_sale_price=position.last_sale_price ) - if total_cash < self.portfolio.cash: + if cash is None: + total_cash = self.portfolio.cash + + elif total_cash < self.portfolio.cash: raise ValueError('Cash on exchanges is lower than the algo.') return total_cash, total_positions_value diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index f5a10a37..f649b3b6 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -237,6 +237,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase): """ exchange = self.exchanges[exchange_name] + df = exchange.get_history_window( assets, end_dt, diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 7744aa44..bb393721 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -263,3 +263,9 @@ class NotEnoughCapitalError(ZiplineError): 'exchange should contain at least as much {base_currency} ' 'as the specified `capital_base`. The current balance {balance} is ' 'lower than the `capital_base`: {capital_base}').strip() + +class LastCandleTooEarlyError(ZiplineError): + msg = ( + 'The trade date of the last candle {last_traded} is before the ' + 'specified end date minus one candle {end_dt}. Please verify how ' + '{exchange} calculates the start date of OHLCV candles.').strip() diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index a3442ec9..ec20ad85 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -155,7 +155,7 @@ def _run(handle_data, exchanges[exchange_name] = get_exchange( exchange_name=exchange_name, base_currency=base_currency, - must_authenticate=live, + must_authenticate=(live and not simulate_orders), ) open_calendar = get_calendar('OPEN')