From 753881bade68c972306e54411b8073f75711b311 Mon Sep 17 00:00:00 2001 From: fredfortier Date: Mon, 28 Aug 2017 22:00:31 -0400 Subject: [PATCH] Bug fixes and polishing stats --- catalyst/assets/_assets.pyx | 27 ++++++++++++-- catalyst/examples/buy_the_dip_live.py | 3 +- catalyst/exchange/algorithm_exchange.py | 31 +++++++++++----- catalyst/exchange/bitfinex/bitfinex.py | 4 +-- catalyst/exchange/bitfinex/symbols.json | 4 +++ catalyst/exchange/exchange.py | 17 ++++----- catalyst/exchange/exchange_errors.py | 6 ++++ catalyst/exchange/stats_utils.py | 47 +++++++++++++++++++++++++ 8 files changed, 113 insertions(+), 26 deletions(-) create mode 100644 catalyst/exchange/stats_utils.py diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index b2e9a86d..58de3116 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -37,7 +37,7 @@ import warnings cimport numpy as np from catalyst.utils.calendars import get_calendar -from catalyst.exchange.exchange_errors import InvalidSymbolError +from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError # IMPORTANT NOTE: You must change this template if you change # Asset.__reduce__, or else we'll attempt to unpickle an old version of this @@ -477,8 +477,11 @@ cdef class TradingPair(Asset): except Exception as e: raise InvalidSymbolError(symbol=symbol, error=e) - if sid == 0: - sid = abs(hash(symbol)) % (10 ** 4) + if sid == 0 or sid is None: + try: + sid = abs(hash(symbol)) % (10 ** 4) + except Exception as e: + raise SidHashError(symbol=symbol) if asset_name is None: asset_name = ' / '.join(symbol.split('_')).upper() @@ -518,6 +521,24 @@ cdef class TradingPair(Asset): leverage=self.leverage ) + cpdef __reduce__(self): + """ + Function used by pickle to determine how to serialize/deserialize this + class. Should return a tuple whose first element is self.__class__, + and whose second element is a tuple of all the attributes that should + be serialized/deserialized during pickling. + """ + return (self.__class__, (self.symbol, + self.exchange, + self.start_date, + self.asset_name, + self.sid, + self.leverage, + self.end_date, + self.first_traded, + self.auto_close_date, + self.exchange_full)) + def make_asset_array(int size, Asset asset): cdef np.ndarray out = np.empty([size], dtype=object) out.fill(asset) diff --git a/catalyst/examples/buy_the_dip_live.py b/catalyst/examples/buy_the_dip_live.py index ff09a824..e3469079 100644 --- a/catalyst/examples/buy_the_dip_live.py +++ b/catalyst/examples/buy_the_dip_live.py @@ -8,6 +8,7 @@ from catalyst.api import ( record, get_open_orders, ) +from catalyst.exchange.stats_utils import get_pretty_stats from catalyst.utils.run_algo import run_algorithm algo_namespace = 'buy_the_dip_live' @@ -140,7 +141,7 @@ def handle_data(context, data): def analyze(context, stats): - log.info('the full stats:\n{}'.format(stats.head())) + log.info('the daily stats:\n{}'.format(get_pretty_stats(stats))) pass diff --git a/catalyst/exchange/algorithm_exchange.py b/catalyst/exchange/algorithm_exchange.py index 78f41645..265f3e0d 100644 --- a/catalyst/exchange/algorithm_exchange.py +++ b/catalyst/exchange/algorithm_exchange.py @@ -18,6 +18,7 @@ from datetime import timedelta from time import sleep from os import listdir from os.path import isfile, join +from collections import deque import logbook import pandas as pd @@ -35,6 +36,7 @@ from catalyst.exchange.exchange_errors import ( ) from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \ save_algo_object, get_algo_object, get_algo_folder +from catalyst.exchange.stats_utils import get_pretty_stats from catalyst.finance.performance.period import calc_period_stats from catalyst.gens.tradesimulation import AlgorithmSimulator from catalyst.utils.api_support import ( @@ -55,6 +57,7 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): self.exchange = kwargs.pop('exchange', None) self.algo_namespace = kwargs.pop('algo_namespace', None) self.orders = {} + self.minute_stats = deque(maxlen=60) self.is_running = True self.retry_check_open_orders = 5 @@ -63,6 +66,8 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): self.retry_order = 2 self.retry_delay = 5 + self.stats_minutes = 5 + super(self.__class__, self).__init__(*args, **kwargs) self._create_minute_writer() @@ -93,10 +98,14 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): def signal_handler(self, signal, frame): self.is_running = False - log.info('You pressed Ctrl+C!') + if self._analyze is None: + log.info('Interruption signal detected {}, exiting the ' + 'algorithm'.format(signal)) + + else: + log.info('Interruption signal detected {}, calling `analyze()` ' + 'before exiting the algorithm'.format(signal)) - stats = None - try: algo_folder = get_algo_folder(self.algo_namespace) folder = join(algo_folder, 'daily_perf') files = [f for f in listdir(folder) if isfile(join(folder, f))] @@ -108,12 +117,9 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): daily_perf_list.append(pickle.load(handle)) stats = pd.DataFrame(daily_perf_list) - stats.set_index('period_close', drop=True, inplace=True) - except Exception as e: - log.warn('Unable to compute daily stats: {}'.format(e)) + self.analyze(stats) - self.analyze(stats) sys.exit(0) def _create_clock(self): @@ -306,10 +312,17 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): # Performance tracker and keep only minute and cumulative self.perf_tracker.update_performance() - # TODO: save for future use? minute_stats = self.prepare_period_stats( data.current_dt, data.current_dt + timedelta(minutes=1)) - log.debug('the minute performance:\n{}'.format(minute_stats)) + # Saving the last hour in memory + self.minute_stats.append(minute_stats) + + print_df = pd.DataFrame(list(self.minute_stats)) + log.debug( + 'statistics for the last {stats_minutes} minutes:\n{stats}'.format( + stats_minutes=self.stats_minutes, + stats=get_pretty_stats(print_df, self.stats_minutes) + )) today = pd.to_datetime('today', utc=True) daily_stats = self.prepare_period_stats( diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index de3384e2..0afedd41 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -134,7 +134,6 @@ class Bitfinex(Exchange): amount = float(order_status['original_amount']) filled = float(order_status['executed_amount']) - is_buy = (amount > 0) price = float(order_status['price']) order_type = order_status['type'] @@ -153,7 +152,6 @@ class Bitfinex(Exchange): # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it. commission = None - # TODO: zipline likes rounded dates to match statistics, is this ok? date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) date = pytz.utc.localize(date) order = Order( @@ -451,7 +449,7 @@ class Bitfinex(Exchange): orders = list() for order_status in order_statuses: - order, = self._create_order(order_status) + order, executed_price = self._create_order(order_status) if asset is None or asset == order.sid: orders.append(order) diff --git a/catalyst/exchange/bitfinex/symbols.json b/catalyst/exchange/bitfinex/symbols.json index 6543b2d6..8ab44191 100644 --- a/catalyst/exchange/bitfinex/symbols.json +++ b/catalyst/exchange/bitfinex/symbols.json @@ -3,6 +3,10 @@ "symbol": "btc_usd", "start_date": "2010-01-01" }, + "bchusd": { + "symbol": "bch_usd", + "start_date": "2010-01-01" + }, "ltcusd": { "symbol": "ltc_usd", "start_date": "2010-01-01" diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index f1d8ddfe..b002296c 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -368,21 +368,18 @@ class Exchange: bar_count=bar_count, ) - frames = [] + series = dict() for asset in assets: asset_candles = candles[asset] - asset_data = dict() - asset_data[asset] = map(lambda candle: candle[field], - asset_candles) + values = map(lambda candle: candle[field], asset_candles) + dates = map(lambda candle: candle['last_traded'], asset_candles) - dates = map(lambda candle: candle['last_traded'], - asset_candles) + value_series = pd.Series(values, index=dates) + series[asset] = value_series - df = pd.DataFrame(asset_data, index=dates) - frames.append(df) - - return pd.concat(frames) + df = pd.concat(series) + return df @abstractmethod def create_order(self, asset, amount, is_buy, style): diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 29ae7ff6..955c33b8 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -73,3 +73,9 @@ class InvalidOrderType(ZiplineError): msg = ( 'Order type not found.' ).strip() + + +class SidHashError(ZiplineError): + msg = ( + 'Unable to hash sid from symbol {symbol}.' + ).strip() diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/stats_utils.py new file mode 100644 index 00000000..eda1b2fd --- /dev/null +++ b/catalyst/exchange/stats_utils.py @@ -0,0 +1,47 @@ +import pandas as pd + + +def get_pretty_stats(stats_df, num_rows=10): + """ + Format and print the last few rows of a statistics DataFrame. + See the pyfolio project for the data structure. + + :param stats_df: + :param num_rows: + :return: + """ + stats_df.set_index('period_close', drop=True, inplace=True) + stats_df.dropna(axis=1, how='all', inplace=True) + + pd.set_option('display.expand_frame_repr', False) + pd.set_option('precision', 3) + pd.set_option('display.width', 1000) + pd.set_option('display.max_colwidth', 1000) + + columns = ['starting_cash', 'ending_cash', 'portfolio_value', + 'pnl', 'long_exposure', 'short_exposure', 'orders', + 'transactions', 'positions'] + + def format_positions(positions): + parts = [] + for position in positions: + msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format( + amount=position['amount'], + market=position['sid'].market_currency, + cost_basis=position['cost_basis'], + base=position['sid'].base_currency + ) + parts.append(msg) + return ', '.join(parts) + + formatters = { + 'orders': lambda orders: len(orders), + 'transactions': lambda transactions: len(transactions), + 'returns': lambda returns: "{0:.4f}".format(returns), + 'positions': format_positions + } + + return stats_df.tail(num_rows).to_string( + columns=columns, + formatters=formatters + )