From 51cda7b2d4f59354c2111f61f89bd7b3596fe7c0 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 17 Aug 2017 23:53:49 -0400 Subject: [PATCH] Implemented portfolio stats --- catalyst/examples/buy_and_hold_live.py | 32 ++++--- catalyst/exchange/algorithm_exchange.py | 23 ++++- catalyst/exchange/bitfinex.py | 95 +++++++++++-------- catalyst/exchange/exchange.py | 29 ++++++ catalyst/exchange/exchange_order.py | 55 +++++++++++ catalyst/exchange/exchange_portfolio.py | 117 ++++++++++++++++++++++++ catalyst/exchange/portfolio/__init__.py | 0 catalyst/utils/run_algo.py | 34 +++++-- docs/source/live-trading-blueprint.md | 7 +- 9 files changed, 327 insertions(+), 65 deletions(-) create mode 100644 catalyst/exchange/exchange_order.py create mode 100644 catalyst/exchange/exchange_portfolio.py create mode 100644 catalyst/exchange/portfolio/__init__.py diff --git a/catalyst/examples/buy_and_hold_live.py b/catalyst/examples/buy_and_hold_live.py index 46fe467b..25c57533 100644 --- a/catalyst/examples/buy_and_hold_live.py +++ b/catalyst/examples/buy_and_hold_live.py @@ -4,6 +4,7 @@ import pytz from logbook import Logger from catalyst.api import ( + order, order_target_value, order_target_percent, symbol, @@ -37,10 +38,10 @@ def handle_data(context, data): log.info('got price {}'.format(price)) # Stop buying after passing the reserve threshold - orders = get_open_orders(context.asset) or [] - for order in orders: - log.info('cancelling open order {}'.format(order)) - cancel_order(order) + # orders = get_open_orders(context.asset) or [] + # for order in orders: + # log.info('cancelling open order {}'.format(order)) + # cancel_order(order) # Stop buying after passing the reserve threshold cash = context.portfolio.cash @@ -52,18 +53,26 @@ def handle_data(context, data): # Check if still buying and could (approximately) afford another purchase if context.is_buying and cash > price: # Place order to make position in asset equal to target_hodl_value + order(context.asset, 1, limit_price=price + 1.1) # This works # order_target_value( # context.asset, # target_hodl_value, # limit_price=price * 1.1, - # stop_price=price * 0.9, # ) - order_target_percent( - context.asset, - 0.2, - limit_price=price * 1.1 - ) + # order_target_percent( + # context.asset, + # 0.01, + # limit_price=price * 1.1 + # ) + + record( + price=price, + cash=cash, + starting_cash=context.portfolio.starting_cash, + leverage=context.account.leverage, + ) + pass exchange_conn = dict( @@ -77,5 +86,6 @@ run_algorithm( handle_data=handle_data, capital_base=100000, exchange_conn=exchange_conn, - live=True + live=True, + algo_namespace='buy_and_hold_live' ) diff --git a/catalyst/exchange/algorithm_exchange.py b/catalyst/exchange/algorithm_exchange.py index be820098..2e9d4b9f 100644 --- a/catalyst/exchange/algorithm_exchange.py +++ b/catalyst/exchange/algorithm_exchange.py @@ -40,7 +40,8 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): super(self.__class__, self).__init__(*args, **kwargs) - log.info("initialization done") + self.perf_tracker = None + log.info('exchange trading algorithm successfully initialized') def _create_clock(self): # This method is taken from TradingAlgorithm. @@ -103,11 +104,28 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): return self.trading_client.transform() def updated_portfolio(self): + """ + We skip the entire performance tracker business and update the + portfolio directly. + :return: + """ return self.exchange.portfolio def updated_account(self): return self.exchange.account + def handle_data(self, data): + self.exchange.update_portfolio() + self.exchange.check_open_orders() + + if self._handle_data: + self._handle_data(self, data) + + # Unlike trading controls which remain constant unless placing an + # order, account controls can change each bar. Thus, must check + # every bar no matter if the algorithm places an order or not. + self.validate_account_controls() + @api_method @disallowed_in_before_trading_start(OrderInBeforeTradingStart()) def order(self, @@ -119,7 +137,8 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): amount, style = self._calculate_order(asset, amount, limit_price, stop_price, style) - return self.exchange.order(asset, amount, limit_price, stop_price, style) + return self.exchange.order(asset, amount, limit_price, stop_price, + style) @api_method def batch_market_order(self, share_counts): diff --git a/catalyst/exchange/bitfinex.py b/catalyst/exchange/bitfinex.py index de2a06de..e317a52d 100644 --- a/catalyst/exchange/bitfinex.py +++ b/catalyst/exchange/bitfinex.py @@ -1,3 +1,4 @@ +import pytz import six import base64 import hashlib @@ -11,12 +12,14 @@ from catalyst.protocol import Portfolio, Account # from websocket import create_connection from catalyst.exchange.exchange import Exchange from logbook import Logger -from catalyst.finance.order import Order, ORDER_STATUS +from catalyst.finance.order import ORDER_STATUS +from catalyst.exchange.exchange_order import ExchangeOrder from catalyst.finance.execution import (MarketOrder, LimitOrder, StopOrder, StopLimitOrder) from catalyst.data.data_portal import BASE_FIELDS +from catalyst.exchange.exchange_portfolio import ExchangePortfolio BITFINEX_URL = 'https://api.bitfinex.com' ASSETS = '{ "USDT_BTC": {"symbol":"btc_usd", "start_date": "2010-01-01"}, "ltcusd": {"symbol":"ltc_usd", "start_date": "2010-01-01"}, "ltcbtc": {"symbol":"ltc_btc", "start_date": "2010-01-01"}, "ethusd": {"symbol":"eth_usd", "start_date": "2010-01-01"}, "ethbtc": {"symbol":"eth_btc", "start_date": "2010-01-01"}, "etcbtc": {"symbol":"etc_btc", "start_date": "2010-01-01"}, "etcusd": {"symbol":"etc_usd", "start_date": "2010-01-01"}, "rrtusd": {"symbol":"rrt_usd", "start_date": "2010-01-01"}, "rrtbtc": {"symbol":"rrt_btc", "start_date": "2010-01-01"}, "zecusd": {"symbol":"zec_usd", "start_date": "2010-01-01"}, "zecbtc": {"symbol":"zec_btc", "start_date": "2010-01-01"}, "xmrusd": {"symbol":"xmr_usd", "start_date": "2010-01-01"}, "xmrbtc": {"symbol":"xmr_btc", "start_date": "2010-01-01"}, "dshusd": {"symbol":"dsh_usd", "start_date": "2010-01-01"}, "dshbtc": {"symbol":"dsh_btc", "start_date": "2010-01-01"}, "bccbtc": {"symbol":"bcc_btc", "start_date": "2010-01-01"}, "bcubtc": {"symbol":"bcu_btc", "start_date": "2010-01-01"}, "bccusd": {"symbol":"bcc_usd", "start_date": "2010-01-01"}, "bcuusd": {"symbol":"bcu_usd", "start_date": "2010-01-01"}, "xrpusd": {"symbol":"xrp_usd", "start_date": "2010-01-01"}, "xrpbtc": {"symbol":"xrp_btc", "start_date": "2010-01-01"}, "iotusd": {"symbol":"iot_usd", "start_date": "2010-01-01"}, "iotbtc": {"symbol":"iot_btc", "start_date": "2010-01-01"}, "ioteth": {"symbol":"iot_eth", "start_date": "2010-01-01"}, "eosusd": {"symbol":"eos_usd", "start_date": "2010-01-01"}, "eosbtc": {"symbol":"eos_btc", "start_date": "2010-01-01"}, "eoseth": {"symbol":"eos_eth", "start_date": "2010-01-01"} }' @@ -26,17 +29,16 @@ warning_logger = Logger('AlgoWarning') class Bitfinex(Exchange): - def __init__(self, key, secret, base_currency): + def __init__(self, key, secret, base_currency, store): self.url = BITFINEX_URL self.key = key self.secret = secret self.id = 'b' self.name = 'bitfinex' - self.orders = {} self.assets = {} self.load_assets(ASSETS) self.base_currency = base_currency - self._portfolio = None + self.store = store def _request(self, operation, data, version='v1'): payload_object = { @@ -115,7 +117,7 @@ class Bitfinex(Exchange): if order_status['is_cancelled']: status = ORDER_STATUS.CANCELLED elif not order_status['is_live']: - log.info('found executed order %s', order_status) + log.info('found executed order {}'.format(order_status)) status = ORDER_STATUS.FILLED else: status = ORDER_STATUS.OPEN @@ -145,8 +147,10 @@ class Bitfinex(Exchange): else: commission = None - order = Order( - dt=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])), + date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) + date = pytz.utc.localize(date) + order = ExchangeOrder( + dt=date, asset=self.assets[order_status['symbol']], amount=amount, stop=stop_price, @@ -156,14 +160,16 @@ class Bitfinex(Exchange): commission=commission ) order.status = status + order.executed_price = executed_price return order - @property - def portfolio(self): + def update_portfolio(self): """ - TODO: I'm not sure how that's used yet - :return: + Update the portfolio cash and position balances based on the + latest ticker prices. + + :return: """ response = self._request('balances', None) balances = response.json() @@ -183,21 +189,40 @@ class Bitfinex(Exchange): 'Base currency %s not found in portfolio' % self.base_currency ) - base_position_available = float(base_position['available']) - if self._portfolio is None: - portfolio = self._portfolio = Portfolio() - portfolio.starting_cash = portfolio.cash = \ - portfolio.portfolio_value = base_position_available - portfolio.capital_used = 0 - portfolio.pnl = 0 - portfolio.returns = 0 - portfolio.start_date = pd.Timestamp.utcnow() - portfolio.positions = [] - portfolio.positions_value = 0 - portfolio.positions_exposure = 0 + portfolio = self.store.portfolio + portfolio.cash = float(base_position['available']) + + if portfolio.positions: + tickers = self.tickers(portfolio.positions.keys()) + portfolio.positions_value = 0.0 + for ticker in tickers: + # TODO: convert if the position is not in the base currency + position = portfolio.positions[ticker['asset']] + position.last_sale_price = ticker['last_price'] + position.last_sale_date = ticker['timestamp'] + + portfolio.positions_value += \ + position.amount * position.last_sale_price + portfolio.portfolio_value = \ + portfolio.positions_value + portfolio.cash + + @property + def portfolio(self): + """ + TODO: I'm not sure how that's used yet + :return: + """ + if self.store.portfolio is None: + portfolio = ExchangePortfolio( + store=self.store, + start_date=pd.Timestamp.utcnow() + ) + self.store.portfolio = portfolio + self.update_portfolio() + + portfolio.starting_cash = portfolio.cash else: - portfolio = self._portfolio - portfolio.cash = base_position_available + portfolio = self.store.portfolio return portfolio @@ -227,13 +252,7 @@ class Bitfinex(Exchange): @property def positions(self): - response = self._request('positions', None) - positions = response.json() - if 'message' in positions: - raise ValueError( - 'unable to fetch positions %s' % positions['message'] - ) - raise NotImplementedError('positions not implemented') + return self.portfolio.positions @property def time_skew(self): @@ -438,7 +457,7 @@ class Bitfinex(Exchange): ) order_id = exchange_order['id'] - order = Order( + order = ExchangeOrder( dt=pd.Timestamp.utcnow(), asset=asset, amount=amount, @@ -449,7 +468,7 @@ class Bitfinex(Exchange): # TODO: is this required? order.broker_order_id = order_id - self.orders[order_id] = order + self.portfolio.create_order(order) return order_id @@ -518,8 +537,8 @@ class Bitfinex(Exchange): order_param : str or Order The order_id or order object to cancel. """ - order_id = \ - order_param.id if isinstance(order_param, Order) else order_param + order_id = order_param.id \ + if isinstance(order_param, ExchangeOrder) else order_param response = self._request('order/cancel', {'order_id': order_id}) status = response.json() @@ -528,7 +547,7 @@ class Bitfinex(Exchange): 'Unable to cancel order: %s %s' % (order_id, status['message']) ) - def tickers(self, date, assets): + def tickers(self, assets): """ Fetch ticket data for assets https://docs.bitfinex.com/v2/reference#rest-public-tickers @@ -559,7 +578,7 @@ class Bitfinex(Exchange): tick = dict( asset=assets[index], - timestamp=date, + timestamp=pd.Timestamp.utcnow(), bid=ticker[1], ask=ticker[3], last_price=ticker[7], diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 87c33a65..ce8fb5c3 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -4,12 +4,16 @@ from abc import ABCMeta, abstractmethod, abstractproperty import pandas as pd from catalyst.assets._assets import Asset +from catalyst.finance.order import ORDER_STATUS from catalyst.errors import ( MultipleSymbolsFound, SymbolNotFound, ) from datetime import timedelta +from logbook import Logger + +log = Logger('Exchange') class Exchange: @@ -19,6 +23,7 @@ class Exchange: self.name = None self.trading_pairs = None self.assets = {} + self._portfolio = None def get_trading_pairs(self, pairs): return [pair for pair in pairs if pair in self.trading_pairs] @@ -81,6 +86,26 @@ class Exchange: ) self.assets[exchange_symbol] = asset_obj + def check_open_orders(self): + if self.portfolio.open_orders: + for order_id in list(self.portfolio.open_orders): + log.debug('found open order: {}'.format(order_id)) + order = self.get_order(order_id) + log.debug('got updated order {}'.format(order)) + + if order.status == ORDER_STATUS.FILLED: + self.portfolio.execute_order(order) + elif order.status == ORDER_STATUS.CANCELLED: + self.portfolio.remove_order(order) + else: + delta = pd.Timestamp.utcnow() - order.dt + log.info( + 'order {order_id} still open after {delta}'.format( + order_id=order_id, + delta=delta + ) + ) + @abstractmethod def subscribe_to_market_data(self, symbol): pass @@ -89,6 +114,10 @@ class Exchange: def positions(self): pass + @abstractproperty + def update_portfolio(self): + pass + @abstractproperty def portfolio(self): pass diff --git a/catalyst/exchange/exchange_order.py b/catalyst/exchange/exchange_order.py new file mode 100644 index 00000000..e2375ce3 --- /dev/null +++ b/catalyst/exchange/exchange_order.py @@ -0,0 +1,55 @@ +import math + +import catalyst.protocol as zp +from catalyst.assets import Asset +from catalyst.finance.order import Order +from catalyst.utils.enum import enum +from catalyst.utils.input_validation import expect_types + +ORDER_STATUS = enum( + 'OPEN', + 'FILLED', + 'CANCELLED', + 'REJECTED', + 'HELD', +) + +SELL = 1 << 0 +BUY = 1 << 1 +STOP = 1 << 2 +LIMIT = 1 << 3 + +ORDER_FIELDS_TO_IGNORE = {'type', 'direction', '_status', 'asset'} + + +class ExchangeOrder(Order): + @expect_types(asset=Asset) + def __init__(self, dt, asset, amount, stop=None, limit=None, filled=0, + commission=0, id=None, executed_price=None): + """ + @dt - datetime.datetime that the order was placed + @asset - asset for the order. + @amount - the number of shares to buy/sell + a positive sign indicates a buy + a negative sign indicates a sell + @filled - how many shares of the order have been filled so far + """ + + # get a string representation of the uuid. + self.id = self.make_id() if id is None else id + self.dt = dt + self.reason = None + self.created = dt + self.asset = asset + self.amount = amount + self.filled = filled + self.commission = commission + self._status = ORDER_STATUS.OPEN + self.stop = stop + self.limit = limit + self.stop_reached = False + self.limit_reached = False + self.direction = math.copysign(1, self.amount) + self.type = zp.DATASOURCE_TYPE.ORDER + self.broker_order_id = None + self.executed_price = executed_price diff --git a/catalyst/exchange/exchange_portfolio.py b/catalyst/exchange/exchange_portfolio.py new file mode 100644 index 00000000..fa7ce0b8 --- /dev/null +++ b/catalyst/exchange/exchange_portfolio.py @@ -0,0 +1,117 @@ +import numpy as np +from catalyst.protocol import Portfolio, Positions, Position +from logbook import Logger + +log = Logger('ExchangePortfolio') + + +class PortfolioMemoryStore(object): + def __init__(self, algo_namespace): + self.algo_namespace = algo_namespace + self._portfolio = None + + @property + def portfolio(self): + """ + This is a mock store, the portfolio will always be None initially. + The goal is to retrieve a persisted portfolio using the + algo_namespace attribute so the algorithm can resume. + + :return: + """ + if self._portfolio is not None: + return self._portfolio + else: + return None + + @portfolio.setter + def portfolio(self, portfolio): + self._portfolio = portfolio + self.commit() + + def commit(self): + """ + The goal is to persist the portfolio somewhere so that the + algo can resume if it stops during execution. + + :return: + """ + log.debug('persisting updated portfolio') + + +class ExchangePortfolio(Portfolio): + def __init__(self, store, start_date, starting_cash=0.0): + self.capital_used = 0.0 + self.store = store + self.starting_cash = starting_cash + self.portfolio_value = starting_cash + self.pnl = 0.0 + self.returns = 0.0 + self.cash = starting_cash + self.positions = Positions() + self.start_date = start_date + self.positions_value = 0.0 + self.open_orders = dict() + + def calculate_pnl(self): + log.debug('calculating pnl') + + def update(self): + self.store.commit() + + def create_order(self, order): + log.debug('creating order {}'.format(order.id)) + self.open_orders[order.id] = order + + order_position = self.positions[order.asset] \ + if order.asset in self.positions else None + + if order_position is None: + order_position = Position(order.asset) + self.positions[order.asset] = order_position + + order_position.amount += order.amount + log.debug('open order added to portfolio') + self.update() + + def execute_order(self, order): + log.debug('executing order {}'.format(order.id)) + del self.open_orders[order.id] + + order_position = self.positions[order.asset] \ + if order.asset in self.positions else None + + if order_position is None: + raise ValueError( + 'Trying to execute order for a position not held: %s' % order.id + ) + + self.capital_used += order.amount * order.executed_price + + if order_position.cost_basis > 0: + order_position.cost_basis = np.average( + [order_position.cost_basis, order.executed_price], + weights=[order_position.amount, order.amount] + ) + else: + order_position.cost_basis = order.executed_price + + log.debug('updated portfolio with executed order') + self.update() + + def remove_order(self, order): + log.info('removing cancelled order {}'.format(order.id)) + del self.open_orders[order.id] + + order_position = self.positions[order.asset] \ + if order.asset in self.positions else None + + if order_position is None: + raise ValueError( + 'Trying to remove order for a position not held: %s' % order.id + ) + + order_position.amount -= order.amount + + log.debug('removed order from portfolio') + self.update() diff --git a/catalyst/exchange/portfolio/__init__.py b/catalyst/exchange/portfolio/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 86539318..ae1e6fd2 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -38,6 +38,7 @@ from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm from catalyst.exchange.data_portal_exchange import DataPortalExchange from catalyst.exchange.bitfinex import Bitfinex from catalyst.exchange.asset_finder_exchange import AssetFinderExchange +from catalyst.exchange.exchange_portfolio import PortfolioMemoryStore from logbook import Logger log = Logger('run_algo') @@ -82,7 +83,8 @@ def _run(handle_data, print_algo, local_namespace, environ, - exchange): + exchange, + algo_namespace): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. @@ -136,6 +138,13 @@ def _run(handle_data, end = start + pd.Timedelta('365', 'D') open_calendar = get_calendar('OPEN') + sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=capital_base, + data_frequency=data_frequency, + emission_rate=data_frequency, + ) if bundle is not None: bundles = bundle.split(',') @@ -240,6 +249,12 @@ def _run(handle_data, first_trading_day=pd.to_datetime('today', utc=True) ) choose_loader = None + sim_params = create_simulation_parameters( + start=start, + end=end, + capital_base=exchange.portfolio.starting_cash, + ) + # sim_params = None else: env = TradingEnvironment(environ=environ) choose_loader = None @@ -252,13 +267,7 @@ def _run(handle_data, namespace=namespace, env=env, get_pipeline_loader=choose_loader, - sim_params=create_simulation_parameters( - start=start, - end=end, - capital_base=capital_base, - data_frequency=data_frequency, - emission_rate=data_frequency, - ), + sim_params=sim_params, **{ 'initialize': initialize, 'handle_data': handle_data, @@ -350,7 +359,8 @@ def run_algorithm(initialize, strict_extensions=True, environ=os.environ, live=False, - exchange_conn=None): + exchange_conn=None, + algo_namespace=None): """Run a trading algorithm. Parameters @@ -446,11 +456,14 @@ def run_algorithm(initialize, ) else: if exchange_conn is not None: + store = PortfolioMemoryStore(algo_namespace) + if exchange_conn['name'] == 'bitfinex': exchange = Bitfinex( key=exchange_conn['key'], secret=exchange_conn['secret'], - base_currency=exchange_conn['base_currency'] + base_currency=exchange_conn['base_currency'], + store=store ) else: raise NotImplementedError( @@ -476,4 +489,5 @@ def run_algorithm(initialize, local_namespace=False, environ=environ, exchange=exchange, + algo_namespace=algo_namespace ) diff --git a/docs/source/live-trading-blueprint.md b/docs/source/live-trading-blueprint.md index ccceb778..327986e0 100644 --- a/docs/source/live-trading-blueprint.md +++ b/docs/source/live-trading-blueprint.md @@ -7,14 +7,13 @@ At a high level the following components have been implemented to coerce zipline into live trading.

Exchange

+ +*catalyst/exchange* + Exchange is a new package which introduces the concept of cryptocurrency exchanges to zipline. The package contains all new component implementations adapted to characteristics of exchanges. -``` -catalyst/exchange -``` - Here are some key characteristics which makes exchanges different from equity and futures currently implemented in zipline. * They trade around the clock.