wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-08-02 12:30:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implemented portfolio stats

Browse Source
This commit is contained in:
Frederic Fortier
2017-08-17 23:53:49 -04:00
parent 6f591305e3
commit 51cda7b2d4
9 changed files with 327 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/examples/buy_and_hold_live.py
+21 -11
View File
@@ -4,6 +4,7 @@ import pytz
from logbook import Logger
from catalyst.api import (
order,
order_target_value,
order_target_percent,
symbol,
@@ -37,10 +38,10 @@ def handle_data(context, data):
log.info('got price {}'.format(price))
# Stop buying after passing the reserve threshold
orders = get_open_orders(context.asset) or []
for order in orders:
log.info('cancelling open order {}'.format(order))
cancel_order(order)
# orders = get_open_orders(context.asset) or []
# for order in orders:
# log.info('cancelling open order {}'.format(order))
# cancel_order(order)
# Stop buying after passing the reserve threshold
cash = context.portfolio.cash
@@ -52,18 +53,26 @@ def handle_data(context, data):
# Check if still buying and could (approximately) afford another purchase
if context.is_buying and cash > price:
# Place order to make position in asset equal to target_hodl_value
order(context.asset, 1, limit_price=price + 1.1)
# This works
# order_target_value(
# context.asset,
# target_hodl_value,
# limit_price=price * 1.1,
# stop_price=price * 0.9,
# )
order_target_percent(
context.asset,
0.2,
limit_price=price * 1.1
)
# order_target_percent(
# context.asset,
# 0.01,
# limit_price=price * 1.1
# )
record(
price=price,
cash=cash,
starting_cash=context.portfolio.starting_cash,
leverage=context.account.leverage,
)
pass
exchange_conn = dict(
@@ -77,5 +86,6 @@ run_algorithm(
handle_data=handle_data,
capital_base=100000,
exchange_conn=exchange_conn,
live=True
live=True,
algo_namespace='buy_and_hold_live'
)
catalyst/exchange/algorithm_exchange.py
+21 -2
View File
@@ -40,7 +40,8 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
super(self.__class__, self).__init__(*args, **kwargs)
log.info("initialization done")
self.perf_tracker = None
log.info('exchange trading algorithm successfully initialized')
def _create_clock(self):
# This method is taken from TradingAlgorithm.
@@ -103,11 +104,28 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
return self.trading_client.transform()
def updated_portfolio(self):
"""
We skip the entire performance tracker business and update the
portfolio directly.
:return:
"""
return self.exchange.portfolio
def updated_account(self):
return self.exchange.account
def handle_data(self, data):
self.exchange.update_portfolio()
self.exchange.check_open_orders()
if self._handle_data:
self._handle_data(self, data)
# Unlike trading controls which remain constant unless placing an
# order, account controls can change each bar. Thus, must check
# every bar no matter if the algorithm places an order or not.
self.validate_account_controls()
@api_method
@disallowed_in_before_trading_start(OrderInBeforeTradingStart())
def order(self,
@@ -119,7 +137,8 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
amount, style = self._calculate_order(asset, amount,
limit_price, stop_price, style)
return self.exchange.order(asset, amount, limit_price, stop_price, style)
return self.exchange.order(asset, amount, limit_price, stop_price,
style)
@api_method
def batch_market_order(self, share_counts):
catalyst/exchange/bitfinex.py
+57 -38
View File
@@ -1,3 +1,4 @@
import pytz
import six
import base64
import hashlib
@@ -11,12 +12,14 @@ from catalyst.protocol import Portfolio, Account
# from websocket import create_connection
from catalyst.exchange.exchange import Exchange
from logbook import Logger
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.finance.order import ORDER_STATUS
from catalyst.exchange.exchange_order import ExchangeOrder
from catalyst.finance.execution import (MarketOrder,
LimitOrder,
StopOrder,
StopLimitOrder)
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.exchange_portfolio import ExchangePortfolio
BITFINEX_URL = 'https://api.bitfinex.com'
ASSETS = '{ "USDT_BTC": {"symbol":"btc_usd", "start_date": "2010-01-01"}, "ltcusd": {"symbol":"ltc_usd", "start_date": "2010-01-01"}, "ltcbtc": {"symbol":"ltc_btc", "start_date": "2010-01-01"}, "ethusd": {"symbol":"eth_usd", "start_date": "2010-01-01"}, "ethbtc": {"symbol":"eth_btc", "start_date": "2010-01-01"}, "etcbtc": {"symbol":"etc_btc", "start_date": "2010-01-01"}, "etcusd": {"symbol":"etc_usd", "start_date": "2010-01-01"}, "rrtusd": {"symbol":"rrt_usd", "start_date": "2010-01-01"}, "rrtbtc": {"symbol":"rrt_btc", "start_date": "2010-01-01"}, "zecusd": {"symbol":"zec_usd", "start_date": "2010-01-01"}, "zecbtc": {"symbol":"zec_btc", "start_date": "2010-01-01"}, "xmrusd": {"symbol":"xmr_usd", "start_date": "2010-01-01"}, "xmrbtc": {"symbol":"xmr_btc", "start_date": "2010-01-01"}, "dshusd": {"symbol":"dsh_usd", "start_date": "2010-01-01"}, "dshbtc": {"symbol":"dsh_btc", "start_date": "2010-01-01"}, "bccbtc": {"symbol":"bcc_btc", "start_date": "2010-01-01"}, "bcubtc": {"symbol":"bcu_btc", "start_date": "2010-01-01"}, "bccusd": {"symbol":"bcc_usd", "start_date": "2010-01-01"}, "bcuusd": {"symbol":"bcu_usd", "start_date": "2010-01-01"}, "xrpusd": {"symbol":"xrp_usd", "start_date": "2010-01-01"}, "xrpbtc": {"symbol":"xrp_btc", "start_date": "2010-01-01"}, "iotusd": {"symbol":"iot_usd", "start_date": "2010-01-01"}, "iotbtc": {"symbol":"iot_btc", "start_date": "2010-01-01"}, "ioteth": {"symbol":"iot_eth", "start_date": "2010-01-01"}, "eosusd": {"symbol":"eos_usd", "start_date": "2010-01-01"}, "eosbtc": {"symbol":"eos_btc", "start_date": "2010-01-01"}, "eoseth": {"symbol":"eos_eth", "start_date": "2010-01-01"} }'
@@ -26,17 +29,16 @@ warning_logger = Logger('AlgoWarning')
class Bitfinex(Exchange):
def __init__(self, key, secret, base_currency):
def __init__(self, key, secret, base_currency, store):
self.url = BITFINEX_URL
self.key = key
self.secret = secret
self.id = 'b'
self.name = 'bitfinex'
self.orders = {}
self.assets = {}
self.load_assets(ASSETS)
self.base_currency = base_currency
self._portfolio = None
self.store = store
def _request(self, operation, data, version='v1'):
payload_object = {
@@ -115,7 +117,7 @@ class Bitfinex(Exchange):
if order_status['is_cancelled']:
status = ORDER_STATUS.CANCELLED
elif not order_status['is_live']:
log.info('found executed order %s', order_status)
log.info('found executed order {}'.format(order_status))
status = ORDER_STATUS.FILLED
else:
status = ORDER_STATUS.OPEN
@@ -145,8 +147,10 @@ class Bitfinex(Exchange):
else:
commission = None
order = Order(
dt=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])),
date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
date = pytz.utc.localize(date)
order = ExchangeOrder(
dt=date,
asset=self.assets[order_status['symbol']],
amount=amount,
stop=stop_price,
@@ -156,14 +160,16 @@ class Bitfinex(Exchange):
commission=commission
)
order.status = status
order.executed_price = executed_price
return order
@property
def portfolio(self):
def update_portfolio(self):
"""
TODO: I'm not sure how that's used yet
:return:
Update the portfolio cash and position balances based on the
latest ticker prices.
:return:
"""
response = self._request('balances', None)
balances = response.json()
@@ -183,21 +189,40 @@ class Bitfinex(Exchange):
'Base currency %s not found in portfolio' % self.base_currency
)
base_position_available = float(base_position['available'])
if self._portfolio is None:
portfolio = self._portfolio = Portfolio()
portfolio.starting_cash = portfolio.cash = \
portfolio.portfolio_value = base_position_available
portfolio.capital_used = 0
portfolio.pnl = 0
portfolio.returns = 0
portfolio.start_date = pd.Timestamp.utcnow()
portfolio.positions = []
portfolio.positions_value = 0
portfolio.positions_exposure = 0
portfolio = self.store.portfolio
portfolio.cash = float(base_position['available'])
if portfolio.positions:
tickers = self.tickers(portfolio.positions.keys())
portfolio.positions_value = 0.0
for ticker in tickers:
# TODO: convert if the position is not in the base currency
position = portfolio.positions[ticker['asset']]
position.last_sale_price = ticker['last_price']
position.last_sale_date = ticker['timestamp']
portfolio.positions_value += \
position.amount * position.last_sale_price
portfolio.portfolio_value = \
portfolio.positions_value + portfolio.cash
@property
def portfolio(self):
"""
TODO: I'm not sure how that's used yet
:return:
"""
if self.store.portfolio is None:
portfolio = ExchangePortfolio(
store=self.store,
start_date=pd.Timestamp.utcnow()
)
self.store.portfolio = portfolio
self.update_portfolio()
portfolio.starting_cash = portfolio.cash
else:
portfolio = self._portfolio
portfolio.cash = base_position_available
portfolio = self.store.portfolio
return portfolio
@@ -227,13 +252,7 @@ class Bitfinex(Exchange):
@property
def positions(self):
response = self._request('positions', None)
positions = response.json()
if 'message' in positions:
raise ValueError(
'unable to fetch positions %s' % positions['message']
)
raise NotImplementedError('positions not implemented')
return self.portfolio.positions
@property
def time_skew(self):
@@ -438,7 +457,7 @@ class Bitfinex(Exchange):
)
order_id = exchange_order['id']
order = Order(
order = ExchangeOrder(
dt=pd.Timestamp.utcnow(),
asset=asset,
amount=amount,
@@ -449,7 +468,7 @@ class Bitfinex(Exchange):
# TODO: is this required?
order.broker_order_id = order_id
self.orders[order_id] = order
self.portfolio.create_order(order)
return order_id
@@ -518,8 +537,8 @@ class Bitfinex(Exchange):
order_param : str or Order
The order_id or order object to cancel.
"""
order_id = \
order_param.id if isinstance(order_param, Order) else order_param
order_id = order_param.id \
if isinstance(order_param, ExchangeOrder) else order_param
response = self._request('order/cancel', {'order_id': order_id})
status = response.json()
@@ -528,7 +547,7 @@ class Bitfinex(Exchange):
'Unable to cancel order: %s %s' % (order_id, status['message'])
)
def tickers(self, date, assets):
def tickers(self, assets):
"""
Fetch ticket data for assets
https://docs.bitfinex.com/v2/reference#rest-public-tickers
@@ -559,7 +578,7 @@ class Bitfinex(Exchange):
tick = dict(
asset=assets[index],
timestamp=date,
timestamp=pd.Timestamp.utcnow(),
bid=ticker[1],
ask=ticker[3],
last_price=ticker[7],
catalyst/exchange/exchange.py
+29
View File
@@ -4,12 +4,16 @@ from abc import ABCMeta, abstractmethod, abstractproperty
import pandas as pd
from catalyst.assets._assets import Asset
from catalyst.finance.order import ORDER_STATUS
from catalyst.errors import (
MultipleSymbolsFound,
SymbolNotFound,
)
from datetime import timedelta
from logbook import Logger
log = Logger('Exchange')
class Exchange:
@@ -19,6 +23,7 @@ class Exchange:
self.name = None
self.trading_pairs = None
self.assets = {}
self._portfolio = None
def get_trading_pairs(self, pairs):
return [pair for pair in pairs if pair in self.trading_pairs]
@@ -81,6 +86,26 @@ class Exchange:
)
self.assets[exchange_symbol] = asset_obj
def check_open_orders(self):
if self.portfolio.open_orders:
for order_id in list(self.portfolio.open_orders):
log.debug('found open order: {}'.format(order_id))
order = self.get_order(order_id)
log.debug('got updated order {}'.format(order))
if order.status == ORDER_STATUS.FILLED:
self.portfolio.execute_order(order)
elif order.status == ORDER_STATUS.CANCELLED:
self.portfolio.remove_order(order)
else:
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'order {order_id} still open after {delta}'.format(
order_id=order_id,
delta=delta
)
)
@abstractmethod
def subscribe_to_market_data(self, symbol):
pass
@@ -89,6 +114,10 @@ class Exchange:
def positions(self):
pass
@abstractproperty
def update_portfolio(self):
pass
@abstractproperty
def portfolio(self):
pass
catalyst/exchange/exchange_order.py
+55
View File
@@ -0,0 +1,55 @@
import math
import catalyst.protocol as zp
from catalyst.assets import Asset
from catalyst.finance.order import Order
from catalyst.utils.enum import enum
from catalyst.utils.input_validation import expect_types
ORDER_STATUS = enum(
'OPEN',
'FILLED',
'CANCELLED',
'REJECTED',
'HELD',
)
SELL = 1 << 0
BUY = 1 << 1
STOP = 1 << 2
LIMIT = 1 << 3
ORDER_FIELDS_TO_IGNORE = {'type', 'direction', '_status', 'asset'}
class ExchangeOrder(Order):
@expect_types(asset=Asset)
def __init__(self, dt, asset, amount, stop=None, limit=None, filled=0,
commission=0, id=None, executed_price=None):
"""
@dt - datetime.datetime that the order was placed
@asset - asset for the order.
@amount - the number of shares to buy/sell
a positive sign indicates a buy
a negative sign indicates a sell
@filled - how many shares of the order have been filled so far
"""
# get a string representation of the uuid.
self.id = self.make_id() if id is None else id
self.dt = dt
self.reason = None
self.created = dt
self.asset = asset
self.amount = amount
self.filled = filled
self.commission = commission
self._status = ORDER_STATUS.OPEN
self.stop = stop
self.limit = limit
self.stop_reached = False
self.limit_reached = False
self.direction = math.copysign(1, self.amount)
self.type = zp.DATASOURCE_TYPE.ORDER
self.broker_order_id = None
self.executed_price = executed_price
catalyst/exchange/exchange_portfolio.py
+117
View File
@@ -0,0 +1,117 @@
import numpy as np
from catalyst.protocol import Portfolio, Positions, Position
from logbook import Logger
log = Logger('ExchangePortfolio')
class PortfolioMemoryStore(object):
def __init__(self, algo_namespace):
self.algo_namespace = algo_namespace
self._portfolio = None
@property
def portfolio(self):
"""
This is a mock store, the portfolio will always be None initially.
The goal is to retrieve a persisted portfolio using the
algo_namespace attribute so the algorithm can resume.
:return:
"""
if self._portfolio is not None:
return self._portfolio
else:
return None
@portfolio.setter
def portfolio(self, portfolio):
self._portfolio = portfolio
self.commit()
def commit(self):
"""
The goal is to persist the portfolio somewhere so that the
algo can resume if it stops during execution.
:return:
"""
log.debug('persisting updated portfolio')
class ExchangePortfolio(Portfolio):
def __init__(self, store, start_date, starting_cash=0.0):
self.capital_used = 0.0
self.store = store
self.starting_cash = starting_cash
self.portfolio_value = starting_cash
self.pnl = 0.0
self.returns = 0.0
self.cash = starting_cash
self.positions = Positions()
self.start_date = start_date
self.positions_value = 0.0
self.open_orders = dict()
def calculate_pnl(self):
log.debug('calculating pnl')
def update(self):
self.store.commit()
def create_order(self, order):
log.debug('creating order {}'.format(order.id))
self.open_orders[order.id] = order
order_position = self.positions[order.asset] \
if order.asset in self.positions else None
if order_position is None:
order_position = Position(order.asset)
self.positions[order.asset] = order_position
order_position.amount += order.amount
log.debug('open order added to portfolio')
self.update()
def execute_order(self, order):
log.debug('executing order {}'.format(order.id))
del self.open_orders[order.id]
order_position = self.positions[order.asset] \
if order.asset in self.positions else None
if order_position is None:
raise ValueError(
'Trying to execute order for a position not held: %s' % order.id
)
self.capital_used += order.amount * order.executed_price
if order_position.cost_basis > 0:
order_position.cost_basis = np.average(
[order_position.cost_basis, order.executed_price],
weights=[order_position.amount, order.amount]
)
else:
order_position.cost_basis = order.executed_price
log.debug('updated portfolio with executed order')
self.update()
def remove_order(self, order):
log.info('removing cancelled order {}'.format(order.id))
del self.open_orders[order.id]
order_position = self.positions[order.asset] \
if order.asset in self.positions else None
if order_position is None:
raise ValueError(
'Trying to remove order for a position not held: %s' % order.id
)
order_position.amount -= order.amount
log.debug('removed order from portfolio')
self.update()
catalyst/exchange/portfolio/__init__.py
View File
catalyst/utils/run_algo.py
+24 -10
View File
@@ -38,6 +38,7 @@ from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm
from catalyst.exchange.data_portal_exchange import DataPortalExchange
from catalyst.exchange.bitfinex import Bitfinex
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
from catalyst.exchange.exchange_portfolio import PortfolioMemoryStore
from logbook import Logger
log = Logger('run_algo')
@@ -82,7 +83,8 @@ def _run(handle_data,
print_algo,
local_namespace,
environ,
exchange):
exchange,
algo_namespace):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
@@ -136,6 +138,13 @@ def _run(handle_data,
end = start + pd.Timedelta('365', 'D')
open_calendar = get_calendar('OPEN')
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=data_frequency,
)
if bundle is not None:
bundles = bundle.split(',')
@@ -240,6 +249,12 @@ def _run(handle_data,
first_trading_day=pd.to_datetime('today', utc=True)
)
choose_loader = None
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=exchange.portfolio.starting_cash,
)
# sim_params = None
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
@@ -252,13 +267,7 @@ def _run(handle_data,
namespace=namespace,
env=env,
get_pipeline_loader=choose_loader,
sim_params=create_simulation_parameters(
start=start,
end=end,
capital_base=capital_base,
data_frequency=data_frequency,
emission_rate=data_frequency,
),
sim_params=sim_params,
**{
'initialize': initialize,
'handle_data': handle_data,
@@ -350,7 +359,8 @@ def run_algorithm(initialize,
strict_extensions=True,
environ=os.environ,
live=False,
exchange_conn=None):
exchange_conn=None,
algo_namespace=None):
"""Run a trading algorithm.
Parameters
@@ -446,11 +456,14 @@ def run_algorithm(initialize,
)
else:
if exchange_conn is not None:
store = PortfolioMemoryStore(algo_namespace)
if exchange_conn['name'] == 'bitfinex':
exchange = Bitfinex(
key=exchange_conn['key'],
secret=exchange_conn['secret'],
base_currency=exchange_conn['base_currency']
base_currency=exchange_conn['base_currency'],
store=store
)
else:
raise NotImplementedError(
@@ -476,4 +489,5 @@ def run_algorithm(initialize,
local_namespace=False,
environ=environ,
exchange=exchange,
algo_namespace=algo_namespace
)
docs/source/live-trading-blueprint.md
+3 -4
View File
@@ -7,14 +7,13 @@ At a high level the following components have been implemented to coerce
zipline into live trading.
<h3>Exchange</h3>
*catalyst/exchange*
Exchange is a new package which introduces the concept of cryptocurrency
exchanges to zipline. The package contains all new component
implementations adapted to characteristics of exchanges.
```
catalyst/exchange
```
Here are some key characteristics which makes exchanges different from
equity and futures currently implemented in zipline.
* They trade around the clock.