First version which runs end-to-end.

This commit is contained in:
Frederic Fortier
2017-08-17 01:20:46 -04:00
parent fade9758a8
commit 55f898562c
7 changed files with 311 additions and 149 deletions
+73 -16
View File
@@ -1,28 +1,85 @@
# code
from catalyst.api import order, record, symbol
from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm
from datetime import timedelta
from catalyst.exchange.bitfinex import Bitfinex
import pandas as pd
from catalyst.utils.run_algo import run_algorithm
from datetime import datetime
import pytz
from logbook import Logger
bitfinex = Bitfinex()
from catalyst.api import (
order_target_value,
order_target_percent,
symbol,
record,
cancel_order,
get_open_orders,
)
log = Logger('buy_and_hold_live')
def initialize(context):
pass
log.info('initializing algo')
context.asset = symbol('eos_usd')
context.TARGET_HODL_RATIO = 0.8
context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
context.is_buying = True
def handle_data(context, data):
asset = bitfinex.get_asset('eth_usd')
test = data.current(asset, 'close')
order(symbol('AAPL'), 10)
log.info('handling bar {data}'.format(data=data))
starting_cash = context.portfolio.starting_cash
target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
reserve_value = context.RESERVE_RATIO * starting_cash
log.info('starting cash: {}'.format(starting_cash))
price = data.current(context.asset, 'price')
log.info('got price {}'.format(price))
# Stop buying after passing the reserve threshold
orders = get_open_orders(context.asset) or []
for order in orders:
log.info('cancelling open order {}'.format(order))
cancel_order(order)
# Stop buying after passing the reserve threshold
cash = context.portfolio.cash
if cash <= reserve_value:
context.is_buying = False
log.info('cash {}'.format(cash))
# Check if still buying and could (approximately) afford another purchase
if context.is_buying and cash > price:
# Place order to make position in asset equal to target_hodl_value
# This works
# order_target_value(
# context.asset,
# target_hodl_value,
# limit_price=price * 1.1,
# stop_price=price * 0.9,
# )
order_target_percent(
context.asset,
0.2,
limit_price=price * 1.1
)
algo_obj = ExchangeTradingAlgorithm(
start = datetime(2015, 3, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 6, 28, 0, 0, 0, 0, pytz.utc)
exchange_conn = dict(
name='bitfinex',
key='',
secret=b'',
base_currency='usd'
)
run_algorithm(
initialize=initialize,
handle_data=handle_data,
start=pd.Timestamp.utcnow(),
end=pd.Timestamp.utcnow() + timedelta(hours=1),
exchange=bitfinex,
start=start,
end=end,
capital_base=100000,
exchange_conn=exchange_conn,
live=True
)
perf_manual = algo_obj.run()
+11 -69
View File
@@ -1,11 +1,7 @@
# code
import os
import re
from catalyst.api import order, record, symbol
from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm
from datetime import timedelta
from catalyst.exchange.bitfinex import Bitfinex
import pandas as pd
from catalyst.utils.run_algo import run_algorithm
from datetime import datetime
import pytz
from catalyst.api import (
order_target_value,
symbol,
@@ -13,20 +9,6 @@ from catalyst.api import (
cancel_order,
get_open_orders,
)
from catalyst.algorithm import TradingAlgorithm
from catalyst.data.bundles.core import load
from catalyst.data.data_portal import DataPortal
from catalyst.data.loader import load_crypto_market_data
from catalyst.finance.trading import TradingEnvironment
from catalyst.pipeline.data import USEquityPricing, CryptoPricing
from catalyst.pipeline.loaders import (
USEquityPricingLoader,
CryptoPricingLoader,
)
from catalyst.utils.calendars import get_calendar
from functools import partial
bitfinex = Bitfinex()
def initialize(context):
@@ -43,7 +25,6 @@ def initialize(context):
context.asset = symbol(context.ASSET_NAME)
context.i = 0
pass
def handle_data(context, data):
@@ -86,52 +67,13 @@ def handle_data(context, data):
)
b = 'poloniex'
bundle_data = load(
b,
os.environ,
pd.Timestamp.utcnow() - timedelta(days=10),
)
prefix, connstr = re.split(
r'sqlite:///',
str(bundle_data.asset_finder.engine.url),
maxsplit=1,
)
if prefix:
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url),
)
open_calendar = get_calendar('OPEN')
env = TradingEnvironment(
load=partial(load_crypto_market_data, environ=os.environ),
bm_symbol='USDT_BTC',
trading_calendar=open_calendar,
asset_db_path=connstr,
environ=os.environ,
)
first_trading_day = pd.Timestamp.utcnow() - timedelta(days=10)
data = DataPortal(
env.asset_finder,
open_calendar,
first_trading_day=first_trading_day,
minute_reader=bundle_data.minute_bar_reader,
five_minute_reader=bundle_data.five_minute_bar_reader,
daily_reader=bundle_data.daily_bar_reader,
adjustment_reader=bundle_data.adjustment_reader,
)
algo_obj = ExchangeTradingAlgorithm(
start = datetime(2015, 3, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 6, 28, 0, 0, 0, 0, pytz.utc)
run_algorithm(
initialize=initialize,
handle_data=handle_data,
start=first_trading_day,
end=pd.Timestamp.utcnow() - timedelta(days=1),
exchange=bitfinex,
start=start,
end=end,
capital_base=100000,
bundle='poloniex'
)
perf_manual = algo_obj.run(data, overwrite_sim_params=False)
@@ -0,0 +1,86 @@
from logbook import Logger
log = Logger('AssetFinderExchange')
class AssetFinderExchange(object):
def __init__(self, exchange):
self.exchange = exchange
self._asset_cache = {}
@property
def sids(self):
return list()
def retrieve_all(self, sids, default_none=False):
"""
Retrieve all assets in `sids`.
Parameters
----------
sids : iterable of int
Assets to retrieve.
default_none : bool
If True, return None for failed lookups.
If False, raise `SidsNotFound`.
Returns
-------
assets : list[Asset or None]
A list of the same length as `sids` containing Assets (or Nones)
corresponding to the requested sids.
Raises
------
SidsNotFound
When a requested sid is not found and default_none=False.
"""
for sid in sids:
if sid in self._asset_cache:
log.info('got asset from cache: {}'.format(sid))
else:
log.info('fetching asset: {}'.format(sid))
return list()
def lookup_symbol(self, symbol, as_of_date, fuzzy=False):
"""Lookup an asset by symbol.
Parameters
----------
symbol : str
The ticker symbol to resolve.
as_of_date : datetime or None
Look up the last owner of this symbol as of this datetime.
If ``as_of_date`` is None, then this can only resolve the equity
if exactly one equity has ever owned the ticker.
fuzzy : bool, optional
Should fuzzy symbol matching be used? Fuzzy symbol matching
attempts to resolve differences in representations for
shareclasses. For example, some people may represent the ``A``
shareclass of ``BRK`` as ``BRK.A``, where others could write
``BRK_A``.
Returns
-------
equity : Asset
The equity that held ``symbol`` on the given ``as_of_date``, or the
only equity to hold ``symbol`` if ``as_of_date`` is None.
Raises
------
SymbolNotFound
Raised when no equity has ever held the given symbol.
MultipleSymbolsFound
Raised when no ``as_of_date`` is given and more than one equity
has held ``symbol``. This is also raised when ``fuzzy=True`` and
there are multiple candidates for the given ``symbol`` on the
``as_of_date``.
"""
log.info('looking up symbol: {}'.format(symbol))
if symbol in self._asset_cache:
return self._asset_cache[symbol]
else:
asset = self.exchange.get_asset(symbol)
self._asset_cache[symbol] = asset
return asset
+52 -30
View File
@@ -19,25 +19,24 @@ from catalyst.finance.execution import (MarketOrder,
from catalyst.data.data_portal import BASE_FIELDS
BITFINEX_URL = 'https://api.bitfinex.com'
BITFINEX_KEY = ''
BITFINEX_SECRET = b''
ASSETS = '{ "btcusd": {"symbol":"btc_usd", "start_date": "2010-01-01"}, "ltcusd": {"symbol":"ltc_usd", "start_date": "2010-01-01"}, "ltcbtc": {"symbol":"ltc_btc", "start_date": "2010-01-01"}, "ethusd": {"symbol":"eth_usd", "start_date": "2010-01-01"}, "ethbtc": {"symbol":"eth_btc", "start_date": "2010-01-01"}, "etcbtc": {"symbol":"etc_btc", "start_date": "2010-01-01"}, "etcusd": {"symbol":"etc_usd", "start_date": "2010-01-01"}, "rrtusd": {"symbol":"rrt_usd", "start_date": "2010-01-01"}, "rrtbtc": {"symbol":"rrt_btc", "start_date": "2010-01-01"}, "zecusd": {"symbol":"zec_usd", "start_date": "2010-01-01"}, "zecbtc": {"symbol":"zec_btc", "start_date": "2010-01-01"}, "xmrusd": {"symbol":"xmr_usd", "start_date": "2010-01-01"}, "xmrbtc": {"symbol":"xmr_btc", "start_date": "2010-01-01"}, "dshusd": {"symbol":"dsh_usd", "start_date": "2010-01-01"}, "dshbtc": {"symbol":"dsh_btc", "start_date": "2010-01-01"}, "bccbtc": {"symbol":"bcc_btc", "start_date": "2010-01-01"}, "bcubtc": {"symbol":"bcu_btc", "start_date": "2010-01-01"}, "bccusd": {"symbol":"bcc_usd", "start_date": "2010-01-01"}, "bcuusd": {"symbol":"bcu_usd", "start_date": "2010-01-01"}, "xrpusd": {"symbol":"xrp_usd", "start_date": "2010-01-01"}, "xrpbtc": {"symbol":"xrp_btc", "start_date": "2010-01-01"}, "iotusd": {"symbol":"iot_usd", "start_date": "2010-01-01"}, "iotbtc": {"symbol":"iot_btc", "start_date": "2010-01-01"}, "ioteth": {"symbol":"iot_eth", "start_date": "2010-01-01"}, "eosusd": {"symbol":"eos_usd", "start_date": "2010-01-01"}, "eosbtc": {"symbol":"eos_btc", "start_date": "2010-01-01"}, "eoseth": {"symbol":"eos_eth", "start_date": "2010-01-01"} }'
ASSETS = '{ "USDT_BTC": {"symbol":"btc_usd", "start_date": "2010-01-01"}, "ltcusd": {"symbol":"ltc_usd", "start_date": "2010-01-01"}, "ltcbtc": {"symbol":"ltc_btc", "start_date": "2010-01-01"}, "ethusd": {"symbol":"eth_usd", "start_date": "2010-01-01"}, "ethbtc": {"symbol":"eth_btc", "start_date": "2010-01-01"}, "etcbtc": {"symbol":"etc_btc", "start_date": "2010-01-01"}, "etcusd": {"symbol":"etc_usd", "start_date": "2010-01-01"}, "rrtusd": {"symbol":"rrt_usd", "start_date": "2010-01-01"}, "rrtbtc": {"symbol":"rrt_btc", "start_date": "2010-01-01"}, "zecusd": {"symbol":"zec_usd", "start_date": "2010-01-01"}, "zecbtc": {"symbol":"zec_btc", "start_date": "2010-01-01"}, "xmrusd": {"symbol":"xmr_usd", "start_date": "2010-01-01"}, "xmrbtc": {"symbol":"xmr_btc", "start_date": "2010-01-01"}, "dshusd": {"symbol":"dsh_usd", "start_date": "2010-01-01"}, "dshbtc": {"symbol":"dsh_btc", "start_date": "2010-01-01"}, "bccbtc": {"symbol":"bcc_btc", "start_date": "2010-01-01"}, "bcubtc": {"symbol":"bcu_btc", "start_date": "2010-01-01"}, "bccusd": {"symbol":"bcc_usd", "start_date": "2010-01-01"}, "bcuusd": {"symbol":"bcu_usd", "start_date": "2010-01-01"}, "xrpusd": {"symbol":"xrp_usd", "start_date": "2010-01-01"}, "xrpbtc": {"symbol":"xrp_btc", "start_date": "2010-01-01"}, "iotusd": {"symbol":"iot_usd", "start_date": "2010-01-01"}, "iotbtc": {"symbol":"iot_btc", "start_date": "2010-01-01"}, "ioteth": {"symbol":"iot_eth", "start_date": "2010-01-01"}, "eosusd": {"symbol":"eos_usd", "start_date": "2010-01-01"}, "eosbtc": {"symbol":"eos_btc", "start_date": "2010-01-01"}, "eoseth": {"symbol":"eos_eth", "start_date": "2010-01-01"} }'
log = Logger('Bitfinex')
warning_logger = Logger('AlgoWarning')
class Bitfinex(Exchange):
def __init__(self):
def __init__(self, key, secret, base_currency):
self.url = BITFINEX_URL
self.key = BITFINEX_KEY
self.secret = BITFINEX_SECRET
self.key = key
self.secret = secret
self.id = 'b'
self.name = 'bitfinex'
self.orders = {}
self.assets = {}
self.load_assets(ASSETS)
self.base_currency = base_currency
self._portfolio = None
def _request(self, operation, data, version='v1'):
payload_object = {
@@ -166,19 +165,39 @@ class Bitfinex(Exchange):
TODO: I'm not sure how that's used yet
:return:
"""
portfolio = Portfolio()
portfolio.capital_used = None
portfolio.starting_cash = None
response = self._request('balances', None)
positions = response.json()
if 'message' in positions:
raise ValueError(
'unable to fetch balance %s' % positions['message']
)
portfolio.portfolio_value = None
portfolio.pnl = None
portfolio.cash = None
base_position = None
for position in positions:
if not base_position and position['type'] == 'exchange' \
and position['currency'] == self.base_currency:
base_position = position
portfolio.returns = None
portfolio.start_date = None
portfolio.positions = self.positions
portfolio.positions_value = None
portfolio.positions_exposure = None
if position is None:
raise ValueError(
'Base currency %s not found in portfolio' % self.base_currency
)
base_position_available = float(base_position['available'])
if self._portfolio is None:
portfolio = self._portfolio = Portfolio()
portfolio.starting_cash = portfolio.cash = \
portfolio.portfolio_value = base_position_available
portfolio.capital_used = 0
portfolio.pnl = 0
portfolio.returns = 0
portfolio.start_date = pd.Timestamp.utcnow()
portfolio.positions = []
portfolio.positions_value = 0
portfolio.positions_exposure = 0
else:
portfolio = self._portfolio
portfolio.cash = base_position_available
return portfolio
@@ -208,14 +227,7 @@ class Bitfinex(Exchange):
@property
def positions(self):
response = self._request('balances', None)
positions = response.json()
if 'message' in positions:
raise ValueError(
'unable to fetch balance %s' % positions['message']
)
return positions
raise NotImplementedError('positions not implemented')
@property
def time_skew(self):
@@ -279,7 +291,7 @@ class Bitfinex(Exchange):
if data_frequency == 'minute':
frequency = '1m'
elif data_frequency == 'daily':
frequency = '1d'
frequency = '1D'
else:
raise NotImplementedError(
'Unsupported frequency %s' % data_frequency
@@ -372,7 +384,12 @@ class Bitfinex(Exchange):
order_type = 'stop'
price = stop_price
elif isinstance(style, StopLimitOrder):
raise NotImplementedError('Stop/limit orders not available')
log.warn('using limit order instead of stop/limit')
# TODO: Not sure how to do this with the api. Investigate.
order_type = 'limit'
price = limit_price
else:
raise NotImplementedError('%s orders not available' % style)
exchange_symbol = self.get_symbol(asset)
req = dict(
@@ -442,7 +459,9 @@ class Bitfinex(Exchange):
orders = list()
for order_status in order_statuses:
# TODO: filter by asset
orders.append(self._create_order(order_status))
order = self._create_order(order_status)
if asset is None or asset == order.sid:
orders.append(order)
return orders
@@ -469,7 +488,7 @@ class Bitfinex(Exchange):
)
return self._create_order(order_status)
def cancel_order(self, order_id):
def cancel_order(self, order_param):
"""Cancel an open order.
Parameters
@@ -477,6 +496,9 @@ class Bitfinex(Exchange):
order_param : str or Order
The order_id or order object to cancel.
"""
order_id = \
order_param.id if isinstance(order_param, Order) else order_param
response = self._request('order/cancel', {'order_id': order_id})
status = response.json()
if 'message' in status:
+11 -1
View File
@@ -5,6 +5,12 @@ from abc import ABCMeta, abstractmethod, abstractproperty
import pandas as pd
from catalyst.assets._assets import Asset
from catalyst.errors import (
MultipleSymbolsFound,
SymbolNotFound,
)
from datetime import timedelta
class Exchange:
__metaclass__ = ABCMeta
@@ -39,9 +45,12 @@ class Exchange:
asset = None
for key in self.assets:
if not asset and self.assets[key].symbol == symbol:
if not asset and self.assets[key].symbol.lower() == symbol.lower():
asset = self.assets[key]
if not asset:
raise SymbolNotFound('Asset not found: %s' % symbol)
return asset
def get_symbols(self, assets):
@@ -67,6 +76,7 @@ class Exchange:
asset_obj = Asset(
sid=abs(hash(assets[exchange_symbol]['symbol'])) % (10 ** 4),
exchange=self.name,
end_date=pd.Timestamp.utcnow() + timedelta(minutes=300000),
**assets[exchange_symbol]
)
self.assets[exchange_symbol] = asset_obj
+2
View File
@@ -0,0 +1,2 @@
from catalyst.finance.trading import TradingEnvironment
+76 -33
View File
@@ -4,15 +4,16 @@ from runpy import run_path
import sys
import warnings
import pandas as pd
import click
import time
try:
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import TerminalFormatter
PYGMENTS = True
except:
PYGMENTS = False
@@ -35,6 +36,11 @@ import catalyst.utils.paths as pth
from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm
from catalyst.exchange.data_portal_exchange import DataPortalExchange
from catalyst.exchange.bitfinex import Bitfinex
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
from logbook import Logger
log = Logger('run_algo')
class _RunAlgoError(click.ClickException, ValueError):
@@ -95,7 +101,7 @@ def _run(handle_data,
raise ValueError(
'invalid define %r, should be of the form name=value' %
assign,
)
)
try:
# evaluate in the same namespace so names may refer to
# eachother
@@ -103,7 +109,7 @@ def _run(handle_data,
except Exception as e:
raise ValueError(
'failed to execute definition for name %r: %s' % (name, e),
)
)
elif defines:
raise _RunAlgoError(
'cannot pass define without `algotext`',
@@ -125,6 +131,7 @@ def _run(handle_data,
else:
click.echo(algotext)
open_calendar = get_calendar('OPEN')
if bundle is not None:
bundles = bundle.split(',')
@@ -152,9 +159,7 @@ def _run(handle_data,
raise ValueError(
"invalid url %r, must begin with 'sqlite:///'" %
str(bundle_data.asset_finder.engine.url),
)
open_calendar = get_calendar('OPEN')
)
env = TradingEnvironment(
load=partial(load_crypto_market_data, environ=environ),
@@ -166,10 +171,10 @@ def _run(handle_data,
first_trading_day = bundle_data.minute_bar_reader.first_trading_day
DataPortalClass = (partial(DataPortalExchange, exchange)
if exchange
else DataPortal)
data = DataPortalClass(
# DataPortalClass = (partial(DataPortalExchange, exchange)
# if exchange
# else DataPortal)
data = DataPortal(
env.asset_finder,
open_calendar,
first_trading_day=first_trading_day,
@@ -216,15 +221,32 @@ def _run(handle_data,
)
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
if exchange is not None:
env = TradingEnvironment(
environ=environ,
exchange_tz="UTC",
asset_db_path=None
)
env.asset_finder = AssetFinderExchange(exchange)
data = DataPortalExchange(
exchange=exchange,
asset_finder=env.asset_finder,
trading_calendar=open_calendar,
first_trading_day=start
)
choose_loader = None
else:
env = TradingEnvironment(environ=environ)
choose_loader = None
if exchange:
start = pd.Timestamp.utcnow()
end = start + pd.Timedelta('1', 'D')
TradingAlgorithmClass = (partial(ExchangeTradingAlgorithm, exchange=exchange)
if exchange else TradingAlgorithm)
TradingAlgorithmClass = (
partial(ExchangeTradingAlgorithm, exchange=exchange)
if exchange else TradingAlgorithm)
perf = TradingAlgorithmClass(
namespace=namespace,
@@ -327,8 +349,8 @@ def run_algorithm(start,
extensions=(),
strict_extensions=True,
environ=os.environ,
live_trading=False,
tws_uri=None):
live=False,
exchange_conn=None):
"""Run a trading algorithm.
Parameters
@@ -382,6 +404,12 @@ def run_algorithm(start,
environ : mapping[str -> str], optional
The os environment to use. Many extensions use this to get parameters.
This defaults to ``os.environ``.
live: execute live trading
exchange_conn: The exchange connection parameters
Supported Exchanges
-------------------
bitfinex
Returns
-------
@@ -392,26 +420,41 @@ def run_algorithm(start,
--------
catalyst.data.bundles.bundles : The available data bundles.
"""
mode = 'live' if live else 'backtest'
log.info('running algo in {mode} mode'.format(mode=mode))
load_extensions(default_extension, extensions, strict_extensions, environ)
non_none_data = valfilter(bool, {
'data': data is not None,
'bundle': bundle is not None,
})
if not non_none_data:
# if neither data nor bundle are passed use 'quantopian-quandl'
bundle = 'quantopian-quandl'
exchange = None
if mode == 'backtest':
non_none_data = valfilter(bool, {
'data': data is not None,
'bundle': bundle is not None,
})
if not non_none_data:
# if neither data nor bundle are passed use 'quantopian-quandl'
bundle = 'quantopian-quandl'
elif len(non_none_data) != 1:
raise ValueError(
'must specify one of `data`, `data_portal`, or `bundle`,'
' got: %r' % non_none_data,
elif len(non_none_data) != 1:
raise ValueError(
'must specify one of `data`, `data_portal`, or `bundle`,'
' got: %r' % non_none_data,
)
elif 'bundle' not in non_none_data and bundle_timestamp is not None:
raise ValueError(
'cannot specify `bundle_timestamp` without passing `bundle`',
)
elif 'bundle' not in non_none_data and bundle_timestamp is not None:
raise ValueError(
'cannot specify `bundle_timestamp` without passing `bundle`',
)
else:
if exchange_conn is not None:
if exchange_conn['name'] == 'bitfinex':
exchange = Bitfinex(
key=exchange_conn['key'],
secret=exchange_conn['secret'],
base_currency=exchange_conn['base_currency']
)
else:
raise NotImplementedError(
'exchange not supported: %s' % exchange_conn['name'])
return _run(
handle_data=handle_data,
@@ -432,5 +475,5 @@ def run_algorithm(start,
print_algo=False,
local_namespace=False,
environ=environ,
exchange=None,
exchange=exchange,
)