wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-06-27 20:51:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:enigmampc/catalyst into develop

Browse Source
This commit is contained in:
cyzanfar
2017-12-06 18:02:17 -05:00
parent f47aa13dd3 96a27d083c
commit 02875ef7ab
22 changed files with 1209 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/assets/_assets.pyx
+35 -13
View File
@@ -396,11 +396,15 @@ cdef class Future(Asset):
cdef class TradingPair(Asset):
cdef readonly float leverage
cdef readonly object market_currency
cdef readonly object quote_currency
cdef readonly object base_currency
cdef readonly object end_daily
cdef readonly object end_minute
cdef readonly object exchange_symbol
cdef readonly float maker
cdef readonly float taker
cdef readonly int trading_state
cdef readonly object data_source
_kwargnames = frozenset({
'sid',
@@ -413,12 +417,16 @@ cdef class TradingPair(Asset):
'exchange',
'exchange_full',
'leverage',
'market_currency',
'quote_currency',
'base_currency',
'end_daily',
'end_minute',
'exchange_symbol',
'min_trade_size'
'min_trade_size',
'maker',
'taker',
'trading_state',
'data_source'
})
def __init__(self,
object symbol,
@@ -434,10 +442,14 @@ cdef class TradingPair(Asset):
object first_traded=None,
object auto_close_date=None,
object exchange_full=None,
object min_trade_size=None):
float min_trade_size=0.0001,
float maker=0.0015,
float taker=0.0025,
int trading_state=0,
object data_source='catalyst'):
"""
Replicates the Asset constructor with some built-in conventions
and a new 'leverage' attribute.
and adds properties for leverage and fees.
Symbol
------
@@ -469,8 +481,6 @@ cdef class TradingPair(Asset):
highest volume and market cap generally benefit from high leverage.
New currencies from ICO generally cannot be leveraged.
The leverage value is either None or and integer.
Leverage allows you to open a larger position with a smaller amount
of funds. For example, if you open a $5,000 position in BTC/USD
with 5:1 leverage, only one-fifth of this amount, or $1000, will be
@@ -480,6 +490,11 @@ cdef class TradingPair(Asset):
the position. If you open with 1:1 leverage, $5,000 of your balance
will be tied to the position.
Fees
----
Exchanges generally charge a taker (taking from the order book) or
maker (adding to the order book) fee.
:param symbol:
:param exchange:
:param start_date:
@@ -494,11 +509,14 @@ cdef class TradingPair(Asset):
:param auto_close_date:
:param exchange_full:
:param min_trade_size:
:param maker:
:param taker:
:param data_source
"""
symbol = symbol.lower()
try:
self.market_currency, self.base_currency = symbol.split('_')
self.base_currency, self.quote_currency = symbol.split('_')
except Exception as e:
raise InvalidSymbolError(symbol=symbol, error=e)
@@ -512,7 +530,7 @@ cdef class TradingPair(Asset):
asset_name = ' / '.join(symbol.split('_')).upper()
if start_date is None:
start_date = pd.Timestamp.utcnow()
start_date = pd.to_datetime('2009-1-1', utc=True)
if end_date is None:
end_date = pd.Timestamp.utcnow() + timedelta(days=365)
@@ -527,19 +545,23 @@ cdef class TradingPair(Asset):
first_traded=first_traded,
auto_close_date=auto_close_date,
exchange_full=exchange_full,
min_trade_size=min_trade_size
min_trade_size=min_trade_size,
)
self.maker = maker
self.taker = taker
self.leverage = leverage
self.end_daily = end_daily
self.end_minute = end_minute
self.exchange_symbol = exchange_symbol
self.trading_state = trading_state
self.data_source = data_source
def __repr__(self):
return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
'Introduced On: {start_date}, ' \
'Market Currency: {market_currency}, ' \
'Base Currency: {base_currency}, ' \
'Quote Currency: {quote_currency}, ' \
'Exchange Leverage: {leverage}, ' \
'Minimum Trade Size: {min_trade_size} ' \
'Last daily ingestion: {end_daily} ' \
@@ -548,7 +570,7 @@ cdef class TradingPair(Asset):
sid=self.sid,
exchange=self.exchange,
start_date=self.start_date,
market_currency=self.market_currency,
quote_currency=self.quote_currency,
base_currency=self.base_currency,
leverage=self.leverage,
min_trade_size=self.min_trade_size,
@@ -578,7 +600,7 @@ cdef class TradingPair(Asset):
-------
boolean: whether the asset's exchange is open at the given minute.
"""
#TODO: consider implementing to spot holds
#TODO: make more dymanic to catch holds
return True
cpdef __reduce__(self):
catalyst/constants.py
+2 -1
View File
@@ -7,7 +7,8 @@ import logbook
For example, if you want to see the DEBUG messages, run:
$ export CATALYST_LOG_LEVEL=10
'''
LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
# LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
LOG_LEVEL = logbook.DEBUG
SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
'{exchange}/symbols.json'
catalyst/data/loader.py
+4 -2
View File
@@ -142,8 +142,10 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
if exchange is None:
# This is exceptional, since placing the import at the module scope
# breaks things and it's only needed here
from catalyst.exchange.poloniex.poloniex import Poloniex
exchange = Poloniex('', '', '')
from catalyst.exchange.factory import get_exchange
exchange = get_exchange(
exchange_name='poloniex', base_currency='usdt'
)
benchmark_asset = exchange.get_asset(bm_symbol)
catalyst/examples/arbitrage_with_interface.py
+12 -12
View File
@@ -83,15 +83,15 @@ def place_orders(context, amount, buying_price, selling_price, action):
else:
raise ValueError('invalid order action')
base_currency = enter_exchange.base_currency
base_currency_amount = enter_exchange.portfolio.cash
quote_currency = enter_exchange.quote_currency
quote_currency_amount = enter_exchange.portfolio.cash
exit_balances = exit_exchange.get_balances()
exit_currency = context.trading_pairs[
context.selling_exchange].market_currency
context.selling_exchange].quote_currency
if exit_currency in exit_balances:
market_currency_amount = exit_balances[exit_currency]
quote_currency_amount = exit_balances[exit_currency]
else:
log.warn(
'the selling exchange {exchange_name} does not hold '
@@ -102,25 +102,25 @@ def place_orders(context, amount, buying_price, selling_price, action):
)
return
if base_currency_amount < (amount * entry_price):
adj_amount = base_currency_amount / entry_price
if quote_currency_amount < (amount * entry_price):
adj_amount = quote_currency_amount / entry_price
log.warn(
'not enough {base_currency} ({base_currency_amount}) to buy '
'not enough {quote_currency} ({quote_currency_amount}) to buy '
'{amount}, adjusting the amount to {adj_amount}'.format(
base_currency=base_currency,
base_currency_amount=base_currency_amount,
quote_currency=quote_currency,
quote_currency_amount=quote_currency_amount,
amount=amount,
adj_amount=adj_amount
)
)
amount = adj_amount
elif market_currency_amount < amount:
elif quote_currency_amount < amount:
log.warn(
'not enough {currency} ({currency_amount}) to sell '
'{amount}, aborting'.format(
currency=exit_currency,
currency_amount=market_currency_amount,
currency_amount=quote_currency_amount,
amount=amount
)
)
@@ -270,6 +270,6 @@ run_algorithm(
exchange_name='poloniex,bitfinex',
live=True,
algo_namespace=algo_namespace,
base_currency='btc',
quote_currency='btc',
live_graph=False
)
catalyst/examples/mean_reversion_simple.py
+10 -10
View File
@@ -32,14 +32,14 @@ def initialize(context):
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in USD.
context.neo_eth = symbol('neo_usd')
# In our example, we're looking at Neo in Ether.
context.neo_eth = symbol('neo_eth')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 30
context.RSI_OVERBOUGHT = 80
context.CANDLE_SIZE = '15T'
context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 82
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
@@ -239,7 +239,7 @@ def analyze(context=None, perf=None):
if __name__ == '__main__':
# The execution mode: backtest or live
MODE = 'backtest'
MODE = 'live'
if MODE == 'backtest':
folder = os.path.join(
@@ -251,14 +251,14 @@ if __name__ == '__main__':
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=10000,
capital_base=0.1,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
@@ -267,13 +267,13 @@ if __name__ == '__main__':
elif MODE == 'live':
run_algorithm(
capital_base=0.5,
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
live=True,
algo_namespace=NAMESPACE,
base_currency='usd',
base_currency='eth',
live_graph=False
)
catalyst/examples/simple_loop.py
+12 -14
View File
@@ -9,7 +9,7 @@ from catalyst.exchange.stats_utils import get_pretty_stats, \
def initialize(context):
print('initializing')
context.asset = symbol('neo_usd')
context.asset = symbol('neo_eth')
context.base_price = None
@@ -19,17 +19,14 @@ def handle_data(context, data):
price = data.current(context.asset, 'close')
print('got price {price}'.format(price=price))
try:
prices = data.history(
context.asset,
fields='price',
bar_count=14,
frequency='15T'
)
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
print('got rsi: {}'.format(rsi))
except Exception as e:
print(e)
prices = data.history(
context.asset,
fields='price',
bar_count=20,
frequency='15T'
)
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
print('got rsi: {}'.format(rsi))
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
@@ -126,8 +123,9 @@ run_algorithm(
# initialize=initialize,
# handle_data=handle_data,
# analyze=None,
# exchange_name='poloniex',
# exchange_name='binance',
# live=True,
# algo_namespace='simple_loop',
# base_currency='eth',
# live_graph=False
# live_graph=False,
# )
catalyst/exchange/ccxt/__init__.py
View File
catalyst/exchange/ccxt/ccxt_exchange.py
+600
View File
@@ -0,0 +1,600 @@
import re
from collections import defaultdict
import ccxt
import pandas as pd
from ccxt import ExchangeNotAvailable
from six import string_types
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.algorithm import MarketOrder
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange, ExchangeLimitOrder
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
ExchangeNotFoundError
from catalyst.exchange.exchange_utils import mixin_market_params, \
from_ms_timestamp
log = Logger('CCXT', level=LOG_LEVEL)
SUPPORTED_EXCHANGES = dict(
binance=ccxt.binance,
bitfinex=ccxt.bitfinex,
bittrex=ccxt.bittrex,
poloniex=ccxt.poloniex,
bitmex=ccxt.bitmex,
gdax=ccxt.gdax,
)
class CCXT(Exchange):
def __init__(self, exchange_name, key, secret, base_currency,
portfolio=None):
log.debug(
'finding {} in CCXT exchanges:\n{}'.format(
exchange_name, ccxt.exchanges
)
)
try:
# Making instantiation as explicit as possible for code tracking.
if exchange_name in SUPPORTED_EXCHANGES:
exchange_attr = SUPPORTED_EXCHANGES[exchange_name]
else:
exchange_attr = getattr(ccxt, exchange_name)
self.api = exchange_attr({
'apiKey': key,
'secret': secret,
})
except Exception:
raise ExchangeNotFoundError(exchange_name=exchange_name)
self._symbol_maps = [None, None]
markets_symbols = self.api.load_markets()
log.debug('the markets:\n{}'.format(markets_symbols))
self.name = exchange_name
self.markets = self.api.fetch_markets()
self.load_assets()
self.base_currency = base_currency
self._portfolio = portfolio
self.transactions = defaultdict(list)
self.num_candles_limit = 2000
self.max_requests_per_minute = 60
self.request_cpt = dict()
self.bundle = ExchangeBundle(self.name)
def account(self):
return None
def time_skew(self):
return None
def get_market(self, symbol):
"""
The CCXT market.
Parameters
----------
symbol:
The CCXT symbol.
Returns
-------
dict[str, Object]
"""
s = self.get_symbol(symbol)
market = next(
(market for market in self.markets if market['symbol'] == s),
None,
)
return market
def get_symbol(self, asset_or_symbol):
"""
The CCXT symbol.
Parameters
----------
asset_or_symbol
Returns
-------
"""
symbol = asset_or_symbol if isinstance(
asset_or_symbol, string_types
) else asset_or_symbol.symbol
parts = symbol.split('_')
return '{}/{}'.format(parts[0].upper(), parts[1].upper())
def get_catalyst_symbol(self, market_or_symbol):
"""
The Catalyst symbol.
Parameters
----------
market_or_symbol
Returns
-------
"""
if isinstance(market_or_symbol, string_types):
parts = market_or_symbol.split('/')
return '{}_{}'.format(parts[0].lower(), parts[1].lower())
else:
return '{}_{}'.format(
market_or_symbol['base'].lower(),
market_or_symbol['quote'].lower(),
)
def get_timeframe(self, freq):
"""
The CCXT timeframe from the Catalyst frequency.
Parameters
----------
freq: str
The Catalyst frequency (Pandas convention)
Returns
-------
str
"""
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) \
if freq_match.group(1) else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
if unit.lower() == 'd':
timeframe = '{}d'.format(candle_size)
elif unit.lower() == 'm' or unit == 'T':
timeframe = '{}m'.format(candle_size)
elif unit.lower() == 'h' or unit == 'T':
timeframe = '{}h'.format(candle_size)
return timeframe
def get_candles(self, freq, assets, bar_count=None, start_dt=None,
end_dt=None):
symbols = self.get_symbols(assets)
timeframe = self.get_timeframe(freq)
delta = start_dt - pd.to_datetime('1970-1-1', utc=True)
ms = int(delta.total_seconds()) * 1000
candles = dict()
for asset in assets:
ohlcvs = self.api.fetch_ohlcv(
symbol=symbols[0],
timeframe=timeframe,
since=ms,
limit=bar_count,
params={}
)
candles[asset] = []
for ohlcv in ohlcvs:
candles[asset].append(dict(
last_traded=pd.to_datetime(ohlcv[0], unit='ms', utc=True),
open=ohlcv[1],
high=ohlcv[2],
low=ohlcv[3],
close=ohlcv[4],
volume=ohlcv[5]
))
return candles
def _fetch_symbol_map(self, is_local):
try:
return self.fetch_symbol_map(is_local)
except ExchangeSymbolsNotFound:
return None
def get_asset_defs(self, market):
"""
The local and Catalyst definitions of the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dicts.
Returns
-------
dict[str, Object]
The asset definition.
"""
asset_defs = []
for is_local in (False, True):
asset_def = self.get_asset_def(market, is_local)
asset_defs.append((asset_def, is_local))
return asset_defs
def get_asset_def(self, market, is_local=False):
"""
The asset definition (in symbols.json files) corresponding
to the the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dict.
is_local
Whether to search in local or Catalyst asset definitions.
Returns
-------
dict[str, Object]
The asset definition.
"""
exchange_symbol = market['id']
symbol_map = self._fetch_symbol_map(is_local)
if symbol_map is not None:
assets_lower = {k.lower(): v for k, v in symbol_map.items()}
key = exchange_symbol.lower()
asset = assets_lower[key] if key in assets_lower else None
if asset is not None:
return asset
else:
return None
else:
return None
def create_trading_pair(self, market, asset_def, is_local):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
asset_def: dict[str, Object]
is_local: bool
Returns
-------
"""
data_source = 'local' if is_local else 'catalyst'
params = dict(
exchange=self.name,
data_source=data_source,
exchange_symbol=market['id'],
)
mixin_market_params(self.name, params, market)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
params['start_date'] = asset_def['start_date'] \
if 'start_date' in asset_def else None
params['end_date'] = asset_def['end_date'] \
if 'end_date' in asset_def else None
params['leverage'] = asset_def['leverage'] \
if 'leverage' in asset_def else 1.0
params['asset_name'] = asset_def['asset_name'] \
if 'asset_name' in asset_def else None
params['end_daily'] = asset_def['end_daily'] \
if 'end_daily' in asset_def \
and asset_def['end_daily'] != 'N/A' else None
params['end_minute'] = asset_def['end_minute'] \
if 'end_minute' in asset_def \
and asset_def['end_minute'] != 'N/A' else None
else:
params['symbol'] = self.get_catalyst_symbol(market)
# TODO: add as an optional column
params['leverage'] = 1.0
return TradingPair(**params)
def load_assets(self):
self.assets = []
for market in self.markets:
asset_defs = self.get_asset_defs(market)
for asset_def in asset_defs:
if asset_def[0] is not None or not asset_defs[1]:
try:
asset = self.create_trading_pair(
market=market,
asset_def=asset_def[0],
is_local=asset_def[1]
)
self.assets.append(asset)
except TypeError:
pass
def get_balances(self):
try:
log.debug('retrieving wallets balances')
balances = self.api.fetch_balance()
balances_lower = dict()
for key in balances:
balances_lower[key.lower()] = balances[key]
except Exception as e:
log.debug('error retrieving balances: {}', e)
raise ExchangeRequestError(error=e)
return balances_lower
def _create_order(self, order_status):
"""
Create a Catalyst order object from a CCXT order dictionary
Parameters
----------
order_status: dict[str, Object]
The order dict from the CCXT api.
Returns
-------
Order
The Catalyst order object
"""
if order_status['status'] == 'canceled':
status = ORDER_STATUS.CANCELLED
elif order_status['status'] == 'closed' and order_status['filled'] > 0:
log.debug('found executed order {}'.format(order_status))
status = ORDER_STATUS.FILLED
elif order_status['status'] == 'open':
status = ORDER_STATUS.OPEN
else:
raise ValueError('invalid state for order')
amount = order_status['amount']
filled = order_status['filled']
if order_status['side'] == 'sell':
amount = -amount
filled = -filled
price = order_status['price']
order_type = order_status['type']
limit_price = price if order_type == 'limit' else None
stop_price = None # TODO: add support
executed_price = order_status['cost'] / order_status['amount']
commission = order_status['fee']
date = from_ms_timestamp(order_status['timestamp'])
# order_id = str(order_status['info']['clientOrderId'])
order_id = order_status['id']
# TODO: this won't work, redo the packages with a different key.
symbol = order_status['info']['symbol'] \
if 'symbol' in order_status['info'] \
else order_status['info']['Exchange']
order = Order(
dt=date,
asset=self.get_asset(symbol, is_exchange_symbol=True),
amount=amount,
stop=stop_price,
limit=limit_price,
filled=filled,
id=order_id,
commission=commission
)
order.status = status
return order, executed_price
def create_order(self, asset, amount, is_buy, style):
symbol = self.get_symbol(asset)
if isinstance(style, ExchangeLimitOrder):
price = style.get_limit_price(is_buy)
order_type = 'limit'
elif isinstance(style, MarketOrder):
price = None
order_type = 'market'
else:
raise InvalidOrderStyle(
exchange=self.name,
style=style.__class__.__name__
)
side = 'buy' if amount > 0 else 'sell'
try:
result = self.api.create_order(
symbol=symbol,
type=order_type,
side=side,
amount=abs(amount),
price=price
)
except ExchangeNotAvailable as e:
log.debug('unable to create order: {}'.format(e))
raise ExchangeRequestError(error=e)
if 'info' not in result:
raise ValueError('cannot use order without info attribute')
order_id = result['id']
order = Order(
dt=pd.Timestamp.utcnow(),
asset=asset,
amount=amount,
stop=style.get_stop_price(is_buy),
limit=style.get_limit_price(is_buy),
id=order_id
)
return order
def get_open_orders(self, asset):
try:
symbol = self.get_symbol(asset)
result = self.api.fetch_open_orders(
symbol=symbol,
since=None,
limit=None,
params=dict()
)
except Exception as e:
raise ExchangeRequestError(error=e)
orders = []
for order_status in result:
order, executed_price = self._create_order(order_status)
if asset is None or asset == order.sid:
orders.append(order)
return orders
def _get_asset_from_order(self, order_id):
open_orders = self.portfolio.open_orders
order = next(
(open_orders[id] for id in open_orders if id == order_id),
None
) # type: Order
return order.asset if order is not None else None
def get_order(self, order_id, asset_or_symbol=None):
if asset_or_symbol is None and self.portfolio is not None:
asset_or_symbol = self._get_asset_from_order(order_id)
if asset_or_symbol is None:
log.debug(
'order not found in memory, the request might fail '
'on some exchanges.'
)
try:
symbol = self.get_symbol(asset_or_symbol) \
if asset_or_symbol is not None else None
order_status = self.api.fetch_order(id=order_id, symbol=symbol)
order, executed_price = self._create_order(order_status)
except Exception as e:
raise ExchangeRequestError(error=e)
return order, executed_price
def cancel_order(self, order_param, asset_or_symbol=None):
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
if asset_or_symbol is None and self.portfolio is not None:
asset_or_symbol = self._get_asset_from_order(order_id)
if asset_or_symbol is None:
log.debug(
'order not found in memory, cancelling order might fail '
'on some exchanges.'
)
try:
symbol = self.get_symbol(asset_or_symbol) \
if asset_or_symbol is not None else None
self.api.cancel_order(id=order_id, symbol=symbol)
except Exception as e:
raise ExchangeRequestError(error=e)
def tickers(self, assets):
"""
Retrieve current tick data for the given assets
Parameters
----------
assets: list[TradingPair]
Returns
-------
list[dict[str, float]
"""
tickers = dict()
for asset in assets:
ccxt_symbol = self.get_symbol(asset)
ticker = self.api.fetch_ticker(ccxt_symbol)
ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
if 'last_price' not in ticker:
# TODO: any more exceptions?
ticker['last_price'] = ticker['last']
# Using the volume represented in the base currency
ticker['volume'] = ticker['baseVolume'] \
if 'baseVolume' in ticker else 0
tickers[asset] = ticker
return tickers
def get_account(self):
return None
def get_orderbook(self, asset, order_type='all', limit=None):
ccxt_symbol = self.get_symbol(asset)
params = dict()
if limit is not None:
params['depth'] = limit
order_book = self.api.fetch_order_book(ccxt_symbol, params)
order_types = ['bids', 'asks'] if order_type == 'all' else [order_type]
result = dict(last_traded=from_ms_timestamp(order_book['timestamp']))
for index, order_type in enumerate(order_types):
if limit is not None and index > limit - 1:
break
result[order_type] = []
for entry in order_book[order_type]:
result[order_type].append(dict(
rate=float(entry[0]),
quantity=float(entry[1])
))
return result
catalyst/exchange/exchange.py
+118 -123
View File
@@ -8,15 +8,16 @@ import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.algorithm import MarketOrder
from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.bundle_utils import get_start_dt, \
get_delta, get_periods, get_periods_range
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
PricingDataNotLoadedError, \
NoDataAvailableOnExchange, ExchangeSymbolsNotFound
NoDataAvailableOnExchange, NoValueForField
from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
ExchangeLimitOrder, ExchangeStopOrder
from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -33,8 +34,8 @@ class Exchange:
def __init__(self):
self.name = None
self.assets = dict()
self.local_assets = dict()
self.assets = []
self._symbol_maps = [None, None]
self._portfolio = None
self.minute_writer = None
self.minute_reader = None
@@ -145,9 +146,9 @@ class Exchange:
"""
symbol = None
for key in self.assets:
if not symbol and self.assets[key].symbol == asset.symbol:
symbol = key
for a in self.assets:
if not symbol and a.symbol == asset.symbol:
symbol = a.symbol
if not symbol:
raise ValueError('Currency %s not supported by exchange %s' %
@@ -174,72 +175,103 @@ class Exchange:
return symbols
def get_assets(self, symbols=None, data_frequency=None):
def get_assets(self, symbols=None, data_frequency=None,
is_exchange_symbol=False,
is_local=None):
"""
The list of markets for the specified symbols.
Parameters
----------
symbols: list[str]
data_frequency: str
is_exchange_symbol: bool
is_local: bool
Returns
-------
list[TradingPair]
A list of asset objects.
Notes
-----
See get_asset for details of each parameter.
"""
if symbols is None:
# Make a distinct list of all symbols
symbols = list(set([asset.symbol for asset in self.assets]))
is_exchange_symbol = False
assets = []
if symbols is not None:
for symbol in symbols:
asset = self.get_asset(symbol, data_frequency)
assets.append(asset)
else:
for key in self.assets:
assets.append(self.assets[key])
for symbol in symbols:
asset = self.get_asset(
symbol, data_frequency, is_exchange_symbol, is_local
)
assets.append(asset)
return assets
def _find_asset(self, asset, symbol, data_frequency, is_local=False):
assets = self.assets if not is_local else self.local_assets
for key in assets:
has_data = (data_frequency == 'minute'
and assets[key].end_minute is not None) \
or (data_frequency == 'daily'
and assets[key].end_daily is not None)
if not asset and assets[key].symbol.lower() == symbol.lower() \
and (not data_frequency or has_data):
asset = assets[key]
return asset
def get_asset(self, symbol, data_frequency=None):
def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
is_local=None):
"""
The market for the specified symbol.
Parameters
----------
symbol: str
The Catalyst or exchange symbol.
data_frequency: str
Check for asset corresponding to the specified data_frequency.
The same asset might exist in the Catalyst repository or
locally (following a CSV ingestion). Filtering by
data_frequency picks the right asset.
is_exchange_symbol: bool
Whether the symbol uses the Catalyst or exchange convention.
is_local: bool
For the local or Catalyst asset.
Returns
-------
TradingPair
The asset object.
"""
asset = None
log.debug('searching asset {} on the server'.format(symbol))
asset = self._find_asset(asset, symbol, data_frequency, False)
log.debug(
'searching assets for: {} {}'.format(
self.name, symbol
)
)
for a in self.assets:
if asset is not None:
break
log.debug('asset {} not found on the server, searching local '
'assets'.format(symbol))
asset = self._find_asset(asset, symbol, data_frequency, True)
if is_local is not None:
data_source = 'local' if is_local else 'catalyst'
applies = (a.data_source == data_source)
if not asset:
all_values = list(self.assets.values()) + \
list(self.local_assets.values())
elif data_frequency is not None:
applies = (
(data_frequency == 'minute' and a.end_minute is not None)
or (data_frequency == 'daily' and a.end_daily is not None)
)
else:
applies = True
# The symbol provided may use the Catalyst or the exchange
# convention
key = a.exchange_symbol if is_exchange_symbol else a.symbol
if not asset and key.lower() == symbol.lower() and applies:
asset = a
if asset is None:
supported_symbols = sorted([
asset.symbol for asset in all_values
asset.symbol for asset in self.assets
])
raise SymbolNotFoundOnExchange(
@@ -248,11 +280,20 @@ class Exchange:
supported_symbols=supported_symbols
)
log.debug('found asset: {}'.format(asset))
return asset
def fetch_symbol_map(self, is_local=False):
return get_exchange_symbols(self.name, is_local)
index = 1 if is_local else 0
if self._symbol_maps[index] is not None:
return self._symbol_maps[index]
else:
symbol_map = get_exchange_symbols(self.name, is_local)
self._symbol_maps[index] = symbol_map
return symbol_map
@abstractmethod
def load_assets(self, is_local=False):
"""
Populate the 'assets' attribute with a dictionary of Assets.
@@ -270,66 +311,7 @@ class Exchange:
via its api.
"""
try:
symbol_map = self.fetch_symbol_map(is_local)
except ExchangeSymbolsNotFound:
return None
for exchange_symbol in symbol_map:
asset = symbol_map[exchange_symbol]
if 'start_date' in asset:
start_date = pd.to_datetime(asset['start_date'], utc=True)
else:
start_date = None
if 'end_date' in asset:
end_date = pd.to_datetime(asset['end_date'], utc=True)
else:
end_date = None
if 'leverage' in asset:
leverage = asset['leverage']
else:
leverage = 1.0
if 'asset_name' in asset:
asset_name = asset['asset_name']
else:
asset_name = None
if 'min_trade_size' in asset:
min_trade_size = asset['min_trade_size']
else:
min_trade_size = 0.0000001
if 'end_daily' in asset and asset['end_daily'] != 'N/A':
end_daily = pd.to_datetime(asset['end_daily'], utc=True)
else:
end_daily = None
if 'end_minute' in asset and asset['end_minute'] != 'N/A':
end_minute = pd.to_datetime(asset['end_minute'], utc=True)
else:
end_minute = None
trading_pair = TradingPair(
symbol=asset['symbol'],
exchange=self.name,
start_date=start_date,
end_date=end_date,
leverage=leverage,
asset_name=asset_name,
min_trade_size=min_trade_size,
end_daily=end_daily,
end_minute=end_minute,
exchange_symbol=exchange_symbol
)
if is_local:
self.local_assets[exchange_symbol] = trading_pair
else:
self.assets[exchange_symbol] = trading_pair
pass
def check_open_orders(self):
"""
@@ -348,9 +330,11 @@ class Exchange:
log.debug('found open order: {}'.format(order_id))
order, executed_price = self.get_order(order_id)
log.debug('got updated order {} {}'.format(
order, executed_price))
log.debug(
'got updated order {} {}'.format(
order, executed_price
)
)
if order.status == ORDER_STATUS.FILLED:
transaction = Transaction(
asset=order.asset,
@@ -412,12 +396,15 @@ class Exchange:
if field not in BASE_FIELDS:
raise KeyError('Invalid column: {}'.format(field))
values = []
for asset in assets:
value = self.get_single_spot_value(asset, field, data_frequency)
values.append(value)
tickers = self.tickers(assets)
if field == 'close' or field == 'price':
return [tickers[asset]['last'] for asset in tickers]
return values
elif field == 'volume':
return [tickers[asset]['volume'] for asset in tickers]
else:
raise NoValueForField(field=field)
def get_single_spot_value(self, asset, field, data_frequency):
"""
@@ -691,7 +678,7 @@ class Exchange:
log.debug('synchronizing portfolio with exchange {}'.format(self.name))
balances = self.get_balances()
base_position_available = balances[self.base_currency] \
base_position_available = balances[self.base_currency]['free'] \
if self.base_currency in balances else None
if base_position_available is None:
@@ -716,8 +703,9 @@ class Exchange:
# TODO: convert if the position is not in the base currency
ticker = tickers[asset]
position = portfolio.positions[asset]
position.last_sale_price = ticker['last_price']
position.last_sale_date = ticker['timestamp']
position.last_sale_date = ticker['last_traded']
portfolio.positions_value += \
position.amount * position.last_sale_price
@@ -774,28 +762,30 @@ class Exchange:
log.warn('skipping order amount of 0')
return None
if asset.base_currency != self.base_currency.lower():
if self.base_currency is None:
raise ValueError('no base_currency defined for this exchange')
if asset.quote_currency != self.base_currency.lower():
raise MismatchingBaseCurrencies(
base_currency=asset.base_currency,
base_currency=asset.quote_currency,
algo_currency=self.base_currency
)
is_buy = (amount > 0)
if limit_price is not None and stop_price is not None:
style = ExchangeStopLimitOrder(limit_price, stop_price,
exchange=self.name)
style = ExchangeStopLimitOrder(
limit_price, stop_price, exchange=self.name
)
elif limit_price is not None:
style = ExchangeLimitOrder(limit_price, exchange=self.name)
elif stop_price is not None:
style = ExchangeStopOrder(stop_price, exchange=self.name)
elif style is not None:
raise InvalidOrderStyle(exchange=self.name.title(),
style=style.__class__.__name__)
else:
raise ValueError('Incomplete order data.')
style = MarketOrder(exchange=self.name)
display_price = limit_price if limit_price is not None else stop_price
log.debug(
@@ -804,9 +794,10 @@ class Exchange:
amount=amount,
symbol=asset.symbol,
type=style.__class__.__name__,
price='{}{}'.format(display_price, asset.base_currency)
price='{}{}'.format(display_price, asset.quote_currency)
)
)
order = self.create_order(asset, amount, is_buy, style)
if order:
self._portfolio.create_order(order)
@@ -875,7 +866,7 @@ class Exchange:
pass
@abstractmethod
def get_order(self, order_id):
def get_order(self, order_id, symbol_or_asset=None):
"""Lookup an order based on the order id returned from one of the
order functions.
@@ -883,6 +874,8 @@ class Exchange:
----------
order_id : str
The unique identifier for the order.
symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this
Returns
-------
@@ -894,13 +887,15 @@ class Exchange:
pass
@abstractmethod
def cancel_order(self, order_param):
def cancel_order(self, order_param, symbol_or_asset=None):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this
"""
pass
catalyst/exchange/exchange_algorithm.py
+1
View File
@@ -289,6 +289,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
def __init__(self, *args, **kwargs):
self.algo_namespace = kwargs.pop('algo_namespace', None)
self.live_graph = kwargs.pop('live_graph', None)
self.simulate_orders = kwargs.pop('simulate_orders', None)
self._clock = None
self.frame_stats = deque(maxlen=60)
catalyst/exchange/exchange_bundle.py
+37 -35
View File
@@ -1,5 +1,4 @@
import os
import os
import shutil
from datetime import datetime, timedelta
from functools import partial
@@ -28,10 +27,9 @@ from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
TempBundleNotFoundError, \
NoDataAvailableOnExchange, \
PricingDataNotLoadedError, DataCorruptionError, ExchangeSymbolsNotFound, \
PricingDataValueError
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
from catalyst.exchange.exchange_utils import get_exchange_folder, \
get_exchange_symbols, save_exchange_symbols
save_exchange_symbols, mixin_market_params
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
@@ -667,12 +665,11 @@ class ExchangeBundle:
"""
log.info('ingesting csv file: {}'.format(path))
try:
symbols_def = get_exchange_symbols(
self.exchange_name, is_local=True
)
except ExchangeSymbolsNotFound:
symbols_def = dict()
if self.exchange is None:
# Avoid circular dependencies
from catalyst.exchange.factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
problems = []
df = pd.read_csv(
@@ -705,24 +702,40 @@ class ExchangeBundle:
end_dt = df.index.get_level_values(1).max()
end_dt_key = 'end_{}'.format(data_frequency)
if symbol is symbols_def:
symbol_def = symbols_def[symbol]
market = self.exchange.get_market(symbol)
if market is None:
raise ValueError('symbol not available in the exchange.')
start_dt = symbol_def['start_date'] \
if symbol_def['start_date'] < start_dt else start_dt
params = dict(
exchange=self.exchange.name,
data_source='local',
exchange_symbol=market['id'],
)
mixin_market_params(self.exchange_name, params, market)
end_dt = symbol_def[end_dt_key] \
if symbol_def[end_dt_key] > end_dt else end_dt
asset_def = self.exchange.get_asset_def(market, True)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
end_daily = end_dt \
if data_frequency == 'daily' else symbol_def['end_daily']
params['start_date'] = asset_def['start_date'] \
if asset_def['start_date'] < start_dt else start_dt
end_minute = end_dt \
if data_frequency == 'minute' else symbol_def['end_minute']
params['end_date'] = asset_def[end_dt_key] \
if asset_def[end_dt_key] > end_dt else end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else asset_def['end_daily']
params['end_minute'] = end_dt \
if data_frequency == 'minute' else asset_def['end_minute']
else:
end_daily = end_dt if data_frequency == 'daily' else 'N/A'
end_minute = end_dt if data_frequency == 'minute' else 'N/A'
params['symbol'] = self.exchange.get_catalyst_symbol(market)
params['end_daily'] = end_dt \
if data_frequency == 'daily' else 'N/A'
params['end_minute'] = end_dt \
if data_frequency == 'minute' else 'N/A'
if min_start_dt is None or start_dt < min_start_dt:
min_start_dt = start_dt
@@ -730,19 +743,8 @@ class ExchangeBundle:
if max_end_dt is None or end_dt > max_end_dt:
max_end_dt = end_dt
asset = TradingPair(
symbol=symbol,
exchange=self.exchange_name,
start_date=start_dt,
end_date=end_dt,
leverage=0, # TODO: add as an optional column
asset_name=symbol,
min_trade_size=0, # TODO: add as an optional column
end_daily=end_daily,
end_minute=end_minute,
exchange_symbol=symbol
)
assets[symbol] = asset
asset = TradingPair(**params)
assets[market['id']] = asset
save_exchange_symbols(self.exchange_name, assets, True)
catalyst/exchange/exchange_errors.py
+4
View File
@@ -240,3 +240,7 @@ class NoDataAvailableOnExchange(ZiplineError):
'Requested data for trading pair {symbol} is not available on exchange {exchange} '
'in `{data_frequency}` frequency at this time. '
'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
class NoValueForField(ZiplineError):
msg = ('Value not found for field: {field}.').strip()
catalyst/exchange/exchange_execution.py
+1 -1
View File
@@ -1,4 +1,4 @@
from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder
from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder, MarketOrder
class ExchangeLimitOrder(LimitOrder):
catalyst/exchange/exchange_portfolio.py
+2 -28
View File
@@ -3,7 +3,6 @@ from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.protocol import Portfolio, Positions, Position
from catalyst.utils.deprecate import deprecated
log = Logger('ExchangePortfolio', level=LOG_LEVEL)
@@ -11,7 +10,8 @@ log = Logger('ExchangePortfolio', level=LOG_LEVEL)
class ExchangePortfolio(Portfolio):
"""
Since the goal is to support multiple exchanges, it makes sense to
include additional stats in the portfolio object.
include additional stats in the portfolio object. This fills the role
of Blotter and Portfolio in live mode.
Instead of relying on the performance tracker, each exchange portfolio
tracks its own holding. This offers a separation between tracking an
@@ -89,32 +89,6 @@ class ExchangePortfolio(Portfolio):
log.debug('updated portfolio with executed order')
@deprecated
def execute_transaction(self, transaction):
# TODO: almost duplicate of execute_order. Not sure why Poloniex needs this.
log.debug('executing transaction {}'.format(transaction.order_id))
order_position = self.positions[transaction.asset] \
if transaction.asset in self.positions else None
if order_position is None:
raise ValueError(
'Trying to execute transaction for a position not held: %s' % transaction.order_id
)
self.capital_used += transaction.amount * transaction.price
if transaction.amount > 0:
if order_position.cost_basis > 0:
order_position.cost_basis = np.average(
[order_position.cost_basis, transaction.price],
weights=[order_position.amount, transaction.amount]
)
else:
order_position.cost_basis = transaction.price
log.debug('updated portfolio with executed order')
def remove_order(self, order):
"""
Removing an open order.
catalyst/exchange/exchange_utils.py
+58 -2
View File
@@ -8,6 +8,7 @@ from datetime import date, datetime
import pandas as pd
from catalyst.assets._assets import TradingPair
from six import string_types
from six.moves.urllib import request
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
@@ -100,6 +101,20 @@ def download_exchange_symbols(exchange_name, environ=None):
return response
def symbols_parser(asset_def):
for key, value in asset_def.items():
match = isinstance(value, string_types) \
and re.search(r'(\d{4}-\d{2}-\d{2})', value)
if match:
try:
asset_def[key] = pd.to_datetime(value, utc=True)
except ValueError:
pass
return asset_def
def get_exchange_symbols(exchange_name, is_local=False, environ=None):
"""
The de-serialized content of the exchange's symbols.json.
@@ -125,10 +140,10 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
if os.path.isfile(filename):
with open(filename) as data_file:
try:
data = json.load(data_file)
data = json.load(data_file, object_hook=symbols_parser)
return data
except ValueError:
except ValueError as e:
return dict()
else:
raise ExchangeSymbolsNotFound(
@@ -571,3 +586,44 @@ def resample_history_df(df, freq, field):
resampled_df = df.resample(freq).agg(agg)
return resampled_df
def mixin_market_params(exchange_name, params, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
params: dict[Object]
market: dict[Object]
Returns
-------
"""
# TODO: make this more externalized / configurable
if 'lot' in market:
params['min_trade_size'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
def from_ms_timestamp(ms):
return pd.to_datetime(ms, unit='ms', utc=True)
catalyst/exchange/factory.py
+23 -30
View File
@@ -1,38 +1,31 @@
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
from catalyst.exchange.bittrex.bittrex import Bittrex
from catalyst.exchange.exchange_errors import ExchangeNotFoundError
from catalyst.exchange.exchange_utils import get_exchange_auth
from catalyst.exchange.poloniex.poloniex import Poloniex
import os
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.exchange_utils import get_exchange_auth, \
get_exchange_folder
def get_exchange(exchange_name, base_currency=None):
def get_exchange(exchange_name, base_currency=None, portfolio=None,
must_authenticate=False):
exchange_auth = get_exchange_auth(exchange_name)
if exchange_name == 'bitfinex':
return Bitfinex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=None
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name), 'auth.json'
)
)
elif exchange_name == 'bittrex':
return Bittrex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=None
)
elif exchange_name == 'poloniex':
return Poloniex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=None
)
else:
raise ExchangeNotFoundError(exchange_name=exchange_name)
return CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
def get_exchanges(exchange_names):
catalyst/exchange/stats_utils.py
+3 -3
View File
@@ -153,11 +153,11 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
def format_positions(positions):
parts = []
for position in positions:
msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format(
msg = '{amount:.2f}{base} cost basis {cost_basis:.4f}{quote}'.format(
amount=position['amount'],
market=position['sid'].market_currency,
base=position['sid'].base_currency,
cost_basis=position['cost_basis'],
base=position['sid'].base_currency
quote=position['sid'].quote_currency
)
parts.append(msg)
return ', '.join(parts)
catalyst/utils/run_algo.py
+19 -46
View File
@@ -11,9 +11,7 @@ import pandas as pd
from catalyst.data.bundles import load
from catalyst.data.data_portal import DataPortal
from catalyst.exchange.bittrex.bittrex import Bittrex
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
from catalyst.exchange.poloniex.poloniex import Poloniex
from catalyst.exchange.factory import get_exchange
try:
from pygments import highlight
@@ -39,11 +37,9 @@ from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
from catalyst.exchange.exchange_portfolio import ExchangePortfolio
from catalyst.exchange.exchange_errors import (
ExchangeRequestError, ExchangeAuthEmpty,
ExchangeRequestErrorTooManyAttempts,
BaseCurrencyNotFoundError, ExchangeNotFoundError)
from catalyst.exchange.exchange_utils import get_exchange_auth, \
get_algo_object, get_exchange_folder
ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
BaseCurrencyNotFoundError)
from catalyst.exchange.exchange_utils import get_algo_object
from logbook import Logger
from catalyst.constants import LOG_LEVEL
@@ -94,7 +90,8 @@ def _run(handle_data,
exchange,
algo_namespace,
base_currency,
live_graph):
live_graph,
simulate_orders):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
@@ -164,42 +161,15 @@ def _run(handle_data,
if portfolio is None:
portfolio = ExchangePortfolio(
start_date=pd.Timestamp.utcnow()
start if start is not None else pd.Timestamp.utcnow()
)
# This corresponds to the json file containing api token info
exchange_auth = get_exchange_auth(exchange_name)
if live and (exchange_auth['key'] == '' \
or exchange_auth['secret'] == ''):
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name, environ), 'auth.json'))
if exchange_name == 'bitfinex':
exchanges[exchange_name] = Bitfinex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'bittrex':
exchanges[exchange_name] = Bittrex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'poloniex':
exchanges[exchange_name] = Poloniex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
else:
raise ExchangeNotFoundError(exchange_name=exchange_name)
exchanges[exchange_name] = get_exchange(
exchange_name=exchange_name,
base_currency=base_currency,
portfolio=portfolio,
must_authenticate=live,
)
open_calendar = get_calendar('OPEN')
@@ -263,7 +233,7 @@ def _run(handle_data,
)
if base_currency in balances:
base_currency_available = balances[base_currency]
base_currency_available = balances[base_currency]['free']
log.info(
'base currency available in the account: {} {}'.format(
base_currency_available, base_currency
@@ -308,7 +278,8 @@ def _run(handle_data,
ExchangeTradingAlgorithmLive,
exchanges=exchanges,
algo_namespace=algo_namespace,
live_graph=live_graph
live_graph=live_graph,
simulate_orders=simulate_orders
)
elif exchanges:
# Removed the existing Poloniex fork to keep things simple
@@ -470,6 +441,7 @@ def run_algorithm(initialize,
base_currency=None,
algo_namespace=None,
live_graph=False,
simulate_orders=True,
output=os.devnull):
"""Run a trading algorithm.
@@ -591,5 +563,6 @@ def run_algorithm(initialize,
exchange=exchange_name,
algo_namespace=algo_namespace,
base_currency=base_currency,
live_graph=live_graph
live_graph=live_graph,
simulate_orders=simulate_orders
)
docs/source/example-algos.rst
+152
View File
@@ -31,6 +31,13 @@ Overview
`two-part video tutorial <videos.html#backtesting-a-strategy>`_ to show how
to get started in backtesting and live trading with Catalyst.
- :ref:`Portfolio Optimization <portfolio_optimization>`: Use this code to
execute a portfolio optimization model. This strategy will select the
portfolio with the maximum Sharpe Ratio. The parameters are set to use 180
days of historical data and rebalance every 30 days. This code was used in
writting the following article:
`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
.. _buy_btc_simple:
@@ -746,4 +753,149 @@ implemented after the video was recorded, which executes the orders at slighlty
different prices, but resulting in significant changes in performance of our
strategy.
.. _portfolio_optimization:
Portfolio Optimization
~~~~~~~~~~~~~~~~~~~~~~
Use this code to execute a portfolio optimization model. This strategy will
select the portfolio with the maximum Sharpe Ratio. The parameters are set to
use 180 days of historical data and rebalance every 30 days. This code was used
in writting the following article:
`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
.. code-block:: python
'''
You can run this code using the Python interpreter:
$ python portfolio_optimization.py
'''
from __future__ import division
import os
import pytz
import numpy as np
import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import datetime
from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True)
def initialize(context):
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i%context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n+1, frequency='1d')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n+1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val/tminus_val-1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr),xr)/n
# Compute asset correlation matrix (informative only)
corr_m = cov_m/np.dot(np.transpose(stds),stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3+context.nassets,n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w,np.transpose(m)))*365
p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
#store results in results array
results_array[0,p] = p_r
results_array[1,p] = p_std
#store Sharpe Ratio (return / volatility) - risk free rate element
#excluded for simplicity
results_array[2,p] = results_array[0,p] / results_array[1,p]
i = 0
for iw in weights:
results_array[3+i,p] = weights[i]
i += 1
#convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r','stdev','sharpe']+context.assets)
#locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
#locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
#order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
#create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
#plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
#plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None):
# Form DataFrame with selected data
data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
.. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
:align: center
docs/source/videos.rst
+17 -1
View File
@@ -32,7 +32,9 @@ Where things don't:
Backtesting a Strategy
----------------------
This algorithm is based on a simple momentum strategy. When the cryptoasset
This is the first video of a two-part series on using Catalyst for algorithmic
trading. This video implements a simple momentum strategy based on
`mean reversion <example-algos.html#mean-reversion>`_: when the cryptoasset
goes up quickly, we’re going to buy; when it goes down quickly, we’re going to
sell. Hopefully, we’ll ride the waves.
@@ -40,3 +42,17 @@ sell. Hopefully, we’ll ride the waves.
<iframe width="560" height="315" src="https://www.youtube.com/embed/JOBRwst9jUY" frameborder="0" allowfullscreen></iframe>
|
|
Live Trading a Strategy
-----------------------
This is the second part of the two-part series on using Catalyst for algorithmic
trading. Having backtested `our strategy <example-algos.html#mean-reversion>`_
in the previous video, we now take it to trade live against the Bittrex exchange.
.. raw:: html
<iframe width="560" height="315" src="https://www.youtube.com/embed/NupiE-Xuglw" frameborder="0" allowfullscreen></iframe>
|
|
etc/requirements.txt
+3
View File
@@ -80,3 +80,6 @@ empyrical==0.2.1
tables==3.3.0
#Catalyst dependencies
ccxt==1.10.251
tests/exchange/test_ccxt.py
+96
View File
@@ -0,0 +1,96 @@
import os
import tempfile
import pandas as pd
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.finance.order import Order
from base import BaseExchangeTestCase
from logbook import Logger
from catalyst.exchange.exchange_utils import get_exchange_auth
from catalyst.utils.paths import ensure_directory
log = Logger('test_ccxt')
class TestCCXT(BaseExchangeTestCase):
@classmethod
def setup(self):
exchange_name = 'gdax'
auth = get_exchange_auth(exchange_name)
self.exchange = CCXT(
exchange_name=exchange_name,
key=auth['key'],
secret=auth['secret'],
base_currency='eth',
portfolio=None
)
def test_order(self):
log.info('creating order')
asset = self.exchange.get_asset('neo_eth')
order_id = self.exchange.order(
asset=asset,
limit_price=0.07,
amount=1,
)
log.info('order created {}'.format(order_id))
assert order_id is not None
pass
def test_open_orders(self):
log.info('retrieving open orders')
asset = self.exchange.get_asset('neo_eth')
orders = self.exchange.get_open_orders(asset)
pass
def test_get_order(self):
log.info('retrieving order')
order = self.exchange.get_order('2631386', 'neo_eth')
# order = self.exchange.get_order('2631386')
assert isinstance(order, Order)
pass
def test_cancel_order(self, ):
log.info('cancel order')
self.exchange.cancel_order('2631386', 'neo_eth')
pass
def test_get_candles(self):
log.info('retrieving candles')
candles = self.exchange.get_candles(
freq='5T',
assets=[self.exchange.get_asset('eth_btc')],
bar_count=200,
start_dt=pd.to_datetime('2017-01-01', utc=True)
)
for asset in candles:
df = pd.DataFrame(candles[asset])
df.set_index('last_traded', drop=True, inplace=True)
pass
def test_tickers(self):
log.info('retrieving tickers')
tickers = self.exchange.tickers([
self.exchange.get_asset('eth_btc'),
])
assert len(tickers) == 1
pass
def test_get_balances(self):
log.info('testing wallet balances')
balances = self.exchange.get_balances()
pass
def test_get_account(self):
log.info('testing account data')
pass
def test_orderbook(self):
log.info('testing order book for bittrex')
asset = self.exchange.get_asset('eth_btc')
orderbook = self.exchange.get_orderbook(asset, 'all', limit=10)
pass
def test_get_fees(self):
pass