wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-23 12:50:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

BLD: tested blotter changes with live trading

Browse Source
This commit is contained in:
fredfortier
2017-12-06 23:32:26 -05:00
parent e42276affa
commit 9688d71e23
8 changed files with 219 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/assets/_assets.pyx
+27 -2
View File
@@ -405,6 +405,9 @@ cdef class TradingPair(Asset):
cdef readonly float taker
cdef readonly int trading_state
cdef readonly object data_source
cdef readonly float max_trade_size
cdef readonly float lot
cdef readonly int decimals
_kwargnames = frozenset({
'sid',
@@ -423,10 +426,13 @@ cdef class TradingPair(Asset):
'end_minute',
'exchange_symbol',
'min_trade_size',
'max_trade_size',
'lot',
'maker',
'taker',
'trading_state',
'data_source'
'data_source',
'decimals'
})
def __init__(self,
object symbol,
@@ -443,8 +449,11 @@ cdef class TradingPair(Asset):
object auto_close_date=None,
object exchange_full=None,
float min_trade_size=0.0001,
float max_trade_size=1000000,
float maker=0.0015,
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=0,
object data_source='catalyst'):
"""
@@ -509,9 +518,12 @@ cdef class TradingPair(Asset):
:param auto_close_date:
:param exchange_full:
:param min_trade_size:
:param max_trade_size:
:param maker:
:param taker:
:param data_source
:param decimals
:param lot
"""
symbol = symbol.lower()
@@ -535,6 +547,9 @@ cdef class TradingPair(Asset):
if end_date is None:
end_date = pd.Timestamp.utcnow() + timedelta(days=365)
if lot == 0 and min_trade_size > 0:
lot = min_trade_size
super().__init__(
sid,
exchange,
@@ -556,6 +571,9 @@ cdef class TradingPair(Asset):
self.exchange_symbol = exchange_symbol
self.trading_state = trading_state
self.data_source = data_source
self.max_trade_size = max_trade_size
self.lot = lot
self.decimals = decimals
def __repr__(self):
return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
@@ -582,6 +600,7 @@ cdef class TradingPair(Asset):
"""
Convert to a python dict.
"""
#TODO: missing fields
super_dict = super(TradingPair, self).to_dict()
super_dict['end_daily'] = self.end_daily
super_dict['end_minute'] = self.end_minute
@@ -610,6 +629,7 @@ cdef class TradingPair(Asset):
and whose second element is a tuple of all the attributes that should
be serialized/deserialized during pickling.
"""
#TODO: make sure that all fields set there
return (self.__class__, (self.symbol,
self.exchange,
self.start_date,
@@ -620,7 +640,12 @@ cdef class TradingPair(Asset):
self.first_traded,
self.auto_close_date,
self.exchange_full,
self.min_trade_size))
self.min_trade_size,
self.max_trade_size,
self.lot,
self.decimals,
self.taker,
self.maker))
def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object)
catalyst/examples/mean_reversion_simple.py
+19 -16
View File
@@ -12,7 +12,8 @@ from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.stats_utils import extract_transactions
from catalyst.exchange.stats_utils import extract_transactions, \
get_pretty_stats
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
@@ -33,11 +34,11 @@ def initialize(context):
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
context.neo_eth = symbol('neo_eth')
context.market = symbol('rdn_eth')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERSOLD = 65
context.RSI_OVERBOUGHT = 82
context.CANDLE_SIZE = '5T'
@@ -59,14 +60,14 @@ def handle_data(context, data):
context.current_day = today
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.neo_eth variable. For this example, we're
# defined above, in the context.market variable. For this example, we're
# using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention
# for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
context.neo_eth,
context.market,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
@@ -81,7 +82,7 @@ def handle_data(context, data):
# We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(context.neo_eth, fields=['close', 'volume'])
current = data.current(context.market, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
@@ -110,19 +111,20 @@ def handle_data(context, data):
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.neo_eth)
orders = get_open_orders(context.market)
if len(orders) > 0:
log.info('exiting because orders are open: {}'.format(orders))
return
# Exit if we cannot trade
if not data.can_trade(context.neo_eth):
if not data.can_trade(context.market):
return
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.neo_eth].amount
pos_amount = context.portfolio.positions[context.market].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
@@ -133,7 +135,7 @@ def handle_data(context, data):
# Set a style for limit orders,
limit_price = price * 1.005
order_target_percent(
context.neo_eth, 1, limit_price=limit_price
context.market, 1, limit_price=limit_price
)
context.traded_today = True
@@ -145,7 +147,7 @@ def handle_data(context, data):
)
limit_price = price * 0.995
order_target_percent(
context.neo_eth, 0, limit_price=limit_price
context.market, 0, limit_price=limit_price
)
context.traded_today = True
@@ -168,7 +170,7 @@ def analyze(context=None, perf=None):
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.neo_eth.symbol, base=base_currency
asset=context.market.symbol, base=base_currency
))
transaction_df = extract_transactions(perf)
@@ -229,7 +231,7 @@ def analyze(context=None, perf=None):
)
plt.legend(loc=3)
start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end/5))
ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
# Show the plot.
plt.gcf().set_size_inches(18, 8)
@@ -267,14 +269,15 @@ if __name__ == '__main__':
elif MODE == 'live':
run_algorithm(
capital_base=0.1,
capital_base=0.05,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
exchange_name='binance',
live=True,
algo_namespace=NAMESPACE,
base_currency='eth',
live_graph=False,
simulate_orders=False
simulate_orders=False,
stats_output='s3://something'
)
catalyst/exchange/ccxt/ccxt_exchange.py
+36 -13
View File
@@ -4,23 +4,22 @@ from collections import defaultdict
import ccxt
import pandas as pd
import six
from ccxt import ExchangeNotAvailable
from catalyst.assets._assets import TradingPair
from ccxt import ExchangeNotAvailable, InvalidOrder
from logbook import Logger
from six import string_types
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.algorithm import MarketOrder
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange, ExchangeLimitOrder
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
ExchangeNotFoundError
ExchangeNotFoundError, CreateOrderError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.exchange_utils import mixin_market_params, \
from_ms_timestamp, get_epoch
from catalyst.finance.order import Order, ORDER_STATUS
log = Logger('CCXT', level=LOG_LEVEL)
@@ -288,7 +287,7 @@ class CCXT(Exchange):
else:
return None
def create_trading_pair(self, market, asset_def, is_local):
def create_trading_pair(self, market, asset_def=None, is_local=False):
"""
Creating a TradingPair from market and asset data.
@@ -346,6 +345,7 @@ class CCXT(Exchange):
for market in self.markets:
asset_defs = self.get_asset_defs(market)
asset = None
for asset_def in asset_defs:
if asset_def[0] is not None or not asset_defs[1]:
try:
@@ -356,8 +356,12 @@ class CCXT(Exchange):
)
self.assets.append(asset)
except TypeError:
pass
except TypeError as e:
log.warn('unable to add asset: {}'.format(e))
if asset is None:
asset = self.create_trading_pair(market=market)
self.assets.append(asset)
def get_balances(self):
try:
@@ -460,26 +464,45 @@ class CCXT(Exchange):
side = 'buy' if amount > 0 else 'sell'
if hasattr(self.api, 'amount_to_lots'):
adj_amount = self.api.amount_to_lots(
symbol=symbol,
amount=abs(amount),
)
if adj_amount != abs(amount):
log.info(
'adjusted order amount {} to {} based on lot size'.format(
abs(amount), adj_amount,
)
)
else:
adj_amount = abs(amount)
try:
result = self.api.create_order(
symbol=symbol,
type=order_type,
side=side,
amount=abs(amount),
amount=adj_amount,
price=price
)
except ExchangeNotAvailable as e:
log.debug('unable to create order: {}'.format(e))
raise ExchangeRequestError(error=e)
except InvalidOrder as e:
log.warn('the exchange rejected the order: {}'.format(e))
raise CreateOrderError(exchange=self.name, error=e)
if 'info' not in result:
raise ValueError('cannot use order without info attribute')
final_amount = adj_amount if side == 'buy' else -adj_amount
order_id = result['id']
order = Order(
dt=pd.Timestamp.utcnow(),
asset=asset,
amount=amount,
amount=final_amount,
stop=style.get_stop_price(is_buy),
limit=style.get_limit_price(is_buy),
id=order_id
catalyst/exchange/exchange.py
+1 -6
View File
@@ -8,7 +8,6 @@ import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.algorithm import MarketOrder
from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.bundle_utils import get_start_dt, \
@@ -18,12 +17,8 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
PricingDataNotLoadedError, \
NoDataAvailableOnExchange, NoValueForField
from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
ExchangeLimitOrder, ExchangeStopOrder
from catalyst.exchange.exchange_utils import get_exchange_symbols, \
get_frequency, resample_history_df
from catalyst.finance.order import ORDER_STATUS
from catalyst.finance.transaction import Transaction
log = Logger('Exchange', level=LOG_LEVEL)
@@ -618,7 +613,7 @@ class Exchange:
log.debug('found base currency balance: {}'.format(cash))
positions_value = 0.0
if positions is not None:
if positions:
assets = set([position.asset for position in positions])
tickers = self.tickers(assets)
log.debug('got tickers for positions: {}'.format(tickers))
catalyst/exchange/exchange_algorithm.py
+38 -33
View File
@@ -36,7 +36,7 @@ from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object,
save_algo_df, group_assets_by_exchange
from catalyst.exchange.live_graph_clock import LiveGraphClock
from catalyst.exchange.simple_clock import SimpleClock
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3
from catalyst.finance.execution import MarketOrder
from catalyst.finance.performance.period import calc_period_stats
from catalyst.gens.tradesimulation import AlgorithmSimulator
@@ -305,6 +305,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
def __init__(self, *args, **kwargs):
self.algo_namespace = kwargs.pop('algo_namespace', None)
self.live_graph = kwargs.pop('live_graph', None)
self.stats_output = kwargs.pop('stats_output', None)
self._clock = None
self.frame_stats = deque(maxlen=60)
@@ -436,15 +437,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
def updated_account(self):
return self.perf_tracker.get_account(False)
def update_positions(self, attempt_index=0):
def synchronize_portfolio(self, attempt_index=0):
tracker = self.perf_tracker.position_tracker
total_cash = 0.0
total_positions_value = 0.0
try:
# Position keys correspond to assets
assets = list(tracker.positions)
exchange_assets = group_assets_by_exchange(assets)
for exchange_name in exchange_assets:
assets = exchange_assets[exchange_name]
for exchange_name in self.exchanges:
assets = exchange_assets[exchange_name] \
if exchange_name in exchange_assets else []
exchange_positions = \
[tracker.positions[asset] for asset in assets]
@@ -452,6 +457,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
cash, positions_value = \
exchange.calculate_totals(exchange_positions)
total_cash += cash
total_positions_value += total_positions_value
for position in exchange_positions:
tracker.update_position(
asset=position.asset,
@@ -459,13 +467,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
last_sale_price=position.last_sale_price
)
if total_cash < self.portfolio.cash:
raise ValueError('Cash on exchanges is lower than the algo.')
return total_cash, total_positions_value
except ExchangeRequestError as e:
log.warn(
'update portfolio attempt {}: {}'.format(attempt_index, e)
)
if attempt_index < self.retry_synchronize_portfolio:
sleep(self.retry_delay)
self.update_positions(attempt_index + 1)
self.synchronize_portfolio(attempt_index + 1)
else:
raise ExchangePortfolioDataError(
data_type='update-portfolio',
@@ -565,20 +577,15 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
new_transactions, new_commissions, closed_orders = \
self.blotter.get_transactions(data)
self.blotter.prune_orders(closed_orders)
for transaction in new_transactions:
self.perf_tracker.process_transaction(transaction)
# since this order was modified, record it
order = self.blotter.orders[transaction.order_id]
self.perf_tracker.process_order(order)
if len(new_transactions) > 0:
self.perf_tracker.update_performance()
self.update_positions()
cash, positions_value = self.synchronize_portfolio()
log.info(
'got totals from exchanges, cash: {} positions: {}'.format(
cash, positions_value
)
)
if self._handle_data:
self._handle_data(self, data)
@@ -612,12 +619,26 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
stats_minutes=self.stats_minutes,
stats=get_pretty_stats(
stats_df=print_df,
df=print_df,
recorded_cols=recorded_cols,
num_rows=self.stats_minutes
)
))
if self.stats_output is not None:
if 's3://' in self.stats_output:
stats_to_s3(
uri=self.stats_output,
df=print_df,
algo_namespace=self.algo_namespace,
recorded_cols=recorded_cols,
)
else:
raise ValueError(
'Only S3 stats output is supported for now.'
)
today = pd.to_datetime('today', utc=True)
daily_stats = self.prepare_period_stats(
start_dt=today,
@@ -643,22 +664,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
except Exception as e:
log.warn('unable to save minute perfs to disk: {}'.format(e))
try:
blotter_params = dict(
open_orders=self.blotter.open_orders,
orders=self.blotter.orders,
new_orders=self.blotter.new_orders,
data_frequency=self.blotter.data_frequency,
current_dt=self.blotter.current_dt,
)
save_algo_object(
algo_name=self.algo_namespace,
key='blotter',
obj=blotter_params,
)
except Exception as e:
log.warn('unable to save portfolio to disk: {}'.format(e))
@api_method
def batch_market_order(self, share_counts):
raise NotImplementedError()
catalyst/exchange/exchange_utils.py
+4
View File
@@ -604,6 +604,7 @@ def mixin_market_params(exchange_name, params, market):
# TODO: make this more externalized / configurable
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
@@ -624,6 +625,9 @@ def mixin_market_params(exchange_name, params, market):
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def from_ms_timestamp(ms):
return pd.to_datetime(ms, unit='ms', utc=True)
catalyst/exchange/stats_utils.py
+87 -36
View File
@@ -2,6 +2,7 @@ import numbers
import numpy as np
import pandas as pd
import time
def trend_direction(series):
@@ -119,29 +120,20 @@ def vwap(df):
return ret
def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
"""
Format and print the last few rows of a statistics DataFrame.
See the pyfolio project for the data structure.
def format_positions(positions):
parts = []
for position in positions:
msg = '{amount:.2f}{base} cost basis {cost_basis:.8f}{quote}'.format(
amount=position['amount'],
base=position['sid'].base_currency,
cost_basis=position['cost_basis'],
quote=position['sid'].quote_currency
)
parts.append(msg)
return ', '.join(parts)
Parameters
----------
stats_df: DataFrame
num_rows: int
Returns
-------
str
"""
stats_df.set_index('period_close', drop=True, inplace=True)
stats_df.dropna(axis=1, how='all', inplace=True)
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 3)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
def prepare_stats(df, recorded_cols=None):
columns = ['starting_cash', 'ending_cash', 'portfolio_value',
'pnl', 'long_exposure', 'short_exposure', 'orders',
'transactions', 'positions']
@@ -150,31 +142,90 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
for column in recorded_cols:
columns.append(column)
def format_positions(positions):
parts = []
for position in positions:
msg = '{amount:.2f}{base} cost basis {cost_basis:.4f}{quote}'.format(
amount=position['amount'],
base=position['sid'].base_currency,
cost_basis=position['cost_basis'],
quote=position['sid'].quote_currency
)
parts.append(msg)
return ', '.join(parts)
df = df.copy(True)
df.set_index('period_close', drop=True, inplace=True)
df.dropna(axis=1, how='all', inplace=True)
df['orders'] = df['orders'].apply(lambda orders: len(orders))
df['transactions'] = df['transactions'].apply(
lambda transactions: len(transactions)
)
df['positions'] = df['positions'].apply(format_positions)
return df, columns
def get_pretty_stats(df, recorded_cols=None, num_rows=10):
"""
Format and print the last few rows of a statistics DataFrame.
See the pyfolio project for the data structure.
Parameters
----------
df: pd.DataFrame
num_rows: int
Returns
-------
str
"""
df, columns = prepare_stats(df, recorded_cols=recorded_cols)
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 3)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
formatters = {
'orders': lambda orders: len(orders),
'transactions': lambda transactions: len(transactions),
'returns': lambda returns: "{0:.4f}".format(returns),
'positions': format_positions
}
return stats_df.tail(num_rows).to_string(
return df.tail(num_rows).to_string(
columns=columns,
formatters=formatters
)
def get_csv_stats(df, recorded_cols=None):
"""
Create a CSV buffer from the stats DataFrame.
Parameters
----------
path: str
df: pd.DataFrame
recorded_cols: list[str]
Returns
-------
"""
df, columns = prepare_stats(df, recorded_cols=recorded_cols)
return df.to_csv(
None,
columns=columns,
encoding='utf-8',
).encode()
def stats_to_s3(uri, df, algo_namespace, recorded_cols=None):
import boto3
s3 = boto3.resource('s3')
bytes_to_write = get_csv_stats(df, recorded_cols=recorded_cols)
timestr = time.strftime('%Y%m%d-%H%M%S')
parts = uri.split('//')
obj = s3.Object(parts[1], 'stats/{}-{}.csv'.format(
timestr, algo_namespace
))
obj.put(Body=bytes_to_write)
def df_to_string(df):
"""
Create a formatted str representation of the DataFrame.
catalyst/utils/run_algo.py
+7 -4
View File
@@ -91,7 +91,8 @@ def _run(handle_data,
algo_namespace,
base_currency,
live_graph,
simulate_orders):
simulate_orders,
stats_output):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
@@ -151,7 +152,6 @@ def _run(handle_data,
exchanges = dict()
for exchange_name in exchange_list:
exchanges[exchange_name] = get_exchange(
exchange_name=exchange_name,
base_currency=base_currency,
@@ -266,7 +266,8 @@ def _run(handle_data,
exchanges=exchanges,
algo_namespace=algo_namespace,
live_graph=live_graph,
simulate_orders=simulate_orders
simulate_orders=simulate_orders,
stats_output=stats_output,
)
elif exchanges:
# Removed the existing Poloniex fork to keep things simple
@@ -429,6 +430,7 @@ def run_algorithm(initialize,
algo_namespace=None,
live_graph=False,
simulate_orders=True,
stats_output=None,
output=os.devnull):
"""Run a trading algorithm.
@@ -551,5 +553,6 @@ def run_algorithm(initialize,
algo_namespace=algo_namespace,
base_currency=base_currency,
live_graph=live_graph,
simulate_orders=simulate_orders
simulate_orders=simulate_orders,
stats_output=stats_output
)