diff --git a/catalyst/assets/_assets.pyx b/catalyst/assets/_assets.pyx index af1e81a6..07e7e813 100644 --- a/catalyst/assets/_assets.pyx +++ b/catalyst/assets/_assets.pyx @@ -405,6 +405,9 @@ cdef class TradingPair(Asset): cdef readonly float taker cdef readonly int trading_state cdef readonly object data_source + cdef readonly float max_trade_size + cdef readonly float lot + cdef readonly int decimals _kwargnames = frozenset({ 'sid', @@ -423,10 +426,13 @@ cdef class TradingPair(Asset): 'end_minute', 'exchange_symbol', 'min_trade_size', + 'max_trade_size', + 'lot', 'maker', 'taker', 'trading_state', - 'data_source' + 'data_source', + 'decimals' }) def __init__(self, object symbol, @@ -443,8 +449,11 @@ cdef class TradingPair(Asset): object auto_close_date=None, object exchange_full=None, float min_trade_size=0.0001, + float max_trade_size=1000000, float maker=0.0015, float taker=0.0025, + float lot=0, + int decimals = 8, int trading_state=0, object data_source='catalyst'): """ @@ -509,9 +518,12 @@ cdef class TradingPair(Asset): :param auto_close_date: :param exchange_full: :param min_trade_size: + :param max_trade_size: :param maker: :param taker: :param data_source + :param decimals + :param lot """ symbol = symbol.lower() @@ -535,6 +547,9 @@ cdef class TradingPair(Asset): if end_date is None: end_date = pd.Timestamp.utcnow() + timedelta(days=365) + if lot == 0 and min_trade_size > 0: + lot = min_trade_size + super().__init__( sid, exchange, @@ -556,6 +571,9 @@ cdef class TradingPair(Asset): self.exchange_symbol = exchange_symbol self.trading_state = trading_state self.data_source = data_source + self.max_trade_size = max_trade_size + self.lot = lot + self.decimals = decimals def __repr__(self): return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \ @@ -582,6 +600,7 @@ cdef class TradingPair(Asset): """ Convert to a python dict. """ + #TODO: missing fields super_dict = super(TradingPair, self).to_dict() super_dict['end_daily'] = self.end_daily super_dict['end_minute'] = self.end_minute @@ -610,6 +629,7 @@ cdef class TradingPair(Asset): and whose second element is a tuple of all the attributes that should be serialized/deserialized during pickling. """ + #TODO: make sure that all fields set there return (self.__class__, (self.symbol, self.exchange, self.start_date, @@ -620,7 +640,12 @@ cdef class TradingPair(Asset): self.first_traded, self.auto_close_date, self.exchange_full, - self.min_trade_size)) + self.min_trade_size, + self.max_trade_size, + self.lot, + self.decimals, + self.taker, + self.maker)) def make_asset_array(int size, Asset asset): cdef np.ndarray out = np.empty([size], dtype=object) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 3d85e3f5..db0a90c3 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -12,7 +12,8 @@ from logbook import Logger from catalyst import run_algorithm from catalyst.api import symbol, record, order_target_percent, get_open_orders -from catalyst.exchange.stats_utils import extract_transactions +from catalyst.exchange.stats_utils import extract_transactions, \ + get_pretty_stats # We give a name to the algorithm which Catalyst will use to persist its state. # In this example, Catalyst will create the `.catalyst/data/live_algos` # directory. If we stop and start the algorithm, Catalyst will resume its @@ -33,11 +34,11 @@ def initialize(context): # parameters or values you're going to use. # In our example, we're looking at Neo in Ether. - context.neo_eth = symbol('neo_eth') + context.market = symbol('rdn_eth') context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 55 + context.RSI_OVERSOLD = 65 context.RSI_OVERBOUGHT = 82 context.CANDLE_SIZE = '5T' @@ -59,14 +60,14 @@ def handle_data(context, data): context.current_day = today # We're computing the volume-weighted-average-price of the security - # defined above, in the context.neo_eth variable. For this example, we're + # defined above, in the context.market variable. For this example, we're # using three bars on the 15 min bars. # The frequency attribute determine the bar size. We use this convention # for the frequency alias: # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases prices = data.history( - context.neo_eth, + context.market, fields='close', bar_count=50, frequency=context.CANDLE_SIZE @@ -81,7 +82,7 @@ def handle_data(context, data): # We need a variable for the current price of the security to compare to # the average. Since we are requesting two fields, data.current() # returns a DataFrame with - current = data.current(context.neo_eth, fields=['close', 'volume']) + current = data.current(context.market, fields=['close', 'volume']) price = current['close'] # If base_price is not set, we use the current value. This is the @@ -110,19 +111,20 @@ def handle_data(context, data): # Since we are using limit orders, some orders may not execute immediately # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.neo_eth) + orders = get_open_orders(context.market) if len(orders) > 0: + log.info('exiting because orders are open: {}'.format(orders)) return # Exit if we cannot trade - if not data.can_trade(context.neo_eth): + if not data.can_trade(context.market): return # Another powerful built-in feature of the Catalyst backtester is the # portfolio object. The portfolio object tracks your positions, cash, # cost basis of specific holdings, and more. In this line, we calculate # how long or short our position is at this minute. - pos_amount = context.portfolio.positions[context.neo_eth].amount + pos_amount = context.portfolio.positions[context.market].amount if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: log.info( @@ -133,7 +135,7 @@ def handle_data(context, data): # Set a style for limit orders, limit_price = price * 1.005 order_target_percent( - context.neo_eth, 1, limit_price=limit_price + context.market, 1, limit_price=limit_price ) context.traded_today = True @@ -145,7 +147,7 @@ def handle_data(context, data): ) limit_price = price * 0.995 order_target_percent( - context.neo_eth, 0, limit_price=limit_price + context.market, 0, limit_price=limit_price ) context.traded_today = True @@ -168,7 +170,7 @@ def analyze(context=None, perf=None): perf.loc[:, 'price'].plot(ax=ax2, label='Price') ax2.set_ylabel('{asset}\n({base})'.format( - asset=context.neo_eth.symbol, base=base_currency + asset=context.market.symbol, base=base_currency )) transaction_df = extract_transactions(perf) @@ -229,7 +231,7 @@ def analyze(context=None, perf=None): ) plt.legend(loc=3) start, end = ax6.get_ylim() - ax6.yaxis.set_ticks(np.arange(0, end, end/5)) + ax6.yaxis.set_ticks(np.arange(0, end, end / 5)) # Show the plot. plt.gcf().set_size_inches(18, 8) @@ -267,14 +269,15 @@ if __name__ == '__main__': elif MODE == 'live': run_algorithm( - capital_base=0.1, + capital_base=0.05, initialize=initialize, handle_data=handle_data, analyze=analyze, - exchange_name='bittrex', + exchange_name='binance', live=True, algo_namespace=NAMESPACE, base_currency='eth', live_graph=False, - simulate_orders=False + simulate_orders=False, + stats_output='s3://something' ) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 09879796..6724a8c7 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -4,23 +4,22 @@ from collections import defaultdict import ccxt import pandas as pd import six -from ccxt import ExchangeNotAvailable +from catalyst.assets._assets import TradingPair +from ccxt import ExchangeNotAvailable, InvalidOrder +from logbook import Logger from six import string_types -from catalyst.finance.order import Order, ORDER_STATUS - from catalyst.algorithm import MarketOrder -from catalyst.assets._assets import TradingPair -from logbook import Logger - from catalyst.constants import LOG_LEVEL -from catalyst.exchange.exchange import Exchange, ExchangeLimitOrder +from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \ - ExchangeNotFoundError + ExchangeNotFoundError, CreateOrderError +from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.exchange_utils import mixin_market_params, \ from_ms_timestamp, get_epoch +from catalyst.finance.order import Order, ORDER_STATUS log = Logger('CCXT', level=LOG_LEVEL) @@ -288,7 +287,7 @@ class CCXT(Exchange): else: return None - def create_trading_pair(self, market, asset_def, is_local): + def create_trading_pair(self, market, asset_def=None, is_local=False): """ Creating a TradingPair from market and asset data. @@ -346,6 +345,7 @@ class CCXT(Exchange): for market in self.markets: asset_defs = self.get_asset_defs(market) + asset = None for asset_def in asset_defs: if asset_def[0] is not None or not asset_defs[1]: try: @@ -356,8 +356,12 @@ class CCXT(Exchange): ) self.assets.append(asset) - except TypeError: - pass + except TypeError as e: + log.warn('unable to add asset: {}'.format(e)) + + if asset is None: + asset = self.create_trading_pair(market=market) + self.assets.append(asset) def get_balances(self): try: @@ -460,26 +464,45 @@ class CCXT(Exchange): side = 'buy' if amount > 0 else 'sell' + if hasattr(self.api, 'amount_to_lots'): + adj_amount = self.api.amount_to_lots( + symbol=symbol, + amount=abs(amount), + ) + if adj_amount != abs(amount): + log.info( + 'adjusted order amount {} to {} based on lot size'.format( + abs(amount), adj_amount, + ) + ) + else: + adj_amount = abs(amount) + try: result = self.api.create_order( symbol=symbol, type=order_type, side=side, - amount=abs(amount), + amount=adj_amount, price=price ) except ExchangeNotAvailable as e: log.debug('unable to create order: {}'.format(e)) raise ExchangeRequestError(error=e) + except InvalidOrder as e: + log.warn('the exchange rejected the order: {}'.format(e)) + raise CreateOrderError(exchange=self.name, error=e) + if 'info' not in result: raise ValueError('cannot use order without info attribute') + final_amount = adj_amount if side == 'buy' else -adj_amount order_id = result['id'] order = Order( dt=pd.Timestamp.utcnow(), asset=asset, - amount=amount, + amount=final_amount, stop=style.get_stop_price(is_buy), limit=style.get_limit_price(is_buy), id=order_id diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 072f4701..642d3612 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -8,7 +8,6 @@ import pandas as pd from catalyst.assets._assets import TradingPair from logbook import Logger -from catalyst.algorithm import MarketOrder from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange.bundle_utils import get_start_dt, \ @@ -18,12 +17,8 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \ PricingDataNotLoadedError, \ NoDataAvailableOnExchange, NoValueForField -from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \ - ExchangeLimitOrder, ExchangeStopOrder from catalyst.exchange.exchange_utils import get_exchange_symbols, \ get_frequency, resample_history_df -from catalyst.finance.order import ORDER_STATUS -from catalyst.finance.transaction import Transaction log = Logger('Exchange', level=LOG_LEVEL) @@ -618,7 +613,7 @@ class Exchange: log.debug('found base currency balance: {}'.format(cash)) positions_value = 0.0 - if positions is not None: + if positions: assets = set([position.asset for position in positions]) tickers = self.tickers(assets) log.debug('got tickers for positions: {}'.format(tickers)) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 8cc84224..013fc618 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -36,7 +36,7 @@ from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, save_algo_df, group_assets_by_exchange from catalyst.exchange.live_graph_clock import LiveGraphClock from catalyst.exchange.simple_clock import SimpleClock -from catalyst.exchange.stats_utils import get_pretty_stats +from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3 from catalyst.finance.execution import MarketOrder from catalyst.finance.performance.period import calc_period_stats from catalyst.gens.tradesimulation import AlgorithmSimulator @@ -305,6 +305,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): def __init__(self, *args, **kwargs): self.algo_namespace = kwargs.pop('algo_namespace', None) self.live_graph = kwargs.pop('live_graph', None) + self.stats_output = kwargs.pop('stats_output', None) self._clock = None self.frame_stats = deque(maxlen=60) @@ -436,15 +437,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): def updated_account(self): return self.perf_tracker.get_account(False) - def update_positions(self, attempt_index=0): + def synchronize_portfolio(self, attempt_index=0): tracker = self.perf_tracker.position_tracker + total_cash = 0.0 + total_positions_value = 0.0 try: # Position keys correspond to assets assets = list(tracker.positions) exchange_assets = group_assets_by_exchange(assets) - for exchange_name in exchange_assets: - assets = exchange_assets[exchange_name] + for exchange_name in self.exchanges: + assets = exchange_assets[exchange_name] \ + if exchange_name in exchange_assets else [] + exchange_positions = \ [tracker.positions[asset] for asset in assets] @@ -452,6 +457,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): cash, positions_value = \ exchange.calculate_totals(exchange_positions) + total_cash += cash + total_positions_value += total_positions_value + for position in exchange_positions: tracker.update_position( asset=position.asset, @@ -459,13 +467,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): last_sale_price=position.last_sale_price ) + if total_cash < self.portfolio.cash: + raise ValueError('Cash on exchanges is lower than the algo.') + + return total_cash, total_positions_value except ExchangeRequestError as e: log.warn( 'update portfolio attempt {}: {}'.format(attempt_index, e) ) if attempt_index < self.retry_synchronize_portfolio: sleep(self.retry_delay) - self.update_positions(attempt_index + 1) + self.synchronize_portfolio(attempt_index + 1) else: raise ExchangePortfolioDataError( data_type='update-portfolio', @@ -565,20 +577,15 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): new_transactions, new_commissions, closed_orders = \ self.blotter.get_transactions(data) - self.blotter.prune_orders(closed_orders) - - for transaction in new_transactions: - self.perf_tracker.process_transaction(transaction) - - # since this order was modified, record it - order = self.blotter.orders[transaction.order_id] - self.perf_tracker.process_order(order) - if len(new_transactions) > 0: self.perf_tracker.update_performance() - self.update_positions() - + cash, positions_value = self.synchronize_portfolio() + log.info( + 'got totals from exchanges, cash: {} positions: {}'.format( + cash, positions_value + ) + ) if self._handle_data: self._handle_data(self, data) @@ -612,12 +619,26 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): 'statistics for the last {stats_minutes} minutes:\n{stats}'.format( stats_minutes=self.stats_minutes, stats=get_pretty_stats( - stats_df=print_df, + df=print_df, recorded_cols=recorded_cols, num_rows=self.stats_minutes ) )) + if self.stats_output is not None: + if 's3://' in self.stats_output: + stats_to_s3( + uri=self.stats_output, + df=print_df, + algo_namespace=self.algo_namespace, + recorded_cols=recorded_cols, + ) + + else: + raise ValueError( + 'Only S3 stats output is supported for now.' + ) + today = pd.to_datetime('today', utc=True) daily_stats = self.prepare_period_stats( start_dt=today, @@ -643,22 +664,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): except Exception as e: log.warn('unable to save minute perfs to disk: {}'.format(e)) - try: - blotter_params = dict( - open_orders=self.blotter.open_orders, - orders=self.blotter.orders, - new_orders=self.blotter.new_orders, - data_frequency=self.blotter.data_frequency, - current_dt=self.blotter.current_dt, - ) - save_algo_object( - algo_name=self.algo_namespace, - key='blotter', - obj=blotter_params, - ) - except Exception as e: - log.warn('unable to save portfolio to disk: {}'.format(e)) - @api_method def batch_market_order(self, share_counts): raise NotImplementedError() diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index e4f4cd7f..af7f2fee 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -604,6 +604,7 @@ def mixin_market_params(exchange_name, params, market): # TODO: make this more externalized / configurable if 'lot' in market: params['min_trade_size'] = market['lot'] + params['lot'] = market['lot'] if exchange_name == 'bitfinex': params['maker'] = 0.001 @@ -624,6 +625,9 @@ def mixin_market_params(exchange_name, params, market): if 'minimum_order_size' in info: params['min_trade_size'] = float(info['minimum_order_size']) + if 'lot' not in params: + params['lot'] = params['min_trade_size'] + def from_ms_timestamp(ms): return pd.to_datetime(ms, unit='ms', utc=True) diff --git a/catalyst/exchange/stats_utils.py b/catalyst/exchange/stats_utils.py index 845fccc8..77f4d283 100644 --- a/catalyst/exchange/stats_utils.py +++ b/catalyst/exchange/stats_utils.py @@ -2,6 +2,7 @@ import numbers import numpy as np import pandas as pd +import time def trend_direction(series): @@ -119,29 +120,20 @@ def vwap(df): return ret -def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10): - """ - Format and print the last few rows of a statistics DataFrame. - See the pyfolio project for the data structure. +def format_positions(positions): + parts = [] + for position in positions: + msg = '{amount:.2f}{base} cost basis {cost_basis:.8f}{quote}'.format( + amount=position['amount'], + base=position['sid'].base_currency, + cost_basis=position['cost_basis'], + quote=position['sid'].quote_currency + ) + parts.append(msg) + return ', '.join(parts) - Parameters - ---------- - stats_df: DataFrame - num_rows: int - - Returns - ------- - str - - """ - stats_df.set_index('period_close', drop=True, inplace=True) - stats_df.dropna(axis=1, how='all', inplace=True) - - pd.set_option('display.expand_frame_repr', False) - pd.set_option('precision', 3) - pd.set_option('display.width', 1000) - pd.set_option('display.max_colwidth', 1000) +def prepare_stats(df, recorded_cols=None): columns = ['starting_cash', 'ending_cash', 'portfolio_value', 'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions', 'positions'] @@ -150,31 +142,90 @@ def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10): for column in recorded_cols: columns.append(column) - def format_positions(positions): - parts = [] - for position in positions: - msg = '{amount:.2f}{base} cost basis {cost_basis:.4f}{quote}'.format( - amount=position['amount'], - base=position['sid'].base_currency, - cost_basis=position['cost_basis'], - quote=position['sid'].quote_currency - ) - parts.append(msg) - return ', '.join(parts) + df = df.copy(True) + + df.set_index('period_close', drop=True, inplace=True) + df.dropna(axis=1, how='all', inplace=True) + + df['orders'] = df['orders'].apply(lambda orders: len(orders)) + df['transactions'] = df['transactions'].apply( + lambda transactions: len(transactions) + ) + df['positions'] = df['positions'].apply(format_positions) + + return df, columns + + +def get_pretty_stats(df, recorded_cols=None, num_rows=10): + """ + Format and print the last few rows of a statistics DataFrame. + See the pyfolio project for the data structure. + + Parameters + ---------- + df: pd.DataFrame + num_rows: int + + Returns + ------- + str + + """ + df, columns = prepare_stats(df, recorded_cols=recorded_cols) + + pd.set_option('display.expand_frame_repr', False) + pd.set_option('precision', 3) + pd.set_option('display.width', 1000) + pd.set_option('display.max_colwidth', 1000) formatters = { - 'orders': lambda orders: len(orders), - 'transactions': lambda transactions: len(transactions), 'returns': lambda returns: "{0:.4f}".format(returns), - 'positions': format_positions } - return stats_df.tail(num_rows).to_string( + return df.tail(num_rows).to_string( columns=columns, formatters=formatters ) +def get_csv_stats(df, recorded_cols=None): + """ + Create a CSV buffer from the stats DataFrame. + + Parameters + ---------- + path: str + df: pd.DataFrame + recorded_cols: list[str] + + Returns + ------- + + """ + df, columns = prepare_stats(df, recorded_cols=recorded_cols) + + return df.to_csv( + None, + columns=columns, + encoding='utf-8', + ).encode() + + +def stats_to_s3(uri, df, algo_namespace, recorded_cols=None): + import boto3 + s3 = boto3.resource('s3') + + bytes_to_write = get_csv_stats(df, recorded_cols=recorded_cols) + + timestr = time.strftime('%Y%m%d-%H%M%S') + + parts = uri.split('//') + obj = s3.Object(parts[1], 'stats/{}-{}.csv'.format( + timestr, algo_namespace + )) + obj.put(Body=bytes_to_write) + + def df_to_string(df): """ Create a formatted str representation of the DataFrame. diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index f7219330..92a5e335 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -91,7 +91,8 @@ def _run(handle_data, algo_namespace, base_currency, live_graph, - simulate_orders): + simulate_orders, + stats_output): """Run a backtest for the given algorithm. This is shared between the cli and :func:`catalyst.run_algo`. @@ -151,7 +152,6 @@ def _run(handle_data, exchanges = dict() for exchange_name in exchange_list: - exchanges[exchange_name] = get_exchange( exchange_name=exchange_name, base_currency=base_currency, @@ -266,7 +266,8 @@ def _run(handle_data, exchanges=exchanges, algo_namespace=algo_namespace, live_graph=live_graph, - simulate_orders=simulate_orders + simulate_orders=simulate_orders, + stats_output=stats_output, ) elif exchanges: # Removed the existing Poloniex fork to keep things simple @@ -429,6 +430,7 @@ def run_algorithm(initialize, algo_namespace=None, live_graph=False, simulate_orders=True, + stats_output=None, output=os.devnull): """Run a trading algorithm. @@ -551,5 +553,6 @@ def run_algorithm(initialize, algo_namespace=algo_namespace, base_currency=base_currency, live_graph=live_graph, - simulate_orders=simulate_orders + simulate_orders=simulate_orders, + stats_output=stats_output )