wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-06-27 17:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Frederic Fortier
2018-01-10 13:33:35 -05:00
parent 790ac22f8d b60b50e99a
commit ff7d7c5256
23 changed files with 582 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/data/loader.py
+1
View File
@@ -146,6 +146,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
exchange = get_exchange(
exchange_name='poloniex', base_currency='usdt'
)
exchange.init()
benchmark_asset = exchange.get_asset(bm_symbol)
catalyst/examples/mean_reversion_simple.py
+6 -6
View File
@@ -38,13 +38,13 @@ def initialize(context):
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 65
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
# context.set_commission(maker=0.1, taker=0.2)
context.set_slippage(spread=0.0001)
context.set_commission(maker=0.001, taker=0.002)
context.set_slippage(spread=0.001)
def handle_data(context, data):
@@ -244,11 +244,11 @@ def analyze(context=None, perf=None):
if __name__ == '__main__':
# The execution mode: backtest or live
live = True
live = False
if live:
run_algorithm(
capital_base=0.03,
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
@@ -280,7 +280,7 @@ if __name__ == '__main__':
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='eth',
base_currency='btc',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
catalyst/examples/mean_reversion_simple_custom_fees.py
+288
View File
@@ -0,0 +1,288 @@
# For this example, we're going to write a simple momentum script. When the
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
# going to sell. Hopefully we'll ride the waves.
import os
import tempfile
import time
import numpy as np
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.utils.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
context.market = symbol('eth_btc')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 50
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
context.set_commission(maker=0.001, taker=0.002)
# context.set_slippage(spread=0.001)
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = False
context.current_day = today
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.market variable. For this example, we're
# using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention
# for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
context.market,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(context.market, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
cash = context.portfolio.cash
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
volume=current['volume'],
price=price,
price_change=price_change,
rsi=rsi[-1],
cash=cash
)
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if context.traded_today:
return
# TODO: retest with open orders
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.market)
if len(orders) > 0:
log.info('exiting because orders are open: {}'.format(orders))
return
# Exit if we cannot trade
if not data.can_trade(context.market):
return
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.market].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
order_target_percent(
context.market, 1, limit_price=limit_price
)
context.traded_today = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
context.market, 0, limit_price=limit_price
)
context.traded_today = True
def analyze(context=None, perf=None):
end = time.time()
log.info('elapsed time: {}'.format(end - context.start_time))
import matplotlib.pyplot as plt
# The base currency of the algo exchange
base_currency = context.exchanges.values()[0].base_currency.upper()
# Plot the portfolio value over time.
ax1 = plt.subplot(611)
perf.loc[:, 'portfolio_value'].plot(ax=ax1)
ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
# Plot the price increase or decrease over time.
ax2 = plt.subplot(612, sharex=ax1)
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.market.symbol, base=base_currency
))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'price'],
marker='v',
s=100,
c='red',
label=''
)
ax4 = plt.subplot(613, sharex=ax1)
perf.loc[:, 'cash'].plot(
ax=ax4, label='Base Currency ({})'.format(base_currency)
)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
ax5 = plt.subplot(614, sharex=ax1)
perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
ax5.set_ylabel('Percent\nChange')
ax6 = plt.subplot(615, sharex=ax1)
perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
ax6.set_ylabel('RSI')
ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
if not transaction_df.empty:
ax6.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'rsi'],
marker='^',
s=100,
c='green',
label=''
)
ax6.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'rsi'],
marker='v',
s=100,
c='red',
label=''
)
plt.legend(loc=3)
start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
pass
if __name__ == '__main__':
# The execution mode: backtest or live
live = False
if live:
run_algorithm(
capital_base=0.025,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
live=True,
algo_namespace=NAMESPACE,
base_currency='btc',
live_graph=False,
simulate_orders=False,
stats_output=None,
)
else:
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=0.1,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
)
log.info('saved perf stats: {}'.format(out))
catalyst/exchange/ccxt/ccxt_exchange.py
+47 -18
View File
@@ -6,12 +6,8 @@ from collections import defaultdict
import ccxt
import pandas as pd
import six
from catalyst.assets._assets import TradingPair
from ccxt import ExchangeNotAvailable, InvalidOrder
from logbook import Logger
from six import string_types
from catalyst.algorithm import MarketOrder
from catalyst.assets._assets import TradingPair
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -23,6 +19,10 @@ from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \
get_exchange_auth
from catalyst.finance.order import Order, ORDER_STATUS
from ccxt import InvalidOrder, NetworkError, \
ExchangeError
from logbook import Logger
from six import string_types
log = Logger('CCXT', level=LOG_LEVEL)
@@ -105,7 +105,12 @@ class CCXT(Exchange):
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except ExchangeNotAvailable as e:
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch markets {}: {}'.format(
self.name, e
)
)
raise ExchangeRequestError(error=e)
self.load_assets()
@@ -408,6 +413,7 @@ class CCXT(Exchange):
def _fetch_symbol_map(self, is_local):
try:
return self.fetch_symbol_map(is_local)
except ExchangeSymbolsNotFound:
return None
@@ -559,8 +565,12 @@ class CCXT(Exchange):
for key in balances:
balances_lower[key.lower()] = balances[key]
except Exception as e:
log.debug('error retrieving balances: {}', e)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch balance {}: {}'.format(
self.name, e
)
)
raise ExchangeRequestError(error=e)
return balances_lower
@@ -703,14 +713,18 @@ class CCXT(Exchange):
amount=adj_amount,
price=price
)
except ExchangeNotAvailable as e:
log.debug('unable to create order: {}'.format(e))
raise ExchangeRequestError(error=e)
except InvalidOrder as e:
log.warn('the exchange rejected the order: {}'.format(e))
raise CreateOrderError(exchange=self.name, error=e)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to create order {} / {}: {}'.format(
self.name, symbol, e
)
)
raise ExchangeRequestError(error=e)
if 'info' not in result:
raise ValueError('cannot use order without info attribute')
@@ -735,7 +749,12 @@ class CCXT(Exchange):
limit=None,
params=dict()
)
except Exception as e:
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch open orders {} / {}: {}'.format(
self.name, asset.symbol, e
)
)
raise ExchangeRequestError(error=e)
orders = []
@@ -758,7 +777,12 @@ class CCXT(Exchange):
order_status = self.api.fetch_order(id=order_id, symbol=symbol)
order, executed_price = self._create_order(order_status)
except Exception as e:
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch order {} / {}: {}'.format(
self.name, order_id, e
)
)
raise ExchangeRequestError(error=e)
return order, executed_price
@@ -777,7 +801,12 @@ class CCXT(Exchange):
if asset_or_symbol is not None else None
self.api.cancel_order(id=order_id, symbol=symbol)
except Exception as e:
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to cancel order {} / {}: {}'.format(
self.name, order_id, e
)
)
raise ExchangeRequestError(error=e)
def tickers(self, assets):
@@ -828,10 +857,10 @@ class CCXT(Exchange):
tickers[asset] = ticker
except ExchangeNotAvailable as e:
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch ticker: {} {}'.format(
self.name, asset.symbol
'unable to fetch ticker {} / {}: {}'.format(
self.name, asset.symbol, e
)
)
raise ExchangeRequestError(error=e)
catalyst/exchange/exchange.py
+1 -2
View File
@@ -5,8 +5,6 @@ from time import sleep
import numpy as np
import pandas as pd
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -19,6 +17,7 @@ from catalyst.exchange.utils.bundle_utils import get_start_dt, \
get_delta, get_periods, get_periods_range
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
get_frequency, resample_history_df, has_bundle
from logbook import Logger
log = Logger('Exchange', level=LOG_LEVEL)
catalyst/exchange/exchange_algorithm.py
+82 -44
View File
@@ -18,11 +18,9 @@ from datetime import timedelta
from os import listdir
from os.path import isfile, join
import catalyst.protocol as zp
import logbook
import pandas as pd
from redo import retry
import catalyst.protocol as zp
from catalyst.algorithm import TradingAlgorithm
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange_blotter import ExchangeBlotter
@@ -49,6 +47,7 @@ from catalyst.utils.api_support import api_method
from catalyst.utils.input_validation import error_keywords, ensure_upper_case
from catalyst.utils.math_utils import round_nearest
from catalyst.utils.preprocess import preprocess
from redo import retry
log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
@@ -130,7 +129,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
@api_method
def set_commission(self, maker=None, taker=None):
key = self.blotter.commission_models.keys()[0]
key = list(self.blotter.commission_models.keys())[0]
if maker is not None:
self.blotter.commission_models[key].maker = maker
@@ -139,7 +138,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
@api_method
def set_slippage(self, spread=None):
key = self.blotter.slippage_models.keys()[0]
key = list(self.blotter.slippage_models.keys())[0]
if spread is not None:
self.blotter.slippage_models[key].spread = spread
@@ -271,9 +270,9 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
# Merging latest recorded variables
stats.update(self.recorded_vars)
stats['positions'] = cum.position_tracker.get_positions_list()
period = tracker.todays_performance
stats['positions'] = period.position_tracker.get_positions_list()
# we want the key to be absent, not just empty
# Only include transactions for given dt
stats['transactions'] = []
@@ -391,16 +390,20 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
'before exiting the algorithm.')
algo_folder = get_algo_folder(self.algo_namespace)
folder = join(algo_folder, 'daily_perf')
folder = join(algo_folder, 'daily_performance')
files = [f for f in listdir(folder) if isfile(join(folder, f))]
daily_perf_list = []
for item in files:
filename = join(folder, item)
with open(filename, 'rb') as handle:
daily_perf_list.append(pickle.load(handle))
perf_period = pickle.load(handle)
perf_period_dict = perf_period.to_dict()
daily_perf_list.append(perf_period_dict)
stats = pd.DataFrame(daily_perf_list)
stats.set_index('period_close', drop=False, inplace=True)
self.analyze(stats)
@@ -460,43 +463,62 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
return self._clock
def get_generator(self):
if self.trading_client is not None:
return self.trading_client.transform()
def _init_trading_client(self):
"""
This replaces Ziplines `_create_generator` method. The main difference
is that we are restoring performance tracker objects if available.
This allows us to stop/start algos without loosing their state.
perf = None
"""
if self.perf_tracker is None:
# Note from the Zipline dev:
# HACK: When running with the `run` method, we set perf_tracker to
# None so that it will be overwritten here.
tracker = self.perf_tracker = PerformanceTracker(
sim_params=self.sim_params,
trading_calendar=self.trading_calendar,
env=self.trading_environment,
)
# Set the dt initially to the period start by forcing it to change.
self.on_dt_changed(self.sim_params.start_session)
new_position_tracker = tracker.position_tracker
tracker.position_tracker = None
# Unpacking the perf_tracker and positions if available
perf = get_algo_object(
cum_perf = get_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance',
)
if cum_perf is not None:
tracker.cumulative_performance = cum_perf
# Ensure single common position tracker
tracker.position_tracker = cum_perf.position_tracker
today = pd.Timestamp.utcnow().floor('1D')
todays_perf = get_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
rel_path='daily_performance',
)
if todays_perf is not None:
# Ensure single common position tracker
if tracker.position_tracker is not None:
todays_perf.position_tracker = tracker.position_tracker
else:
tracker.position_tracker = todays_perf.position_tracker
tracker.todays_performance = todays_perf
if tracker.position_tracker is None:
# Use a new position_tracker if not is found in the state
tracker.position_tracker = new_position_tracker
if not self.initialized:
# Calls the initialize function of the algorithm
self.initialize(*self.initialize_args, **self.initialize_kwargs)
self.initialized = True
# Call the simulation trading algorithm for side-effects:
# it creates the perf tracker
# TradingAlgorithm._create_generator(self, self.sim_params)
if perf is not None:
tracker.cumulative_performance = perf
period = self.perf_tracker.todays_performance
period.starting_cash = perf.ending_cash
period.starting_exposure = perf.ending_exposure
period.starting_value = perf.ending_value
period.position_tracker = perf.position_tracker
self.trading_client = ExchangeAlgorithmExecutor(
algo=self,
sim_params=self.sim_params,
@@ -506,6 +528,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
restrictions=self.restrictions,
universe_func=self._calculate_universe,
)
def get_generator(self):
if self.trading_client is None:
self._init_trading_client()
return self.trading_client.transform()
def updated_portfolio(self):
@@ -523,10 +550,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
positions, returning the available cash, and raising error
if the data goes out of sync.
Parameters
----------
attempt_index: int
Returns
-------
float
@@ -559,10 +582,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
if base_currency is None:
base_currency = exchange.base_currency
# Don't check the cash if there are open orders. This could
# results in false positives.
orders = []
for asset in self.blotter.open_orders:
asset_orders = self.blotter.open_orders[asset]
if asset_orders:
orders += asset_orders
required_cash = self.portfolio.cash if not orders else None
cash, positions_value = exchange.sync_positions(
positions=exchange_positions,
check_balances=check_balances,
cash=self.portfolio.cash,
cash=required_cash,
)
total_cash += cash
total_positions_value += positions_value
@@ -676,11 +708,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.frame_stats = list()
self.performance_needs_update = False
new_orders = self.perf_tracker.todays_performance.orders_by_id.keys()
if new_orders != self._last_orders:
orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
if orders != self._last_orders:
self.performance_needs_update = True
self._last_orders = copy.deepcopy(new_orders)
# Saving current orders to detect changes in the next frame
self._last_orders = copy.deepcopy(orders)
if self.performance_needs_update:
self.perf_tracker.update_performance()
@@ -697,7 +730,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.portfolio_needs_update = False
log.info(
'got totals from exchanges, cash: {} positions: {}'.format(
'portfolio balances, cash: {}, positions: {}'.format(
cash, positions_value
)
)
@@ -709,18 +742,29 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# every bar no matter if the algorithm places an order or not.
self.validate_account_controls()
self._save_algo_state(data)
self.current_day = data.current_dt.floor('1D')
def _save_algo_state(self, data):
today = data.current_dt.floor('1D')
try:
self._save_stats_csv(self._process_stats(data))
except Exception as e:
log.warn('unable to calculate performance: {}'.format(e))
log.debug('saving cumulative performance object')
save_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance',
obj=self.perf_tracker.cumulative_performance,
)
self.current_day = data.current_dt.floor('1D')
log.debug('saving todays performance object')
save_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
obj=self.perf_tracker.todays_performance,
rel_path='daily_performance'
)
def _process_stats(self, data):
today = data.current_dt.floor('1D')
@@ -763,12 +807,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
start_dt=today,
end_dt=data.current_dt
)
save_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
obj=daily_stats,
rel_path='daily_perf'
)
return recorded_cols
catalyst/exchange/exchange_asset_finder.py
+1 -2
View File
@@ -1,8 +1,7 @@
import pandas as pd
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.utils.factory import find_exchanges
from logbook import Logger
log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
catalyst/exchange/exchange_blotter.py
+9 -5
View File
@@ -1,8 +1,5 @@
import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from redo import retry
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange_errors import ExchangeRequestError
from catalyst.finance.blotter import Blotter
@@ -11,6 +8,8 @@ from catalyst.finance.order import ORDER_STATUS
from catalyst.finance.slippage import SlippageModel
from catalyst.finance.transaction import create_transaction, Transaction
from catalyst.utils.input_validation import expect_types
from logbook import Logger
from redo import retry
log = Logger('exchange_blotter', level=LOG_LEVEL)
@@ -42,6 +41,11 @@ class TradingPairFeeSchedule(CommissionModel):
)
)
def get_maker_taker(self, asset):
maker = self.maker if self.maker is not None else asset.maker
taker = self.taker if self.taker is not None else asset.taker
return maker, taker
def calculate(self, order, transaction):
"""
Calculate the final fee based on the order parameters.
@@ -55,8 +59,7 @@ class TradingPairFeeSchedule(CommissionModel):
cost = abs(transaction.amount) * transaction.price
asset = order.asset
maker = self.maker if self.maker is not None else asset.maker
taker = self.taker if self.taker is not None else asset.taker
maker, taker = self.get_maker_taker(asset)
multiplier = taker
if order.limit is not None:
@@ -251,6 +254,7 @@ class ExchangeBlotter(Blotter):
for order, txn in self.check_open_orders():
order.dt = txn.dt
# TODO: is the commission already on the order object?
transactions.append(txn)
if not order.open:
catalyst/exchange/exchange_bundle.py
+4 -5
View File
@@ -8,12 +8,8 @@ from operator import is_not
import numpy as np
import pandas as pd
import pytz
from catalyst.assets._assets import TradingPair
from logbook import Logger
from pytz import UTC
from six import itervalues
from catalyst import get_calendar
from catalyst.assets._assets import TradingPair
from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
from catalyst.constants import LOG_LEVEL
from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
@@ -32,6 +28,9 @@ from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
from logbook import Logger
from pytz import UTC
from six import itervalues
log = Logger('exchange_bundle', level=LOG_LEVEL)
catalyst/exchange/exchange_data_portal.py
+2 -3
View File
@@ -3,9 +3,6 @@ import abc
import numpy as np
import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from redo import retry
from catalyst.constants import LOG_LEVEL, AUTO_INGEST
from catalyst.data.data_portal import DataPortal
from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -14,6 +11,8 @@ from catalyst.exchange.exchange_errors import (
PricingDataNotLoadedError)
from catalyst.exchange.utils.exchange_utils import get_frequency, \
resample_history_df, group_assets_by_exchange
from logbook import Logger
from redo import retry
log = Logger('DataPortalExchange', level=LOG_LEVEL)
catalyst/exchange/exchange_portfolio.py
+1 -2
View File
@@ -1,8 +1,7 @@
import numpy as np
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.protocol import Portfolio, Positions, Position
from logbook import Logger
log = Logger('ExchangePortfolio', level=LOG_LEVEL)
catalyst/exchange/exchange_pricing_loader.py
+5 -6
View File
@@ -11,12 +11,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from logbook import Logger
from numpy import (
iinfo,
uint32,
)
from catalyst.constants import LOG_LEVEL
from catalyst.data.us_equity_pricing import BcolzDailyBarReader
from catalyst.errors import NoFurtherDataError
@@ -26,6 +20,11 @@ from catalyst.pipeline.data import DataSet, Column
from catalyst.pipeline.loaders.base import PipelineLoader
from catalyst.utils.calendars import get_calendar
from catalyst.utils.numpy_utils import float64_dtype
from logbook import Logger
from numpy import (
iinfo,
uint32,
)
UINT32_MAX = iinfo(uint32).max
catalyst/exchange/live_graph_clock.py
+2 -3
View File
@@ -1,13 +1,12 @@
import pandas as pd
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.utils.stats_utils import prepare_stats
from catalyst.gens.sim_engine import (
BAR,
SESSION_START
)
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.utils.stats_utils import prepare_stats
log = Logger('LiveGraphClock', level=LOG_LEVEL)
catalyst/exchange/simple_clock.py
+1 -2
View File
@@ -14,14 +14,13 @@
from time import sleep
import pandas as pd
from catalyst.constants import LOG_LEVEL
from catalyst.gens.sim_engine import (
BAR,
SESSION_START
)
from logbook import Logger
from catalyst.constants import LOG_LEVEL
log = Logger('ExchangeClock', level=LOG_LEVEL)
catalyst/exchange/utils/bundle_utils.py
-1
View File
@@ -6,7 +6,6 @@ from datetime import timedelta, datetime, date
import numpy as np
import pandas as pd
import pytz
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
catalyst/exchange/utils/exchange_utils.py
+2 -3
View File
@@ -8,9 +8,6 @@ from datetime import date, datetime
import pandas as pd
from catalyst.assets._assets import TradingPair
from six import string_types
from six.moves.urllib import request
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
@@ -18,6 +15,8 @@ from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
ExchangeJSONDecoder
from catalyst.utils.paths import data_root, ensure_directory, \
last_modified_time
from six import string_types
from six.moves.urllib import request
def get_sid(symbol):
catalyst/exchange/utils/factory.py
+1 -2
View File
@@ -1,13 +1,12 @@
import os
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
get_exchange_folder, is_blacklist
from logbook import Logger
log = Logger('factory', level=LOG_LEVEL)
exchange_cache = dict()
catalyst/exchange/utils/serialization_utils.py
+1 -2
View File
@@ -3,9 +3,8 @@ import re
from json import JSONEncoder
import pandas as pd
from six import string_types
from catalyst.constants import DATE_TIME_FORMAT
from six import string_types
class ExchangeJSONEncoder(json.JSONEncoder):
catalyst/exchange/utils/stats_utils.py
+27 -9
View File
@@ -8,9 +8,9 @@ import time
import numpy as np
import pandas as pd
from catalyst.assets._assets import TradingPair
from catalyst.exchange.utils.exchange_utils import get_algo_folder
from catalyst.utils.paths import data_root, ensure_directory
from operator import itemgetter
s3_conn = []
mailgun = []
@@ -261,7 +261,14 @@ def prepare_stats(stats, recorded_cols=list()):
return df, columns
def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
def set_print_settings():
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 8)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
"""
Format and print the last few rows of a statistics DataFrame.
See the pyfolio project for the data structure.
@@ -281,17 +288,17 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
"""
if isinstance(stats, pd.DataFrame):
stats = stats.T.to_dict().values()
stats.sort(key=itemgetter('period_close'))
if len(stats) > num_rows:
display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
else:
display_stats = stats
display_stats = stats[-num_rows:] if len(stats) > num_rows else stats
df, columns = prepare_stats(
display_stats, recorded_cols=recorded_cols
)
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 8)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
set_print_settings()
return df.to_string(columns=columns)
@@ -439,6 +446,17 @@ def df_to_string(df):
return df.to_string()
def extract_orders(perf):
order_list = perf.orders.values
all_orders = [t for sublist in order_list for t in sublist]
all_orders.sort(key=lambda o: o['dt'])
orders = pd.DataFrame(all_orders)
if not orders.empty:
orders.set_index('dt', inplace=True, drop=True)
return orders
def extract_transactions(perf):
"""
Compute indexes for buy and sell transactions
catalyst/exchange/utils/test_utils.py
-1
View File
@@ -3,7 +3,6 @@ import random
import tempfile
from catalyst.assets._assets import TradingPair
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import find_exchanges
from catalyst.utils.paths import ensure_directory
catalyst/utils/run_algo.py
+17 -67
View File
@@ -8,13 +8,12 @@ from time import sleep
import click
import pandas as pd
from logbook import Logger
from catalyst.data.bundles import load
from catalyst.data.data_portal import DataPortal
from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
TradingPairPricing
from catalyst.exchange.utils.factory import get_exchange
from logbook import Logger
try:
from pygments import highlight
@@ -40,9 +39,6 @@ from catalyst.exchange.exchange_algorithm import (
from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
DataPortalExchangeBacktest
from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder
from catalyst.exchange.exchange_errors import (
ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
BaseCurrencyNotFoundError, NotEnoughCapitalError)
from catalyst.constants import LOG_LEVEL
@@ -144,8 +140,22 @@ def _run(handle_data,
else:
click.echo(algotext)
mode = 'paper-trading' if simulate_orders else 'live-trading' \
if live else 'backtest'
log.warn(
'Catalyst is currently in ALPHA. It is going through rapid '
'development and it is subject to errors. Please use carefully. '
'We encourage you to report any issue on GitHub: '
'https://github.com/enigmampc/catalyst/issues'
)
sleep(3)
if live:
if simulate_orders:
mode = 'paper-trading'
else:
mode = 'live-trading'
else:
mode = 'backtest'
log.info('running algo in {mode} mode'.format(mode=mode))
exchange_name = exchange
@@ -199,66 +209,6 @@ def _run(handle_data,
first_trading_day=pd.to_datetime('today', utc=True)
)
def fetch_capital_base(exchange, attempt_index=0):
"""
Fetch the base currency amount required to bootstrap
the algorithm against the exchange.
The algorithm cannot continue without this value.
:param exchange: the targeted exchange
:param attempt_index:
:return capital_base: the amount of base currency available for
trading
"""
try:
log.debug('retrieving capital base in {} to bootstrap '
'exchange {}'.format(base_currency, exchange_name))
balances = exchange.get_balances()
except ExchangeRequestError as e:
if attempt_index < 20:
log.warn(
'could not retrieve balances on {}: {}'.format(
exchange.name, e
)
)
sleep(5)
return fetch_capital_base(exchange, attempt_index + 1)
else:
raise ExchangeRequestErrorTooManyAttempts(
attempts=attempt_index,
error=e
)
if base_currency in balances:
base_currency_available = balances[base_currency]['free']
log.info(
'base currency available in the account: {} {}'.format(
base_currency_available, base_currency
)
)
return base_currency_available
else:
raise BaseCurrencyNotFoundError(
base_currency=base_currency,
exchange=exchange_name
)
if not simulate_orders:
for exchange_name in exchanges:
exchange = exchanges[exchange_name]
balance = fetch_capital_base(exchange)
if balance < capital_base:
raise NotEnoughCapitalError(
exchange=exchange_name,
base_currency=base_currency,
balance=balance,
capital_base=capital_base,
)
sim_params = create_simulation_parameters(
start=start,
end=end,
docs/source/releases.rst
+12 -2
View File
@@ -2,9 +2,19 @@
Release Notes
=============
Version 0.4.4
^^^^^^^^^^^^^
**Release Date**: 2018-01-09
Bug Fixes
~~~~~~~~~
- Removed redundant capital_base validation (:issue:`142`)
- Fixed portfolio update issue with restored state (:issue:`111`)
- Skipping cash validation where there are open orders (:issue:`144`)
Version 0.4.3
^^^^^^^^^^^^^
**Release Date**: 2017-01-05
**Release Date**: 2018-01-05
Bug Fixes
~~~~~~~~~
@@ -13,7 +23,7 @@ Bug Fixes
Version 0.4.2
^^^^^^^^^^^^^
**Release Date**: 2017-01-03
**Release Date**: 2018-01-03
Bug Fixes
~~~~~~~~~
tests/exchange/test_suites/test_suite_algo.py
+72
View File
@@ -0,0 +1,72 @@
import importlib
from os.path import join, isfile
import pandas as pd
import os
from catalyst import run_algorithm
from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
extract_transactions, set_print_settings, extract_orders
from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
from logbook import TestHandler, WARNING
from pathtools.path import listdir
filter_algos = [
'mean_reversion_simple_custom_fees.py',
]
class TestSuiteAlgo(WithLogger, ZiplineTestCase):
@staticmethod
def analyze(context, perf):
set_print_settings()
transaction_df = extract_transactions(perf)
print('the transactions:\n{}'.format(transaction_df))
orders_df = extract_orders(perf)
print('the orders:\n{}'.format(orders_df))
stats = get_pretty_stats(perf, show_tail=False, num_rows=5)
print('the stats:\n{}'.format(stats))
pass
def test_run_examples(self):
folder = join('..', '..', '..', 'catalyst', 'examples')
files = [f for f in listdir(folder) if isfile(join(folder, f))]
algo_list = []
for filename in files:
name = os.path.basename(filename)
if filter_algos and name not in filter_algos:
continue
module_name = 'catalyst.examples.{}'.format(
name.replace('.py', '')
)
algo_list.append(module_name)
for module_name in algo_list:
algo = importlib.import_module(module_name)
namespace = module_name.replace('.', '_')
log_catcher = TestHandler()
with log_catcher:
run_algorithm(
capital_base=0.1,
data_frequency='minute',
initialize=algo.initialize,
handle_data=algo.handle_data,
analyze=TestSuiteAlgo.analyze,
exchange_name='bitfinex',
algo_namespace='test_{}'.format(namespace),
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-10-02', utc=True),
# output=out
)
warnings = [record for record in log_catcher.records if
record.level == WARNING]
self.assertEqual(0, len(warnings))
pass