wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-10 01:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

BLD: added a cmd for running on the cloud (WIP)

Browse Source
This commit is contained in:
AvishaiW
2018-03-01 01:30:18 +02:00
parent fec829b82e
commit 8a3cc7e5fa
24 changed files with 879 additions and 1851 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/__main__.py
+174
View File
@@ -14,6 +14,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.utils.exchange_utils import delete_algo_folder
from catalyst.utils.cli import Date, Timestamp
from catalyst.utils.run_algo import _run, load_extensions
from catalyst.utils.run_server import run_server
try:
__IPYTHON__
@@ -505,6 +506,179 @@ def live(ctx,
return perf
@main.command(name='serve-live')
@click.option(
'-f',
'--algofile',
default=None,
type=click.File('r'),
help='The file that contains the algorithm to run.',
)
@click.option(
'--capital-base',
type=float,
show_default=True,
help='The amount of capital (in base_currency) allocated to trading.',
)
@click.option(
'-t',
'--algotext',
help='The algorithm script to run.',
)
@click.option(
'-D',
'--define',
multiple=True,
help="Define a name to be bound in the namespace before executing"
" the algotext. For example '-Dname=value'. The value may be"
" any python expression. These are evaluated in order so they"
" may refer to previously defined names.",
)
@click.option(
'-o',
'--output',
default='-',
metavar='FILENAME',
show_default=True,
help="The location to write the perf data. If this is '-' the perf will"
" be written to stdout.",
)
@click.option(
'--print-algo/--no-print-algo',
is_flag=True,
default=False,
help='Print the algorithm to stdout.',
)
@ipython_only(click.option(
'--local-namespace/--no-local-namespace',
is_flag=True,
default=None,
help='Should the algorithm methods be resolved in the local namespace.'
))
@click.option(
'-x',
'--exchange-name',
help='The name of the targeted exchange.',
)
@click.option(
'-n',
'--algo-namespace',
help='A label assigned to the algorithm for data storage purposes.'
)
@click.option(
'-c',
'--base-currency',
help='The base currency used to calculate statistics '
'(e.g. usd, btc, eth).',
)
@click.option(
'-e',
'--end',
type=Date(tz='utc', as_timestamp=True),
help='An optional end date at which to stop the execution.',
)
@click.option(
'--live-graph/--no-live-graph',
is_flag=True,
default=False,
help='Display live graph.',
)
@click.option(
'--simulate-orders/--no-simulate-orders',
is_flag=True,
default=True,
help='Simulating orders enable the paper trading mode. No orders will be '
'sent to the exchange unless set to false.',
)
@click.option(
'--auth-aliases',
default=None,
help='Authentication file aliases for the specified exchanges. By default,'
'each exchange uses the "auth.json" file in the exchange folder. '
'Specifying an "auth2" alias would use "auth2.json". It should be '
'specified like this: "[exchange_name],[alias],..." For example, '
'"binance,auth2" or "binance,auth2,bittrex,auth2".',
)
@click.pass_context
def serve_live(ctx,
algofile,
capital_base,
algotext,
define,
output,
print_algo,
local_namespace,
exchange_name,
algo_namespace,
base_currency,
end,
live_graph,
auth_aliases,
simulate_orders):
"""Trade live with the given algorithm on the server.
"""
if (algotext is not None) == (algofile is not None):
ctx.fail(
"must specify exactly one of '-f' / '--algofile' or"
" '-t' / '--algotext'",
)
if exchange_name is None:
ctx.fail("must specify an exchange name '-x'")
if algo_namespace is None:
ctx.fail("must specify an algorithm name '-n' in live execution mode")
if base_currency is None:
ctx.fail("must specify a base currency '-c' in live execution mode")
if capital_base is None:
ctx.fail("must specify a capital base with '--capital-base'")
if simulate_orders:
click.echo('Running in paper trading mode.', sys.stdout)
else:
click.echo('Running in live trading mode.', sys.stdout)
perf = run_server(
initialize=None,
handle_data=None,
before_trading_start=None,
analyze=None,
algofile=algofile,
algotext=algotext,
defines=define,
data_frequency=None,
capital_base=capital_base,
data=None,
bundle=None,
bundle_timestamp=None,
start=None,
end=end,
output=output,
print_algo=print_algo,
local_namespace=local_namespace,
environ=os.environ,
live=True,
exchange=exchange_name,
algo_namespace=algo_namespace,
base_currency=base_currency,
live_graph=live_graph,
analyze_live=None,
simulate_orders=simulate_orders,
auth_aliases=auth_aliases,
stats_output=None,
)
if output == '-':
click.echo(str(perf), sys.stdout)
elif output != os.devnull: # make the catalyst magic not write any data
perf.to_pickle(output)
return perf
@main.command(name='ingest-exchange')
@click.option(
'-x',
catalyst/assets/_assets.pyx
+10 -59
View File
@@ -433,7 +433,7 @@ cdef class TradingPair(Asset):
'taker',
'trading_state',
'data_source',
'decimals',
'decimals'
})
def __init__(self,
object symbol,
@@ -455,7 +455,7 @@ cdef class TradingPair(Asset):
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=1,
int trading_state=0,
object data_source='catalyst'):
"""
Replicates the Asset constructor with some built-in conventions
@@ -600,51 +600,14 @@ cdef class TradingPair(Asset):
cpdef to_dict(self):
"""
Convert to a python dict.
Repeat constructor params:
object symbol,
object exchange,
object start_date=None,
object asset_name=None,
int sid=0,
float leverage=1.0,
object end_daily=None,
object end_minute=None,
object end_date=None,
object exchange_symbol=None,
object first_traded=None,
object auto_close_date=None,
object exchange_full=None,
float min_trade_size=0.0001,
float max_trade_size=1000000,
float maker=0.0015,
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=1,
object data_source='catalyst',
"""
trading_pair_dict = dict(
symbol=self.symbol,
exchange=self.exchange,
start_date=self.start_date,
asset_name=self.asset_name,
leverage=self.leverage,
end_daily=self.end_daily,
end_minute=self.end_minute,
end_date=self.end_date,
exchange_symbol=self.exchange_symbol,
exchange_full=self.exchange_full,
min_trade_size=self.min_trade_size,
max_trade_size=self.max_trade_size,
maker=self.maker,
taker=self.taker,
lot=self.lot,
decimals=self.decimals,
trading_state=self.trading_state,
data_source=self.data_source,
)
return trading_pair_dict
#TODO: missing fields
super_dict = super(TradingPair, self).to_dict()
super_dict['end_daily'] = self.end_daily
super_dict['end_minute'] = self.end_minute
super_dict['leverage'] = self.leverage
super_dict['min_trade_size'] = self.min_trade_size
return super_dict
def is_exchange_open(self, dt_minute):
"""
@@ -660,16 +623,6 @@ cdef class TradingPair(Asset):
#TODO: make more dymanic to catch holds
return True
def set_end_date(self, dt, data_frequency):
if data_frequency == 'minute':
self.end_minute = dt
else:
self.end_daily = dt
def set_start_date(self, dt):
self.start_date = dt
cpdef __reduce__(self):
"""
Function used by pickle to determine how to serialize/deserialize this
@@ -693,9 +646,7 @@ cdef class TradingPair(Asset):
self.lot,
self.decimals,
self.taker,
self.maker,
self.trading_state,
self.data_source))
self.maker))
def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object)
catalyst/constants.py
+1 -2
View File
@@ -11,8 +11,7 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
'{exchange}/symbols.json'
EXCHANGE_CONFIG_URL = 'https://s3.amazonaws.com/enigmaco/exchanges/' \
'{exchange}/config.json'
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
DATE_FORMAT = '%Y-%m-%d'
catalyst/examples/dual_moving_average.py
+16 -28
View File
@@ -4,8 +4,7 @@ import pandas as pd
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import (record, symbol, order_target_percent,
get_open_orders)
from catalyst.api import (record, symbol, order_target_percent,)
from catalyst.exchange.utils.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
@@ -20,8 +19,8 @@ def initialize(context):
def handle_data(context, data):
# define the windows for the moving averages
short_window = 2
long_window = 3
short_window = 50
long_window = 200
# Skip as many bars as long_window to properly compute the average
context.i += 1
@@ -63,7 +62,7 @@ def handle_data(context, data):
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
orders = context.blotter.open_orders
if len(orders) > 0:
return
@@ -150,27 +149,16 @@ def analyze(context, perf):
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
simulate_orders=True,
live=True,
)
# run_algorithm(
# capital_base=1000,
# data_frequency='minute',
# initialize=initialize,
# handle_data=handle_data,
# analyze=analyze,
# exchange_name='bitfinex',
# algo_namespace=NAMESPACE,
# base_currency='usd',
# start=pd.to_datetime('2017-9-22', utc=True),
# end=pd.to_datetime('2017-9-23', utc=True),
# )
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
catalyst/examples/mean_reversion_simple.py
+7 -7
View File
@@ -33,12 +33,12 @@ def initialize(context):
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
context.market = symbol('eth_btc')
context.market = symbol('bnb_eth')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 60
context.RSI_OVERSOLD = 60
context.RSI_OVERBOUGHT = 70
context.CANDLE_SIZE = '15T'
context.start_time = time.time()
@@ -248,14 +248,14 @@ if __name__ == '__main__':
if live:
run_algorithm(
capital_base=0.03,
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
exchange_name='binance',
live=True,
algo_namespace=NAMESPACE,
base_currency='btc',
base_currency='eth',
live_graph=False,
simulate_orders=False,
stats_output=None,
@@ -274,7 +274,7 @@ if __name__ == '__main__':
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=0.1,
capital_base=0.035,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
catalyst/exchange/ccxt/ccxt_exchange.py
+269 -152
View File
@@ -1,33 +1,34 @@
import json
import os
import re
from collections import defaultdict
import ccxt
import pandas as pd
import six
from catalyst.assets._assets import TradingPair
from redo import retry
from ccxt import InvalidOrder, NetworkError, \
ExchangeError
from logbook import Logger
from six import string_types
from catalyst.algorithm import MarketOrder
from catalyst.assets._assets import TradingPair
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
UnsupportedHistoryFrequencyError, \
ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
MarketsNotFoundError, InvalidMarketError
UnsupportedHistoryFrequencyError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.utils.ccxt_utils import get_exchange_config
from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
get_exchange_folder, get_catalyst_symbol, \
get_exchange_auth
from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \
get_epoch, \
get_periods_range
from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.finance.transaction import Transaction
from ccxt import InvalidOrder, NetworkError, \
ExchangeError
from logbook import Logger
from six import string_types
log = Logger('CCXT', level=LOG_LEVEL)
@@ -43,7 +44,7 @@ SUPPORTED_EXCHANGES = dict(
class CCXT(Exchange):
def __init__(self, exchange_name, key,
secret, password, base_currency, config=None):
secret, password, base_currency):
log.debug(
'finding {} in CCXT exchanges:\n{}'.format(
exchange_name, ccxt.exchanges
@@ -63,8 +64,6 @@ class CCXT(Exchange):
'password': password,
})
self.api.enableRateLimit = True
self.has = self.api.has
self.fees = self.api.fees
except Exception:
raise ExchangeNotFoundError(exchange_name=exchange_name)
@@ -72,7 +71,6 @@ class CCXT(Exchange):
self._symbol_maps = [None, None]
self.name = exchange_name
self.assets = []
self.base_currency = base_currency
self.transactions = defaultdict(list)
@@ -84,123 +82,97 @@ class CCXT(Exchange):
self._common_symbols = dict()
self.bundle = ExchangeBundle(self.name)
self.markets = None
self._is_init = False
self._config = config
def init(self):
if self._is_init:
return
if self._config is None:
self._config = get_exchange_config(self.name)
log.debug(
'got exchange config {}:\n{}'.format(
self.name, self._config
exchange_folder = get_exchange_folder(self.name)
filename = os.path.join(exchange_folder, 'cctx_markets.json')
if os.path.exists(filename):
timestamp = os.path.getmtime(filename)
dt = pd.to_datetime(timestamp, unit='s', utc=True)
if dt >= pd.Timestamp.utcnow().floor('1D'):
with open(filename) as f:
self.markets = json.load(f)
log.debug('loaded markets for {}'.format(self.name))
if self.markets is None:
try:
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
)
self.markets = self.api.fetch_markets()
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch markets {}: {}'.format(
self.name, e
)
)
raise ExchangeRequestError(error=e)
self.load_assets()
self._is_init = True
def load_assets(self):
if self._config is None:
raise ValueError('Exchange config not available.')
@staticmethod
def find_exchanges(features=None, is_authenticated=False):
ccxt_features = []
if features is not None:
for feature in features:
if not feature.endswith('Bundle'):
ccxt_features.append(feature)
self.assets = []
for asset_dict in self._config['assets']:
asset = TradingPair(**asset_dict)
self.assets.append(asset)
exchange_names = []
for exchange_name in ccxt.exchanges:
if is_authenticated:
exchange_auth = get_exchange_auth(exchange_name)
def _fetch_markets(self):
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
try:
markets = self.api.fetch_markets()
has_auth = (exchange_auth['key'] != ''
and exchange_auth['secret'] != '')
except NetworkError as e:
raise ExchangeRequestError(error=e)
if not has_auth:
continue
if not markets:
raise MarketsNotFoundError(
exchange=self.name,
)
log.debug('loading exchange: {}'.format(exchange_name))
exchange = getattr(ccxt, exchange_name)()
for market in markets:
if 'id' not in market:
raise InvalidMarketError(
exchange=self.name,
market=market,
)
return markets
if ccxt_features is None:
has_feature = True
def create_exchange_config(self):
config = dict(
name=self.name,
features=[feature for feature in self.has if self.has[feature]]
)
markets = retry(
action=self._fetch_markets,
attempts=5,
sleeptime=5,
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn(
'fetching markets again for {}'.format(self.name)
),
)
else:
try:
has_feature = all(
[exchange.has[feature] for feature in ccxt_features]
)
config['assets'] = []
for market in markets:
asset = self.create_trading_pair(market=market)
config['assets'].append(asset)
except Exception:
has_feature = False
return config
if has_feature:
try:
log.info('initializing {}'.format(exchange_name))
exchange_names.append(exchange_name)
def create_trading_pair(self, market, start_dt=None, end_dt=None,
leverage=1, end_daily=None, end_minute=None):
"""
Creating a TradingPair from market and asset data.
except Exception as e:
log.warn(
'unable to initialize exchange {}: {}'.format(
exchange_name, e
)
)
Parameters
----------
market: dict[str, Object]
start_dt
end_dt
leverage
end_daily
end_minute
Returns
-------
"""
params = dict(
exchange=self.name,
data_source='catalyst',
exchange_symbol=market['id'],
symbol=get_catalyst_symbol(market),
start_date=start_dt,
end_date=end_dt,
leverage=leverage,
asset_name=market['symbol'],
end_daily=end_daily,
end_minute=end_minute,
)
self.apply_conditional_market_params(params, market)
return TradingPair(**params)
def load_assets(self):
if self._config is None or 'error' in self._config:
raise ValueError('Exchange config not available.')
self.assets = []
for asset_dict in self._config['assets']:
asset = TradingPair(**asset_dict)
self.assets.append(asset)
return exchange_names
def account(self):
return None
@@ -232,6 +204,42 @@ class CCXT(Exchange):
return frequencies
def get_market(self, symbol):
"""
The CCXT market.
Parameters
----------
symbol:
The CCXT symbol.
Returns
-------
dict[str, Object]
"""
s = self.get_symbol(symbol)
market = next(
(market for market in self.markets if market['symbol'] == s),
None,
)
return market
def substitute_currency_code(self, currency, source='catalyst'):
if source == 'catalyst':
currency = currency.upper()
key = self.api.common_currency_code(currency)
self._common_symbols[key] = currency.lower()
return key
else:
if currency in self._common_symbols:
return self._common_symbols[currency]
else:
return currency.lower()
def get_symbol(self, asset_or_symbol, source='catalyst'):
"""
The CCXT symbol.
@@ -249,7 +257,13 @@ class CCXT(Exchange):
if source == 'ccxt':
if isinstance(asset_or_symbol, string_types):
parts = asset_or_symbol.split('/')
return '{}_{}'.format(parts[0].lower(), parts[1].lower())
base_currency = self.substitute_currency_code(
parts[0], source
)
quote_currency = self.substitute_currency_code(
parts[1], source
)
return '{}_{}'.format(base_currency, quote_currency)
else:
return asset_or_symbol.symbol
@@ -260,7 +274,13 @@ class CCXT(Exchange):
) else asset_or_symbol.symbol
parts = symbol.split('_')
return '{}/{}'.format(parts[0].upper(), parts[1].upper())
base_currency = self.substitute_currency_code(
parts[0], source
)
quote_currency = self.substitute_currency_code(
parts[1], source
)
return '{}/{}'.format(base_currency, quote_currency)
@staticmethod
def map_frequency(value, source='ccxt', raise_error=True):
@@ -460,53 +480,144 @@ class CCXT(Exchange):
except ExchangeSymbolsNotFound:
return None
def apply_conditional_market_params(self, params, market):
def get_asset_defs(self, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
The local and Catalyst definitions of the specified market.
Parameters
----------
params: dict[Object]
market: dict[Object]
market: dict[str, Object]
The CCXT market dicts.
Returns
-------
dict[str, Object]
The asset definition.
"""
asset_defs = []
for is_local in (False, True):
asset_def = self.get_asset_def(market, is_local)
asset_defs.append((asset_def, is_local))
return asset_defs
def get_asset_def(self, market, is_local=False):
"""
The asset definition (in symbols.json files) corresponding
to the the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dict.
is_local
Whether to search in local or Catalyst asset definitions.
Returns
-------
dict[str, Object]
The asset definition.
"""
exchange_symbol = market['id']
symbol_map = self._fetch_symbol_map(is_local)
if symbol_map is not None:
assets_lower = {k.lower(): v for k, v in symbol_map.items()}
key = exchange_symbol.lower()
asset = assets_lower[key] if key in assets_lower else None
if asset is not None:
return asset
else:
return None
else:
return None
def create_trading_pair(self, market, asset_def=None, is_local=False):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
asset_def: dict[str, Object]
is_local: bool
Returns
-------
"""
# TODO: make this more externalized / configurable
# Consider representing in some type of JSON structure
if 'active' in market:
params['trading_state'] = 1 if market['active'] else 0
data_source = 'local' if is_local else 'catalyst'
params = dict(
exchange=self.name,
data_source=data_source,
exchange_symbol=market['id'],
)
mixin_market_params(self.name, params, market)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
params['start_date'] = asset_def['start_date'] \
if 'start_date' in asset_def else None
params['end_date'] = asset_def['end_date'] \
if 'end_date' in asset_def else None
params['leverage'] = asset_def['leverage'] \
if 'leverage' in asset_def else 1.0
params['asset_name'] = asset_def['asset_name'] \
if 'asset_name' in asset_def else None
params['end_daily'] = asset_def['end_daily'] \
if 'end_daily' in asset_def \
and asset_def['end_daily'] != 'N/A' else None
params['end_minute'] = asset_def['end_minute'] \
if 'end_minute' in asset_def \
and asset_def['end_minute'] != 'N/A' else None
else:
params['trading_state'] = 1
params['symbol'] = get_catalyst_symbol(market)
# TODO: add as an optional column
params['leverage'] = 1.0
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
return TradingPair(**params)
if self.name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
def load_assets(self):
log.debug('loading assets for {}'.format(self.name))
self.assets = []
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None \
and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
for market in self.markets:
if 'id' not in market:
log.warn('invalid market: {}'.format(market))
continue
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
asset_defs = self.get_asset_defs(market)
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
asset = None
for asset_def in asset_defs:
if asset_def[0] is not None or not asset_defs[1]:
try:
asset = self.create_trading_pair(
market=market,
asset_def=asset_def[0],
is_local=asset_def[1]
)
self.assets.append(asset)
if 'lot' not in params:
params['lot'] = params['min_trade_size']
except TypeError as e:
log.warn('unable to add asset: {}'.format(e))
if asset is None:
asset = self.create_trading_pair(market=market)
self.assets.append(asset)
def get_balances(self):
try:
@@ -644,14 +755,18 @@ class CCXT(Exchange):
side = 'buy' if amount > 0 else 'sell'
if hasattr(self.api, 'amount_to_lots'):
adj_amount = self.api.amount_to_lots(
symbol=symbol,
amount=abs(amount),
)
if adj_amount != abs(amount):
log.info(
'adjusted order amount {} to {} based on lot size'.format(
abs(amount), adj_amount,
# TODO: is this right?
if self.api.markets is None:
self.api.load_markets()
# https://github.com/ccxt/ccxt/issues/1483
adj_amount = round(abs(amount), asset.decimals)
market = self.api.markets[symbol]
if 'lots' in market and market['lots'] > amount:
raise CreateOrderError(
exchange=self.name,
e='order amount lower than the smallest lot: {}'.format(
amount
)
)
@@ -865,7 +980,8 @@ class CCXT(Exchange):
)
raise ExchangeRequestError(error=e)
def cancel_order(self, order_param, asset_or_symbol=None):
def cancel_order(self, order_param,
asset_or_symbol=None, params={}):
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
@@ -877,7 +993,8 @@ class CCXT(Exchange):
try:
symbol = self.get_symbol(asset_or_symbol) \
if asset_or_symbol is not None else None
self.api.cancel_order(id=order_id, symbol=symbol)
self.api.cancel_order(id=order_id,
symbol=symbol, params= params)
except (ExchangeError, NetworkError) as e:
log.warn(
catalyst/exchange/exchange.py
+60 -36
View File
@@ -1,12 +1,11 @@
import abc
import pytz
from abc import ABCMeta, abstractmethod, abstractproperty
from datetime import timedelta
from time import sleep
import numpy as np
import pandas as pd
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.exchange_bundle import ExchangeBundle
@@ -18,8 +17,9 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
from catalyst.exchange.utils.datetime_utils import get_delta, \
get_periods_range, \
get_periods, get_start_dt, get_frequency
from catalyst.exchange.utils.exchange_utils import \
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
resample_history_df, has_bundle
from logbook import Logger
log = Logger('Exchange', level=LOG_LEVEL)
@@ -291,6 +291,16 @@ class Exchange:
log.debug('found asset: {}'.format(asset))
return asset
def fetch_symbol_map(self, is_local=False):
index = 1 if is_local else 0
if self._symbol_maps[index] is not None:
return self._symbol_maps[index]
else:
symbol_map = get_exchange_symbols(self.name, is_local)
self._symbol_maps[index] = symbol_map
return symbol_map
@abstractmethod
def init(self):
"""
@@ -302,13 +312,24 @@ class Exchange:
"""
@abstractmethod
def create_exchange_config(self):
def load_assets(self, is_local=False):
"""
Fetch the exchange market data and generate a config object
Returns
-------
Populate the 'assets' attribute with a dictionary of Assets.
The key of the resulting dictionary is the exchange specific
currency pair symbol. The universal symbol is contained in the
'symbol' attribute of each asset.
Notes
-----
The sid of each asset is calculated based on a numeric hash of the
universal symbol. This simple approach avoids maintaining a mapping
of sids.
This method can be omerridden if an exchange offers equivalent data
via its api.
"""
pass
def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
"""
@@ -494,32 +515,37 @@ class Exchange:
series = dict()
for asset in candles:
first_candle = candles[asset][0]
asset_series = self.get_series_from_candles(
candles=candles[asset],
start_dt=first_candle['last_traded'],
end_dt=end_dt,
data_frequency=frequency,
field=field,
)
delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
delta = get_delta(delta_candle_size, data_frequency)
adj_end_dt = end_dt - delta
last_traded = asset_series.index[-1]
if last_traded < adj_end_dt:
raise LastCandleTooEarlyError(
last_traded=last_traded,
end_dt=adj_end_dt,
exchange=self.name,
if candles[asset]:
first_candle = candles[asset][0]
asset_series = self.get_series_from_candles(
candles=candles[asset],
start_dt=first_candle['last_traded'],
end_dt=end_dt,
data_frequency=frequency,
field=field,
)
delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
delta = get_delta(delta_candle_size, data_frequency)
adj_end_dt = end_dt - delta
last_traded = asset_series.index[-1]
if last_traded < adj_end_dt:
raise LastCandleTooEarlyError(
last_traded=last_traded,
end_dt=adj_end_dt,
exchange=self.name,
)
else: # empty candle received
# because other assets are tz-aware, we need its tz to be set as well
asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc))
series[asset] = asset_series
df = pd.DataFrame(series)
df.dropna(inplace=True)
#df.dropna(inplace=True) # commented out due to issue 236
return df
@@ -636,20 +662,16 @@ class Exchange:
return df
def _check_low_balance(self, currency, balances, amount, open_orders=None):
def _check_low_balance(self, currency, balances, amount):
free = balances[currency]['free'] if currency in balances else 0.0
if open_orders:
# TODO: make sure that this works
free += sum([order.amount for order in open_orders])
if free < amount:
return free, True
else:
return free, False
def sync_positions(self, positions, open_orders=None, cash=None,
def sync_positions(self, positions, cash=None,
check_balances=False):
"""
Update the portfolio cash and position balances based on the
@@ -679,7 +701,7 @@ class Exchange:
balances=balances,
amount=cash,
)
if is_lower and not open_orders:
if is_lower:
raise NotEnoughCashError(
currency=self.base_currency,
exchange=self.name,
@@ -906,7 +928,8 @@ class Exchange:
"""
@abstractmethod
def cancel_order(self, order_param, symbol_or_asset=None):
def cancel_order(self, order_param,
symbol_or_asset=None, params={}):
"""Cancel an open order.
Parameters
@@ -915,6 +938,7 @@ class Exchange:
The order_id or order object to cancel.
symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this
params:
"""
pass
catalyst/exchange/exchange_algorithm.py
+56 -25
View File
@@ -18,11 +18,9 @@ from datetime import timedelta
from os import listdir
from os.path import isfile, join, exists
import catalyst.protocol as zp
import logbook
import pandas as pd
from redo import retry
import catalyst.protocol as zp
from catalyst.algorithm import TradingAlgorithm
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange_blotter import ExchangeBlotter
@@ -52,6 +50,7 @@ from catalyst.utils.api_support import api_method
from catalyst.utils.input_validation import error_keywords, ensure_upper_case
from catalyst.utils.math_utils import round_nearest
from catalyst.utils.preprocess import preprocess
from redo import retry
log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
@@ -376,19 +375,30 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
if error:
log.warning(error)
self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
# in order to save paper & live files separately
self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live'
self.custom_signals_stats = \
get_algo_df(self.algo_namespace, 'custom_signals_stats')
self.pnl_stats = get_algo_df(
self.algo_namespace,
'pnl_stats_{}'.format(self.mode_name),
)
self.exposure_stats = \
get_algo_df(self.algo_namespace, 'exposure_stats')
self.custom_signals_stats = get_algo_df(
self.algo_namespace,
'custom_signals_stats_{}'.format(self.mode_name)
)
self.exposure_stats = get_algo_df(
self.algo_namespace,
'exposure_stats_{}'.format(self.mode_name)
)
self.is_running = True
self.stats_minutes = 1
self._last_orders = []
self._last_open_orders = []
self.trading_client = None
super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
@@ -515,7 +525,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
"""
self.state = get_algo_object(
algo_name=self.algo_namespace,
key='context.state',
key='context.state_{}'.format(self.mode_name),
)
if self.state is None:
self.state = {}
@@ -538,7 +548,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# Unpacking the perf_tracker and positions if available
cum_perf = get_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance',
key='cumulative_performance_{}'.format(self.mode_name),
)
if cum_perf is not None:
tracker.cumulative_performance = cum_perf
@@ -549,7 +559,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
todays_perf = get_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
rel_path='daily_performance',
rel_path='daily_performance_{}'.format(self.mode_name),
)
if todays_perf is not None:
# Ensure single common position tracker
@@ -641,7 +651,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
required_cash = self.portfolio.cash if not orders else None
cash, positions_value = exchange.sync_positions(
positions=exchange_positions,
open_orders=orders,
check_balances=check_balances,
cash=required_cash,
)
@@ -687,7 +696,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
)
self.pnl_stats = pd.concat([self.pnl_stats, df])
save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats)
save_algo_df(
self.algo_namespace,
'pnl_stats_{}'.format(self.mode_name),
self.pnl_stats,
)
def add_custom_signals_stats(self, period_stats):
"""
@@ -708,8 +721,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
)
self.custom_signals_stats = pd.concat([self.custom_signals_stats, df])
save_algo_df(self.algo_namespace, 'custom_signals_stats',
self.custom_signals_stats)
save_algo_df(
self.algo_namespace,
'custom_signals_stats_{}'.format(self.mode_name),
self.custom_signals_stats,
)
def add_exposure_stats(self, period_stats):
"""
@@ -736,7 +752,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.exposure_stats = pd.concat([self.exposure_stats, df])
save_algo_df(
self.algo_namespace, 'exposure_stats', self.exposure_stats
self.algo_namespace,
'exposure_stats_{}'.format(self.mode_name),
self.exposure_stats
)
def nullify_frame_stats(self, now):
@@ -760,6 +778,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
obj=self.frame_stats,
rel_path='frame_stats'
)
error = remove_old_files(
algo_name=self.algo_namespace,
today=now,
@@ -792,12 +811,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.nullify_frame_stats(now=data.current_dt)
self.performance_needs_update = False
orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
if orders != self._last_orders:
last_orders_list = list(self.blotter.orders.keys())
open_orders_list = list(self.blotter.open_orders.keys())
if last_orders_list != self._last_orders or \
open_orders_list != self._last_open_orders:
self.performance_needs_update = True
# Saving current orders to detect changes in the next frame
self._last_orders = copy.deepcopy(orders)
# Saving current order positions
# to detect changes in the next frame
self._last_orders = copy.deepcopy(last_orders_list)
self._last_open_orders = copy.deepcopy(open_orders_list)
if self.performance_needs_update:
self.perf_tracker.update_performance()
@@ -839,7 +863,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
log.debug('saving cumulative performance object')
save_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance',
key='cumulative_performance_{}'.format(self.mode_name),
obj=self.perf_tracker.cumulative_performance,
)
log.debug('saving todays performance object')
@@ -847,12 +871,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
obj=self.perf_tracker.todays_performance,
rel_path='daily_performance'
rel_path='daily_performance_{}'.format(self.mode_name)
)
log.debug('saving context.state object')
save_algo_object(
algo_name=self.algo_namespace,
key='context.state',
key='context.state_{}'.format(self.mode_name),
obj=self.state)
def _process_stats(self, data):
@@ -908,6 +932,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
csv_bytes = stats_to_algo_folder(
stats=self.frame_stats,
algo_namespace=self.algo_namespace,
folder_name='stats_{}'.format(self.mode_name),
recorded_cols=recorded_cols,
)
except Exception as e:
@@ -1012,13 +1037,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
args=(order_id,))
@api_method
def cancel_order(self, order_param, exchange_name):
def cancel_order(self, order_param, exchange_name,
symbol=None, params={}):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
exchange_name: name of exchange from
which you want to cancel the order
symbol:
params:
"""
exchange = self.exchanges[exchange_name]
@@ -1032,4 +1063,4 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('cancelling order again.'),
args=(order_id,))
args=(order_id, symbol, params))
catalyst/exchange/exchange_bundle.py
+47 -40
View File
@@ -1,4 +1,3 @@
import copy
import os
import shutil
from datetime import timedelta
@@ -9,12 +8,8 @@ from operator import is_not
import numpy as np
import pandas as pd
import pytz
from catalyst.assets._assets import TradingPair
from logbook import Logger
from pytz import UTC
from six import itervalues
from catalyst import get_calendar
from catalyst.assets._assets import TradingPair
from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
from catalyst.constants import LOG_LEVEL
from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
@@ -27,11 +22,15 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
get_bcolz_chunk, get_df_from_arrays, get_assets
from catalyst.exchange.utils.datetime_utils import get_start_dt, \
from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
get_period_label, get_month_start_end, get_year_start_end
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
from logbook import Logger
from pytz import UTC
from six import itervalues
log = Logger('exchange_bundle', level=LOG_LEVEL)
@@ -626,13 +625,13 @@ class ExchangeBundle:
key=lambda chunk: pd.to_datetime(chunk['period'])
)
with maybe_show_progress(
all_chunks,
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
)) as it:
all_chunks,
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
)) as it:
for chunk in it:
problems += self.ingest_ctable(
asset=chunk['asset'],
@@ -701,36 +700,42 @@ class ExchangeBundle:
for symbol in symbols:
start_dt = df.index.get_level_values(1).min()
end_dt = df.index.get_level_values(1).max()
end_dt_key = 'end_{}'.format(data_frequency)
try:
asset = self.exchange.get_asset(symbol, is_local=True)
except:
asset = copy.deepcopy(self.exchange.get_asset(symbol))
market = self.exchange.get_market(symbol)
if market is None:
raise ValueError('symbol not available in the exchange.')
if asset.data_source == 'local':
asset.start_date = asset.start_date \
if asset.start_date < start_dt else start_dt
params = dict(
exchange=self.exchange.name,
data_source='local',
exchange_symbol=market['id'],
)
mixin_market_params(self.exchange_name, params, market)
if data_frequency == 'daily':
asset.end_date = asset.end_daily = asset.end_daily \
if asset.end_daily > end_dt else end_dt
asset_def = self.exchange.get_asset_def(market, True)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
else:
asset.end_date = asset.end_minute = asset.end_minute \
if asset.end_minute > end_dt else end_dt
params['start_date'] = asset_def['start_date'] \
if asset_def['start_date'] < start_dt else start_dt
params['end_date'] = asset_def[end_dt_key] \
if asset_def[end_dt_key] > end_dt else end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else asset_def['end_daily']
params['end_minute'] = end_dt \
if data_frequency == 'minute' else asset_def['end_minute']
else:
asset.data_source = 'local'
asset.start_date = start_dt
asset.end_dt = end_dt
params['symbol'] = get_catalyst_symbol(market)
if data_frequency == 'daily':
asset.end_daily = end_dt
asset.end_minute = None
else:
asset.end_daily = None
asset.end_minute = end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else 'N/A'
params['end_minute'] = end_dt \
if data_frequency == 'minute' else 'N/A'
if min_start_dt is None or start_dt < min_start_dt:
min_start_dt = start_dt
@@ -738,9 +743,11 @@ class ExchangeBundle:
if max_end_dt is None or end_dt > max_end_dt:
max_end_dt = end_dt
assets[symbol] = asset
asset = TradingPair(**params)
assets[market['id']] = asset
save_exchange_symbols(self.exchange_name, assets, True)
# TODO: update config.json
writer = self.get_writer(
start_dt=min_start_dt.replace(hour=00, minute=00),
end_dt=max_end_dt.replace(hour=23, minute=59),
catalyst/exchange/exchange_errors.py
-14
View File
@@ -322,17 +322,3 @@ class BalanceTooLowError(ZiplineError):
'add positions to hold a free amount greater than {amount}, or clean '
'the state of this algo and restart.'
).strip()
class MarketsNotFoundError(ZiplineError):
msg = (
'Exchange {exchange} contains no valid market so it is unusable in '
'Catalyst.'
).strip()
class InvalidMarketError(ZiplineError):
msg = (
'Exchange {exchange} contains at least one incorrectly structured '
'market: {market}, so it is unusable in Catalyst.'
).strip()
catalyst/exchange/utils/ccxt_utils.py
-307
View File
@@ -1,307 +0,0 @@
import json
import os
import pandas as pd
from six.moves.urllib import request
from catalyst.assets._assets import TradingPair
from ccxt import NetworkError
from catalyst.constants import LOG_LEVEL, EXCHANGE_CONFIG_URL
from catalyst.exchange.exchange_errors import MarketsNotFoundError, \
InvalidMarketError
from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol, \
get_exchange_folder, get_exchange_auth
from catalyst.exchange.utils.serialization_utils import ExchangeJSONDecoder, \
ExchangeJSONEncoder
from logbook import Logger
from redo import retry
from ccxt.base.exchange import Exchange
from catalyst.utils.paths import last_modified_time, data_root, \
ensure_directory
import ccxt
log = Logger('ccxt_utils', level=LOG_LEVEL)
def scan_exchange_configs(features=None, history=None, is_authenticated=False,
path=None):
"""
Finding exchanges from their config files
Parameters
----------
features
is_authenticated
Returns
-------
"""
for exchange_name in ccxt.exchanges:
config = get_exchange_config(exchange_name, path)
if not config or 'error' in config:
log.info(
'skipping invalid exchange {}'.format(exchange_name)
)
# Check if the exchange has an auth.json file
if is_authenticated:
exchange_auth = get_exchange_auth(exchange_name)
has_auth = (exchange_auth['key'] != ''
and exchange_auth['secret'] != '')
if not has_auth:
continue
if features is None:
has_features = True
else:
try:
supported_features = [
feature for feature in features if
feature in config['features']
]
has_features = len(supported_features) > 0
except Exception:
has_features = False
# TODO: filter by history
if has_features:
yield config
def get_exchange_config(exchange_name, path=None, environ=None,
expiry='2H'):
"""
The de-serialized content of the exchange's config.json.
Parameters
----------
exchange_name: str
The exchange name
filename: str
The target file
environ:
Returns
-------
config: dict[srt, Object]
The config dictionary.
"""
try:
if path is None:
root = data_root(environ)
path = os.path.join(root, 'exchanges')
folder = os.path.join(path, exchange_name)
ensure_directory(folder)
filename = os.path.join(folder, 'config.json')
url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
if os.path.isfile(filename):
# If the file exists, only update periodically to avoid
# unnecessary calls
now = pd.Timestamp.utcnow()
limit = pd.Timedelta(expiry)
if pd.Timedelta(now - last_modified_time(filename)) > limit:
try:
request.urlretrieve(url=url, filename=filename)
except Exception as e:
log.warn(
'unable to update config {} => {}: {}'.format(
url, filename, e
)
)
else:
request.urlretrieve(url=url, filename=filename)
with open(filename) as data_file:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
except Exception as e:
log.warn(
'unable to download {} config: {}'.format(
exchange_name, e
)
)
return dict(error=e)
def save_exchange_config(config, filename=None, environ=None):
"""
Save assets into an exchange_config file.
Parameters
----------
exchange_name: str
config
environ
Returns
-------
"""
if filename is None:
name = 'config.json'
exchange_folder = get_exchange_folder(config['id'], environ)
filename = os.path.join(exchange_folder, name)
with open(filename, 'w+') as handle:
json.dump(config, handle, indent=4, cls=ExchangeJSONEncoder)
def fetch_markets(ccxt_exchange):
"""
Fetches CCXT market objects.
Parameters
----------
ccxt_exchange: Exchange
Returns
-------
"""
markets_symbols = ccxt_exchange.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
ccxt_exchange.name, markets_symbols
)
)
markets = ccxt_exchange.fetch_markets()
if not markets:
raise MarketsNotFoundError(
exchange=ccxt_exchange.name,
)
for market in markets:
if 'id' not in market:
raise InvalidMarketError(
exchange=ccxt_exchange.name,
market=market,
)
return markets
def create_exchange_config(ccxt_exchange):
"""
Creates an exchange config structure.
Parameters
----------
ccxt_exchange: Exchange
Returns
-------
"""
exchange_name = ccxt_exchange.__class__.__name__
config = dict(
id=exchange_name,
name=ccxt_exchange.name,
features=[
feature for feature in ccxt_exchange.has if
ccxt_exchange.has[feature]
]
)
markets = retry(
action=fetch_markets,
attempts=5,
sleeptime=5,
retry_exceptions=(NetworkError,),
cleanup=lambda: log.warn(
'fetching markets again for {}'.format(exchange_name)
),
args=(ccxt_exchange,)
)
config['assets'] = []
for market in markets:
asset = create_trading_pair(exchange_name, market)
config['assets'].append(asset)
return config
def create_trading_pair(exchange_name, market, start_dt=None, end_dt=None,
leverage=1, end_daily=None, end_minute=None):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
start_dt
end_dt
leverage
end_daily
end_minute
Returns
-------
"""
params = dict(
exchange=exchange_name,
data_source='catalyst',
exchange_symbol=market['id'],
symbol=get_catalyst_symbol(market),
start_date=start_dt,
end_date=end_dt,
leverage=leverage,
asset_name=market['symbol'],
end_daily=end_daily,
end_minute=end_minute,
)
apply_conditional_market_params(exchange_name, params, market)
return TradingPair(**params)
def apply_conditional_market_params(exchange_name, params, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
params: dict[Object]
market: dict[Object]
Returns
-------
"""
# TODO: make this more externalized / configurable
# Consider representing in some type of JSON structure
if 'active' in market:
params['trading_state'] = 1 if market['active'] else 0
else:
params['trading_state'] = 1
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None \
and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
catalyst/exchange/utils/exchange_utils.py
+115 -49
View File
@@ -11,14 +11,11 @@ from six import string_types
from six.moves.urllib import request
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
ExchangeJSONDecoder, ConfigJSONEncoder
ExchangeJSONDecoder
from catalyst.utils.paths import data_root, ensure_directory, \
last_modified_time
from six import string_types
from six.moves.urllib import request
def get_sid(symbol):
@@ -72,7 +69,7 @@ def is_blacklist(exchange_name, environ=None):
return os.path.exists(filename)
def get_exchange_config_filename(exchange_name, environ=None):
def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
"""
The absolute path of the exchange's symbol.json file.
@@ -86,12 +83,12 @@ def get_exchange_config_filename(exchange_name, environ=None):
str
"""
name = 'config.json'
name = 'symbols.json' if not is_local else 'symbols_local.json'
exchange_folder = get_exchange_folder(exchange_name, environ)
return os.path.join(exchange_folder, name)
def download_exchange_config(exchange_name, filename, environ=None):
def download_exchange_symbols(exchange_name, environ=None):
"""
Downloads the exchange's symbols.json from the repository.
@@ -105,13 +102,15 @@ def download_exchange_config(exchange_name, filename, environ=None):
str
"""
url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
request.urlretrieve(url=url, filename=filename)
filename = get_exchange_symbols_filename(exchange_name)
url = SYMBOLS_URL.format(exchange=exchange_name)
response = request.urlretrieve(url=url, filename=filename)
return response
def get_exchange_config(exchange_name, filename=None, environ=None):
def get_exchange_symbols(exchange_name, is_local=False, environ=None):
"""
The de-serialized content of the exchange's config.json.
The de-serialized content of the exchange's symbols.json.
Parameters
----------
@@ -124,47 +123,55 @@ def get_exchange_config(exchange_name, filename=None, environ=None):
Object
"""
if filename is None:
filename = get_exchange_config_filename(exchange_name)
filename = get_exchange_symbols_filename(exchange_name, is_local)
if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
pd.Timestamp('now', tz='UTC') - last_modified_time(
filename)).days > 1):
try:
download_exchange_symbols(exchange_name, environ)
except Exception:
pass
if os.path.isfile(filename):
now = pd.Timestamp.utcnow()
limit = pd.Timedelta('2H')
if pd.Timedelta(now - last_modified_time(filename)) > limit:
download_exchange_config(exchange_name, filename, environ)
with open(filename) as data_file:
try:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
except ValueError:
return dict()
else:
download_exchange_config(exchange_name, filename, environ)
raise ExchangeSymbolsNotFound(
exchange=exchange_name,
filename=filename
)
with open(filename) as data_file:
try:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
except ValueError:
return dict()
def save_exchange_config(exchange_name, config, filename=None, environ=None):
def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
"""
Save assets into an exchange_config file.
Save assets into an exchange_symbols file.
Parameters
----------
exchange_name: str
config
assets: list[dict[str, object]]
is_local: bool
environ
Returns
-------
"""
if filename is None:
name = 'config.json'
exchange_folder = get_exchange_folder(exchange_name, environ)
filename = os.path.join(exchange_folder, name)
asset_dicts = dict()
for symbol in assets:
asset_dicts[symbol] = assets[symbol].to_dict()
with open(filename, 'w+') as handle:
json.dump(config, handle, indent=4, cls=ConfigJSONEncoder)
filename = get_exchange_symbols_filename(
exchange_name, is_local, environ
)
with open(filename, 'wt') as handle:
json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
def get_symbols_string(assets):
@@ -413,7 +420,7 @@ def clear_frame_stats_directory(algo_name):
return error
def remove_old_files(algo_name, today, rel_path):
def remove_old_files(algo_name, today, rel_path, environ=None):
"""
remove old files from a directory
to avoid overloading the disk
@@ -423,27 +430,31 @@ def remove_old_files(algo_name, today, rel_path):
algo_name: str
today: Timestamp
rel_path: str
environ:
Returns
-------
error: str
"""
error = None
algo_folder = get_algo_folder(algo_name)
algo_folder = get_algo_folder(algo_name, environ)
folder = os.path.join(algo_folder, rel_path)
ensure_directory(folder)
# run on all files in the folder
for f in os.listdir(folder):
creation_unix = os.path.getctime(f)
creation_time = pd.to_datetime(creation_unix, unit='s', )
try:
file_path = os.path.join(folder, f)
creation_unix = os.path.getctime(file_path)
creation_time = pd.to_datetime(creation_unix, unit='s', utc=True)
# if the file is older than 30 days erase it
if today - pd.DateOffset(30) > creation_time:
try:
os.unlink(f)
except OSError:
error = 'unable to erase files in {}'.format(folder)
# if the file is older than 30 days erase it
if today - pd.DateOffset(30) > creation_time:
os.unlink(file_path)
except OSError:
error = 'unable to erase files in {}'.format(folder)
return error
@@ -501,6 +512,25 @@ def has_bundle(exchange_name, data_frequency, environ=None):
return os.path.isdir(folder)
def symbols_serial(obj):
"""
JSON serializer for objects not serializable by default json code
Parameters
----------
obj: Object
Returns
-------
str
"""
if isinstance(obj, (datetime, date)):
return obj.floor('1D').strftime(DATE_FORMAT)
raise TypeError("Type %s not serializable" % type(obj))
def perf_serial(obj):
"""
JSON serializer for objects not serializable by default json code
@@ -590,12 +620,46 @@ def resample_history_df(df, freq, field, start_dt=None):
return resampled_df
def from_ms_timestamp(ms):
return pd.to_datetime(ms, unit='ms', utc=True)
def mixin_market_params(exchange_name, params, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
params: dict[Object]
market: dict[Object]
def get_epoch():
return pd.to_datetime('1970-1-1', utc=True)
Returns
-------
"""
# TODO: make this more externalized / configurable
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market and \
market['maker'] is not None and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def group_assets_by_exchange(assets):
@@ -662,12 +726,14 @@ def get_candles_df(candles, field, freq, bar_count, end_dt,
values = [candle[field] for candle in candles[asset]]
series = pd.Series(values, index=dates)
"""
series = series.reindex(
periods,
method='ffill',
fill_value=previous_value,
)
series.sort_index(inplace=True)
"""
all_series[asset] = series
df = pd.DataFrame(all_series)
catalyst/exchange/utils/factory.py
+23 -30
View File
@@ -4,9 +4,8 @@ from catalyst.constants import LOG_LEVEL
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.utils.ccxt_utils import scan_exchange_configs
from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
get_exchange_folder
get_exchange_folder, is_blacklist
from logbook import Logger
log = Logger('factory', level=LOG_LEVEL)
@@ -14,7 +13,7 @@ exchange_cache = dict()
def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
skip_init=False, auth_alias=None, config=None):
skip_init=False, auth_alias=None):
key = (exchange_name, base_currency)
if key in exchange_cache:
return exchange_cache[key]
@@ -37,7 +36,6 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
password=exchange_auth['password'] if 'password'
in exchange_auth.keys() else '',
base_currency=base_currency,
config=config,
)
exchange_cache[key] = exchange
@@ -55,8 +53,8 @@ def get_exchanges(exchange_names):
return exchanges
def find_exchanges(features=None, history=None, skip_blacklist=True, path=None,
is_authenticated=False, base_currency=None):
def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
base_currency=None):
"""
Find exchanges filtered by a list of feature.
@@ -74,33 +72,28 @@ def find_exchanges(features=None, history=None, skip_blacklist=True, path=None,
list[Exchange]
"""
exchange_names = CCXT.find_exchanges(features, is_authenticated)
return list(
scan_exchanges(
features,
history,
skip_blacklist,
path,
is_authenticated,
base_currency
)
)
def scan_exchanges(features=None, history=None, skip_blacklist=True, path=None,
is_authenticated=False, base_currency=None):
for config in scan_exchange_configs(
features=features,
history=history,
is_authenticated=is_authenticated,
path=path,
):
if skip_blacklist and (config is None or 'error' in config):
exchanges = []
for exchange_name in exchange_names:
if skip_blacklist and is_blacklist(exchange_name):
continue
yield get_exchange(
exchange_name=config['id'],
exchange = get_exchange(
exchange_name=exchange_name,
skip_init=True,
base_currency=base_currency,
config=config,
)
if features is not None:
if 'dailyBundle' in features \
and not exchange.has_bundle('daily'):
continue
elif 'minuteBundle' in features \
and not exchange.has_bundle('minute'):
continue
exchanges.append(exchange)
return exchanges
catalyst/exchange/utils/serialization_utils.py
+1 -28
View File
@@ -3,42 +3,15 @@ import re
from json import JSONEncoder
import pandas as pd
from catalyst.constants import DATE_TIME_FORMAT
from six import string_types
from datetime import date, datetime
from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT
from catalyst.assets._assets import TradingPair
class ConfigJSONEncoder(json.JSONEncoder):
def default(self, obj):
"""
JSON serializer for objects not serializable by default json code
Parameters
----------
obj: Object
Returns
-------
str
"""
if isinstance(obj, (datetime, date)):
return obj.floor('1D').strftime(DATE_FORMAT)
elif isinstance(obj, TradingPair):
return obj.to_dict()
class ExchangeJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, pd.Timestamp):
return obj.strftime(DATE_TIME_FORMAT)
elif isinstance(obj, TradingPair):
return obj.to_dict()
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, obj)
catalyst/exchange/utils/stats_utils.py
+4 -2
View File
@@ -396,7 +396,8 @@ def email_error(algo_name, dt, e, environ=None):
)})
def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
def stats_to_algo_folder(stats, algo_namespace,
folder_name, recorded_cols=None):
"""
Saves the performance stats to the algo local folder.
@@ -404,6 +405,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
----------
stats: list[Object]
algo_namespace: str
folder_name: str
recorded_cols: list[str]
Returns
@@ -416,7 +418,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
timestr = time.strftime('%Y%m%d')
folder = get_algo_folder(algo_namespace)
stats_folder = os.path.join(folder, 'stats')
stats_folder = os.path.join(folder, folder_name)
ensure_directory(stats_folder)
filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
docs/source/beginner-tutorial.rst
+2 -156
View File
@@ -580,162 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in
the ``examples`` directory:
`dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_.
.. code-block:: python
import numpy as np
import pandas as pd
from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order, record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.utils.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
def initialize(context):
context.i = 0
context.asset = symbol('ltc_usd')
context.base_price = None
def handle_data(context, data):
# define the windows for the moving averages
short_window = 50
long_window = 200
# Skip as many bars as long_window to properly compute the average
context.i += 1
if context.i < long_window:
return
# Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_mavg = data.history(context.asset, 'price',
bar_count=short_window, frequency="1m").mean()
long_mavg = data.history(context.asset, 'price',
bar_count=long_window, frequency="1m").mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
# Save values for later inspection
record(price=price,
cash=context.portfolio.cash,
price_change=price_change,
short_mavg=short_mavg,
long_mavg=long_mavg)
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.asset):
return
# We check what's our position on our portfolio and trade accordingly
pos_amount = context.portfolio.positions[context.asset].amount
# Trading logic
if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation
exchange = list(context.exchanges.values())[0]
base_currency = exchange.base_currency.upper()
# First chart: Plot portfolio value using base_currency
ax1 = plt.subplot(411)
perf.loc[:, ['portfolio_value']].plot(ax=ax1)
ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format(
asset = context.asset.symbol,
base = base_currency
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index, 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index, 'price'],
marker='v',
s=100,
c='red',
label=''
)
# Third chart: Compare percentage change between our portfolio
# and the price of the asset
ax3 = plt.subplot(413, sharex=ax1)
perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
ax3.legend_.remove()
ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
:language: python
In order to run the code above, you have to ingest the needed data first:
docs/source/example-algos.rst
+17 -881
View File
@@ -52,35 +52,8 @@ Buy BTC Simple Algorithm
Source code: `examples/buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_
.. code-block:: python
'''
Run this example, by executing the following from your terminal:
catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
If you want to run this code using another exchange, make sure that
the asset is available on that exchange. For example, if you were to run
it for exchange Poloniex, you would need to edit the following line:
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
and specify exchange poloniex as follows:
catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
To see which assets are available on each exchange, visit:
https://www.enigma.co/catalyst/status
'''
from catalyst.api import order, record, symbol
def initialize(context):
context.asset = symbol('btc_usd')
def handle_data(context, data):
order(context.asset, 1)
record(btc = data.current(context.asset, 'price'))
.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py
:language: python
This simple algorithm does not produce any output nor displays any chart.
@@ -90,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart.
Buy and Hodl Algorithm
~~~~~~~~~~~~~~~~~~~~~~
Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
First ingest the historical pricing data needed to run this algorithm:
.. code-block:: bash
@@ -119,157 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an
earlier date, you'll get an error), and the most recent date you can choose is
one day prior to the current date.
Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
.. code-block:: python
#!/usr/bin/env python
#
# Copyright 2017 Enigma MPC, Inc.
# Copyright 2015 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pandas as pd
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order_target_value, symbol, record,
cancel_order, get_open_orders, )
def initialize(context):
context.ASSET_NAME = 'btc_usd'
context.TARGET_HODL_RATIO = 0.8
context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
context.is_buying = True
context.asset = symbol(context.ASSET_NAME)
context.i = 0
def handle_data(context, data):
context.i += 1
starting_cash = context.portfolio.starting_cash
target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
reserve_value = context.RESERVE_RATIO * starting_cash
# Cancel any outstanding orders
orders = get_open_orders(context.asset) or []
for order in orders:
cancel_order(order)
# Stop buying after passing the reserve threshold
cash = context.portfolio.cash
if cash <= reserve_value:
context.is_buying = False
# Retrieve current asset price from pricing data
price = data.current(context.asset, 'price')
# Check if still buying and could (approximately) afford another purchase
if context.is_buying and cash > price:
print('buying')
# Place order to make position in asset equal to target_hodl_value
order_target_value(
context.asset,
target_hodl_value,
limit_price=price * 1.1,
)
record(
price=price,
volume=data.current(context.asset, 'volume'),
cash=cash,
starting_cash=context.portfolio.starting_cash,
leverage=context.account.leverage,
)
def analyze(context=None, results=None):
# Plot the portfolio and asset data.
ax1 = plt.subplot(611)
results[['portfolio_value']].plot(ax=ax1)
ax1.set_ylabel('Portfolio Value (USD)')
ax2 = plt.subplot(612, sharex=ax1)
ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
results[['price']].plot(ax=ax2)
trans = results.ix[[t != [] for t in results.transactions]]
buys = trans.ix[
[t[0]['amount'] > 0 for t in trans.transactions]
]
ax2.scatter(
buys.index.to_pydatetime(),
results.price[buys.index],
marker='^',
s=100,
c='g',
label=''
)
ax3 = plt.subplot(613, sharex=ax1)
results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
ax3.set_ylabel('Leverage ')
ax4 = plt.subplot(614, sharex=ax1)
results[['starting_cash', 'cash']].plot(ax=ax4)
ax4.set_ylabel('Cash (USD)')
results[[
'treasury',
'algorithm',
'benchmark',
]] = results[[
'treasury_period_return',
'algorithm_period_return',
'benchmark_period_return',
]]
ax5 = plt.subplot(615, sharex=ax1)
results[[
'treasury',
'algorithm',
'benchmark',
]].plot(ax=ax5)
ax5.set_ylabel('Percent Change')
ax6 = plt.subplot(616, sharex=ax1)
results[['volume']].plot(ax=ax6)
ax6.set_ylabel('Volume (mCoins/5min)')
plt.legend(loc=3)
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=10000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace='buy_and_hodl',
base_currency='usd',
start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True),
)
.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py
:language: python
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png
@@ -278,166 +102,13 @@ one day prior to the current date.
Dual Moving Average Crossover
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
This strategy is covered in detail in the last part of
`this tutorial <beginner-tutorial.html#history>`_.
.. code-block:: python
Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
import numpy as np
import pandas as pd
from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order, record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
def initialize(context):
context.i = 0
context.asset = symbol('ltc_usd')
context.base_price = None
def handle_data(context, data):
# define the windows for the moving averages
short_window = 50
long_window = 200
# Skip as many bars as long_window to properly compute the average
context.i += 1
if context.i < long_window:
return
# Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_mavg = data.history(context.asset, 'price',
bar_count=short_window, frequency="1m").mean()
long_mavg = data.history(context.asset, 'price',
bar_count=long_window, frequency="1m").mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
# Save values for later inspection
record(price=price,
cash=context.portfolio.cash,
price_change=price_change,
short_mavg=short_mavg,
long_mavg=long_mavg)
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.asset):
return
# We check what's our position on our portfolio and trade accordingly
pos_amount = context.portfolio.positions[context.asset].amount
# Trading logic
if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation
base_currency = context.exchanges.values()[0].base_currency.upper()
# First chart: Plot portfolio value using base_currency
ax1 = plt.subplot(411)
perf.loc[:, ['portfolio_value']].plot(ax=ax1)
ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format(
asset = context.asset.symbol,
base = base_currency
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index, 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index, 'price'],
marker='v',
s=100,
c='red',
label=''
)
# Third chart: Compare percentage change between our portfolio
# and the price of the asset
ax3 = plt.subplot(413, sharex=ax1)
perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
ax3.legend_.remove()
ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
:language: python
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
@@ -447,8 +118,6 @@ This strategy is covered in detail in the last part of
Mean Reversion Algorithm
~~~~~~~~~~~~~~~~~~~~~~~~
Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
This algorithm is based on a simple momentum strategy. When the cryptoasset goes
up quickly, we're going to buy; when it goes down quickly, we're going to sell.
Hopefully, we'll ride the waves.
@@ -469,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type:
python mean_reversion_simple.py
.. code-block:: python
Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
import os
import tempfile
import time
import numpy as np
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in USD.
context.neo_eth = symbol('neo_usd')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 30
context.RSI_OVERBOUGHT = 80
context.CANDLE_SIZE = '15T'
context.start_time = time.time()
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = False
context.current_day = today
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.neo_eth variable. For this example, we're
# using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention
# for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
context.neo_eth,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(context.neo_eth, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
cash = context.portfolio.cash
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
price=price,
volume=current['volume'],
price_change=price_change,
rsi=rsi[-1],
cash=cash
)
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if context.traded_today:
return
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.neo_eth)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.neo_eth):
return
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.neo_eth].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
order_target_percent(
context.neo_eth, 1, limit_price=limit_price
)
context.traded_today = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
context.neo_eth, 0, limit_price=limit_price
)
context.traded_today = True
def analyze(context=None, perf=None):
end = time.time()
log.info('elapsed time: {}'.format(end - context.start_time))
import matplotlib.pyplot as plt
# The base currency of the algo exchange
base_currency = context.exchanges.values()[0].base_currency.upper()
# Plot the portfolio value over time.
ax1 = plt.subplot(611)
perf.loc[:, 'portfolio_value'].plot(ax=ax1)
ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
# Plot the price increase or decrease over time.
ax2 = plt.subplot(612, sharex=ax1)
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.neo_eth.symbol, base=base_currency
))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'price'],
marker='v',
s=100,
c='red',
label=''
)
ax4 = plt.subplot(613, sharex=ax1)
perf.loc[:, 'cash'].plot(
ax=ax4, label='Base Currency ({})'.format(base_currency)
)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
ax5 = plt.subplot(614, sharex=ax1)
perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
ax5.set_ylabel('Percent\nChange')
ax6 = plt.subplot(615, sharex=ax1)
perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
ax6.set_ylabel('RSI')
ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
if not transaction_df.empty:
ax6.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'rsi'],
marker='^',
s=100,
c='green',
label=''
)
ax6.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'rsi'],
marker='v',
s=100,
c='red',
label=''
)
plt.legend(loc=3)
start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end/5))
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
pass
if __name__ == '__main__':
# The execution mode: backtest or live
MODE = 'backtest'
if MODE == 'backtest':
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=10000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
)
log.info('saved perf stats: {}'.format(out))
elif MODE == 'live':
run_algorithm(
capital_base=0.5,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
live=True,
algo_namespace=NAMESPACE,
base_currency='usd',
live_graph=False
)
.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py
:language: python
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png
@@ -763,8 +158,6 @@ strategy.
Simple Universe
~~~~~~~~~~~~~~~
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
This example aims to provide an easy way for users to learn how to
collect data from any given exchange and select a subset of the available
currency pairs for trading. You simply need to specify the exchange and
@@ -791,142 +184,10 @@ of the file:
catalyst ingest-exchange -x bitfinex -f minute
.. code-block:: bash
python simple_universe.py
Credits: This code was originally submitted by `Abner Ayala-Acevedo
<https://github.com/abnera>`_. Thank you!
.. code-block:: python
from datetime import timedelta
import numpy as np
import pandas as pd
from catalyst import run_algorithm
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
from catalyst.api import (symbols, )
def initialize(context):
context.i = -1 # minute counter
context.exchange = context.exchanges.values()[0].name.lower()
context.base_currency = context.exchanges.values()[0].base_currency.lower()
def handle_data(context, data):
context.i += 1
lookback_days = 7 # 7 days
# current date & time in each iteration formatted into a string
now = data.current_dt
date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = now - timedelta(days=lookback_days)
# keep only the date as a string, discard the time
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute'
# update universe everyday at midnight
if not context.i % one_day_in_minutes:
context.universe = universe(context, lookback_date, date)
# get data every 30 minutes
minutes = 30
# get lookback_days of history data: that is 'lookback' number of bins
lookback = one_day_in_minutes / minutes * lookback_days
if not context.i % minutes and context.universe:
# we iterate for every pair in the current universe
for coin in context.coins:
pair = str(coin.symbol)
# Get 30 minute interval OHLCV data. This is the standard data
# required for candlestick or indicators/signals. Return Pandas
# DataFrames. 30T means 30-minute re-sampling of one minute data.
# Adjust it to your desired time interval as needed.
opened = fill(data.history(coin, 'open',
bar_count=lookback, frequency='30T')).values
high = fill(data.history(coin, 'high',
bar_count=lookback, frequency='30T')).values
low = fill(data.history(coin, 'low',
bar_count=lookback, frequency='30T')).values
close = fill(data.history(coin, 'price',
bar_count=lookback, frequency='30T')).values
volume = fill(data.history(coin, 'volume',
bar_count=lookback, frequency='30T')).values
# close[-1] is the last value in the set, which is the equivalent
# to current price (as in the most recent value)
# displays the minute price for each pair every 30 minutes
print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format(
now=now,
pair=pair,
o=opened[-1],
h=high[-1],
l=low[-1],
c=close[-1],
v=volume[-1],
))
# -------------------------------------------------------------
# --------------- Insert Your Strategy Here -------------------
# -------------------------------------------------------------
def analyze(context=None, results=None):
pass
# Get the universe for a given exchange and a given base_currency market
# Example: Poloniex BTC Market
def universe(context, lookback_date, current_date):
# get all the pairs for the given exchange
json_symbols = get_exchange_symbols(context.exchange)
# convert into a DataFrame for easier processing
df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1)
df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1)
# Filter all the pairs to get only the ones for a given base_currency
df = df[df['base_currency'] == context.base_currency]
# Filter all the pairs to ensure that pair existed in the current date range
df = df[df.start_date < lookback_date]
df = df[df.end_daily >= current_date]
context.coins = symbols(*df.symbol) # convert all the pairs to symbols
return df.symbol.tolist()
# Replace all NA, NAN or infinite values with its nearest value
def fill(series):
if isinstance(series, pd.Series):
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
elif isinstance(series, np.ndarray):
return pd.Series(series).replace(
[np.inf, -np.inf], np.nan
).ffill().bfill().values
else:
return series
if __name__ == '__main__':
start_date = pd.to_datetime('2017-11-10', utc=True)
end_date = pd.to_datetime('2017-11-13', utc=True)
performance = run_algorithm(start=start_date, end=end_date,
capital_base=100.0, # amount of base_currency
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
data_frequency='minute',
base_currency='btc',
live=False,
live_graph=False,
algo_namespace='simple_universe')
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
.. literalinclude:: ../../catalyst/examples/simple_universe.py
:language: python
.. _portfolio_optimization:
@@ -940,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used
in writting the following article:
`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
.. code-block:: python
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/portfolio_optimization.py>`_
'''
You can run this code using the Python interpreter:
$ python portfolio_optimization.py
'''
from __future__ import division
import os
import pytz
import numpy as np
import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import datetime
from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True)
def initialize(context):
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i%context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n+1, frequency='1d')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n+1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val/tminus_val-1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr),xr)/n
# Compute asset correlation matrix (informative only)
corr_m = cov_m/np.dot(np.transpose(stds),stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3+context.nassets,n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w,np.transpose(m)))*365
p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
#store results in results array
results_array[0,p] = p_r
results_array[1,p] = p_std
#store Sharpe Ratio (return / volatility) - risk free rate element
#excluded for simplicity
results_array[2,p] = results_array[0,p] / results_array[1,p]
i = 0
for iw in weights:
results_array[3+i,p] = weights[i]
i += 1
#convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r','stdev','sharpe']+context.assets)
#locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
#locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
#order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
#create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
#plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
#plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None):
# Form DataFrame with selected data
data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py
:language: python
.. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
:align: center
docs/source/install.rst
+10 -1
View File
@@ -89,7 +89,7 @@ Once either Conda or MiniConda has been set up you can install Catalyst:
.. code-block:: bash
conda env create -f python2.7-environment.yml
conda env create -f python2.7-environment.yml
4. Activate the environment (which you need to do every time you start a new
session to run Catalyst):
@@ -132,10 +132,19 @@ with the following steps:
conda env remove --name catalyst
2. Create the environment:
for python 2.7:
.. code-block:: bash
conda create --name catalyst python=2.7 scipy zlib
or for python 3.6:
.. code-block:: bash
conda create --name catalyst python=3.6 scipy zlib
3. Activate the environment:
docs/source/live-trading.rst
+9
View File
@@ -184,5 +184,14 @@ Here is the breakdown of the new arguments:
essentially sleep and when the predefined time comes, it would start executing.
The `catalyst live` command offers additional parameters.
You can learn more by running the following from the command line:
.. code-block:: bash
catalyst live --help
Here is a complete algorithm for reference:
`Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_
etc/python2.7-environment.yml
+2
View File
@@ -1,9 +1,11 @@
name: catalyst
channels:
- defaults
- conda-forge
dependencies:
- certifi=2016.2.28=py27_0
- mkl=2017.0.3
- matplotlib=2.1.2=py36_0
- numpy=1.13.1=py27_0
- openssl=1.0.2l
- pip=9.0.1=py27_1
etc/python3.6-environment.yml
+16 -21
View File
@@ -1,29 +1,24 @@
name: catalyst
channels:
- defaults
- conda-forge
dependencies:
- ca-certificates=2017.08.26=ha1e5d58_0
- certifi=2018.1.18=py36_0
- intel-openmp=2018.0.0=h8158457_8
- libcxx=4.0.1=h579ed51_0
- libcxxabi=4.0.1=hebd6815_0
- libedit=3.1=hb4e282d_0
- libffi=3.2.1=h475c297_4
- libgfortran=3.0.1=h93005f0_2
- mkl=2018.0.1=hfbd8650_4
- ncurses=6.0=hd04f020_2
- numpy=1.14.0=py36h8a80b8c_1
- openssl=1.0.2n=hdbc3d79_0
- pip=9.0.1=py36h1555ced_4
- python=3.6.4=hc167b69_1
- readline=7.0=hc1231fa_4
- scipy=1.0.0=py36h1de22e9_0
- ca-certificates=2017.08.26
- certifi=2018.1.18
- intel-openmp=2018.0.0
- mkl=2018.0.1
- numpy=1.14.0
- openssl=1.0.2n
- matplotlib=2.1.2=py36_0
- pip=9.0.1
- python=3.6.4
- scipy=1.0.0
- setuptools=38.4.0=py36_0
- sqlite=3.22.0=h3efe00b_0
- tk=8.6.7=h35a86e2_3
- wheel=0.30.0=py36h5eb2c71_1
- xz=5.2.3=h0278029_2
- zlib=1.2.11=hf3cbc9b_2
- sqlite=3.22.0
- tk=8.6.7
- wheel=0.30.0
- xz=5.2.3
- zlib=1.2.11
- pip:
- aiodns==1.1.1
- aiohttp==3.0.1
tests/exchange/test_config.py
-8
View File
@@ -1,8 +0,0 @@
from catalyst.exchange.utils.factory import get_exchange
class TestConfig:
def test_create_config(self):
exchange = get_exchange('binance', skip_init=True)
config = exchange.create_exchange_config()
pass
tests/exchange/test_suites/test_suite_bundle.py
+1 -1
View File
@@ -197,7 +197,7 @@ class TestSuiteBundle:
# population=exchange_population,
# features=[bundle],
# ) # Type: list[Exchange]
exchanges = [get_exchange('binance', skip_init=True)]
exchanges = [get_exchange('poloniex', skip_init=True)]
data_portal = TestSuiteBundle.get_data_portal(exchanges)
for exchange in exchanges:
tests/exchange/test_suites/test_suite_exchange.py
+39 -4
View File
@@ -5,28 +5,63 @@ from logging import Logger, WARNING
from time import sleep
import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import TestHandler
from catalyst.assets._assets import TradingPair
from catalyst.exchange.exchange_errors import ExchangeRequestError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import get_exchanges, get_exchange
from catalyst.exchange.utils.test_utils import select_random_exchanges, \
select_random_assets
handle_exchange_error, select_random_assets
from catalyst.testing import ZiplineTestCase
from catalyst.testing.fixtures import WithLogger
from catalyst.exchange.utils.factory import get_exchanges, get_exchange
log = Logger('TestSuiteExchange')
class TestSuiteExchange(WithLogger, ZiplineTestCase):
def _test_markets_exchange(self, exchange, attempts=0):
assets = None
try:
exchange.init()
# Verify that the assets and markets are populated
if not exchange.markets:
raise ValueError(
'no markets found'
)
if not exchange.assets:
raise ValueError(
'no assets derived from markets'
)
assets = exchange.assets
except ExchangeRequestError as e:
sleep(5)
if attempts > 5:
handle_exchange_error(exchange, e)
else:
print(
're-trying an exchange request {} {}'.format(
exchange.name, attempts
)
)
self._test_markets_exchange(exchange, attempts + 1)
except Exception as e:
handle_exchange_error(exchange, e)
return assets
def test_markets(self):
population = 3
results = dict()
exchanges = select_random_exchanges(population) # Type: list[Exchange]
for exchange in exchanges:
exchange.init()
assets = self._test_markets_exchange(exchange)
if assets is not None: