wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-07 19:55:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

merged from develop

Browse Source
This commit is contained in:
Frederic Fortier
2017-12-12 15:48:35 -05:00
parent 60747082b6 3014651ac2
commit 263a2ba547
89 changed files with 19079 additions and 18266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/__init__.py
+4 -10
View File
@@ -29,11 +29,14 @@ from ._version import get_versions
from . algorithm import TradingAlgorithm
from . import api
from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
__version__ = get_versions()['version']
del get_versions
# PERF: Fire a warning if calendars were instantiated during catalyst import.
# Having calendars doesn't break anything per-se, but it makes catalyst imports
# noticeably slower, which becomes particularly noticeable in the Zipline CLI.
from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
if global_calendar_dispatcher._calendars:
import warnings
warnings.warn(
@@ -44,10 +47,6 @@ if global_calendar_dispatcher._calendars:
del global_calendar_dispatcher
__version__ = get_versions()['version']
del get_versions
def load_ipython_extension(ipython):
from .__main__ import catalyst_magic
ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst')
@@ -69,7 +68,6 @@ if os.name == 'nt':
_()
del _
__all__ = [
'TradingAlgorithm',
'api',
@@ -80,7 +78,3 @@ __all__ = [
'run_algorithm',
'utils',
]
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
catalyst/__main__.py
+49 -28
View File
@@ -10,7 +10,6 @@ from six import text_type
from catalyst.data import bundles as bundles_module
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_utils import delete_algo_folder
from catalyst.exchange.factory import get_exchange
from catalyst.utils.cli import Date, Timestamp
from catalyst.utils.run_algo import _run, load_extensions
@@ -194,9 +193,7 @@ def ipython_only(option):
@click.option(
'-x',
'--exchange-name',
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the targeted exchange (supported: bitfinex,'
' bittrex, poloniex).',
help='The name of the targeted exchange.',
)
@click.option(
'-n',
@@ -258,8 +255,9 @@ def run(ctx,
ctx.fail("must specify a base currency with '-c' in backtest mode")
if capital_base is None:
ctx.fail("must specify a capital base with '--capital-base'"
" in backtest mode")
ctx.fail("must specify a capital base with '--capital-base'")
click.echo('Running in backtesting mode.')
perf = _run(
initialize=None,
@@ -284,7 +282,9 @@ def run(ctx,
exchange=exchange_name,
algo_namespace=algo_namespace,
base_currency=base_currency,
live_graph=False
live_graph=False,
simulate_orders=True,
stats_output=None,
)
if output == '-':
@@ -312,11 +312,11 @@ def catalyst_magic(line, cell=None):
'--algotext', cell,
'--output', os.devnull, # don't write the results by default
] + ([
# these options are set when running in line magic mode
# set a non None algo text to use the ipython user_ns
'--algotext', '',
'--local-namespace',
] if cell is None else []) + line.split(),
# these options are set when running in line magic mode
# set a non None algo text to use the ipython user_ns
'--algotext', '',
'--local-namespace',
] if cell is None else []) + line.split(),
'%s%%catalyst' % ((cell or '') and '%'),
# don't use system exit and propogate errors to the caller
standalone_mode=False,
@@ -336,6 +336,12 @@ def catalyst_magic(line, cell=None):
type=click.File('r'),
help='The file that contains the algorithm to run.',
)
@click.option(
'--capital-base',
type=float,
show_default=True,
help='The amount of capital (in base_currency) allocated to trading.',
)
@click.option(
'-t',
'--algotext',
@@ -374,9 +380,7 @@ def catalyst_magic(line, cell=None):
@click.option(
'-x',
'--exchange-name',
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the targeted exchange (supported: bitfinex,'
' bittrex, poloniex).',
help='The name of the targeted exchange.',
)
@click.option(
'-n',
@@ -395,9 +399,17 @@ def catalyst_magic(line, cell=None):
default=False,
help='Display live graph.',
)
@click.option(
'--simulate-orders/--no-simulate-orders',
is_flag=True,
default=True,
help='Simulating orders enable the paper trading mode. No orders will be '
'sent to the exchange unless set to false.',
)
@click.pass_context
def live(ctx,
algofile,
capital_base,
algotext,
define,
output,
@@ -406,7 +418,8 @@ def live(ctx,
exchange_name,
algo_namespace,
base_currency,
live_graph):
live_graph,
simulate_orders):
"""Trade live with the given algorithm.
"""
if (algotext is not None) == (algofile is not None):
@@ -417,11 +430,22 @@ def live(ctx,
if exchange_name is None:
ctx.fail("must specify an exchange name '-x'")
if algo_namespace is None:
ctx.fail("must specify an algorithm name '-n' in live execution mode")
if base_currency is None:
ctx.fail("must specify a base currency '-c' in live execution mode")
if capital_base is None:
ctx.fail("must specify a capital base with '--capital-base'")
if simulate_orders:
click.echo('Running in paper trading mode.')
else:
click.echo('Running in live trading mode.')
perf = _run(
initialize=None,
handle_data=None,
@@ -431,7 +455,7 @@ def live(ctx,
algotext=algotext,
defines=define,
data_frequency=None,
capital_base=None,
capital_base=capital_base,
data=None,
bundle=None,
bundle_timestamp=None,
@@ -445,7 +469,9 @@ def live(ctx,
exchange=exchange_name,
algo_namespace=algo_namespace,
base_currency=base_currency,
live_graph=live_graph
live_graph=live_graph,
simulate_orders=simulate_orders,
stats_output=None,
)
if output == '-':
@@ -460,9 +486,7 @@ def live(ctx,
@click.option(
'-x',
'--exchange-name',
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the exchange bundle to ingest (supported: bitfinex,'
' bittrex, poloniex).',
help='The name of the exchange bundle to ingest.',
)
@click.option(
'-f',
@@ -520,7 +544,8 @@ def live(ctx,
default=False,
help='Report potential anomalies found in data bundles.'
)
def ingest_exchange(exchange_name, data_frequency, start, end,
@click.pass_context
def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
include_symbols, exclude_symbols, csv, show_progress,
verbose, validate):
"""
@@ -565,9 +590,7 @@ def clean_algo(ctx, algo_namespace):
@click.option(
'-x',
'--exchange-name',
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the exchange bundle to ingest (supported: bitfinex,'
' bittrex, poloniex).',
help='The name of the exchange bundle to ingest.',
)
@click.option(
'-f',
@@ -606,9 +629,7 @@ def clean_exchange(ctx, exchange_name, data_frequency):
@click.option(
'-x',
'--exchange-name',
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the exchange bundle to ingest (supported: bitfinex,'
' bittrex, poloniex).',
help='The name of the exchange bundle to ingest.',
)
@click.option(
'-c',
catalyst/algorithm.py
+1 -2
View File
@@ -124,7 +124,6 @@ from catalyst.utils.events import (
from catalyst.utils.factory import create_simulation_parameters
from catalyst.utils.math_utils import (
tolerant_equals,
round_if_near_integer,
round_nearest
)
from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
@@ -1485,7 +1484,6 @@ class TradingAlgorithm(object):
"""
Converts the number of shares to the smallest tradable lot size for
the asset being ordered.
"""
return round_nearest(amount, asset.min_trade_size)
@@ -1523,6 +1521,7 @@ class TradingAlgorithm(object):
self.updated_portfolio(),
self.get_datetime(),
self.trading_client.current_data)
@staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style):
"""
catalyst/assets/_assets.pyx
+61 -14
View File
@@ -396,11 +396,18 @@ cdef class Future(Asset):
cdef class TradingPair(Asset):
cdef readonly float leverage
cdef readonly object market_currency
cdef readonly object quote_currency
cdef readonly object base_currency
cdef readonly object end_daily
cdef readonly object end_minute
cdef readonly object exchange_symbol
cdef readonly float maker
cdef readonly float taker
cdef readonly int trading_state
cdef readonly object data_source
cdef readonly float max_trade_size
cdef readonly float lot
cdef readonly int decimals
_kwargnames = frozenset({
'sid',
@@ -413,12 +420,19 @@ cdef class TradingPair(Asset):
'exchange',
'exchange_full',
'leverage',
'market_currency',
'quote_currency',
'base_currency',
'end_daily',
'end_minute',
'exchange_symbol',
'min_trade_size'
'min_trade_size',
'max_trade_size',
'lot',
'maker',
'taker',
'trading_state',
'data_source',
'decimals'
})
def __init__(self,
object symbol,
@@ -434,10 +448,17 @@ cdef class TradingPair(Asset):
object first_traded=None,
object auto_close_date=None,
object exchange_full=None,
object min_trade_size=None):
float min_trade_size=0.0001,
float max_trade_size=1000000,
float maker=0.0015,
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=0,
object data_source='catalyst'):
"""
Replicates the Asset constructor with some built-in conventions
and a new 'leverage' attribute.
and adds properties for leverage and fees.
Symbol
------
@@ -469,8 +490,6 @@ cdef class TradingPair(Asset):
highest volume and market cap generally benefit from high leverage.
New currencies from ICO generally cannot be leveraged.
The leverage value is either None or and integer.
Leverage allows you to open a larger position with a smaller amount
of funds. For example, if you open a $5,000 position in BTC/USD
with 5:1 leverage, only one-fifth of this amount, or $1000, will be
@@ -480,6 +499,11 @@ cdef class TradingPair(Asset):
the position. If you open with 1:1 leverage, $5,000 of your balance
will be tied to the position.
Fees
----
Exchanges generally charge a taker (taking from the order book) or
maker (adding to the order book) fee.
:param symbol:
:param exchange:
:param start_date:
@@ -494,11 +518,17 @@ cdef class TradingPair(Asset):
:param auto_close_date:
:param exchange_full:
:param min_trade_size:
:param max_trade_size:
:param maker:
:param taker:
:param data_source
:param decimals
:param lot
"""
symbol = symbol.lower()
try:
self.market_currency, self.base_currency = symbol.split('_')
self.base_currency, self.quote_currency = symbol.split('_')
except Exception as e:
raise InvalidSymbolError(symbol=symbol, error=e)
@@ -512,11 +542,14 @@ cdef class TradingPair(Asset):
asset_name = ' / '.join(symbol.split('_')).upper()
if start_date is None:
start_date = pd.Timestamp.utcnow()
start_date = pd.to_datetime('2009-1-1', utc=True)
if end_date is None:
end_date = pd.Timestamp.utcnow() + timedelta(days=365)
if lot == 0 and min_trade_size > 0:
lot = min_trade_size
super().__init__(
sid,
exchange,
@@ -527,19 +560,26 @@ cdef class TradingPair(Asset):
first_traded=first_traded,
auto_close_date=auto_close_date,
exchange_full=exchange_full,
min_trade_size=min_trade_size
min_trade_size=min_trade_size,
)
self.maker = maker
self.taker = taker
self.leverage = leverage
self.end_daily = end_daily
self.end_minute = end_minute
self.exchange_symbol = exchange_symbol
self.trading_state = trading_state
self.data_source = data_source
self.max_trade_size = max_trade_size
self.lot = lot
self.decimals = decimals
def __repr__(self):
return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
'Introduced On: {start_date}, ' \
'Market Currency: {market_currency}, ' \
'Base Currency: {base_currency}, ' \
'Quote Currency: {quote_currency}, ' \
'Exchange Leverage: {leverage}, ' \
'Minimum Trade Size: {min_trade_size} ' \
'Last daily ingestion: {end_daily} ' \
@@ -548,7 +588,7 @@ cdef class TradingPair(Asset):
sid=self.sid,
exchange=self.exchange,
start_date=self.start_date,
market_currency=self.market_currency,
quote_currency=self.quote_currency,
base_currency=self.base_currency,
leverage=self.leverage,
min_trade_size=self.min_trade_size,
@@ -560,6 +600,7 @@ cdef class TradingPair(Asset):
"""
Convert to a python dict.
"""
#TODO: missing fields
super_dict = super(TradingPair, self).to_dict()
super_dict['end_daily'] = self.end_daily
super_dict['end_minute'] = self.end_minute
@@ -578,7 +619,7 @@ cdef class TradingPair(Asset):
-------
boolean: whether the asset's exchange is open at the given minute.
"""
#TODO: consider implementing to spot holds
#TODO: make more dymanic to catch holds
return True
cpdef __reduce__(self):
@@ -588,6 +629,7 @@ cdef class TradingPair(Asset):
and whose second element is a tuple of all the attributes that should
be serialized/deserialized during pickling.
"""
#TODO: make sure that all fields set there
return (self.__class__, (self.symbol,
self.exchange,
self.start_date,
@@ -598,7 +640,12 @@ cdef class TradingPair(Asset):
self.first_traded,
self.auto_close_date,
self.exchange_full,
self.min_trade_size))
self.min_trade_size,
self.max_trade_size,
self.lot,
self.decimals,
self.taker,
self.maker))
def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object)
catalyst/constants.py
+1 -1
View File
@@ -15,4 +15,4 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
DATE_FORMAT = '%Y-%m-%d'
AUTO_INGEST = False
AUTO_INGEST = False
catalyst/curate/poloniex.py
+141 -136
View File
@@ -1,25 +1,33 @@
import json, time, csv
import os
import time
import shutil
import json
import csv
from datetime import datetime
import pandas as pd
import os, time, shutil, requests, logbook
import requests
import logbook
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
DT_END = pd.to_datetime('today').value // 10 ** 9
CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
CONN_RETRIES = 2
DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
DT_END = pd.to_datetime('today').value // 10 ** 9
CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
CONN_RETRIES = 2
logbook.StderrHandler().push_application()
log = logbook.Logger(__name__)
class PoloniexCurator(object):
'''
OHLCV data feed generator for crypto data. Based on Poloniex market data
'''
_api_path = 'https://poloniex.com/public?'
currency_pairs = []
_api_path = 'https://poloniex.com/public?'
currency_pairs = []
def __init__(self):
if not os.path.exists(CSV_OUT_FOLDER):
@@ -30,10 +38,9 @@ class PoloniexCurator(object):
CSV_OUT_FOLDER))
log.exception(e)
def get_currency_pairs(self):
'''
Retrieves and returns all currency pairs from the exchange
Retrieves and returns all currency pairs from the exchange
'''
url = self._api_path + 'command=returnTicker'
@@ -45,7 +52,7 @@ class PoloniexCurator(object):
return None
data = response.json()
self.currency_pairs = []
self.currency_pairs = []
for ticker in data:
self.currency_pairs.append(ticker)
self.currency_pairs.sort()
@@ -54,54 +61,60 @@ class PoloniexCurator(object):
len(self.currency_pairs)
))
def _retrieve_tradeID_date(self, row):
'''
Helper function that reads tradeID and date fields from CSV readline
'''
tId = int(row.split(',')[0])
d = pd.to_datetime(row.split(',')[1],
infer_datetime_format=True).value // 10 ** 9
d = pd.to_datetime(row.split(',')[1],
infer_datetime_format=True).value // 10 ** 9
return tId, d
def retrieve_trade_history(self, currencyPair, start=DT_START,
def retrieve_trade_history(self, currencyPair, start=DT_START,
end=DT_END, temp=None):
'''
Retrieves TradeHistory from exchange for a given currencyPair
between start and end dates. If no start date is provided, uses
Retrieves TradeHistory from exchange for a given currencyPair
between start and end dates. If no start date is provided, uses
a system-wide one (beginning of time for cryptotrading).
If no end date is provided, 'now' is used.
Stores results in CSV file on disk.
This function is called recursively to work around the
This function is called recursively to work around the
limitations imposed by the provider API.
'''
csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
'''
Check what data we already have on disk, reading first and last
Check what data we already have on disk, reading first and last
lines from file. Data is stored on file from NEWEST to OLDEST.
'''
try:
with open(csv_fn, 'ab+') as f:
with open(csv_fn, 'ab+') as f:
f.seek(0, os.SEEK_END)
if(f.tell() > 2): # Check file size is not 0
f.seek(0) # Go to start to read
last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
f.seek(0) # Go to start to read
last_tradeID, end_file = self._retrieve_tradeID_date(
f.readline())
f.seek(-2, os.SEEK_END) # Jump to the 2nd last byte
while f.read(1) != b"\n": # Until EOL is found...
f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
# ...jump back the read byte plus one more.
f.seek(-2, os.SEEK_CUR)
first_tradeID, start_file = self._retrieve_tradeID_date(
f.readline())
if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
or (currencyPair == 'BTC_HUC' and first_tradeID == 2)
or (currencyPair == 'BTC_RIC' and first_tradeID == 2)
or (currencyPair == 'BTC_XCP' and first_tradeID == 2)
or (currencyPair == 'BTC_NAV' and first_tradeID == 4569)
or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
if(end_file + 3600 * 6 > DT_END
and (first_tradeID == 1
or (currencyPair == 'BTC_HUC'
and first_tradeID == 2)
or (currencyPair == 'BTC_RIC'
and first_tradeID == 2)
or (currencyPair == 'BTC_XCP'
and first_tradeID == 2)
or (currencyPair == 'BTC_NAV'
and first_tradeID == 4569)
or (currencyPair == 'BTC_POT'
and first_tradeID == 23511))):
return
except Exception as e:
@@ -109,11 +122,11 @@ class PoloniexCurator(object):
log.exception(e)
'''
Poloniex API limits querying TradeHistory to intervals smaller
Poloniex API limits querying TradeHistory to intervals smaller
than 1 month, so we make sure that start date is never more than
1 month apart from end date
'''
if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days
if(end - start > 2419200): # 60s/min * 60min/hr * 24hr/day * 28days
newstart = end - 2419200
else:
newstart = start
@@ -124,12 +137,11 @@ class PoloniexCurator(object):
url = '{path}command=returnTradeHistory&currencyPair={pair}' \
'&start={start}&end={end}'.format(
path = self._api_path,
pair = currencyPair,
start = str(newstart),
end = str(end)
path=self._api_path,
pair=currencyPair,
start=str(newstart),
end=str(end)
)
print url
attempts = 0
success = 0
@@ -137,14 +149,14 @@ class PoloniexCurator(object):
try:
response = requests.get(url)
except Exception as e:
log.error('Failed to retrieve trade history data for {}'.format(
currencyPair
))
log.error('Failed to retrieve trade history data'
'for {}'.format(currencyPair))
log.exception(e)
attempts += 1
else:
try:
if isinstance(response.json(), dict) and response.json()['error']:
if(isinstance(response.json(), dict)
and response.json()['error']):
log.error('Failed to to retrieve trade history data '
'for {}: {}'.format(
currencyPair,
@@ -161,33 +173,32 @@ class PoloniexCurator(object):
if not success:
return None
'''
If we get to transactionId == 1, and we already have that on
If we get to transactionId == 1, and we already have that on
disk, we got to the end of TradeHistory for this coin.
'''
if('first_tradeID' in locals()
and response.json()[-1]['tradeID'] == first_tradeID):
if('first_tradeID' in locals()
and response.json()[-1]['tradeID'] == first_tradeID):
return
'''
There are primarily two scenarios:
a) There is newer data available that we need to add at
the beginning of the file. We'll retrieve all what we
need until we get to what we already have, writing it
to a temporary file; and we will write that at the
a) There is newer data available that we need to add at
the beginning of the file. We'll retrieve all what we
need until we get to what we already have, writing it
to a temporary file; and we will write that at the
beginning of our existing file.
b) We are going back in time, appending at the end of
our existing TradeHistory until the first transaction
b) We are going back in time, appending at the end of
our existing TradeHistory until the first transaction
for this currencyPair
'''
try:
if( 'end_file' in locals() and end_file + 3600 < end):
try:
if('end_file' in locals() and end_file + 3600 < end):
if (temp is None):
temp = os.tmpfile()
tempcsv = csv.writer(temp)
for item in response.json():
if( item['tradeID'] <= last_tradeID ):
if(item['tradeID'] <= last_tradeID):
continue
tempcsv.writerow([
item['tradeID'],
@@ -196,27 +207,28 @@ class PoloniexCurator(object):
item['rate'],
item['amount'],
item['total'],
item['globalTradeID']
item['globalTradeID'],
])
if( response.json()[-1]['tradeID'] > last_tradeID ):
end = pd.to_datetime( response.json()[-1]['date'],
infer_datetime_format=True).value // 10 ** 9
self.retrieve_trade_history(currencyPair, start,
end, temp=temp)
if(response.json()[-1]['tradeID'] > last_tradeID):
end = pd.to_datetime(response.json()[-1]['date'],
infer_datetime_format=True
).value // 10**9
self.retrieve_trade_history(currencyPair, start,
end, temp=temp)
else:
with open(csv_fn,'rb+') as f:
shutil.copyfileobj(f,temp)
with open(csv_fn, 'rb+') as f:
shutil.copyfileobj(f, temp)
f.seek(0)
temp.seek(0)
shutil.copyfileobj(temp,f)
shutil.copyfileobj(temp, f)
temp.close()
end = start_file
else:
with open(csv_fn, 'ab') as csvfile:
csvwriter = csv.writer(csvfile)
for item in response.json():
if( 'first_tradeID' in locals()
and item['tradeID'] >= first_tradeID ):
if('first_tradeID' in locals()
and item['tradeID'] >= first_tradeID):
continue
csvwriter.writerow([
item['tradeID'],
@@ -227,70 +239,66 @@ class PoloniexCurator(object):
item['total'],
item['globalTradeID']
])
end = pd.to_datetime(response.json()[-1]['date'],
infer_datetime_format=True).value // 10 ** 9
end = pd.to_datetime(response.json()[-1]['date'],
infer_datetime_format=True).value//10**9
except Exception as e:
log.error('Error opening {}'.format(csv_fn))
log.exception(e)
'''
If we got here, we aren't done yet. Call recursively with
If we got here, we aren't done yet. Call recursively with
'end' times that go sequentially back in time.
'''
self.retrieve_trade_history(currencyPair, start, end)
def generate_ohlcv(self, df):
'''
Generates OHLCV dataframe from a dataframe containing all TradeHistory
by resampling with 1-minute period
'''
df.set_index('date', inplace=True) # Index by date
vol = df['total'].to_frame('volume') # set Vol aside
df.drop('total', axis=1, inplace=True) # Drop volume data
ohlc = df.resample('T').ohlc() # Resample OHLC 1min
ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Raname columns by dropping 'rate'
closes = ohlc['close'].fillna(method='pad') # Pad fwd missing 'close'
ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill N/A with last close
vol = vol.resample('T').sum().fillna(0) # Add volumes by bin
ohlcv = pd.concat([ohlc,vol], axis=1) # Concatenate OHLC + Vol
df.set_index('date', inplace=True) # Index by date
vol = df['total'].to_frame('volume') # set Vol aside
df.drop('total', axis=1, inplace=True) # Drop volume data
ohlc = df.resample('T').ohlc() # Resample OHLC 1min
ohlc.cols = ohlc.cols.map(lambda t: t[1]) # Raname cols
closes = ohlc['close'].fillna(method='pad') # Pad fwd missing close
ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill NA w/ last close
vol = vol.resample('T').sum().fillna(0) # Add volumes by bin
ohlcv = pd.concat([ohlc, vol], axis=1) # Concat OHLC + Vol
return ohlcv
def write_ohlcv_file(self, currencyPair):
def write_ohlcv_file(self, currencyPair):
'''
Generates OHLCV data file with 1minute bars from TradeHistory on disk
'''
'''
csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
if( os.path.getmtime(csv_1min) > time.time() - 7200 ):
csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
if(os.path.getmtime(csv_1min) > time.time() - 7200):
log.debug(currencyPair+': 1min data file already up to date. '
'Delete the file if you want to rebuild it.')
else:
df = pd.read_csv(csv_trades,
names=['tradeID',
'date',
'type',
'rate',
'amount',
'total',
'globalTradeID'],
dtype = {'tradeID': int,
'date': str,
'type': str,
'rate': float,
'amount': float,
'total': float,
'globalTradeID': int }
)
df.drop(['tradeID','type','amount','globalTradeID'],
df = pd.read_csv(csv_trades,
names=['tradeID',
'date',
'type',
'rate',
'amount',
'total',
'globalTradeID'],
dtype={'tradeID': int,
'date': str,
'type': str,
'rate': float,
'amount': float,
'total': float,
'globalTradeID': int}
)
df.drop(['tradeID', 'type', 'amount', 'globalTradeID'],
axis=1, inplace=True)
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
ohlcv = self.generate_ohlcv(df)
try:
try:
with open(csv_1min, 'w') as csvfile:
csvwriter = csv.writer(csvfile)
for item in ohlcv.itertuples():
@@ -305,32 +313,28 @@ class PoloniexCurator(object):
item.volume,
])
except Exception as e:
log.error('Error opening {}'.format(csv_fn))
log.error('Error opening {}'.format(csv_1min))
log.exception(e)
log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
def onemin_to_dataframe(self, currencyPair, start, end):
'''
Returns a data frame for a given currencyPair from data on disk
'''
csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
df = pd.read_csv(csv_fn, names=['date',
'open',
'high',
'low',
'close',
'volume']
)
df['date'] = pd.to_datetime(df['date'],unit='s')
csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
df = pd.read_csv(csv_fn, names=['date',
'open',
'high',
'low',
'close',
'volume'])
df['date'] = pd.to_datetime(df['date'], unit='s')
df.set_index('date', inplace=True)
return df[start : end]
return df[start:end]
def generate_symbols_json(self, filename=None):
'''
Generates a symbols.json file with corresponding start_date
Generates a symbols.json file with corresponding start_date
for each currencyPair
'''
symbol_map = {}
@@ -341,36 +345,37 @@ class PoloniexCurator(object):
with open(filename, 'w') as symbols:
for currencyPair in self.currency_pairs:
start = None
csv_fn = '{}crypto_trades-{}.csv'.format(
CSV_OUT_FOLDER, currencyPair)
with open(csv_fn, 'r') as f:
csv_fn = '{}crypto_trades-{}.csv'.format(
CSV_OUT_FOLDER,
currencyPair)
with open(csv_fn, 'r') as f:
f.seek(0, os.SEEK_END)
if(f.tell() > 2): # Check file size is not 0
f.seek(-2, os.SEEK_END) # Jump to 2nd last byte
while f.read(1) != b"\n": # Until EOL is found...
f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
start = pd.to_datetime( f.readline().split(',')[1],
infer_datetime_format=True)
# ...jump back the read byte plus one more.
f.seek(-2, os.SEEK_CUR)
start = pd.to_datetime(f.readline().split(',')[1],
infer_datetime_format=True)
if(start is None):
start = time.gmtime()
base, market = currencyPair.lower().split('_')
symbol = '{market}_{base}'.format( market=market, base=base )
symbol = '{market}_{base}'.format(market=market, base=base)
symbol_map[currencyPair] = dict(
symbol = symbol,
start_date = start.strftime("%Y-%m-%d")
symbol=symbol,
start_date=start.strftime("%Y-%m-%d")
)
json.dump(symbol_map, symbols, sort_keys=True, indent=2,
separators=(',',':'))
json.dump(symbol_map, symbols, sort_keys=True, indent=2,
separators=(',', ':'))
if __name__ == '__main__':
pc = PoloniexCurator()
pc.get_currency_pairs()
#pc.generate_symbols_json()
# pc.generate_symbols_json()
for currencyPair in pc.currency_pairs:
pc.retrieve_trade_history(currencyPair)
log.debug('{} up to date.'.format(currencyPair))
pc.write_ohlcv_file(currencyPair)
catalyst/data/bundles/__init__.py
-1
View File
@@ -1,6 +1,5 @@
# These imports are necessary to force module-scope register calls to happen.
from . import quandl # noqa
from . import poloniex
from .core import (
UnknownBundle,
bundles,
catalyst/data/bundles/base.py
+35 -36
View File
@@ -13,10 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from itertools import count
import tarfile
from time import time, sleep
from time import sleep
from abc import abstractmethod, abstractproperty
import logbook
@@ -37,6 +36,7 @@ log = logbook.Logger(__name__, level=LOG_LEVEL)
DEFAULT_RETRIES = 5
class BaseBundle(object):
def __init__(self, asset_filter=[]):
self._asset_filter = asset_filter
@@ -104,11 +104,11 @@ class BaseBundle(object):
def post_process_symbol_metadata(self, metadata, data):
return metadata
@abstractmethod
def fetch_raw_symbol_frame(self, api_key, symbol, start_date, end_date):
raise NotImplementedError()
def ingest(self,
environ,
asset_db_writer,
@@ -128,7 +128,7 @@ class BaseBundle(object):
retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5)
if is_compile:
# User has instructed local compilation and ingestion of bundle.
# User has instructed local compilation & ingestion of bundle.
# Fetch raw metadata for all symbols.
raw_metadata = self._fetch_metadata_frame(
api_key,
@@ -157,9 +157,9 @@ class BaseBundle(object):
show_progress=show_progress,
)
# Post-process metadata using cached symbol frames, and write to
# disk. This metadata must be written before any attempt to write
# minute data.
# Post-process metadata using cached symbol frames, and write
# to disk. This metadata must be written before any attempt
# to write minute data.
metadata = self._post_process_metadata(
raw_metadata,
cache,
@@ -184,10 +184,11 @@ class BaseBundle(object):
show_progress=show_progress,
)
# For legacy purposes, this call is required to ensure the database
# contains an appropriately initialized file structure. We don't
# forsee a usecase for adjustments at this time, but may later
# choose to expose this functionality in the future.
# For legacy purposes, this call is required to ensure the
# database contains an appropriately initialized file
# structure. We don't forsee a usecase for adjustments at
# this time, but may later choose to expose this functionality
# in the future.
adjustment_writer.write(
splits=(
pd.concat(self.splits, ignore_index=True)
@@ -232,12 +233,12 @@ class BaseBundle(object):
tar.extractall(output_dir)
def _fetch_metadata_frame(self,
api_key,
cache,
retries=DEFAULT_RETRIES,
environ=None,
show_progress=False):
api_key,
cache,
retries=DEFAULT_RETRIES,
environ=None,
show_progress=False):
# Setup raw metadata iterator to fetch pages if necessary.
raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ)
@@ -251,7 +252,7 @@ class BaseBundle(object):
show_percent=False,
) as blocks:
metadata = pd.concat(blocks, ignore_index=True)
return metadata
def _fetch_metadata_iter(self, api_key, cache, retries, environ):
@@ -269,21 +270,20 @@ class BaseBundle(object):
page_number,
)
break
except ValueError as e:
except ValueError:
raw = pd.DataFrame([])
break
except Exception as e:
except Exception:
log.exception(
'Failed to load metadata from {}. '
'Retrying.'.format(self.name)
)
)
else:
raise ValueError(
'Failed to download metadata page {} after {} '
'attempts.'.format(page_number, retries)
)
if raw.empty:
# Empty DataFrame signals completion.
break
@@ -305,7 +305,7 @@ class BaseBundle(object):
columns=self.md_column_names,
index=metadata.index,
)
# Iterate over the available symbols, loading the asset's raw symbol
# data from the cache. The final metadata is computed and recorded in
# the appropriate row depending on the asset's id.
@@ -318,22 +318,22 @@ class BaseBundle(object):
show_percent=False,
) as symbols_map:
for asset_id, symbol in symbols_map:
# Attempt to load data from disk, the cache should have an entry
# for each symbol at this point of the execution. If one does
# not exist, we should fail.
# Attempt to load data from disk, the cache should have an
# entry for each symbol at this point of the execution. If one
# does not exist, we should fail.
key = '{sym}.daily.frame'.format(sym=symbol)
try:
raw_data = cache[key]
except KeyError:
raise ValueError(
'Unable to find cached data for symbol: {0}'.format(symbol)
)
'Unable to find cached data for symbol:'
' {0}'.format(symbol))
# Perform and require post-processing of metadata.
final_symbol_metadata = self.post_process_symbol_metadata(
asset_id,
metadata.iloc[asset_id],
raw_data,
raw_data,
)
# Record symbol's final metadata.
@@ -363,8 +363,8 @@ class BaseBundle(object):
# returns the cached data unaltered. The `should_sleep` flag
# indicates that an API call was attempted, and that we should be
# ensure aren't exceeding our rate limit before proceeding to the
# next symbol. If the raw_data is updated, it is cached before being
# returned.
# next symbol. If the raw_data is updated, it is cached before
# being returned.
raw_data, should_sleep = self._maybe_update_symbol_frame(
start_time,
api_key,
@@ -414,7 +414,7 @@ class BaseBundle(object):
last = start_session
if raw_data is not None and len(raw_data) > 0:
last = raw_data.index[-1].tz_localize('UTC')
should_sleep = False
# Determine time at which cached data will be considered stale.
@@ -455,7 +455,7 @@ class BaseBundle(object):
retries=DEFAULT_RETRIES):
# Data for symbol is old enough to attempt an update or is not
# present in the cache. Fetch raw data for a single symbol
# present in the cache. Fetch raw data for a single symbol
# with requested intervals and frequency. Retry as necessary.
for _ in range(retries):
try:
@@ -468,7 +468,6 @@ class BaseBundle(object):
data_frequency,
)
raw_data.index = pd.to_datetime(raw_data.index, utc=True)
#raw_data.index = raw_data.index.tz_localize('UTC')
# Filter incoming data to fit start and end sessions.
raw_data = raw_data[
@@ -482,7 +481,7 @@ class BaseBundle(object):
return raw_data
except Exception as e:
except Exception:
log.exception(
'Exception raised fetching {name} data. Retrying.'
.format(name=self.name)
catalyst/data/bundles/base_pricing.py
+3
View File
@@ -16,6 +16,7 @@
from catalyst.data.bundles.base import BaseBundle
from catalyst.utils.memoize import lazyval
class BasePricingBundle(BaseBundle):
@lazyval
def md_dtypes(self):
@@ -38,6 +39,7 @@ class BasePricingBundle(BaseBundle):
('volume', 'float64'),
]
class BaseCryptoPricingBundle(BasePricingBundle):
@lazyval
def calendar_name(self):
@@ -55,6 +57,7 @@ class BaseCryptoPricingBundle(BasePricingBundle):
def dividends(self):
return []
class BaseEquityPricingBundle(BasePricingBundle):
@lazyval
def calendar_name(self):
catalyst/data/bundles/core.py
+4 -1
View File
@@ -37,6 +37,7 @@ from catalyst.utils.cli import maybe_show_progress
ONE_MEGABYTE = 1024 * 1024
def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
return pth.data_path(
asset_db_relative(bundle_name, timestr, environ, db_version),
@@ -135,6 +136,7 @@ def ingestions_for_bundle(bundle, environ=None):
reverse=True,
)
def download_with_progress(url, chunk_size, **progress_kwargs):
"""
Download streaming data from a URL, printing progress information to the
@@ -705,4 +707,5 @@ def _make_bundle_core():
)
bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core()
bundles, register_bundle, register, unregister, ingest, load, clean = \
_make_bundle_core()
catalyst/data/bundles/poloniex.py
+16 -18
View File
@@ -14,19 +14,17 @@
# limitations under the License.
import sys
from datetime import datetime
from six.moves.urllib.parse import urlencode
import pandas as pd
from six.moves.urllib.parse import urlencode
from catalyst.data.bundles.core import register_bundle
from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
from catalyst.utils.memoize import lazyval
from catalyst.curate.poloniex import PoloniexCurator
class PoloniexBundle(BaseCryptoPricingBundle):
@lazyval
def name(self):
@@ -46,7 +44,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
@lazyval
def tar_url(self):
return (
'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
'https://s3.amazonaws.com/enigmaco/catalyst-bundles/'
'poloniex/poloniex-bundle.tar.gz'
)
@lazyval
@@ -67,12 +66,11 @@ class PoloniexBundle(BaseCryptoPricingBundle):
raw = raw.sort_index().reset_index()
raw.rename(
columns={'index':'symbol'},
columns={'index': 'symbol'},
inplace=True,
)
raw = raw[raw['isFrozen'] == 0]
return raw
def post_process_symbol_metadata(self, asset_id, sym_md, sym_data):
@@ -98,7 +96,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
frequency):
# TODO: replace this with direct exchange call
# The end date and frequency should be used to calculate the number of bars
# The end date and frequency should be used to
# calculate the number of bars
if(frequency == 'minute'):
pc = PoloniexCurator()
raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
@@ -116,8 +115,9 @@ class PoloniexBundle(BaseCryptoPricingBundle):
)
raw.set_index('date', inplace=True)
# BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
# on disk, which we compensate here to get the right pricing amounts
# BcolzDailyBarReader introduces a 1/1000 factor in the way
# pricing is stored on disk, which we compensate here to get
# the right pricing amounts
# ref: data/us_equity_pricing.py
scale = 1
raw.loc[:, 'open'] /= scale
@@ -139,7 +139,6 @@ class PoloniexBundle(BaseCryptoPricingBundle):
return self._format_polo_query(query_params)
def _format_data_url(self,
api_key,
symbol,
@@ -162,27 +161,26 @@ class PoloniexBundle(BaseCryptoPricingBundle):
('end', end_date.value / 10**9),
('period', period),
]
return self._format_polo_query(query_params)
def _format_polo_query(self, query_params):
# TODO: got against the exchange object
return 'https://poloniex.com/public?{query}'.format(
query=urlencode(query_params),
)
'''
As a second parameter, you can pass an array of currency pairs
that will be processed as an asset_filter to only process that
'''
As a second parameter, you can pass an array of currency pairs
that will be processed as an asset_filter to only process that
subset of assets in the bundle, such as:
register_bundle(PoloniexBundle, ['USDT_BTC',])
For a production environment make sure to use (to bundle all pairs):
register_bundle(PoloniexBundle)
'''
if 'ingest' in sys.argv and '-c' in sys.argv:
register_bundle(PoloniexBundle)
else:
register_bundle(PoloniexBundle, create_writers=False)
catalyst/data/bundles/quandl.py
+8 -19
View File
@@ -16,7 +16,6 @@
from datetime import datetime
import pandas as pd
from six.moves.urllib.parse import urlencode
from catalyst.data.bundles.core import register_bundle
@@ -26,25 +25,16 @@ from catalyst.utils.memoize import lazyval
"""
Module for building a complete daily dataset from Quandl's WIKI dataset.
"""
from itertools import count
import tarfile
from time import time, sleep
from datetime import datetime
from logbook import Logger
import pandas as pd
from six.moves.urllib.parse import urlencode
from catalyst.utils.calendars import register_calendar_alias
from catalyst.utils.cli import maybe_show_progress
from . import core as bundles
from catalyst.constants import LOG_LEVEL
from catalyst.utils.calendars import register_calendar_alias
log = Logger(__name__, level=LOG_LEVEL)
seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
class QuandlBundle(BaseEquityPricingBundle):
@lazyval
def name(self):
@@ -109,8 +99,8 @@ class QuandlBundle(BaseEquityPricingBundle):
# Filter out invalid symbols
raw = raw[~raw.symbol.isin(self._excluded_symbols)]
# cut out all the other stuff in the name column
# we need to escape the paren because it is actually splitting on a regex
# cut out all the other stuff in the name column. We need to
# escape the paren because it is actually splitting on a regex
raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0)
return raw
@@ -175,7 +165,6 @@ class QuandlBundle(BaseEquityPricingBundle):
df['sid'] = asset_id
self.splits.append(df)
def _update_dividends(self, asset_id, raw_data):
divs = raw_data.ex_dividend
df = pd.DataFrame({'amount': divs[divs != 0]})
@@ -186,7 +175,6 @@ class QuandlBundle(BaseEquityPricingBundle):
df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
self.dividends.append(df)
def _format_metadata_url(self, api_key, page_number):
"""Build the query RL for the quandl WIKI metadata.
"""
@@ -200,10 +188,10 @@ class QuandlBundle(BaseEquityPricingBundle):
query_params = [('api_key', api_key)] + query_params
return (
'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
'https://www.quandl.com/api/v3/datasets.csv?'
+ urlencode(query_params)
)
def _format_wiki_url(self,
api_key,
symbol,
@@ -229,5 +217,6 @@ class QuandlBundle(BaseEquityPricingBundle):
)
)
register_calendar_alias('QUANDL', 'NYSE')
register_bundle(QuandlBundle)
catalyst/data/data_portal.py
+6 -6
View File
@@ -656,11 +656,11 @@ class DataPortal(object):
return spot_value
def _get_minutely_spot_value(self,
asset,
column,
dt,
data_frequency,
ffill=False):
asset,
column,
dt,
data_frequency,
ffill=False):
reader = self._get_pricing_reader(data_frequency)
@@ -706,7 +706,7 @@ class DataPortal(object):
asset,
column,
dt,
ffill,
ffill,
'minute',
)
catalyst/data/dispatch_bar_reader.py
+2
View File
@@ -133,11 +133,13 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
return results
class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
def _dt_window_size(self, start_dt, end_dt):
return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
class AssetDispatchSessionBarReader(AssetDispatchBarReader):
def _dt_window_size(self, start_dt, end_dt):
catalyst/data/loader.py
+23 -80
View File
@@ -12,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
import os
from collections import OrderedDict
@@ -129,11 +128,13 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
# before this date.
'''
if(bundle_data):
# If we are using the bundle to retrieve the cryptobenchmark, find the last
# date for which there is trading data in the bundle
asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
# If we are using the bundle to retrieve the cryptobenchmark, find
# the last date for which there is trading data in the bundle
asset = bundle_data.asset_finder.lookup_symbol(
symbol=bm_symbol,as_of_date=None)
ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
last_date = pd.to_datetime(
bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
else:
last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
'''
@@ -142,8 +143,10 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
if exchange is None:
# This is exceptional, since placing the import at the module scope
# breaks things and it's only needed here
from catalyst.exchange.poloniex.poloniex import Poloniex
exchange = Poloniex('', '', '')
from catalyst.exchange.factory import get_exchange
exchange = get_exchange(
exchange_name='poloniex', base_currency='usdt'
)
benchmark_asset = exchange.get_asset(bm_symbol)
@@ -162,8 +165,8 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
br.loc[start_dt] = 0
br = br.sort_index()
# Override first_date for treasury data since we have it for many more years
# and is independent of crypto data
# Override first_date for treasury data since we have it for many more
# years and is independent of crypto data
first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC')
tc = ensure_treasury_data(
bm_symbol,
@@ -299,14 +302,14 @@ def ensure_crypto_benchmark_data(symbol,
if (bundle == 'poloniex'):
'''
If we're using the Poloniex bundle, we'll get the benchmark from the bundle
instead of downloading it from Poloniex every time we need it.
Poloniex has a captcha for API queries originating from outside the US that
prevents users abroad from getting Catalyst to work
If we're using the Poloniex bundle, we'll get the benchmark from the
bundle instead of downloading it from Poloniex every time we need it.
Poloniex has a captcha for API queries originating from outside the US
that prevents users abroad from getting Catalyst to work
'''
logger.info(
(
'Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
('Retrieving benchmark data from bundle for {symbol!r}'
' from {first_date} to {last_date}'),
symbol=symbol, first_date=first_date, last_date=last_date)
asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,
@@ -328,11 +331,12 @@ def ensure_crypto_benchmark_data(symbol,
last_date)]
else:
# This is how it used to be: downloading the benchmark everytime.
# Leaving this code here to be repurposed in the future for other bundles.
# This is how it used to be: downloading the benchmark everytime.
# Leaving this code here to be repurposed in the future for
# other bundles.
logger.info(
(
'Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
('Downloading benchmark data for {symbol!r}'
' from {first_date} to {last_date}'),
symbol=symbol, first_date=first_date, last_date=last_date)
raise DeprecationWarning('poloniex bundle deprecated')
@@ -429,67 +433,6 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
return data
def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
environ=None):
"""
Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
Parameters
----------
symbol : str
The symbol for the benchmark to load.
first_date : pd.Timestamp
First required date for the cache.
last_date : pd.Timestamp
Last required date for the cache.
now : pd.Timestamp
The current time. This is used to prevent repeated attempts to
re-download data that isn't available due to scheduling quirks or other
failures.
trading_day : pd.CustomBusinessDay
A trading day delta. Used to find the day before first_date so we can
get the close of the day prior to first_date.
We attempt to download data unless we already have data stored at the data
cache for `symbol` whose first entry is before or on `first_date` and whose
last entry is on or after `last_date`.
If we perform a download and the cache criteria are not satisfied, we wait
at least one hour before attempting a redownload. This is determined by
comparing the current time to the result of os.path.getmtime on the cache
path.
"""
filename = get_benchmark_filename(symbol)
data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
environ)
if data is not None:
return data
# If no cached data was found or it was missing any dates then download the
# necessary data.
logger.info(
('Downloading benchmark data for {symbol!r} '
'from {first_date} to {last_date}'),
symbol=symbol,
first_date=first_date - trading_day,
last_date=last_date
)
try:
data = get_benchmark_returns(
symbol,
first_date - trading_day,
last_date,
)
data.to_csv(get_data_filepath(filename, environ))
except (OSError, IOError, HTTPError):
logger.exception('Failed to cache the new benchmark returns')
raise
if not has_data_for_dates(data, first_date, last_date):
logger.warn("Still don't have expected data after redownload!")
return data
def ensure_treasury_data(symbol, first_date, last_date, now, environ=None):
"""
Ensure we have treasury data from treasury module associated with
catalyst/data/minute_bars.py
+8 -11
View File
@@ -341,12 +341,10 @@ class BcolzMinuteBarMetadata(object):
'end_session': str(self.end_session.date()),
# Write these values for backwards compatibility
'first_trading_day': str(self.start_session.date()),
'market_opens': (
market_opens.values.astype('datetime64[m]').
astype(np.int64).tolist()),
'market_closes': (
market_closes.values.astype('datetime64[m]').
astype(np.int64).tolist()),
'market_opens': (market_opens.values.astype('datetime64[m]').
astype(np.int64).tolist()),
'market_closes': (market_closes.values.astype('datetime64[m]').
astype(np.int64).tolist()),
}
with open(self.metadata_path(rootdir), 'w+') as fp:
json.dump(metadata, fp)
@@ -1256,8 +1254,8 @@ class BcolzMinuteBarReader(MinuteBarReader):
values = carray[start_idx:end_idx + 1]
if indices_to_exclude is not None:
for excl_start, excl_stop in indices_to_exclude[::-1]:
excl_slice = np.s_[
excl_start - start_idx:excl_stop - start_idx + 1]
excl_slice = np.s_[excl_start - start_idx:excl_stop
- start_idx + 1]
values = np.delete(values, excl_slice)
where = values != 0
@@ -1320,9 +1318,8 @@ class H5MinuteBarUpdateWriter(object):
def __init__(self, path, complevel=None, complib=None):
self._complevel = complevel if complevel \
is not None else self._COMPLEVEL
self._complib = complib if complib \
is not None else self._COMPLIB
is not None else self._COMPLEVEL
self._complib = complib if complib is not None else self._COMPLIB
self._path = path
def write(self, frames):
catalyst/data/us_equity_pricing.py
+11 -7
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import division # Python2 req to have division of ints yield float
from __future__ import division # Python2 req for division of ints yield float
from errno import ENOENT
from functools import partial
@@ -120,7 +120,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
UINT32_MAX = iinfo(uint32).max
UINT64_MAX = iinfo(uint64).max
PRICE_ADJUSTMENT_FACTOR = 1000000000 # Provides 9 decimals resolution. Also affects _equities.pyx L220
# Provides 9 decimals resolution. Also affects _equities.pyx L220
PRICE_ADJUSTMENT_FACTOR = 1000000000
def check_uint32_safe(value, colname):
@@ -130,6 +131,7 @@ def check_uint32_safe(value, colname):
"for uint32" % (value, colname)
)
def check_uint64_safe(value, colname):
if value >= UINT64_MAX:
raise ValueError(
@@ -322,8 +324,8 @@ class BcolzDailyBarWriter(object):
# Maps column name -> output carray.
columns = {
k: carray(array([], dtype=uint64))
if k in OHLCV
else carray(array([], dtype=uint32))
if k in OHLCV
else carray(array([], dtype=uint32))
for k in US_EQUITY_PRICING_BCOLZ_COLUMNS
}
@@ -439,11 +441,13 @@ class BcolzDailyBarWriter(object):
return raw_data
winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
processed = (raw_data[list(OHLC)]
* PRICE_ADJUSTMENT_FACTOR).astype('uint64')
dates = raw_data.index.values.astype('datetime64[s]')
check_uint32_safe(dates.max().view(np.int64), 'day')
processed['day'] = dates.astype('uint32')
processed['volume'] = (raw_data.volume * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
processed['volume'] = (raw_data.volume
* PRICE_ADJUSTMENT_FACTOR).astype('uint64')
return ctable.fromdataframe(processed)
@@ -496,7 +500,7 @@ class BcolzDailyBarReader(SessionBarReader):
The data in these columns is interpreted as follows:
- Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
- Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
as 10^9 * as-traded dollar value.
- Day is interpreted as seconds since midnight UTC, Jan 1, 1970.
- Id is the asset id of the row.
catalyst/examples/arbitrage_with_interface.py
+28 -21
View File
@@ -83,15 +83,15 @@ def place_orders(context, amount, buying_price, selling_price, action):
else:
raise ValueError('invalid order action')
base_currency = enter_exchange.base_currency
base_currency_amount = enter_exchange.portfolio.cash
quote_currency = enter_exchange.quote_currency
quote_currency_amount = enter_exchange.portfolio.cash
exit_balances = exit_exchange.get_balances()
exit_currency = context.trading_pairs[
context.selling_exchange].market_currency
context.selling_exchange].quote_currency
if exit_currency in exit_balances:
market_currency_amount = exit_balances[exit_currency]
quote_currency_amount = exit_balances[exit_currency]
else:
log.warn(
'the selling exchange {exchange_name} does not hold '
@@ -102,25 +102,25 @@ def place_orders(context, amount, buying_price, selling_price, action):
)
return
if base_currency_amount < (amount * entry_price):
adj_amount = base_currency_amount / entry_price
if quote_currency_amount < (amount * entry_price):
adj_amount = quote_currency_amount / entry_price
log.warn(
'not enough {base_currency} ({base_currency_amount}) to buy '
'not enough {quote_currency} ({quote_currency_amount}) to buy '
'{amount}, adjusting the amount to {adj_amount}'.format(
base_currency=base_currency,
base_currency_amount=base_currency_amount,
quote_currency=quote_currency,
quote_currency_amount=quote_currency_amount,
amount=amount,
adj_amount=adj_amount
)
)
amount = adj_amount
elif market_currency_amount < amount:
elif quote_currency_amount < amount:
log.warn(
'not enough {currency} ({currency_amount}) to sell '
'{amount}, aborting'.format(
currency=exit_currency,
currency_amount=market_currency_amount,
currency_amount=quote_currency_amount,
amount=amount
)
)
@@ -263,13 +263,20 @@ def analyze(context, stats):
pass
run_algorithm(
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex,bitfinex',
live=True,
algo_namespace=algo_namespace,
base_currency='btc',
live_graph=False
)
if __name__ == '__main__':
# The execution mode: backtest or live
MODE = 'live'
if MODE == 'live':
run_algorithm(
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex,bitfinex',
live=True,
algo_namespace=algo_namespace,
base_currency='btc',
live_graph=False,
simulate_orders=True,
stats_output=None,
)
catalyst/examples/buy_and_hodl.py
+1 -2
View File
@@ -19,7 +19,7 @@ import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order_target_value, symbol, record,
cancel_order, get_open_orders, )
cancel_order, get_open_orders, )
def initialize(context):
@@ -61,7 +61,6 @@ def handle_data(context, data):
context.asset,
target_hodl_value,
limit_price=price * 1.1,
stop_price=price * 0.9,
)
record(
catalyst/examples/buy_btc_simple.py
+35 -16
View File
@@ -1,30 +1,49 @@
'''
This is a very simple example referenced in the beginner's tutorial:
https://enigmampc.github.io/catalyst/beginner-tutorial.html
This is a very simple example referenced in the beginner's tutorial:
https://enigmampc.github.io/catalyst/beginner-tutorial.html
Run this example, by executing the following from your terminal:
catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
Run this example, by executing the following from your terminal:
catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \
--end 2017-9-30 -o buy_btc_simple_out.pickle
If you want to run this code using another exchange, make sure that
the asset is available on that exchange. For example, if you were to run
it for exchange Poloniex, you would need to edit the following line:
If you want to run this code using another exchange, make sure that
the asset is available on that exchange. For example, if you were to run
it for exchange Poloniex, you would need to edit the following line:
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
and specify exchange poloniex as follows:
catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
and specify exchange poloniex as follows:
catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \
--end 2017-9-30 -o buy_btc_simple_out.pickle
To see which assets are available on each exchange, visit:
https://www.enigma.co/catalyst/status
To see which assets are available on each exchange, visit:
https://www.enigma.co/catalyst/status
'''
from catalyst import run_algorithm
from catalyst.api import order, record, symbol
import pandas as pd
def initialize(context):
context.asset = symbol('btc_usd')
def handle_data(context, data):
order(context.asset, 1)
record(btc = data.current(context.asset, 'price'))
record(btc=data.current(context.asset, 'price'))
if __name__ == '__main__':
run_algorithm(
capital_base=10000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
exchange_name='bitfinex',
algo_namespace='buy_and_hodl',
base_currency='usd',
start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True),
)
catalyst/examples/buy_low_sell_high.py
+26 -8
View File
@@ -1,17 +1,19 @@
'''
This algorithm requires an additional library (ta-lib) beyond those required by catalyst.
Install it first by running:
This algorithm requires an additional library (ta-lib) beyond those
required by catalyst. Install it first by running:
$ pip install TA-Lib
If you get build errors like "fatal error: ta-lib/ta_libc.h: No such file or directory"
it typically means that it can't find the underlying TA-Lib library and needs to be installed.
See https://mrjbq7.github.io/ta-lib/install.html for instructions on how to install
the required dependencies.
If you get build errors like:
"fatal error: ta-lib/ta_libc.h: No such file or directory"
it typically means that it can't find the underlying TA-Lib library and it
needs to be installed. See https://mrjbq7.github.io/ta-lib/install.html for
instructions on how to install the required dependencies.
'''
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import (
order,
order_target_percent,
@@ -20,6 +22,7 @@ from catalyst.api import (
get_open_orders,
)
from catalyst.exchange.stats_utils import get_pretty_stats
import pandas as pd
algo_namespace = 'buy_low_sell_high_xrp'
log = Logger(algo_namespace)
@@ -100,8 +103,8 @@ def _handle_data(context, data):
if price < cost_basis:
is_buy = True
elif position.amount > 0 and \
price > cost_basis * (1 + context.PROFIT_TARGET):
elif (position.amount > 0
and price > cost_basis * (1 + context.PROFIT_TARGET)):
profit = (price * position.amount) - (cost_basis * position.amount)
log.info('closing position, taking profit: {}'.format(profit))
order_target_percent(
@@ -156,3 +159,18 @@ def handle_data(context, data):
def analyze(context, stats):
log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
pass
if __name__ == '__main__':
run_algorithm(
capital_base=10000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace='buy_and_hodl',
base_currency='usd',
start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True),
)
catalyst/examples/buy_low_sell_high_live.py
+15 -23
View File
@@ -41,7 +41,7 @@ def _handle_data(context, data):
context.asset,
fields='price',
bar_count=20,
frequency='1d'
frequency='1D'
)
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
log.info('got rsi: {}'.format(rsi))
@@ -88,8 +88,8 @@ def _handle_data(context, data):
if price < cost_basis:
is_buy = True
elif position.amount > 0 and \
price > cost_basis * (1 + context.PROFIT_TARGET):
elif (position.amount > 0
and price > cost_basis * (1 + context.PROFIT_TARGET)):
profit = (price * position.amount) - (cost_basis * position.amount)
log.info('closing position, taking profit: {}'.format(profit))
order_target_percent(
@@ -146,23 +146,15 @@ def analyze(context, stats):
pass
run_algorithm(
capital_base=100000,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
start=pd.to_datetime('2017-5-01', utc=True),
end=pd.to_datetime('2017-10-16', utc=True),
base_currency='usdt',
data_frequency='daily'
)
# run_algorithm(
# initialize=initialize,
# handle_data=handle_data,
# analyze=analyze,
# exchange_name='poloniex',
# live=True,
# algo_namespace=algo_namespace,
# base_currency='btc'
# )
if __name__ == '__main__':
run_algorithm(
capital_base=0.001,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='binance',
live=True,
algo_namespace=algo_namespace,
base_currency='btc',
simulate_orders=True,
)
catalyst/examples/dual_moving_average.py
+24 -15
View File
@@ -4,13 +4,14 @@ from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order, record, symbol, order_target_percent,
get_open_orders)
from catalyst.api import (record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
def initialize(context):
context.i = 0
context.asset = symbol('ltc_usd')
@@ -25,16 +26,22 @@ def handle_data(context, data):
# Skip as many bars as long_window to properly compute the average
context.i += 1
if context.i < long_window:
return
return
# Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_mavg = data.history(context.asset, 'price',
bar_count=short_window, frequency="1m").mean()
long_mavg = data.history(context.asset, 'price',
bar_count=long_window, frequency="1m").mean()
short_mavg = data.history(context.asset,
'price',
bar_count=short_window,
frequency="1m",
).mean()
long_mavg = data.history(context.asset,
'price',
bar_count=long_window,
frequency="1m",
).mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
@@ -67,11 +74,11 @@ def handle_data(context, data):
# Trading logic
if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
def analyze(context, perf):
@@ -89,11 +96,13 @@ def analyze(context, perf):
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot(
ax=ax2,
label='Price')
ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format(
asset = context.asset.symbol,
base = base_currency
asset=context.asset.symbol,
base=base_currency
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
@@ -150,4 +159,4 @@ if __name__ == '__main__':
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
)
catalyst/examples/dual_vwap.py
-196
View File
@@ -1,196 +0,0 @@
''' Catalyst currently does not support the Pipeline implementation
from Zipline, see Issue #96:
https://github.com/enigmampc/catalyst/issues/96
Until the above issue is resolved, this example is non-functional.
We are keeping this script here for when the issue is resolved
'''
#!/usr/bin/env python
#
# Copyright 2017 Enigma MPC, Inc.
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from catalyst.api import (
order_target_percent,
record,
symbol,
get_open_orders,
set_max_leverage,
schedule_function,
date_rules,
attach_pipeline,
pipeline_output,
)
from catalyst.pipeline import Pipeline
from catalyst.pipeline.data import CryptoPricing
from catalyst.pipeline.factors.crypto import VWAP
def initialize(context):
context.ASSET_NAME = 'USDT_BTC'
context.TARGET_INVESTMENT_RATIO = 0.8
context.SHORT_WINDOW = 30
context.LONG_WINDOW = 100
# For all trading pairs in the poloniex bundle, the default denomination
# currently supported by Catalyst is 1/1000th of a full coin. Use this
# constant to scale the price of up to that of a full coin if desired.
context.TICK_SIZE = 1000.0
context.i = 0
context.asset = symbol(context.ASSET_NAME)
set_max_leverage(1.0)
attach_pipeline(make_pipeline(context), 'vwap_pipeline')
schedule_function(
rebalance,
time_rules=times_rules.every_minute(),
)
def before_trading_start(context, data):
context.pipeline_data = pipeline_output('vwap_pipeline')
def make_pipeline(context):
return Pipeline(
columns={
'price': CryptoPricing.open.latest,
'volume': CryptoPricing.volume.latest,
'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
'long_mavg': VWAP(window_length=context.LONG_WINDOW),
}
)
def rebalance(context, data):
context.i += 1
# skip first LONG_WINDOW bars to fill windows
if context.i < context.LONG_WINDOW:
return
# get pipeline data for asset of interest
pipeline_data = context.pipeline_data
pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]
# retrieve long and short moving averages from pipeline
short_mavg = pipeline_data.short_mavg
long_mavg = pipeline_data.long_mavg
price = pipeline_data.price
volume = pipeline_data.volume
# check that order has not already been placed
open_orders = get_open_orders()
if context.asset not in open_orders:
# check that the asset of interest can currently be traded
if data.can_trade(context.asset):
# adjust portfolio based on comparison of long and short vwap
if short_mavg > long_mavg:
order_target_percent(
context.asset,
context.TARGET_INVESTMENT_RATIO,
)
elif short_mavg < long_mavg:
order_target_percent(
context.asset,
0.0,
)
record(
price=price,
cash=context.portfolio.cash,
leverage=context.account.leverage,
short_mavg=short_mavg,
long_mavg=long_mavg,
volume=volume,
)
def analyze(context=None, results=None):
import matplotlib.pyplot as plt
# Plot the portfolio and asset data.
ax1 = plt.subplot(611)
results[['portfolio_value']].plot(ax=ax1)
ax1.set_ylabel('Portfolio value (USD)')
ax2 = plt.subplot(612, sharex=ax1)
ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
(context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
trans = results.ix[[t != [] for t in results.transactions]]
amounts = [t[0]['amount'] for t in trans.transactions]
buys = trans.ix[
[t[0]['amount'] > 0 for t in trans.transactions]
]
sells = trans.ix[
[t[0]['amount'] < 0 for t in trans.transactions]
]
ax2.plot(
buys.index,
context.TICK_SIZE * results.price[buys.index],
'^',
markersize=10,
color='g',
)
ax2.plot(
sells.index,
context.TICK_SIZE * results.price[sells.index],
'v',
markersize=10,
color='r',
)
ax3 = plt.subplot(613, sharex=ax1)
results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
ax3.set_ylabel('Leverage (USD)')
ax4 = plt.subplot(614, sharex=ax1)
results[['cash']].plot(ax=ax4)
ax4.set_ylabel('Cash (USD)')
results[[
'treasury',
'algorithm',
'benchmark',
]] = results[[
'treasury_period_return',
'algorithm_period_return',
'benchmark_period_return',
]]
ax5 = plt.subplot(615, sharex=ax1)
results[[
'treasury',
'algorithm',
'benchmark',
]].plot(ax=ax5)
ax5.set_ylabel('Percent Change')
ax6 = plt.subplot(616, sharex=ax1)
results[['volume']].plot(ax=ax6)
ax6.set_ylabel('Volume (mBTC/day)')
plt.legend(loc=3)
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
catalyst/examples/mean_reversion_simple.py
+34 -24
View File
@@ -1,4 +1,4 @@
# For this example, we're going to write a simple momentum script. When the
# For this example, we're going to write a simple momentum script. When the
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
# going to sell. Hopefully we'll ride the waves.
import os
@@ -13,6 +13,7 @@ from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
@@ -32,17 +33,20 @@ def initialize(context):
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in USD.
context.neo_eth = symbol('neo_usd')
# In our example, we're looking at Neo in Ether.
context.market = symbol('neo_eth')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 30
context.RSI_OVERBOUGHT = 80
context.CANDLE_SIZE = '15T'
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
# context.set_commission(maker=0.1, taker=0.2)
context.set_slippage(spread=0.0001)
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
@@ -59,14 +63,14 @@ def handle_data(context, data):
context.current_day = today
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.neo_eth variable. For this example, we're
# defined above, in the context.market variable. For this example, we're
# using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention
# for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
context.neo_eth,
context.market,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
@@ -81,7 +85,7 @@ def handle_data(context, data):
# We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(context.neo_eth, fields=['close', 'volume'])
current = data.current(context.market, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
@@ -95,34 +99,36 @@ def handle_data(context, data):
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
price=price,
volume=current['volume'],
price=price,
price_change=price_change,
rsi=rsi[-1],
cash=cash
)
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if context.traded_today:
return
# TODO: retest with open orders
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.neo_eth)
orders = get_open_orders(context.market)
if len(orders) > 0:
log.info('exiting because orders are open: {}'.format(orders))
return
# Exit if we cannot trade
if not data.can_trade(context.neo_eth):
if not data.can_trade(context.market):
return
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.neo_eth].amount
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.market].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
@@ -133,7 +139,7 @@ def handle_data(context, data):
# Set a style for limit orders,
limit_price = price * 1.005
order_target_percent(
context.neo_eth, 1, limit_price=limit_price
context.market, 1, limit_price=limit_price
)
context.traded_today = True
@@ -145,7 +151,7 @@ def handle_data(context, data):
)
limit_price = price * 0.995
order_target_percent(
context.neo_eth, 0, limit_price=limit_price
context.market, 0, limit_price=limit_price
)
context.traded_today = True
@@ -168,7 +174,7 @@ def analyze(context=None, perf=None):
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.neo_eth.symbol, base=base_currency
asset=context.market.symbol, base=base_currency
))
transaction_df = extract_transactions(perf)
@@ -229,7 +235,7 @@ def analyze(context=None, perf=None):
)
plt.legend(loc=3)
start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end/5))
ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
# Show the plot.
plt.gcf().set_size_inches(18, 8)
@@ -249,16 +255,18 @@ if __name__ == '__main__':
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=10000,
capital_base=0.1,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
@@ -267,13 +275,15 @@ if __name__ == '__main__':
elif MODE == 'live':
run_algorithm(
capital_base=0.5,
capital_base=0.05,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
exchange_name='binance',
live=True,
algo_namespace=NAMESPACE,
base_currency='usd',
live_graph=False
base_currency='eth',
live_graph=False,
simulate_orders=True,
stats_output=None
)
catalyst/examples/portfolio_optimization.py
+121 -105
View File
@@ -1,7 +1,7 @@
'''Use this code to execute a portfolio optimization model. This code
will select the portfolio with the maximum Sharpe Ratio. The parameters
'''Use this code to execute a portfolio optimization model. This code
will select the portfolio with the maximum Sharpe Ratio. The parameters
are set to use 180 days of historical data and rebalance every 30 days.
This is the code used in the following article:
https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556
@@ -15,119 +15,135 @@ import os
import pytz
import numpy as np
import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import datetime
from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.api import record, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True)
def initialize(context):
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i%context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n+1, frequency='1d')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n+1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val/tminus_val-1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr),xr)/n
# Compute asset correlation matrix (informative only)
corr_m = cov_m/np.dot(np.transpose(stds),stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3+context.nassets,n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w,np.transpose(m)))*365
p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
#store results in results array
results_array[0,p] = p_r
results_array[1,p] = p_std
#store Sharpe Ratio (return / volatility) - risk free rate element
#excluded for simplicity
results_array[2,p] = results_array[0,p] / results_array[1,p]
i = 0
for iw in weights:
results_array[3+i,p] = weights[i]
i += 1
#convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r','stdev','sharpe']+context.assets)
#locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
#locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
#order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
#create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
#plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
#plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
context.i += 1
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i % context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n + 1, frequency='1d')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n + 1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val / tminus_val - 1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr), xr) / n
# Compute asset correlation matrix (informative only)
corr_m = cov_m / np.dot(np.transpose(stds), stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3 + context.nassets, n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w, np.transpose(m))) * 365
p_std = np.sqrt(np.dot(np.dot(w, cov_m),
np.transpose(w))) * np.sqrt(365)
# store results in results array
results_array[0, p] = p_r
results_array[1, p] = p_std
# store Sharpe Ratio (return / volatility) - risk free rate element
# excluded for simplicity
results_array[2, p] = results_array[0, p] / results_array[1, p]
i = 0
for iw in weights:
results_array[3 + i, p] = weights[i]
i += 1
# convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r', 'stdev', 'sharpe']
+ context.assets)
# locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
# locate positon of portfolio with minimum standard deviation
# min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
# order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
# create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,
results_frame.r,
c=results_frame.sharpe,
cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
# plot red star to highlight position of portfolio
# with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],
max_sharpe_port[0],
marker='o',
color='b',
s=200)
# plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,
r=r,
m=m,
stds=stds,
max_sharpe_port=max_sharpe_port,
corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None):
# Form DataFrame with selected data
data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Form DataFrame with selected data
data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
'portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
if __name__ == '__main__':
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
catalyst/examples/rsi_profit_target.py
+19 -30
View File
@@ -11,7 +11,6 @@ from catalyst.api import (
record,
get_open_orders,
)
from catalyst.exchange.stats_utils import crossover, crossunder
from catalyst.utils.run_algo import run_algorithm
algo_namespace = 'rsi'
@@ -55,7 +54,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
stop=None
)
action = None
# action = None
if context.position is not None:
cost_basis = context.position['cost_basis']
amount = context.position['amount']
@@ -80,7 +79,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
amount=-amount,
limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
)
action = 0
# action = 0
context.position = None
else:
@@ -97,7 +96,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
amount=buy_amount,
stop=None
)
action = 0
# action = 0
def _handle_data_rsi_only(context, data):
@@ -115,7 +114,7 @@ def _handle_data_rsi_only(context, data):
prices = data.history(
context.asset,
fields='price',
bar_count=17,
bar_count=20,
frequency='30T'
)
except Exception as e:
@@ -157,7 +156,7 @@ def handle_data(context, data):
dt = data.current_dt
if context.last_bar is None or (
context.last_bar + timedelta(minutes=15)) <= dt:
context.last_bar + timedelta(minutes=15)) <= dt:
context.last_bar = dt
else:
return
@@ -250,27 +249,17 @@ def analyze(context=None, results=None):
pass
# run_algorithm(
# initialize=initialize,
# handle_data=handle_data,
# analyze=analyze,
# exchange_name='bittrex',
# live=True,
# algo_namespace=algo_namespace,
# base_currency='btc',
# live_graph=False
# )
# Backtest
run_algorithm(
capital_base=0.5,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace=algo_namespace,
base_currency='btc',
start=pd.to_datetime('2017-9-1', utc=True),
end=pd.to_datetime('2017-10-1', utc=True),
)
if __name__ == '__main__':
# Backtest
run_algorithm(
capital_base=0.5,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace=algo_namespace,
base_currency='btc',
start=pd.to_datetime('2017-9-1', utc=True),
end=pd.to_datetime('2017-10-1', utc=True),
)
catalyst/examples/simple_loop.py
+25 -33
View File
@@ -9,7 +9,7 @@ from catalyst.exchange.stats_utils import get_pretty_stats, \
def initialize(context):
print('initializing')
context.asset = symbol('neo_usd')
context.asset = symbol('eth_btc')
context.base_price = None
@@ -19,17 +19,17 @@ def handle_data(context, data):
price = data.current(context.asset, 'close')
print('got price {price}'.format(price=price))
try:
prices = data.history(
context.asset,
fields='price',
bar_count=14,
frequency='15T'
)
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
print('got rsi: {}'.format(rsi))
except Exception as e:
print(e)
prices = data.history(
context.asset,
fields='price',
bar_count=20,
frequency='30T'
)
last_traded = prices.index[-1]
print('last candle date: {}'.format(last_traded))
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
print('got rsi: {}'.format(rsi))
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
@@ -110,24 +110,16 @@ def analyze(context, perf):
pass
run_algorithm(
capital_base=250,
start=pd.to_datetime('2017-11-1 0:00', utc=True),
end=pd.to_datetime('2017-11-10 23:59', utc=True),
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace='simple_loop',
base_currency='usd'
)
# run_algorithm(
# initialize=initialize,
# handle_data=handle_data,
# analyze=None,
# exchange_name='poloniex',
# live=True,
# algo_namespace='simple_loop',
# base_currency='eth',
# live_graph=False
if __name__ == '__main__':
run_algorithm(
capital_base=1,
initialize=initialize,
handle_data=handle_data,
analyze=None,
exchange_name='poloniex',
live=True,
algo_namespace='simple_loop',
base_currency='eth',
live_graph=False,
simulate_orders=True
)
catalyst/examples/simple_universe.py
+102 -60
View File
@@ -2,73 +2,117 @@
Requires Catalyst version 0.3.0 or above
Tested on Catalyst version 0.3.3
These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
You simply need to specify the exchange and the market that you want to focus on.
You will all see how to create a universe and filter it base on the exchange and the market you desire.
This example aims to provide an easy way for users to learn how to
collect data from any given exchange and select a subset of the available
currency pairs for trading. You simply need to specify the exchange and
the market (base_currency) that you want to focus on. You will then see
how to create a universe of assets, and filter it based the market you
desire.
The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
Use this as the backbone to create your own trading strategies.
The example prints out the closing price of all the pairs for a given
market in a given exchange every 30 minutes. The example also contains
the OHLCV data with minute-resolution for the past seven days which
could be used to create indicators. Use this code as the backbone to
create your own trading strategy.
The lookback_date variable is used to ensure data for a coin existed on
the lookback period specified.
To run, execute the following two commands in a terminal (inside catalyst
environment). The first one retrieves all the pricing data needed for this
script to run (only needs to be run once), and the second one executes this
script with the parameters specified in the run_algorithm() call at the end
of the file:
catalyst ingest-exchange -x bitfinex -f minute
python simple_universe.py
Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
"""
from datetime import timedelta
import numpy as np
import pandas as pd
from datetime import timedelta
from catalyst import run_algorithm
from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.api import (
symbols,
)
from catalyst.api import (symbols, )
def initialize(context):
context.i = -1 # counts the minutes
context.exchange = context.exchanges.values()[0].name.lower() # exchange name
context.base_currency = context.exchanges.values()[0].base_currency.lower() # market base currency
context.i = -1 # minute counter
context.exchange = context.exchanges.values()[0].name.lower()
context.base_currency = context.exchanges.values()[0].base_currency.lower()
def handle_data(context, data):
context.i += 1
lookback_days = 7 # 7 days
# current date formatted into a string
today = data.current_dt
date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = today - timedelta(days=lookback_days) # subtract the amount of days specified in lookback
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] # get only the date as a string
# current date & time in each iteration formatted into a string
now = data.current_dt
date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = now - timedelta(days=lookback_days)
# keep only the date as a string, discard the time
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
# update universe everyday
new_day = 60 * 24 # assuming data_frequency='minute'
if not context.i % new_day:
one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute'
# update universe everyday at midnight
if not context.i % one_day_in_minutes:
context.universe = universe(context, lookback_date, date)
# get data every 30 minutes
minutes = 30
one_day_in_minutes = 1440 # 1440 assumes data_frequency='minute'
lookback = one_day_in_minutes / minutes * lookback_days # get N lookback_days of history data
if not ((context.i % minutes) - minutes + 1) and context.universe: # fetch data at last minute of the candle
# get lookback_days of history data: that is 'lookback' number of bins
lookback = one_day_in_minutes / minutes * lookback_days
if not context.i % minutes and context.universe:
# we iterate for every pair in the current universe
for coin in context.coins:
pair = str(coin.symbol)
# 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
# 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
opened = fill(data.history(coin, 'open', bar_count=lookback, frequency='30T')).values
high = fill(data.history(coin, 'high', bar_count=lookback, frequency='30T')).values
low = fill(data.history(coin, 'low', bar_count=lookback, frequency='30T')).values
close = fill(data.history(coin, 'price', bar_count=lookback, frequency='30T')).values
volume = fill(data.history(coin, 'volume', bar_count=lookback, frequency='30T')).values
# Get 30 minute interval OHLCV data. This is the standard data
# required for candlestick or indicators/signals. Return Pandas
# DataFrames. 30T means 30-minute re-sampling of one minute data.
# Adjust it to your desired time interval as needed.
opened = fill(data.history(coin,
'open',
bar_count=lookback,
frequency='30T')).values
high = fill(data.history(coin,
'high',
bar_count=lookback,
frequency='30T')).values
low = fill(data.history(coin,
'low',
bar_count=lookback,
frequency='30T')).values
close = fill(data.history(coin,
'price',
bar_count=lookback,
frequency='30T')).values
volume = fill(data.history(coin,
'volume',
bar_count=lookback,
frequency='30T')).values
# close[-1] is the equivalent to current price
# close[-1] is the last value in the set, which is the equivalent
# to current price (as in the most recent value)
# displays the minute price for each pair every 30 minutes
print(today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},'
'\tV:{v}'.format(
now=now,
pair=pair,
o=opened[-1],
h=high[-1],
l=low[-1],
c=close[-1],
v=volume[-1],
))
# ----------------------------------------------------------------------------------------------------------
# -------------------------------------- Insert Your Strategy Here -----------------------------------------
# ----------------------------------------------------------------------------------------------------------
# -------------------------------------------------------------
# --------------- Insert Your Strategy Here -------------------
# -------------------------------------------------------------
def analyze(context=None, results=None):
@@ -78,23 +122,24 @@ def analyze(context=None, results=None):
# Get the universe for a given exchange and a given base_currency market
# Example: Poloniex BTC Market
def universe(context, lookback_date, current_date):
json_symbols = get_exchange_symbols(context.exchange) # get all the pairs for the exchange
universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) # convert into a dataframe
universe_df['base_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[1],
axis=1)
universe_df['market_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[0],
axis=1)
# get all the pairs for the given exchange
json_symbols = get_exchange_symbols(context.exchange)
# convert into a DataFrame for easier processing
df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],
axis=1)
df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],
axis=1)
# Filter all the exchange pairs to only the ones for a give base currency
universe_df = universe_df[universe_df['base_currency'] == context.base_currency]
# Filter all the pairs to get only the ones for a given base_currency
df = df[df['base_currency'] == context.base_currency]
# Filter all the pairs to ensure that pair existed in the current date range
universe_df = universe_df[universe_df.start_date < lookback_date]
universe_df = universe_df[universe_df.end_daily >= current_date]
context.coins = symbols(*universe_df.symbol) # convert all the pairs to symbols
# Filter all pairs to ensure that pair existed in the current date range
df = df[df.start_date < lookback_date]
df = df[df.end_daily >= current_date]
context.coins = symbols(*df.symbol) # convert all the pairs to symbols
# print(universe_df.symbol.tolist())
return universe_df.symbol.tolist()
return df.symbol.tolist()
# Replace all NA, NAN or infinite values with its nearest value
@@ -102,7 +147,9 @@ def fill(series):
if isinstance(series, pd.Series):
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
elif isinstance(series, np.ndarray):
return pd.Series(series).replace([np.inf, -np.inf], np.nan).ffill().bfill().values
return pd.Series(series).replace(
[np.inf, -np.inf], np.nan
).ffill().bfill().values
else:
return series
@@ -112,18 +159,13 @@ if __name__ == '__main__':
end_date = pd.to_datetime('2017-11-13', utc=True)
performance = run_algorithm(start=start_date, end=end_date,
capital_base=100.0, # amount of base_currency, not always in dollars unless usd
capital_base=100.0, # amount of base_currency
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
exchange_name='poloniex',
data_frequency='minute',
base_currency='btc',
live=False,
live_graph=False,
algo_namespace='simple_universe')
"""
Run in Terminal (inside catalyst environment):
python simple_universe.py
"""
catalyst/examples/talib_simple.py
+7 -5
View File
@@ -1,9 +1,11 @@
# Run Command
# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle -f talib_simple.py -x poloniex
#
# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \
# -f talib_simple.py -x poloniex
#
# Description
# Simple TALib Example showing how to use various indicators in you strategy
# Based loosly on https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
# Simple TALib Example showing how to use various indicators
# in you strategy. Based loosly on
# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
import os
@@ -88,7 +90,7 @@ def _handle_data(context, data):
prices.close.as_matrix(), fastperiod=context.MACD_FAST,
slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL)
# Stochastics %K %D
# Stochastics %K %D
# %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100
# %D = 3-day SMA of %K
analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH(
catalyst/exchange/bitfinex/bitfinex.py
+14 -10
View File
@@ -14,6 +14,7 @@ import six
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import (
@@ -29,16 +30,17 @@ from catalyst.protocol import Account
# Trying to account for REST api instability
# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
from catalyst.utils.deprecate import deprecated
requests.adapters.DEFAULT_RETRIES = 20
BITFINEX_URL = 'https://api.bitfinex.com'
from catalyst.constants import LOG_LEVEL
log = Logger('Bitfinex', level=LOG_LEVEL)
warning_logger = Logger('AlgoWarning')
@deprecated
class Bitfinex(Exchange):
def __init__(self, key, secret, base_currency, portfolio=None):
self.url = BITFINEX_URL
@@ -172,7 +174,8 @@ class Bitfinex(Exchange):
executed_price = float(order_status['avg_execution_price'])
# TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
# TODO: bitfinex does not specify comission.
# I could calculate it but not sure if it's worth it.
commission = None
date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
@@ -599,17 +602,17 @@ class Bitfinex(Exchange):
else:
try:
start_date = cached_symbols[symbol]['start_date']
except KeyError as e:
except KeyError:
start_date = time.strftime('%Y-%m-%d')
try:
end_daily = cached_symbols[symbol]['end_daily']
except KeyError as e:
except KeyError:
end_daily = 'N/A'
try:
end_minute = cached_symbols[symbol]['end_minute']
except KeyError as e:
except KeyError:
end_minute = 'N/A'
symbol_map[symbol] = dict(
@@ -660,15 +663,16 @@ class Bitfinex(Exchange):
"""
Query again with daily resolution setting the start and end around
the startmonth we got above. Avoid end dates greater than now: time.time()
the startmonth we got above. Avoid end dates greater than
now: time.time()
"""
url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format(
url = ('{url}/v2/candles/trade:1D:{symbol}/hist?start={start}'
'&end={end}').format(
url=self.url,
symbol=symbol_v2,
start=startmonth - 3600 * 24 * 31 * 1000,
end=min(startmonth + 3600 * 24 * 31 * 1000,
int(time.time() * 1000))
)
int(time.time() * 1000)))
try:
self.ask_request()
catalyst/exchange/bittrex/bittrex.py
+8 -7
View File
@@ -19,12 +19,14 @@ from catalyst.finance.execution import LimitOrder, StopLimitOrder
from catalyst.finance.order import Order, ORDER_STATUS
# TODO: consider using this: https://github.com/mondeja/bittrex_v2
from catalyst.utils.deprecate import deprecated
log = Logger('Bittrex', level=LOG_LEVEL)
URL2 = 'https://bittrex.com/Api/v2.0'
@deprecated
class Bittrex(Exchange):
def __init__(self, key, secret, base_currency, portfolio=None):
self.api = Bittrex_api(key=key, secret=secret)
@@ -262,11 +264,10 @@ class Bittrex(Exchange):
end = int(time.mktime(end_dt.timetuple()))
url = '{url}/pub/market/GetTicks?marketName={symbol}' \
'&tickInterval={frequency}&_={end}'.format(
url=URL2,
symbol=self.get_symbol(asset),
frequency=frequency,
end=end
)
url=URL2,
symbol=self.get_symbol(asset),
frequency=frequency,
end=end, )
try:
data = json.loads(urllib.request.urlopen(url).read().decode())
@@ -359,12 +360,12 @@ class Bittrex(Exchange):
try:
end_daily = cached_symbols[exchange_symbol]['end_daily']
except KeyError as e:
except KeyError:
end_daily = 'N/A'
try:
end_minute = cached_symbols[exchange_symbol]['end_minute']
except KeyError as e:
except KeyError:
end_minute = 'N/A'
symbol_map[exchange_symbol] = dict(
catalyst/exchange/bittrex/extensions-example.py
+1 -1
View File
@@ -4,4 +4,4 @@ from catalyst.exchange.exchange_bundle import exchange_bundle
symbols = (
'neo_btc',
)
register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))
register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))
catalyst/exchange/bundle_utils.py
+8 -9
View File
@@ -6,11 +6,9 @@ from datetime import timedelta, datetime, date
import numpy as np
import pandas as pd
import pytz
from catalyst.assets._assets import TradingPair
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.exchange_utils import get_exchange_bundles_folder, \
get_exchange_symbols
from catalyst.exchange.exchange_utils import get_exchange_bundles_folder
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
API_URL = 'http://data.enigma.co/api/v1'
@@ -80,9 +78,8 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
if not os.path.isdir(path):
url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
'exchange-{exchange}/{name}.tar.gz'.format(
exchange=exchange_name,
name=name
)
exchange=exchange_name,
name=name)
bytes = download_without_progress(url)
with tarfile.open('r', fileobj=bytes) as tar:
@@ -193,8 +190,10 @@ def get_period_label(dt, data_frequency):
str
"""
return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \
else '{}'.format(dt.year)
if data_frequency == 'minute':
return '{}-{:02d}'.format(dt.year, dt.month)
else:
return '{}'.format(dt.year)
def get_month_start_end(dt, first_day=None, last_day=None):
@@ -315,7 +314,7 @@ def range_in_bundle(asset, start_dt, end_dt, reader):
if np.isnan(close):
has_data = False
except Exception as e:
except Exception:
has_data = False
return has_data
catalyst/exchange/ccxt/__init__.py
View File
catalyst/exchange/ccxt/ccxt_exchange.py
+638
View File
@@ -0,0 +1,638 @@
import re
from collections import defaultdict
import ccxt
import pandas as pd
import six
from ccxt import ExchangeNotAvailable, InvalidOrder
from logbook import Logger
from six import string_types
from catalyst.algorithm import MarketOrder
from catalyst.assets._assets import TradingPair
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
ExchangeNotFoundError, CreateOrderError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.exchange_utils import mixin_market_params, \
from_ms_timestamp, get_epoch
from catalyst.finance.order import Order, ORDER_STATUS
log = Logger('CCXT', level=LOG_LEVEL)
SUPPORTED_EXCHANGES = dict(
binance=ccxt.binance,
bitfinex=ccxt.bitfinex,
bittrex=ccxt.bittrex,
poloniex=ccxt.poloniex,
bitmex=ccxt.bitmex,
gdax=ccxt.gdax,
)
class CCXT(Exchange):
def __init__(self, exchange_name, key, secret, base_currency):
log.debug(
'finding {} in CCXT exchanges:\n{}'.format(
exchange_name, ccxt.exchanges
)
)
try:
# Making instantiation as explicit as possible for code tracking.
if exchange_name in SUPPORTED_EXCHANGES:
exchange_attr = SUPPORTED_EXCHANGES[exchange_name]
else:
exchange_attr = getattr(ccxt, exchange_name)
self.api = exchange_attr({
'apiKey': key,
'secret': secret,
})
except Exception:
raise ExchangeNotFoundError(exchange_name=exchange_name)
self._symbol_maps = [None, None]
try:
markets_symbols = self.api.load_markets()
log.debug('the markets:\n{}'.format(markets_symbols))
except ExchangeNotAvailable as e:
raise ExchangeRequestError(error=e)
self.name = exchange_name
self.markets = self.api.fetch_markets()
self.load_assets()
self.base_currency = base_currency
self.transactions = defaultdict(list)
self.num_candles_limit = 2000
self.max_requests_per_minute = 60
self.request_cpt = dict()
self.bundle = ExchangeBundle(self.name)
def account(self):
return None
def time_skew(self):
return None
def get_market(self, symbol):
"""
The CCXT market.
Parameters
----------
symbol:
The CCXT symbol.
Returns
-------
dict[str, Object]
"""
s = self.get_symbol(symbol)
market = next(
(market for market in self.markets if market['symbol'] == s),
None,
)
return market
def get_symbol(self, asset_or_symbol):
"""
The CCXT symbol.
Parameters
----------
asset_or_symbol
Returns
-------
"""
symbol = asset_or_symbol if isinstance(
asset_or_symbol, string_types
) else asset_or_symbol.symbol
parts = symbol.split('_')
return '{}/{}'.format(parts[0].upper(), parts[1].upper())
def get_catalyst_symbol(self, market_or_symbol):
"""
The Catalyst symbol.
Parameters
----------
market_or_symbol
Returns
-------
"""
if isinstance(market_or_symbol, string_types):
parts = market_or_symbol.split('/')
return '{}_{}'.format(parts[0].lower(), parts[1].lower())
else:
return '{}_{}'.format(
market_or_symbol['base'].lower(),
market_or_symbol['quote'].lower(),
)
def get_timeframe(self, freq):
"""
The CCXT timeframe from the Catalyst frequency.
Parameters
----------
freq: str
The Catalyst frequency (Pandas convention)
Returns
-------
str
"""
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) \
if freq_match.group(1) else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
if unit.lower() == 'd':
timeframe = '{}d'.format(candle_size)
elif unit.lower() == 'm' or unit == 'T':
timeframe = '{}m'.format(candle_size)
elif unit.lower() == 'h' or unit == 'T':
timeframe = '{}h'.format(candle_size)
return timeframe
def get_candles(self, freq, assets, bar_count=None, start_dt=None,
end_dt=None):
is_single = (isinstance(assets, TradingPair))
if is_single:
assets = [assets]
symbols = self.get_symbols(assets)
timeframe = self.get_timeframe(freq)
ms = None
if start_dt is not None:
delta = start_dt - get_epoch()
ms = int(delta.total_seconds()) * 1000
candles = dict()
for asset in assets:
try:
ohlcvs = self.api.fetch_ohlcv(
symbol=symbols[0],
timeframe=timeframe,
since=ms,
limit=bar_count,
params={}
)
candles[asset] = []
for ohlcv in ohlcvs:
candles[asset].append(dict(
last_traded=pd.to_datetime(
ohlcv[0], unit='ms', utc=True
),
open=ohlcv[1],
high=ohlcv[2],
low=ohlcv[3],
close=ohlcv[4],
volume=ohlcv[5]
))
except Exception as e:
raise ExchangeRequestError(error=e)
if is_single:
return six.next(six.itervalues(candles))
else:
return candles
def _fetch_symbol_map(self, is_local):
try:
return self.fetch_symbol_map(is_local)
except ExchangeSymbolsNotFound:
return None
def get_asset_defs(self, market):
"""
The local and Catalyst definitions of the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dicts.
Returns
-------
dict[str, Object]
The asset definition.
"""
asset_defs = []
for is_local in (False, True):
asset_def = self.get_asset_def(market, is_local)
asset_defs.append((asset_def, is_local))
return asset_defs
def get_asset_def(self, market, is_local=False):
"""
The asset definition (in symbols.json files) corresponding
to the the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dict.
is_local
Whether to search in local or Catalyst asset definitions.
Returns
-------
dict[str, Object]
The asset definition.
"""
exchange_symbol = market['id']
symbol_map = self._fetch_symbol_map(is_local)
if symbol_map is not None:
assets_lower = {k.lower(): v for k, v in symbol_map.items()}
key = exchange_symbol.lower()
asset = assets_lower[key] if key in assets_lower else None
if asset is not None:
return asset
else:
return None
else:
return None
def create_trading_pair(self, market, asset_def=None, is_local=False):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
asset_def: dict[str, Object]
is_local: bool
Returns
-------
"""
data_source = 'local' if is_local else 'catalyst'
params = dict(
exchange=self.name,
data_source=data_source,
exchange_symbol=market['id'],
)
mixin_market_params(self.name, params, market)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
params['start_date'] = asset_def['start_date'] \
if 'start_date' in asset_def else None
params['end_date'] = asset_def['end_date'] \
if 'end_date' in asset_def else None
params['leverage'] = asset_def['leverage'] \
if 'leverage' in asset_def else 1.0
params['asset_name'] = asset_def['asset_name'] \
if 'asset_name' in asset_def else None
params['end_daily'] = asset_def['end_daily'] \
if 'end_daily' in asset_def \
and asset_def['end_daily'] != 'N/A' else None
params['end_minute'] = asset_def['end_minute'] \
if 'end_minute' in asset_def \
and asset_def['end_minute'] != 'N/A' else None
else:
params['symbol'] = self.get_catalyst_symbol(market)
# TODO: add as an optional column
params['leverage'] = 1.0
return TradingPair(**params)
def load_assets(self):
self.assets = []
for market in self.markets:
asset_defs = self.get_asset_defs(market)
asset = None
for asset_def in asset_defs:
if asset_def[0] is not None or not asset_defs[1]:
try:
asset = self.create_trading_pair(
market=market,
asset_def=asset_def[0],
is_local=asset_def[1]
)
self.assets.append(asset)
except TypeError as e:
log.warn('unable to add asset: {}'.format(e))
if asset is None:
asset = self.create_trading_pair(market=market)
self.assets.append(asset)
def get_balances(self):
try:
log.debug('retrieving wallets balances')
balances = self.api.fetch_balance()
balances_lower = dict()
for key in balances:
balances_lower[key.lower()] = balances[key]
except Exception as e:
log.debug('error retrieving balances: {}', e)
raise ExchangeRequestError(error=e)
return balances_lower
def _create_order(self, order_status):
"""
Create a Catalyst order object from a CCXT order dictionary
Parameters
----------
order_status: dict[str, Object]
The order dict from the CCXT api.
Returns
-------
Order
The Catalyst order object
"""
if order_status['status'] == 'canceled':
status = ORDER_STATUS.CANCELLED
elif order_status['status'] == 'closed' and order_status['filled'] > 0:
log.debug('found executed order {}'.format(order_status))
status = ORDER_STATUS.FILLED
elif order_status['status'] == 'open':
status = ORDER_STATUS.OPEN
else:
raise ValueError('invalid state for order')
amount = order_status['amount']
filled = order_status['filled']
if order_status['side'] == 'sell':
amount = -amount
filled = -filled
price = order_status['price']
order_type = order_status['type']
limit_price = price if order_type == 'limit' else None
stop_price = None # TODO: add support
executed_price = order_status['cost'] / order_status['amount']
commission = order_status['fee']
date = from_ms_timestamp(order_status['timestamp'])
# order_id = str(order_status['info']['clientOrderId'])
order_id = order_status['id']
# TODO: this won't work, redo the packages with a different key.
symbol = order_status['info']['symbol'] \
if 'symbol' in order_status['info'] \
else order_status['info']['Exchange']
order = Order(
dt=date,
asset=self.get_asset(symbol, is_exchange_symbol=True),
amount=amount,
stop=stop_price,
limit=limit_price,
filled=filled,
id=order_id,
commission=commission
)
order.status = status
return order, executed_price
def create_order(self, asset, amount, is_buy, style):
symbol = self.get_symbol(asset)
if isinstance(style, ExchangeLimitOrder):
price = style.get_limit_price(is_buy)
order_type = 'limit'
elif isinstance(style, MarketOrder):
price = None
order_type = 'market'
else:
raise InvalidOrderStyle(
exchange=self.name,
style=style.__class__.__name__
)
side = 'buy' if amount > 0 else 'sell'
if hasattr(self.api, 'amount_to_lots'):
adj_amount = self.api.amount_to_lots(
symbol=symbol,
amount=abs(amount),
)
if adj_amount != abs(amount):
log.info(
'adjusted order amount {} to {} based on lot size'.format(
abs(amount), adj_amount,
)
)
else:
adj_amount = abs(amount)
try:
result = self.api.create_order(
symbol=symbol,
type=order_type,
side=side,
amount=adj_amount,
price=price
)
except ExchangeNotAvailable as e:
log.debug('unable to create order: {}'.format(e))
raise ExchangeRequestError(error=e)
except InvalidOrder as e:
log.warn('the exchange rejected the order: {}'.format(e))
raise CreateOrderError(exchange=self.name, error=e)
if 'info' not in result:
raise ValueError('cannot use order without info attribute')
final_amount = adj_amount if side == 'buy' else -adj_amount
order_id = result['id']
order = Order(
dt=pd.Timestamp.utcnow(),
asset=asset,
amount=final_amount,
stop=style.get_stop_price(is_buy),
limit=style.get_limit_price(is_buy),
id=order_id
)
return order
def get_open_orders(self, asset):
try:
symbol = self.get_symbol(asset)
result = self.api.fetch_open_orders(
symbol=symbol,
since=None,
limit=None,
params=dict()
)
except Exception as e:
raise ExchangeRequestError(error=e)
orders = []
for order_status in result:
order, executed_price = self._create_order(order_status)
if asset is None or asset == order.sid:
orders.append(order)
return orders
def get_order(self, order_id, asset_or_symbol=None):
if asset_or_symbol is None:
log.debug(
'order not found in memory, the request might fail '
'on some exchanges.'
)
try:
symbol = self.get_symbol(asset_or_symbol) \
if asset_or_symbol is not None else None
order_status = self.api.fetch_order(id=order_id, symbol=symbol)
order, executed_price = self._create_order(order_status)
except Exception as e:
raise ExchangeRequestError(error=e)
return order, executed_price
def cancel_order(self, order_param, asset_or_symbol=None):
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
if asset_or_symbol is None:
log.debug(
'order not found in memory, cancelling order might fail '
'on some exchanges.'
)
try:
symbol = self.get_symbol(asset_or_symbol) \
if asset_or_symbol is not None else None
self.api.cancel_order(id=order_id, symbol=symbol)
except Exception as e:
raise ExchangeRequestError(error=e)
def tickers(self, assets):
"""
Retrieve current tick data for the given assets
Parameters
----------
assets: list[TradingPair]
Returns
-------
list[dict[str, float]
"""
tickers = dict()
for asset in assets:
try:
ccxt_symbol = self.get_symbol(asset)
ticker = self.api.fetch_ticker(ccxt_symbol)
ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
if 'last_price' not in ticker:
# TODO: any more exceptions?
ticker['last_price'] = ticker['last']
# Using the volume represented in the base currency
ticker['volume'] = ticker['baseVolume'] \
if 'baseVolume' in ticker else 0
tickers[asset] = ticker
except ExchangeNotAvailable as e:
log.warn(
'unable to fetch ticker: {} {}'.format(
self.name, asset.symbol
)
)
raise ExchangeRequestError(error=e)
return tickers
def get_account(self):
return None
def get_orderbook(self, asset, order_type='all', limit=None):
ccxt_symbol = self.get_symbol(asset)
params = dict()
if limit is not None:
params['depth'] = limit
order_book = self.api.fetch_order_book(ccxt_symbol, params)
order_types = ['bids', 'asks'] if order_type == 'all' else [order_type]
result = dict(last_traded=from_ms_timestamp(order_book['timestamp']))
for index, order_type in enumerate(order_types):
if limit is not None and index > limit - 1:
break
result[order_type] = []
for entry in order_book[order_type]:
result[order_type].append(dict(
rate=float(entry[0]),
quantity=float(entry[1])
))
return result
catalyst/exchange/exchange.py
+169 -245
View File
@@ -5,7 +5,6 @@ from time import sleep
import numpy as np
import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.constants import LOG_LEVEL
@@ -14,16 +13,11 @@ from catalyst.exchange.bundle_utils import get_start_dt, \
get_delta, get_periods, get_periods_range
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
PricingDataNotLoadedError, \
NoDataAvailableOnExchange, ExchangeSymbolsNotFound
from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
ExchangeLimitOrder, ExchangeStopOrder
from catalyst.exchange.exchange_portfolio import ExchangePortfolio
NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError
from catalyst.exchange.exchange_utils import get_exchange_symbols, \
get_frequency, resample_history_df
from catalyst.finance.order import ORDER_STATUS
from catalyst.finance.transaction import Transaction
log = Logger('Exchange', level=LOG_LEVEL)
@@ -33,9 +27,8 @@ class Exchange:
def __init__(self):
self.name = None
self.assets = dict()
self.local_assets = dict()
self._portfolio = None
self.assets = []
self._symbol_maps = [None, None]
self.minute_writer = None
self.minute_reader = None
self.base_currency = None
@@ -45,27 +38,6 @@ class Exchange:
self.request_cpt = None
self.bundle = ExchangeBundle(self.name)
@property
def positions(self):
return self.portfolio.positions
@property
def portfolio(self):
"""
The exchange portfolio
Returns
-------
ExchangePortfolio
"""
if self._portfolio is None:
self._portfolio = ExchangePortfolio(
start_date=pd.Timestamp.utcnow()
)
self.synchronize_portfolio()
return self._portfolio
@abstractproperty
def account(self):
pass
@@ -145,9 +117,9 @@ class Exchange:
"""
symbol = None
for key in self.assets:
if not symbol and self.assets[key].symbol == asset.symbol:
symbol = key
for a in self.assets:
if not symbol and a.symbol == asset.symbol:
symbol = a.symbol
if not symbol:
raise ValueError('Currency %s not supported by exchange %s' %
@@ -174,73 +146,112 @@ class Exchange:
return symbols
def get_assets(self, symbols=None, data_frequency=None):
def get_assets(self, symbols=None, data_frequency=None,
is_exchange_symbol=False,
is_local=None):
"""
The list of markets for the specified symbols.
Parameters
----------
symbols: list[str]
data_frequency: str
is_exchange_symbol: bool
is_local: bool
Returns
-------
list[TradingPair]
A list of asset objects.
Notes
-----
See get_asset for details of each parameter.
"""
if symbols is None:
# Make a distinct list of all symbols
symbols = list(set([asset.symbol for asset in self.assets]))
is_exchange_symbol = False
assets = []
if symbols is not None:
for symbol in symbols:
asset = self.get_asset(symbol, data_frequency)
for symbol in symbols:
try:
asset = self.get_asset(
symbol, data_frequency, is_exchange_symbol, is_local
)
assets.append(asset)
else:
for key in self.assets:
assets.append(self.assets[key])
except SymbolNotFoundOnExchange:
log.debug(
'skipping non-existent market {} {}'.format(
self.name, symbol
)
)
return assets
def _find_asset(self, asset, symbol, data_frequency, is_local=False):
assets = self.assets if not is_local else self.local_assets
for key in assets:
has_data = (data_frequency == 'minute'
and assets[key].end_minute is not None) \
or (data_frequency == 'daily'
and assets[key].end_daily is not None)
if not asset and assets[key].symbol.lower() == symbol.lower() \
and (not data_frequency or has_data):
asset = assets[key]
return asset
def get_asset(self, symbol, data_frequency=None):
def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
is_local=None):
"""
The market for the specified symbol.
Parameters
----------
symbol: str
The Catalyst or exchange symbol.
data_frequency: str
Check for asset corresponding to the specified data_frequency.
The same asset might exist in the Catalyst repository or
locally (following a CSV ingestion). Filtering by
data_frequency picks the right asset.
is_exchange_symbol: bool
Whether the symbol uses the Catalyst or exchange convention.
is_local: bool
For the local or Catalyst asset.
Returns
-------
TradingPair
The asset object.
"""
asset = None
log.debug('searching asset {} on the server'.format(symbol))
asset = self._find_asset(asset, symbol, data_frequency, False)
log.debug(
'searching assets for: {} {}'.format(
self.name, symbol
)
)
for a in self.assets:
if asset is not None:
break
log.debug('asset {} not found on the server, searching local '
'assets'.format(symbol))
asset = self._find_asset(asset, symbol, data_frequency, True)
if is_local is not None:
data_source = 'local' if is_local else 'catalyst'
applies = (a.data_source == data_source)
if not asset:
all_values = list(self.assets.values()) + \
list(self.local_assets.values())
supported_symbols = sorted([
asset.symbol for asset in all_values
])
elif data_frequency is not None:
applies = (
(
data_frequency == 'minute' and a.end_minute is not None)
or (
data_frequency == 'daily' and a.end_daily is not None)
)
else:
applies = True
# The symbol provided may use the Catalyst or the exchange
# convention
key = a.exchange_symbol if is_exchange_symbol else a.symbol
if not asset and key.lower() == symbol.lower() and applies:
asset = a
if asset is None:
supported_symbols = sorted([a.symbol for a in self.assets])
raise SymbolNotFoundOnExchange(
symbol=symbol,
@@ -248,11 +259,20 @@ class Exchange:
supported_symbols=supported_symbols
)
log.debug('found asset: {}'.format(asset))
return asset
def fetch_symbol_map(self, is_local=False):
return get_exchange_symbols(self.name, is_local)
index = 1 if is_local else 0
if self._symbol_maps[index] is not None:
return self._symbol_maps[index]
else:
symbol_map = get_exchange_symbols(self.name, is_local)
self._symbol_maps[index] = symbol_map
return symbol_map
@abstractmethod
def load_assets(self, is_local=False):
"""
Populate the 'assets' attribute with a dictionary of Assets.
@@ -270,112 +290,7 @@ class Exchange:
via its api.
"""
try:
symbol_map = self.fetch_symbol_map(is_local)
except ExchangeSymbolsNotFound:
return None
for exchange_symbol in symbol_map:
asset = symbol_map[exchange_symbol]
if 'start_date' in asset:
start_date = pd.to_datetime(asset['start_date'], utc=True)
else:
start_date = None
if 'end_date' in asset:
end_date = pd.to_datetime(asset['end_date'], utc=True)
else:
end_date = None
if 'leverage' in asset:
leverage = asset['leverage']
else:
leverage = 1.0
if 'asset_name' in asset:
asset_name = asset['asset_name']
else:
asset_name = None
if 'min_trade_size' in asset:
min_trade_size = asset['min_trade_size']
else:
min_trade_size = 0.0000001
if 'end_daily' in asset and asset['end_daily'] != 'N/A':
end_daily = pd.to_datetime(asset['end_daily'], utc=True)
else:
end_daily = None
if 'end_minute' in asset and asset['end_minute'] != 'N/A':
end_minute = pd.to_datetime(asset['end_minute'], utc=True)
else:
end_minute = None
trading_pair = TradingPair(
symbol=asset['symbol'],
exchange=self.name,
start_date=start_date,
end_date=end_date,
leverage=leverage,
asset_name=asset_name,
min_trade_size=min_trade_size,
end_daily=end_daily,
end_minute=end_minute,
exchange_symbol=exchange_symbol
)
if is_local:
self.local_assets[exchange_symbol] = trading_pair
else:
self.assets[exchange_symbol] = trading_pair
def check_open_orders(self):
"""
Loop through the list of open orders in the Portfolio object.
For each executed order found, create a transaction and apply to the
Portfolio.
Returns
-------
list[Transaction]
"""
transactions = list()
if self.portfolio.open_orders:
for order_id in list(self.portfolio.open_orders):
log.debug('found open order: {}'.format(order_id))
order, executed_price = self.get_order(order_id)
log.debug('got updated order {} {}'.format(
order, executed_price))
if order.status == ORDER_STATUS.FILLED:
transaction = Transaction(
asset=order.asset,
amount=order.amount,
dt=pd.Timestamp.utcnow(),
price=executed_price,
order_id=order.id,
commission=order.commission
)
transactions.append(transaction)
self.portfolio.execute_order(order, transaction)
elif order.status == ORDER_STATUS.CANCELLED:
self.portfolio.remove_order(order)
else:
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'order {order_id} still open after {delta}'.format(
order_id=order_id,
delta=delta
)
)
return transactions
pass
def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
"""
@@ -412,12 +327,15 @@ class Exchange:
if field not in BASE_FIELDS:
raise KeyError('Invalid column: {}'.format(field))
values = []
for asset in assets:
value = self.get_single_spot_value(asset, field, data_frequency)
values.append(value)
tickers = self.tickers(assets)
if field == 'close' or field == 'price':
return [tickers[asset]['last'] for asset in tickers]
return values
elif field == 'volume':
return [tickers[asset]['volume'] for asset in tickers]
else:
raise NoValueForField(field=field)
def get_single_spot_value(self, asset, field, data_frequency):
"""
@@ -491,7 +409,7 @@ class Exchange:
method='ffill',
fill_value=previous_value,
)
series.sort_index(inplace=True)
return series
def get_history_window(self,
@@ -501,7 +419,7 @@ class Exchange:
frequency,
field,
data_frequency=None,
ffill=True):
is_current=False):
"""
Public API method that returns a dataframe containing the requested
@@ -528,10 +446,15 @@ class Exchange:
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
# TODO: fill how?
ffill: boolean
Forward-fill missing values. Only has effect if field
is 'price'.
is_current: bool
Skip date filters when current data is requested (last few bars
until now).
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns
-------
@@ -543,6 +466,7 @@ class Exchange:
frequency, data_frequency
)
adj_bar_count = candle_size * bar_count
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
# The get_history method supports multiple asset
@@ -550,8 +474,8 @@ class Exchange:
freq=freq,
assets=assets,
bar_count=bar_count,
start_dt=start_dt,
end_dt=end_dt
start_dt=start_dt if not is_current else None,
end_dt=end_dt if not is_current else None,
)
series = dict()
@@ -563,6 +487,17 @@ class Exchange:
data_frequency=frequency,
field=field,
)
if end_dt is not None:
delta = get_delta(candle_size, data_frequency)
adj_end_dt = end_dt - delta
last_traded = asset_series.index[-1]
if last_traded < adj_end_dt:
raise LastCandleTooEarlyError(
last_traded=last_traded,
end_dt=adj_end_dt,
exchange=self.name,
)
series[asset] = asset_series
df = pd.DataFrame(series)
@@ -620,6 +555,7 @@ class Exchange:
frequency, data_frequency
)
adj_bar_count = candle_size * bar_count
try:
series = self.bundle.get_history_window_series_and_load(
assets=assets,
@@ -629,6 +565,7 @@ class Exchange:
data_frequency=data_frequency,
force_auto_ingest=force_auto_ingest
)
except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
series = dict()
@@ -682,50 +619,48 @@ class Exchange:
return df
def synchronize_portfolio(self):
def calculate_totals(self, check_cash=False, positions=None):
"""
Update the portfolio cash and position balances based on the
latest ticker prices.
"""
log.debug('synchronizing portfolio with exchange {}'.format(self.name))
balances = self.get_balances()
base_position_available = balances[self.base_currency] \
if self.base_currency in balances else None
cash = None
if check_cash:
balances = self.get_balances()
if base_position_available is None:
raise BaseCurrencyNotFoundError(
base_currency=self.base_currency,
exchange=self.name.title()
)
cash = balances[self.base_currency]['free'] \
if self.base_currency in balances else None
portfolio = self._portfolio
portfolio.cash = base_position_available
log.debug('found base currency balance: {}'.format(portfolio.cash))
if cash is None:
raise BaseCurrencyNotFoundError(
base_currency=self.base_currency,
exchange=self.name
)
log.debug('found base currency balance: {}'.format(cash))
if portfolio.starting_cash is None:
portfolio.starting_cash = portfolio.cash
if portfolio.positions:
assets = list(portfolio.positions.keys())
positions_value = 0.0
if positions:
assets = set([position.asset for position in positions])
tickers = self.tickers(assets)
log.debug('got tickers for positions: {}'.format(tickers))
portfolio.positions_value = 0.0
for asset in tickers:
# TODO: convert if the position is not in the base currency
ticker = tickers[asset]
position = portfolio.positions[asset]
position.last_sale_price = ticker['last_price']
position.last_sale_date = ticker['timestamp']
positions = [p for p in positions if p.asset == asset]
portfolio.positions_value += \
position.amount * position.last_sale_price
portfolio.portfolio_value = \
portfolio.positions_value + portfolio.cash
for position in positions:
position.last_sale_price = ticker['last_price']
position.last_sale_date = ticker['last_traded']
def order(self, asset, amount, limit_price=None, stop_price=None,
style=None):
positions_value += \
position.amount * position.last_sale_price
return cash, positions_value
def order(self, asset, amount, style):
"""Place an order.
Parameters
@@ -774,45 +709,30 @@ class Exchange:
log.warn('skipping order amount of 0')
return None
if asset.base_currency != self.base_currency.lower():
if self.base_currency is None:
raise ValueError('no base_currency defined for this exchange')
if asset.quote_currency != self.base_currency.lower():
raise MismatchingBaseCurrencies(
base_currency=asset.base_currency,
base_currency=asset.quote_currency,
algo_currency=self.base_currency
)
is_buy = (amount > 0)
display_price = style.get_limit_price(is_buy)
if limit_price is not None and stop_price is not None:
style = ExchangeStopLimitOrder(limit_price, stop_price,
exchange=self.name)
elif limit_price is not None:
style = ExchangeLimitOrder(limit_price, exchange=self.name)
elif stop_price is not None:
style = ExchangeStopOrder(stop_price, exchange=self.name)
elif style is not None:
raise InvalidOrderStyle(exchange=self.name.title(),
style=style.__class__.__name__)
else:
raise ValueError('Incomplete order data.')
display_price = limit_price if limit_price is not None else stop_price
log.debug(
'issuing {side} order of {amount} {symbol} for {type}: {price}'.format(
'issuing {side} order of {amount} {symbol} for {type}:'
' {price}'.format(
side='buy' if is_buy else 'sell',
amount=amount,
symbol=asset.symbol,
type=style.__class__.__name__,
price='{}{}'.format(display_price, asset.base_currency)
price='{}{}'.format(display_price, asset.quote_currency)
)
)
order = self.create_order(asset, amount, is_buy, style)
if order:
self._portfolio.create_order(order)
return order.id
else:
return None
return self.create_order(asset, amount, is_buy, style)
# The methods below must be implemented for each exchange.
@abstractmethod
@@ -875,7 +795,7 @@ class Exchange:
pass
@abstractmethod
def get_order(self, order_id):
def get_order(self, order_id, symbol_or_asset=None):
"""Lookup an order based on the order id returned from one of the
order functions.
@@ -883,6 +803,8 @@ class Exchange:
----------
order_id : str
The unique identifier for the order.
symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this
Returns
-------
@@ -894,13 +816,15 @@ class Exchange:
pass
@abstractmethod
def cancel_order(self, order_param):
def cancel_order(self, order_param, symbol_or_asset=None):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this
"""
pass
catalyst/exchange/exchange_algorithm.py
+244 -234
View File
@@ -13,7 +13,6 @@
import pickle
import signal
import sys
from collections import deque
from datetime import timedelta
from os import listdir
from os.path import isfile, join
@@ -21,34 +20,32 @@ from time import sleep
import logbook
import pandas as pd
from catalyst.assets._assets import TradingPair
import catalyst.protocol as zp
from catalyst.algorithm import TradingAlgorithm
from catalyst.constants import LOG_LEVEL
from catalyst.errors import OrderInBeforeTradingStart
from catalyst.exchange.exchange_blotter import ExchangeBlotter
from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
ExchangePortfolioDataError,
ExchangeTransactionError,
OrphanOrderError)
from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
ExchangeLimitOrder, ExchangeStopOrder
from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object, \
get_algo_folder, get_algo_df, \
save_algo_df
OrderTypeNotSupported, )
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.exchange_utils import (
save_algo_object,
get_algo_object,
get_algo_folder,
get_algo_df,
save_algo_df,
group_assets_by_exchange, )
from catalyst.exchange.live_graph_clock import LiveGraphClock
from catalyst.exchange.simple_clock import SimpleClock
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.exchange.stats_utils import get_pretty_stats, stats_to_s3, \
stats_to_algo_folder
from catalyst.finance.execution import MarketOrder
from catalyst.finance.performance.period import calc_period_stats
from catalyst.gens.tradesimulation import AlgorithmSimulator
from catalyst.utils.api_support import (
api_method,
disallowed_in_before_trading_start)
from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
expect_types
from catalyst.utils.api_support import api_method
from catalyst.utils.input_validation import error_keywords, ensure_upper_case
from catalyst.utils.math_utils import round_nearest
from catalyst.utils.preprocess import preprocess
@@ -63,9 +60,90 @@ class ExchangeAlgorithmExecutor(AlgorithmSimulator):
class ExchangeTradingAlgorithmBase(TradingAlgorithm):
def __init__(self, *args, **kwargs):
self.exchanges = kwargs.pop('exchanges', None)
self.simulate_orders = kwargs.pop('simulate_orders', None)
super(ExchangeTradingAlgorithmBase, self).__init__(*args, **kwargs)
self.current_day = None
if self.simulate_orders is None \
and self.sim_params.arena == 'backtest':
self.simulate_orders = True
self.blotter = ExchangeBlotter(
data_frequency=self.data_frequency,
# Default to NeverCancel in catalyst
cancel_policy=self.cancel_policy,
simulate_orders=self.simulate_orders,
exchanges=self.exchanges
)
@staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style):
"""
Helper method for converting deprecated limit_price and stop_price
arguments into ExecutionStyle instances.
This function assumes that either style == None or (limit_price,
stop_price) == (None, None).
"""
if stop_price:
raise OrderTypeNotSupported(order_type='stop')
if style:
if limit_price is not None:
raise ValueError(
'An order style and a limit price was included in the '
'order. Please pick one to avoid any possible conflict.'
)
# Currently limiting order types or limit and market to
# be in-line with CXXT and many exchanges. We'll consider
# adding more order types in the future.
if not isinstance(style, ExchangeLimitOrder) or \
not isinstance(style, MarketOrder):
raise OrderTypeNotSupported(
order_type=style.__class__.__name__
)
return style
if limit_price:
return ExchangeLimitOrder(limit_price)
else:
return MarketOrder()
@api_method
def set_commission(self, maker=None, taker=None):
key = self.blotter.commission_models.keys()[0]
if maker is not None:
self.blotter.commission_models[key].maker = maker
if taker is not None:
self.blotter.commission_models[key].taker = taker
@api_method
def set_slippage(self, spread=None):
key = self.blotter.slippage_models.keys()[0]
if spread is not None:
self.blotter.slippage_models[key].spread = spread
def _calculate_order(self, asset, amount,
limit_price=None, stop_price=None, style=None):
# Raises a ZiplineError if invalid parameters are detected.
self.validate_order_params(asset,
amount,
limit_price,
stop_price,
style)
# Convert deprecated limit_price and stop_price parameters to use
# ExecutionStyle objects.
style = self.__convert_order_params_for_blotter(limit_price,
stop_price,
style)
return amount, style
def round_order(self, amount, asset):
"""
We need fractions with cryptocurrencies
@@ -204,50 +282,8 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs)
self.frame_stats = list()
self.blotter = ExchangeBlotter(
data_frequency=self.data_frequency,
# Default to NeverCancel in catalyst
cancel_policy=self.cancel_policy,
)
log.info('initialized trading algorithm in backtest mode')
def _calculate_order(self, asset, amount,
limit_price=None, stop_price=None, style=None):
# Raises a ZiplineError if invalid parameters are detected.
self.validate_order_params(asset,
amount,
limit_price,
stop_price,
style)
# Convert deprecated limit_price and stop_price parameters to use
# ExecutionStyle objects.
style = self.__convert_order_params_for_blotter(limit_price,
stop_price,
style)
return amount, style
@staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style):
"""
Helper method for converting deprecated limit_price and stop_price
arguments into ExecutionStyle instances.
This function assumes that either style == None or (limit_price,
stop_price) == (None, None).
"""
if style:
assert (limit_price, stop_price) == (None, None)
return style
if limit_price and stop_price:
return ExchangeStopLimitOrder(limit_price, stop_price)
if limit_price:
return ExchangeLimitOrder(limit_price)
if stop_price:
return ExchangeStopOrder(stop_price)
else:
return MarketOrder()
def is_last_frame_of_day(self, data):
# TODO: adjust here to support more intervals
next_frame_dt = data.current_dt + timedelta(minutes=1)
@@ -265,6 +301,8 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase):
)
self.frame_stats.append(frame_stats)
self.current_day = data.current_dt.floor('1D')
def _create_stats_df(self):
stats = pd.DataFrame(self.frame_stats)
stats.set_index('period_close', inplace=True, drop=False)
@@ -289,9 +327,10 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
def __init__(self, *args, **kwargs):
self.algo_namespace = kwargs.pop('algo_namespace', None)
self.live_graph = kwargs.pop('live_graph', None)
self.stats_output = kwargs.pop('stats_output', None)
self._clock = None
self.frame_stats = deque(maxlen=60)
self.frame_stats = list()
self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
@@ -309,7 +348,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.retry_order = 2
self.retry_delay = 5
self.stats_minutes = 5
self.stats_minutes = 10
super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
@@ -377,7 +416,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# This method is taken from TradingAlgorithm.
# The clock has been replaced to use RealtimeClock
# TODO: should we apply a time skew? not sure to understand the utility.
# TODO: should we apply time skew? not sure to understand the utility.
log.debug('creating clock')
if self.live_graph:
@@ -415,47 +454,83 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
return self.trading_client.transform()
def updated_portfolio(self):
"""
We skip the entire performance tracker business and update the
portfolio directly.
Returns
-------
ExchangePortfolio
"""
# TODO: build cumulative portfolio
return self.perf_tracker.get_portfolio(False)
def updated_account(self):
return self.perf_tracker.get_account(False)
def _synchronize_portfolio(self, attempt_index=0):
def synchronize_portfolio(self, attempt_index=0):
"""
Synchronizes the portfolio tracked by the algorithm to refresh
its current value.
This includes updating the last_sale_price of all tracked
positions, returning the available cash, and raising error
if the data goes out of sync.
Parameters
----------
attempt_index: int
Returns
-------
float
The amount of base currency available for trading.
float
The total value of all tracked positions.
"""
tracker = self.perf_tracker.position_tracker
total_cash = 0.0
total_positions_value = 0.0
try:
# Position keys correspond to assets
positions = self.portfolio.positions
assets = list(positions)
exchange_assets = group_assets_by_exchange(assets)
for exchange_name in self.exchanges:
exchange = self.exchanges[exchange_name]
assets = exchange_assets[exchange_name] \
if exchange_name in exchange_assets else []
exchange.synchronize_portfolio()
exchange_positions = \
[positions[asset] for asset in assets]
# Applying the updated last_sales_price to the positions
# in the performance tracker. This seems a bit redundant
# but it will make sense when we have multiple exchange portfolios
# feeding into the same performance tracker.
tracker = self.perf_tracker.todays_performance.position_tracker
for asset in exchange.portfolio.positions:
position = exchange.portfolio.positions[asset]
check_cash = (not self.simulate_orders)
exchange = self.exchanges[exchange_name] # Type: Exchange
cash, positions_value = exchange.calculate_totals(
positions=exchange_positions,
check_cash=check_cash,
)
total_positions_value += positions_value
if cash is not None:
total_cash += cash
for position in exchange_positions:
tracker.update_position(
asset=asset,
asset=position.asset,
last_sale_date=position.last_sale_date,
last_sale_price=position.last_sale_price
)
if cash is None:
total_cash = self.portfolio.cash
elif total_cash < self.portfolio.cash:
raise ValueError('Cash on exchanges is lower than the algo.')
return total_cash, total_positions_value
except ExchangeRequestError as e:
log.warn(
'update portfolio attempt {}: {}'.format(attempt_index, e)
)
if attempt_index < self.retry_synchronize_portfolio:
sleep(self.retry_delay)
self._synchronize_portfolio(attempt_index + 1)
return self.synchronize_portfolio(attempt_index + 1)
else:
raise ExchangePortfolioDataError(
data_type='update-portfolio',
@@ -463,30 +538,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
error=e
)
def _check_open_orders(self, attempt_index=0):
try:
orders = list()
for exchange_name in self.exchanges:
exchange = self.exchanges[exchange_name]
exchange_orders = exchange.check_open_orders()
orders += exchange_orders
return orders
except ExchangeRequestError as e:
log.warn(
'check open orders attempt {}: {}'.format(attempt_index, e)
)
if attempt_index < self.retry_check_open_orders:
sleep(self.retry_delay)
return self._check_open_orders(attempt_index + 1)
else:
raise ExchangePortfolioDataError(
data_type='order-status',
attempts=attempt_index,
error=e
)
def add_pnl_stats(self, period_stats):
"""
Save p&l stats.
@@ -576,15 +627,23 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
if not self.is_running:
return
self._synchronize_portfolio()
# Resetting the frame stats every day to minimize memory footprint
today = data.current_dt.floor('1D')
if self.current_day is not None and today > self.current_day:
self.frame_stats = list()
transactions = self._check_open_orders()
if len(transactions) > 0:
for transaction in transactions:
self.perf_tracker.process_transaction(transaction)
new_transactions, new_commissions, closed_orders = \
self.blotter.get_transactions(data)
if len(new_transactions) > 0:
self.perf_tracker.update_performance()
cash, positions_value = self.synchronize_portfolio()
log.info(
'got totals from exchanges, cash: {} positions: {}'.format(
cash, positions_value
)
)
if self._handle_data:
self._handle_data(self, data)
@@ -594,48 +653,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.validate_account_controls()
try:
# Since the clock runs 24/7, I trying to disable the daily
# Performance tracker and keep only minute and cumulative
self.perf_tracker.update_performance()
frame_stats = self.prepare_period_stats(
data.current_dt, data.current_dt + timedelta(minutes=1))
# Saving the last hour in memory
self.frame_stats.append(frame_stats)
self.add_pnl_stats(frame_stats)
if self.recorded_vars:
self.add_custom_signals_stats(frame_stats)
recorded_cols = list(self.recorded_vars.keys())
else:
recorded_cols = None
self.add_exposure_stats(frame_stats)
print_df = pd.DataFrame(list(self.frame_stats))
log.info(
'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
stats_minutes=self.stats_minutes,
stats=get_pretty_stats(
stats_df=print_df,
recorded_cols=recorded_cols,
num_rows=self.stats_minutes
)
))
today = pd.to_datetime('today', utc=True)
daily_stats = self.prepare_period_stats(
start_dt=today,
end_dt=pd.Timestamp.utcnow()
)
save_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
obj=daily_stats,
rel_path='daily_perf'
)
self._save_stats_csv(self._process_stats(data))
except Exception as e:
log.warn('unable to calculate performance: {}'.format(e))
@@ -649,93 +667,85 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
except Exception as e:
log.warn('unable to save minute perfs to disk: {}'.format(e))
try:
for exchange_name in self.exchanges:
exchange = self.exchanges[exchange_name]
save_algo_object(
algo_name=self.algo_namespace,
key='portfolio_{}'.format(exchange_name),
obj=exchange.portfolio
)
except Exception as e:
log.warn('unable to save portfolio to disk: {}'.format(e))
self.current_day = data.current_dt.floor('1D')
def _order(self,
asset,
amount,
limit_price=None,
stop_price=None,
style=None,
attempt_index=0):
try:
exchange = self.exchanges[asset.exchange]
return exchange.order(asset, amount, limit_price,
stop_price,
style)
except ExchangeRequestError as e:
log.warn(
'order attempt {}: {}'.format(attempt_index, e)
)
if attempt_index < self.retry_order:
sleep(self.retry_delay)
return self._order(
asset, amount, limit_price, stop_price, style,
attempt_index + 1)
else:
raise ExchangeTransactionError(
transaction_type='order',
attempts=attempt_index,
error=e
)
def _process_stats(self, data):
today = data.current_dt.floor('1D')
@api_method
@disallowed_in_before_trading_start(OrderInBeforeTradingStart())
@expect_types(asset=TradingPair)
def order(self,
asset,
amount,
limit_price=None,
stop_price=None,
style=None):
"""
We use the exchange specific portfolio to place orders.
The cumulative portfolio does not contain open orders but exchange
portfolios do.
# Since the clock runs 24/7, I trying to disable the daily
# Performance tracker and keep only minute and cumulative
self.perf_tracker.update_performance()
Parameters
----------
asset: TradingPair
amount: float
limit_price: float
stop_price: float
style: Style
order: Order
The catalyst order object or None
"""
amount, style = self._calculate_order(asset, amount,
limit_price, stop_price,
style)
frame_stats = self.prepare_period_stats(
data.current_dt, data.current_dt + timedelta(minutes=1))
order_id = self._order(asset, amount, limit_price, stop_price, style)
# Saving the last hour in memory
self.frame_stats.append(frame_stats)
exchange = self.exchanges[asset.exchange]
exchange_portfolio = exchange.portfolio
if order_id is not None:
self.add_pnl_stats(frame_stats)
if self.recorded_vars:
self.add_custom_signals_stats(frame_stats)
recorded_cols = list(self.recorded_vars.keys())
if order_id in exchange_portfolio.open_orders:
order = exchange_portfolio.open_orders[order_id]
self.perf_tracker.process_order(order)
return order
else:
raise OrphanOrderError(
order_id=order_id,
exchange=exchange.name
)
else:
log.warn('unable to order {} {} on exchange {}'.format(
amount, asset.symbol, asset.exchange))
return None
recorded_cols = None
self.add_exposure_stats(frame_stats)
log.info(
'statistics for the last {stats_minutes} minutes:\n'
'{stats}'.format(
stats_minutes=self.stats_minutes,
stats=get_pretty_stats(
stats=self.frame_stats,
recorded_cols=recorded_cols,
num_rows=self.stats_minutes
)
))
# Saving the daily stats in a format usable for performance
# analysis.
daily_stats = self.prepare_period_stats(
start_dt=today,
end_dt=data.current_dt
)
save_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
obj=daily_stats,
rel_path='daily_perf'
)
return recorded_cols
def _save_stats_csv(self, recorded_cols):
# Writing the stats output
csv_bytes = None
try:
csv_bytes = stats_to_algo_folder(
stats=self.frame_stats,
algo_namespace=self.algo_namespace,
recorded_cols=recorded_cols,
)
except Exception as e:
log.warn('unable save stats locally: {}'.format(e))
try:
if self.stats_output is not None:
if 's3://' in self.stats_output:
stats_to_s3(
uri=self.stats_output,
stats=self.frame_stats,
algo_namespace=self.algo_namespace,
recorded_cols=recorded_cols,
bytes_to_write=csv_bytes
)
else:
raise ValueError(
'Only S3 stats output is supported for now.'
)
except Exception as e:
log.warn('unable save stats externally: {}'.format(e))
@api_method
def batch_market_order(self, share_counts):
catalyst/exchange/exchange_blotter.py
+186 -23
View File
@@ -1,21 +1,21 @@
from time import sleep
import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange_errors import ExchangeRequestError, \
ExchangePortfolioDataError, ExchangeTransactionError
from catalyst.finance.blotter import Blotter
from catalyst.finance.commission import CommissionModel
from catalyst.finance.order import ORDER_STATUS, Order
from catalyst.finance.slippage import SlippageModel
from catalyst.finance.transaction import create_transaction
from catalyst.finance.transaction import create_transaction, Transaction
from catalyst.utils.input_validation import expect_types
log = Logger('exchange_blotter', level=LOG_LEVEL)
# It seems like we need to accept greater slippage risk in cryptos
# Orders won't often close at Equity levels.
# TODO: should work with set_commission and set_slippage
DEFAULT_SLIPPAGE_SPREAD = 0.0001
DEFAULT_MAKER_FEE = 0.0015
DEFAULT_TAKER_FEE = 0.0025
class TradingPairFeeSchedule(CommissionModel):
"""
@@ -23,23 +23,24 @@ class TradingPairFeeSchedule(CommissionModel):
Parameters
----------
fee : float, optional
The percentage fee.
maker : float, optional
The percentage maker fee.
taker: float, optional
The percentage taker fee.
"""
def __init__(self,
maker_fee=DEFAULT_MAKER_FEE,
taker_fee=DEFAULT_TAKER_FEE):
self.maker_fee = maker_fee
self.taker_fee = taker_fee
def __init__(self, maker=None, taker=None):
self.maker = maker
self.taker = taker
def __repr__(self):
return (
'{class_name}(maker_fee={maker_fee}, '
'taker_fee={taker_fee})'.format(
'{class_name}(maker={maker}, '
'taker={taker})'.format(
class_name=self.__class__.__name__,
maker_fee=self.maker_fee,
taker_fee=self.taker_fee,
maker=self.maker,
taker=self.taker,
)
)
@@ -47,16 +48,25 @@ class TradingPairFeeSchedule(CommissionModel):
"""
Calculate the final fee based on the order parameters.
:param order:
:param transaction:
:param order: Order
:param transaction: Transaction
:return float:
The total commission.
"""
cost = abs(transaction.amount) * transaction.price
asset = order.asset
maker = self.maker if self.maker is not None else asset.maker
taker = self.taker if self.taker is not None else asset.taker
multiplier = maker \
if ((order.amount > 0 and order.limit < transaction.price)
or (order.amount < 0 and order.limit > transaction.price)) \
and order.limit_reached else taker
# Assuming just the taker fee for now
fee = cost * self.taker_fee
fee = cost * multiplier
return fee
@@ -70,7 +80,7 @@ class TradingPairFixedSlippage(SlippageModel):
spread / 2 will be added to buys and subtracted from sells.
"""
def __init__(self, spread=DEFAULT_SLIPPAGE_SPREAD):
def __init__(self, spread=0.0001):
super(TradingPairFixedSlippage, self).__init__()
self.spread = spread
@@ -121,6 +131,14 @@ class TradingPairFixedSlippage(SlippageModel):
class ExchangeBlotter(Blotter):
def __init__(self, *args, **kwargs):
self.simulate_orders = kwargs.pop('simulate_orders', False)
self.exchanges = kwargs.pop('exchanges', None)
if not self.exchanges:
raise ValueError(
'ExchangeBlotter must have an `exchanges` attribute.'
)
super(ExchangeBlotter, self).__init__(*args, **kwargs)
# Using the equity models for now
@@ -132,3 +150,148 @@ class ExchangeBlotter(Blotter):
self.commission_models = {
TradingPair: TradingPairFeeSchedule()
}
self.retry_delay = 5
self.retry_check_open_orders = 5
def exchange_order(self, asset, amount, style=None, attempt_index=0):
try:
exchange = self.exchanges[asset.exchange]
return exchange.order(
asset, amount, style
)
except ExchangeRequestError as e:
log.warn(
'order attempt {}: {}'.format(attempt_index, e)
)
if attempt_index < self.retry_order:
sleep(self.retry_delay)
return self.exchange_order(
asset, amount, style, attempt_index + 1
)
else:
raise ExchangeTransactionError(
transaction_type='order',
attempts=attempt_index,
error=e
)
@expect_types(asset=TradingPair)
def order(self, asset, amount, style, order_id=None):
log.debug('ordering {} {}'.format(amount, asset.symbol))
if amount == 0:
log.warn('skipping 0 amount orders')
return None
if self.simulate_orders:
return super(ExchangeBlotter, self).order(
asset, amount, style, order_id
)
else:
order = self.exchange_order(
asset, amount, style
)
self.open_orders[order.asset].append(order)
self.orders[order.id] = order
self.new_orders.append(order)
return order.id
def check_open_orders(self):
"""
Loop through the list of open orders in the Portfolio object.
For each executed order found, create a transaction and apply to the
Portfolio.
Returns
-------
list[Transaction]
"""
for asset in self.open_orders:
exchange = self.exchanges[asset.exchange]
for order in self.open_orders[asset]:
log.debug('found open order: {}'.format(order.id))
new_order, executed_price = exchange.get_order(order.id, asset)
log.debug(
'got updated order {} {}'.format(
new_order, executed_price
)
)
order.status = new_order.status
if order.status == ORDER_STATUS.FILLED:
order.commission = new_order.commission
if order.amount != new_order.amount:
log.warn(
'executed order amount {} differs '
'from original'.format(
new_order.amount, order.amount
)
)
order.amount = new_order.amount
transaction = Transaction(
asset=order.asset,
amount=order.amount,
dt=pd.Timestamp.utcnow(),
price=executed_price,
order_id=order.id,
commission=order.commission
)
yield order, transaction
elif order.status == ORDER_STATUS.CANCELLED:
yield order, None
else:
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'order {order_id} still open after {delta}'.format(
order_id=order.id,
delta=delta
)
)
def get_exchange_transactions(self, attempt_index=0):
closed_orders = []
transactions = []
commissions = []
try:
for order, txn in self.check_open_orders():
order.dt = txn.dt
transactions.append(txn)
if not order.open:
closed_orders.append(order)
return transactions, commissions, closed_orders
except ExchangeRequestError as e:
log.warn(
'check open orders attempt {}: {}'.format(attempt_index, e)
)
if attempt_index < self.retry_check_open_orders:
sleep(self.retry_delay)
return self.get_exchange_transactions(attempt_index + 1)
else:
raise ExchangePortfolioDataError(
data_type='order-status',
attempts=attempt_index,
error=e
)
def get_transactions(self, bar_data):
if self.simulate_orders:
return super(ExchangeBlotter, self).get_transactions(bar_data)
else:
return self.get_exchange_transactions()
catalyst/exchange/exchange_bundle.py
+54 -52
View File
@@ -1,7 +1,6 @@
import os
import os
import shutil
from datetime import datetime, timedelta
from datetime import timedelta
from functools import partial
from itertools import chain
from operator import is_not
@@ -28,10 +27,9 @@ from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
TempBundleNotFoundError, \
NoDataAvailableOnExchange, \
PricingDataNotLoadedError, DataCorruptionError, ExchangeSymbolsNotFound, \
PricingDataValueError
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
from catalyst.exchange.exchange_utils import get_exchange_folder, \
get_exchange_symbols, save_exchange_symbols
save_exchange_symbols, mixin_market_params
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
@@ -235,11 +233,13 @@ class ExchangeBundle:
problem = '{name} ({start_dt} to {end_dt}) has empty ' \
'periods: {dates}'.format(
name=asset.symbol,
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
dates=[date.strftime(DATE_TIME_FORMAT) for date in dates]
)
name=asset.symbol,
start_dt=asset.start_date.strftime(
DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
dates=[date.strftime(
DATE_TIME_FORMAT) for date in dates])
if empty_rows_behavior == 'warn':
log.warn(problem)
@@ -247,8 +247,7 @@ class ExchangeBundle:
raise EmptyValuesInBundleError(
name=asset.symbol,
end_minute=end_dt,
dates=dates
)
dates=dates, )
else:
ohlcv_df.dropna(inplace=True)
@@ -288,13 +287,12 @@ class ExchangeBundle:
problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
'identical close values on: {dates}'.format(
name=asset.symbol,
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
threshold=threshold,
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
for date in dates]
)
name=asset.symbol,
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
threshold=threshold,
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
for date in dates])
problems.append(problem)
@@ -632,8 +630,8 @@ class ExchangeBundle:
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
exchange=self.exchange_name,
frequency=data_frequency,
)) as it:
for chunk in it:
problems += self.ingest_ctable(
@@ -667,12 +665,11 @@ class ExchangeBundle:
"""
log.info('ingesting csv file: {}'.format(path))
try:
symbols_def = get_exchange_symbols(
self.exchange_name, is_local=True
)
except ExchangeSymbolsNotFound:
symbols_def = dict()
if self.exchange is None:
# Avoid circular dependencies
from catalyst.exchange.factory import get_exchange
self.exchange = get_exchange(self.exchange_name)
problems = []
df = pd.read_csv(
@@ -705,24 +702,40 @@ class ExchangeBundle:
end_dt = df.index.get_level_values(1).max()
end_dt_key = 'end_{}'.format(data_frequency)
if symbol is symbols_def:
symbol_def = symbols_def[symbol]
market = self.exchange.get_market(symbol)
if market is None:
raise ValueError('symbol not available in the exchange.')
start_dt = symbol_def['start_date'] \
if symbol_def['start_date'] < start_dt else start_dt
params = dict(
exchange=self.exchange.name,
data_source='local',
exchange_symbol=market['id'],
)
mixin_market_params(self.exchange_name, params, market)
end_dt = symbol_def[end_dt_key] \
if symbol_def[end_dt_key] > end_dt else end_dt
asset_def = self.exchange.get_asset_def(market, True)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
end_daily = end_dt \
if data_frequency == 'daily' else symbol_def['end_daily']
params['start_date'] = asset_def['start_date'] \
if asset_def['start_date'] < start_dt else start_dt
end_minute = end_dt \
if data_frequency == 'minute' else symbol_def['end_minute']
params['end_date'] = asset_def[end_dt_key] \
if asset_def[end_dt_key] > end_dt else end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else asset_def['end_daily']
params['end_minute'] = end_dt \
if data_frequency == 'minute' else asset_def['end_minute']
else:
end_daily = end_dt if data_frequency == 'daily' else 'N/A'
end_minute = end_dt if data_frequency == 'minute' else 'N/A'
params['symbol'] = self.exchange.get_catalyst_symbol(market)
params['end_daily'] = end_dt \
if data_frequency == 'daily' else 'N/A'
params['end_minute'] = end_dt \
if data_frequency == 'minute' else 'N/A'
if min_start_dt is None or start_dt < min_start_dt:
min_start_dt = start_dt
@@ -730,19 +743,8 @@ class ExchangeBundle:
if max_end_dt is None or end_dt > max_end_dt:
max_end_dt = end_dt
asset = TradingPair(
symbol=symbol,
exchange=self.exchange_name,
start_date=start_dt,
end_date=end_dt,
leverage=0, # TODO: add as an optional column
asset_name=symbol,
min_trade_size=0, # TODO: add as an optional column
end_daily=end_daily,
end_minute=end_minute,
exchange_symbol=symbol
)
assets[symbol] = asset
asset = TradingPair(**params)
assets[market['id']] = asset
save_exchange_symbols(self.exchange_name, assets, True)
catalyst/exchange/exchange_data_portal.py
+5 -9
View File
@@ -13,7 +13,8 @@ from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
ExchangeBarDataError,
PricingDataNotLoadedError)
from catalyst.exchange.exchange_utils import get_frequency, resample_history_df
from catalyst.exchange.exchange_utils import get_frequency, \
resample_history_df, group_assets_by_exchange
log = Logger('DataPortalExchange', level=LOG_LEVEL)
@@ -38,13 +39,7 @@ class DataPortalExchangeBase(DataPortal):
ffill=True,
attempt_index=0):
try:
exchange_assets = dict()
for asset in assets:
if asset.exchange not in exchange_assets:
exchange_assets[asset.exchange] = list()
exchange_assets[asset.exchange].append(asset)
exchange_assets = group_assets_by_exchange(assets)
if len(exchange_assets) > 1:
df_list = []
for exchange_name in exchange_assets:
@@ -242,6 +237,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
"""
exchange = self.exchanges[exchange_name]
df = exchange.get_history_window(
assets,
end_dt,
@@ -249,7 +245,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
frequency,
field,
data_frequency,
ffill)
False)
return df
def get_exchange_spot_value(self, exchange_name, assets, field, dt,
catalyst/exchange/exchange_errors.py
+34 -5
View File
@@ -143,7 +143,8 @@ class OrphanOrderError(ZiplineError):
class OrphanOrderReverseError(ZiplineError):
msg = (
'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.'
'Order {order_id} tracked by algorithm, but not found in exchange '
'{exchange}.'
).strip()
@@ -206,8 +207,9 @@ class EmptyValuesInBundleError(ZiplineError):
class PricingDataBeforeTradingError(ZiplineError):
msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} '
'starts on {first_trading_day}, but you are either trying to trade or '
'retrieve pricing data on {dt}. Adjust your dates accordingly.').strip()
'starts on {first_trading_day}, but you are either trying to trade '
'or retrieve pricing data on {dt}. Adjust your dates accordingly.'
).strip()
class PricingDataNotLoadedError(ZiplineError):
@@ -217,6 +219,7 @@ class PricingDataNotLoadedError(ZiplineError):
'{data_frequency} -i {symbol_list}`. See catalyst documentation '
'for details.').strip()
class PricingDataValueError(ZiplineError):
msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
'[{start_dt} - {end_dt}]: {error}').strip()
@@ -237,6 +240,32 @@ class ApiCandlesError(ZiplineError):
class NoDataAvailableOnExchange(ZiplineError):
msg = (
'Requested data for trading pair {symbol} is not available on exchange {exchange} '
'Requested data for trading pair {symbol} is not available on '
'exchange {exchange} '
'in `{data_frequency}` frequency at this time. '
'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
'Check `http://enigma.co/catalyst/status` for market coverage.'
).strip()
class NoValueForField(ZiplineError):
msg = ('Value not found for field: {field}.').strip()
class OrderTypeNotSupported(ZiplineError):
msg = (
'Order type `{order_type}` not currencly supported by Catalyst. '
'Please use `limit` or `market` orders only.').strip()
class NotEnoughCapitalError(ZiplineError):
msg = (
'Not enough capital on exchange {exchange} for trading. Each '
'exchange should contain at least as much {base_currency} '
'as the specified `capital_base`. The current balance {balance} is '
'lower than the `capital_base`: {capital_base}').strip()
class LastCandleTooEarlyError(ZiplineError):
msg = (
'The trade date of the last candle {last_traded} is before the '
'specified end date minus one candle {end_dt}. Please verify how '
'{exchange} calculates the start date of OHLCV candles.').strip()
catalyst/exchange/exchange_portfolio.py
+24 -32
View File
@@ -3,7 +3,6 @@ from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.protocol import Portfolio, Positions, Position
from catalyst.utils.deprecate import deprecated
log = Logger('ExchangePortfolio', level=LOG_LEVEL)
@@ -11,7 +10,8 @@ log = Logger('ExchangePortfolio', level=LOG_LEVEL)
class ExchangePortfolio(Portfolio):
"""
Since the goal is to support multiple exchanges, it makes sense to
include additional stats in the portfolio object.
include additional stats in the portfolio object. This fills the role
of Blotter and Portfolio in live mode.
Instead of relying on the performance tracker, each exchange portfolio
tracks its own holding. This offers a separation between tracking an
@@ -40,7 +40,13 @@ class ExchangePortfolio(Portfolio):
"""
log.debug('creating order {}'.format(order.id))
self.open_orders[order.id] = order
open_orders = self.open_orders[order.asset] \
if order.asset is self.open_orders else []
open_orders.append(order)
self.open_orders[order.asset] = open_orders
order_position = self.positions[order.asset] \
if order.asset in self.positions else None
@@ -52,6 +58,17 @@ class ExchangePortfolio(Portfolio):
order_position.amount += order.amount
log.debug('open order added to portfolio')
def _remove_open_order(self, order):
try:
open_orders = self.open_orders[order.asset]
if order in open_orders:
open_orders.remove(order)
except Exception:
raise ValueError(
'unable to clear order not found in open order list.'
)
def execute_order(self, order, transaction):
"""
Update the open orders and positions to apply an executed order.
@@ -66,14 +83,15 @@ class ExchangePortfolio(Portfolio):
"""
log.debug('executing order {}'.format(order.id))
del self.open_orders[order.id]
self._remove_open_order(order)
order_position = self.positions[order.asset] \
if order.asset in self.positions else None
if order_position is None:
raise ValueError(
'Trying to execute order for a position not held: %s' % order.id
'Trying to execute order for a position not held:'
' {}'.format(order.id)
)
self.capital_used += order.amount * transaction.price
@@ -89,32 +107,6 @@ class ExchangePortfolio(Portfolio):
log.debug('updated portfolio with executed order')
@deprecated
def execute_transaction(self, transaction):
# TODO: almost duplicate of execute_order. Not sure why Poloniex needs this.
log.debug('executing transaction {}'.format(transaction.order_id))
order_position = self.positions[transaction.asset] \
if transaction.asset in self.positions else None
if order_position is None:
raise ValueError(
'Trying to execute transaction for a position not held: %s' % transaction.order_id
)
self.capital_used += transaction.amount * transaction.price
if transaction.amount > 0:
if order_position.cost_basis > 0:
order_position.cost_basis = np.average(
[order_position.cost_basis, transaction.price],
weights=[order_position.amount, transaction.amount]
)
else:
order_position.cost_basis = transaction.price
log.debug('updated portfolio with executed order')
def remove_order(self, order):
"""
Removing an open order.
@@ -125,7 +117,7 @@ class ExchangePortfolio(Portfolio):
"""
log.info('removing cancelled order {}'.format(order.id))
del self.open_orders[order.id]
self._remove_open_order(order)
order_position = self.positions[order.asset] \
if order.asset in self.positions else None
catalyst/exchange/exchange_utils.py
+78 -3
View File
@@ -8,6 +8,7 @@ from datetime import date, datetime
import pandas as pd
from catalyst.assets._assets import TradingPair
from six import string_types
from six.moves.urllib import request
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
@@ -100,6 +101,20 @@ def download_exchange_symbols(exchange_name, environ=None):
return response
def symbols_parser(asset_def):
for key, value in asset_def.items():
match = isinstance(value, string_types) \
and re.search(r'(\d{4}-\d{2}-\d{2})', value)
if match:
try:
asset_def[key] = pd.to_datetime(value, utc=True)
except ValueError:
pass
return asset_def
def get_exchange_symbols(exchange_name, is_local=False, environ=None):
"""
The de-serialized content of the exchange's symbols.json.
@@ -119,13 +134,13 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
pd.Timestamp('now', tz='UTC') - last_modified_time(
filename)).days > 1):
filename)).days > 1):
download_exchange_symbols(exchange_name, environ)
if os.path.isfile(filename):
with open(filename) as data_file:
try:
data = json.load(data_file)
data = json.load(data_file, object_hook=symbols_parser)
return data
except ValueError:
@@ -281,7 +296,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
try:
with open(filename, 'rb') as handle:
return pickle.load(handle)
except Exception as e:
except Exception:
return None
else:
return None
@@ -571,3 +586,63 @@ def resample_history_df(df, freq, field):
resampled_df = df.resample(freq).agg(agg)
return resampled_df
def mixin_market_params(exchange_name, params, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
params: dict[Object]
market: dict[Object]
Returns
-------
"""
# TODO: make this more externalized / configurable
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def from_ms_timestamp(ms):
return pd.to_datetime(ms, unit='ms', utc=True)
def get_epoch():
return pd.to_datetime('1970-1-1', utc=True)
def group_assets_by_exchange(assets):
exchange_assets = dict()
for asset in assets:
if asset.exchange not in exchange_assets:
exchange_assets[asset.exchange] = list()
exchange_assets[asset.exchange].append(asset)
return exchange_assets
catalyst/exchange/factory.py
+21 -30
View File
@@ -1,38 +1,29 @@
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
from catalyst.exchange.bittrex.bittrex import Bittrex
from catalyst.exchange.exchange_errors import ExchangeNotFoundError
from catalyst.exchange.exchange_utils import get_exchange_auth
from catalyst.exchange.poloniex.poloniex import Poloniex
import os
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.exchange_utils import get_exchange_auth, \
get_exchange_folder
def get_exchange(exchange_name, base_currency=None):
def get_exchange(exchange_name, base_currency=None, must_authenticate=False):
exchange_auth = get_exchange_auth(exchange_name)
if exchange_name == 'bitfinex':
return Bitfinex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=None
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name), 'auth.json'
)
)
elif exchange_name == 'bittrex':
return Bittrex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=None
)
elif exchange_name == 'poloniex':
return Poloniex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=None
)
else:
raise ExchangeNotFoundError(exchange_name=exchange_name)
return CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
)
def get_exchanges(exchange_names):
catalyst/exchange/poloniex/poloniex.py
+32 -26
View File
@@ -1,5 +1,4 @@
import json
import json
import time
from collections import defaultdict
@@ -18,7 +17,9 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
InvalidHistoryFrequencyError,
InvalidOrderStyle, OrphanOrderReverseError)
InvalidOrderStyle,
OrphanOrderError,
OrphanOrderReverseError)
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
ExchangeStopLimitOrder
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
@@ -27,10 +28,12 @@ from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.finance.transaction import Transaction
from catalyst.protocol import Account
from catalyst.utils.deprecate import deprecated
log = Logger('Poloniex', level=LOG_LEVEL)
@deprecated
class Poloniex(Exchange):
def __init__(self, key, secret, base_currency, portfolio=None):
self.api = Poloniex_api(key=key, secret=secret)
@@ -87,7 +90,6 @@ class Poloniex(Exchange):
# filled = -filled
price = float(order_status['rate'])
order_type = order_status['type']
stop_price = None
limit_price = None
@@ -101,11 +103,11 @@ class Poloniex(Exchange):
# executed_price = float(order_status['avg_execution_price'])
executed_price = price
# TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
# TODO: Set Poloniex comission
commission = None
# date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
# date = pytz.utc.localize(date)
# date=pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
# date=pytz.utc.localize(date)
date = None
order = Order(
@@ -292,8 +294,8 @@ class Poloniex(Exchange):
"""
exchange_symbol = self.get_symbol(asset)
if isinstance(style, ExchangeLimitOrder) or isinstance(style,
ExchangeStopLimitOrder):
if (isinstance(style, ExchangeLimitOrder)
or isinstance(style, ExchangeStopLimitOrder)):
if isinstance(style, ExchangeStopLimitOrder):
log.warn('{} will ignore the stop price'.format(self.name))
@@ -350,8 +352,8 @@ class Poloniex(Exchange):
return self.portfolio.open_orders
"""
TODO: Why going to the exchange if we already have this info locally?
And why creating all these Orders if we later discard them?
TODO: Why going to the exchange if we already have this info locally?
And why creating all these Orders if we later discard them?
"""
try:
@@ -365,7 +367,7 @@ class Poloniex(Exchange):
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve open orders: {}'.format(
order_statuses['message'])
response['message'])
)
print(self.portfolio.open_orders)
@@ -373,8 +375,8 @@ class Poloniex(Exchange):
# TODO: Need to handle openOrders for 'all'
orders = list()
for order_status in response:
order, executed_price = self._create_order(
order_status) # will Throw error b/c Polo doesn't track order['symbol']
# will Throw error b/c Polo doesn't track order['symbol']
order, executed_price = self._create_order(order_status)
if asset is None or asset == order.sid:
orders.append(order)
@@ -437,7 +439,8 @@ class Poloniex(Exchange):
if 'error' in response:
log.info(
'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format(
'Unable to cancel order {order_id} on exchange {exchange} '
'{error}.'.format(
order_id=order.id,
exchange=self.name,
error=response['error']
@@ -512,17 +515,17 @@ class Poloniex(Exchange):
else:
try:
start_date = cached_symbols[exchange_symbol]['start_date']
except KeyError as e:
except KeyError:
start_date = time.strftime('%Y-%m-%d')
try:
end_daily = cached_symbols[exchange_symbol]['end_daily']
except KeyError as e:
except KeyError:
end_daily = 'N/A'
try:
end_minute = cached_symbols[exchange_symbol]['end_minute']
except KeyError as e:
except KeyError:
end_minute = 'N/A'
symbol_map[exchange_symbol] = dict(
@@ -593,19 +596,21 @@ class Poloniex(Exchange):
else:
for tx in response:
"""
We maintain a list of dictionaries of transactions that correspond to
partially filled orders, indexed by order_id. Every time we query
executed transactions from the exchange, we check if we had that
transaction for that order already. If not, we process it.
We maintain a list of dictionaries of transactions that
correspond to partially filled orders, indexed by
order_id. Every time we query executed transactions
from the exchange, we check if we had that transaction
for that order already. If not, we process it.
When an order if fully filled, we flush the dict of transactions
associated with that order.
When an order if fully filled, we flush the dict of
transactions associated with that order.
"""
if (not filter(
lambda item: item['order_id'] == tx['tradeID'],
self.transactions[order_id])):
log.debug(
'Got new transaction for order {}: amount {}, price {}'.format(
'Got new transaction for order {}: amount {}, '
'price {}'.format(
order_id, tx['amount'], tx['rate']))
tx['amount'] = float(tx['amount'])
if (tx['type'] == 'sell'):
@@ -616,7 +621,7 @@ class Poloniex(Exchange):
dt=pd.to_datetime(tx['date'], utc=True),
price=float(tx['rate']),
order_id=tx['tradeID'],
# it's a misnomer, but keeping it for compatibility
# it's a misnomer, but keep for compatibility
commission=float(tx['fee'])
)
self.transactions[order_id].append(transaction)
@@ -626,7 +631,8 @@ class Poloniex(Exchange):
if (not order_open):
"""
Since transactions have been executed individually
the only thing left to do is remove them from list of open_orders
the only thing left to do is remove them from list
of open_orders
"""
del self.portfolio.open_orders[order_id]
del self.transactions[order_id]
catalyst/exchange/poloniex/poloniex_api.py
+5 -8
View File
@@ -107,8 +107,9 @@ class Poloniex_api(object):
data=post_data,
headers=headers,
)
return json.loads(
urlopen(req, context=ssl._create_unverified_context()).read())
resource = urlopen(req, context=ssl._create_unverified_context())
content = resource.read().decode('utf-8')
return json.loads(content)
def returnticker(self):
return self.query('returnTicker', {})
@@ -160,10 +161,6 @@ class Poloniex_api(object):
def returnopenorders(self, market):
return self.query('returnOpenOrders', {'currencyPair': market})
def returntradehistory(self, market):
# TODO: optional start and/or end and limit
return self.query('returnTradeHistory', {'currencyPair': market})
def returnordertrades(self, ordernumber):
return self.query('returnOrderTrades', {'orderNumber': ordernumber})
@@ -176,7 +173,7 @@ class Poloniex_api(object):
elif (immediateorcancel):
return self.query('buy', {'currencyPair': market, 'rate': rate,
'amount': amount,
'immediateOrCancel': immediateorcancel, })
'immediateOrCancel': immediateorcancel})
elif (postonly):
return self.query('buy', {'currencyPair': market, 'rate': rate,
'amount': amount,
@@ -194,7 +191,7 @@ class Poloniex_api(object):
elif (immediateorcancel):
return self.query('sell', {'currencyPair': market, 'rate': rate,
'amount': amount,
'immediateOrCancel': immediateorcancel, })
'immediateOrCancel': immediateorcancel})
elif (postonly):
return self.query('sell', {'currencyPair': market, 'rate': rate,
'amount': amount,
catalyst/exchange/simple_clock.py
+2 -1
View File
@@ -31,7 +31,8 @@ class SimpleClock(object):
This class is a drop-in replacement for
:class:`zipline.gens.sim_engine.MinuteSimulationClock`.
This is a stripped down version because crypto exchanges run around the clock.
This is a stripped down version because crypto exchanges run
around the clock.
The :param:`time_skew` parameter represents the time difference between
the Broker and the live trading machine's clock.
catalyst/exchange/stats_utils.py
+234 -29
View File
@@ -1,7 +1,18 @@
import csv
import numbers
import copy
import numpy as np
import os
import pandas as pd
import boto3
import time
from catalyst.assets._assets import TradingPair
from catalyst.exchange.exchange_utils import get_algo_folder
s3 = boto3.resource('s3')
def trend_direction(series):
@@ -119,62 +130,256 @@ def vwap(df):
return ret
def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
def set_position_row(row, asset, asset_values=list()):
"""
Apply the position data as individual columns.
Parameters
----------
row: dict[str, Object]
asset: TradingPair
asset_values: list[str]
If a recorded_col contains a tuple which first value is an asset
matching a position, its value will be displayed with the
position and not in the index.
Returns
-------
"""
asset_cols = ['symbol']
row['symbol'] = asset.symbol
position = next((p for p in row['positions'] if p['sid'] == asset), None)
columns = ['amount', 'cost_basis', 'last_sale_price']
for column in columns:
if position is not None:
row[column] = position[column]
else:
row[column] = 0
asset_cols.append(column)
values = asset_values[asset] if asset in asset_values else list()
for column in values:
row[column] = values[column]
asset_cols.append(column)
return asset_cols
def prepare_stats(stats, recorded_cols=list()):
"""
Prepare the stats DataFrame for user-friendly output.
Parameters
----------
stats: list[Object]
recorded_cols: list[str]
Returns
-------
"""
asset_cols = list()
stats = copy.deepcopy(stats)
# Using a copy since we are adding rows inside the loop.
for row_index, row_data in enumerate(list(stats)):
assets = [p['sid'] for p in row_data['positions']]
asset_values = dict()
if recorded_cols is not None:
for column in recorded_cols[:]:
value = row_data[column]
if type(value) is dict:
for asset in value:
if not isinstance(asset, TradingPair):
break
if asset not in assets:
assets.append(asset)
if asset not in asset_values:
asset_values[asset] = dict()
asset_values[asset][column] = value[asset]
if len(assets) == 1:
row = stats[row_index]
asset_cols = set_position_row(row, assets[0], asset_values)
elif len(assets) > 1:
for asset_index, asset in enumerate(assets):
if asset_index > 0:
row = copy.deepcopy(row_data)
stats.append(row)
else:
row = stats[row_index]
asset_cols = set_position_row(row, assets[asset_index],
asset_values)
df = pd.DataFrame(stats)
index_cols = [
'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
]
# Removing the asset specific entries
if recorded_cols is not None:
recorded_cols = [x for x in recorded_cols if x not in asset_cols]
for column in recorded_cols:
index_cols.append(column)
df['orders'] = df['orders'].apply(lambda orders: len(orders))
df['transactions'] = df['transactions'].apply(
lambda transactions: len(transactions)
)
if asset_cols:
columns = asset_cols
df.set_index(index_cols, drop=True, inplace=True)
else:
columns = index_cols
columns.remove('period_close')
df.set_index('period_close', drop=False, inplace=True)
df.dropna(axis=1, how='all', inplace=True)
df.sort_index(axis=0, level=0, inplace=True)
return df, columns
def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
"""
Format and print the last few rows of a statistics DataFrame.
See the pyfolio project for the data structure.
Parameters
----------
stats_df: DataFrame
stats: list[Object]
An array of statistics for the period.
num_rows: int
The number of rows to display on the screen.
Returns
-------
str
"""
stats_df.set_index('period_close', drop=True, inplace=True)
stats_df.dropna(axis=1, how='all', inplace=True)
if isinstance(stats, pd.DataFrame):
stats = stats.T.to_dict().values()
df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 3)
pd.set_option('precision', 8)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
columns = ['starting_cash', 'ending_cash', 'portfolio_value',
'pnl', 'long_exposure', 'short_exposure', 'orders',
'transactions', 'positions']
if recorded_cols is not None:
for column in recorded_cols:
columns.append(column)
def format_positions(positions):
parts = []
for position in positions:
msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format(
amount=position['amount'],
market=position['sid'].market_currency,
cost_basis=position['cost_basis'],
base=position['sid'].base_currency
)
parts.append(msg)
return ', '.join(parts)
formatters = {
'orders': lambda orders: len(orders),
'transactions': lambda transactions: len(transactions),
'returns': lambda returns: "{0:.4f}".format(returns),
'positions': format_positions
}
return stats_df.tail(num_rows).to_string(
return df.tail(num_rows).to_string(
columns=columns,
formatters=formatters
)
def get_csv_stats(stats, recorded_cols=None):
"""
Create a CSV buffer from the stats DataFrame.
Parameters
----------
path: str
stats: list[Object]
recorded_cols: list[str]
Returns
-------
"""
df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
return df.to_csv(
None,
columns=columns,
# encoding='utf-8',
quoting=csv.QUOTE_NONNUMERIC
).encode()
def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
folder='catalyst/stats', bytes_to_write=None):
"""
Uploads the performance stats to a S3 bucket.
Parameters
----------
uri: str
stats: list[Object]
algo_namespace: str
recorded_cols: list[str]
folder: str
bytes_to_write: str
Option to reuse bytes instead of re-computing the csv
Returns
-------
"""
if bytes_to_write is None:
bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
now = pd.Timestamp.utcnow()
timestr = now.strftime('%Y%m%d')
pid = os.getpid()
parts = uri.split('//')
obj = s3.Object(parts[1], '{}/{}-{}-{}.csv'.format(
folder, timestr, algo_namespace, pid
))
obj.put(Body=bytes_to_write)
def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
"""
Saves the performance stats to the algo local folder.
Parameters
----------
stats: list[Object]
algo_namespace: str
recorded_cols: list[str]
Returns
-------
str
"""
bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
timestr = time.strftime('%Y%m%d')
folder = get_algo_folder(algo_namespace)
filename = os.path.join(folder, '{}-{}.csv'.format(timestr, 'frames'))
with open(filename, 'wb') as handle:
handle.write(bytes_to_write)
return bytes_to_write
def df_to_string(df):
"""
Create a formatted str representation of the DataFrame.
catalyst/finance/execution.py
+1 -6
View File
@@ -15,13 +15,8 @@
import abc
from sys import float_info
from six import with_metaclass
import catalyst.utils.math_utils as zp_math
from numpy import isfinite
from six import with_metaclass
from catalyst.errors import BadOrderParameters
catalyst/finance/risk/period.py
+2 -2
View File
@@ -154,8 +154,8 @@ class RiskMetricsPeriod(object):
self.algorithm_returns.values,
self.benchmark_returns.values,
)
self.excess_return = self.algorithm_period_returns - \
self.treasury_period_return
self.excess_return = self.algorithm_period_returns \
- self.treasury_period_return
self.max_drawdown = max_drawdown(self.algorithm_returns.values)
self.max_leverage = self.calculate_max_leverage()
catalyst/finance/risk/risk.py
+2 -1
View File
@@ -160,7 +160,8 @@ def choose_treasury(select_treasury, treasury_curves, start_session,
)
break
if search_day and trading_calendar.name != 'OPEN': # Supress warning for 'OPEN' calendar
# Supress warning for 'OPEN' calendar
if search_day and trading_calendar.name != 'OPEN':
if (search_dist is None or search_dist > 1) and \
search_days[0] <= end_session <= search_days[-1]:
message = "No rate within 1 trading day of end date = \
catalyst/finance/slippage.py
-1
View File
@@ -41,7 +41,6 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025
DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05
class LiquidityExceeded(Exception):
pass
catalyst/pipeline/factors/equity/__init__.py
-3
View File
@@ -1,9 +1,6 @@
from .statistical import (
RollingPearson,
RollingLinearRegression,
RollingLinearRegressionOfReturns,
RollingPearsonOfReturns,
RollingSpearman,
RollingSpearmanOfReturns,
)
from .technical import (
catalyst/pipeline/loaders/equity_pricing_loader.py
+8 -4
View File
@@ -38,9 +38,11 @@ class USEquityPricingLoader(PipelineLoader):
def __init__(self, bundle, data_frequency, dataset):
if data_frequency == 'daily':
reader = bundle.daily_bar_reader
elif daily_bar_reader == 'minute':
# TODO: This is currently broken, No Pipeline support for Catalyst
# if data_frequency == 'daily':
# reader = bundle.daily_bar_reader
# elif daily_bar_reader == 'minute':
if data_frequency == 'minute':
reader = bundle.minute_bar_reader
else:
raise ValueError(
@@ -51,7 +53,9 @@ class USEquityPricingLoader(PipelineLoader):
if data_frequency == 'daily':
all_sessions = cal.all_sessions
elif daily_bar_reader == 'minute':
# TODO: this cannot be right, but no pipeline support at the moment
# elif daily_bar_reader == 'minute':
elif data_frequency == 'minute':
reader = bundle.minute_bar_reader
all_sessions = cal.all_minutes
catalyst/pipeline/loaders/events.py
+1 -1
View File
@@ -231,7 +231,7 @@ class EventsLoader(PipelineLoader):
self.load_next_events(n, dates, sids, mask),
self.load_previous_events(p, dates, sids, mask),
)
@property
def columns(self):
return self._columns
catalyst/pipeline/loaders/frame.py
-1
View File
@@ -180,4 +180,3 @@ class DataFrameLoader(PipelineLoader):
@property
def columns(self):
return self._columns
catalyst/pipeline/loaders/synthetic.py
+1 -1
View File
@@ -163,7 +163,7 @@ class SeededRandomLoader(PrecomputedLoader):
bool_dtype: self._bool_values,
object_dtype: self._object_values,
}[dtype](shape)
@property
def columns(self):
return self._columns
catalyst/support/issue_44.py
-109
View File
@@ -1,109 +0,0 @@
import pandas as pd
from catalyst import run_algorithm
from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.api import (
symbols,
)
def initialize(context):
context.i = -1
context.base_currency = 'btc'
def handle_data(context, data):
lookback = 60 * 24 * 7 # (minutes, hours, days)
context.i += 1
if context.i < lookback:
return
today = context.blotter.current_dt.strftime('%Y-%m-%d %H:%M:%S')
try:
# update universe everyday
new_day = 60 * 24
if not context.i % new_day:
context.universe = universe(context, today)
# get data every 30 minutes
minutes = 30
if not context.i % minutes and context.universe:
for coin in context.coins:
pair = str(coin.symbol)
# ohlcv data
open = data.history(coin, 'open', lookback,
'1m').ffill().bfill().resample(
'30T').first()
high = data.history(coin, 'high', lookback,
'1m').ffill().bfill().resample('30T').max()
low = data.history(coin, 'low', lookback,
'1m').ffill().bfill().resample('30T').min()
close = data.history(coin, 'price', lookback,
'1m').ffill().bfill().resample(
'30T').last()
volume = data.history(coin, 'volume', lookback,
'1m').ffill().bfill().resample(
'30T').sum()
print(today, pair, close[-1])
except Exception as e:
print(e)
def analyze(context=None, results=None):
pass
def universe(context, today):
json_symbols = get_exchange_symbols('poloniex')
poloniex_universe_df = pd.DataFrame.from_dict(
json_symbols).transpose().astype(str)
poloniex_universe_df['base_currency'] = poloniex_universe_df.apply(
lambda row: row.symbol.split('_')[1],
axis=1)
poloniex_universe_df['market_currency'] = poloniex_universe_df.apply(
lambda row: row.symbol.split('_')[0],
axis=1)
poloniex_universe_df = poloniex_universe_df[
poloniex_universe_df['base_currency'] == context.base_currency]
poloniex_universe_df = poloniex_universe_df[
poloniex_universe_df.symbol != 'gas_btc']
# Markets currently not working on Catalyst 0.3.1
# 2017-01-01
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'bcn_btc']
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'burst_btc']
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'dgb_btc']
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'doge_btc']
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'emc2_btc']
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'pink_btc']
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'sc_btc']
print(poloniex_universe_df.head())
date = str(today).split(' ')[0]
poloniex_universe_df = poloniex_universe_df[
poloniex_universe_df.start_date < date]
context.coins = symbols(*poloniex_universe_df.symbol)
print(len(poloniex_universe_df))
return poloniex_universe_df.symbol.tolist()
if __name__ == '__main__':
start_date = pd.to_datetime('2017-01-01', utc=True)
end_date = pd.to_datetime('2017-10-15', utc=True)
performance = run_algorithm(start=start_date, end=end_date,
capital_base=10000.0,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
data_frequency='minute',
base_currency='btc',
live=False,
live_graph=False,
algo_namespace='test')
catalyst/support/issue_47.py
-139
View File
@@ -1,139 +0,0 @@
"""
Requires Catalyst version 0.3.0 or above
Tested on Catalyst version 0.3.3
These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
You simply need to specify the exchange and the market that you want to focus on.
You will all see how to create a universe and filter it base on the exchange and the market you desire.
The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
Use this as the backbone to create your own trading strategies.
Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
"""
import numpy as np
import pandas as pd
from datetime import timedelta
from catalyst import run_algorithm
from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.api import (
symbols,
)
def initialize(context):
context.i = -1 # counts the minutes
context.exchange = 'poloniex' # must match the exchange specified in run_algorithm
context.base_currency = 'btc' # must match the base currency specified in run_algorithm
def handle_data(context, data):
lookback = 60 * 24 * 7 # (minutes, hours, days) of how far to lookback in the data history
context.i += 1
# current date formatted into a string
today = context.blotter.current_dt
date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = today - timedelta(days=(
lookback / (60 * 24))) # subtract the amount of days specified in lookback
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[
0] # get only the date as a string
# update universe everyday
new_day = 60 * 24
if not context.i % new_day:
context.universe = universe(context, lookback_date, date)
# get data every 30 minutes
minutes = 30
if not context.i % minutes and context.universe:
# we iterate for every pair in the current universe
for coin in context.coins:
pair = str(coin.symbol)
# 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
# 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
opened = fill(data.history(coin, 'open', bar_count=lookback,
frequency='30T')).values
high = fill(data.history(coin, 'high', bar_count=lookback,
frequency='30T')).values
low = fill(data.history(coin, 'low', bar_count=lookback,
frequency='30T')).values
close = fill(data.history(coin, 'price', bar_count=lookback,
frequency='30T')).values
volume = fill(data.history(coin, 'volume', bar_count=lookback,
frequency='30T')).values
# close[-1] is the equivalent to current price
# displays the minute price for each pair every 30 minutes
print(
today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
# ----------------------------------------------------------------------------------------------------------
# -------------------------------------- Insert Your Strategy Here -----------------------------------------
# ----------------------------------------------------------------------------------------------------------
def analyze(context=None, results=None):
pass
# Get the universe for a given exchange and a given base_currency market
# Example: Poloniex btc Market
def universe(context, lookback_date, current_date):
json_symbols = get_exchange_symbols(
context.exchange) # get all the pairs for the exchange
universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(
str) # convert into a dataframe
universe_df['base_currency'] = universe_df.apply(
lambda row: row.symbol.split('_')[1],
axis=1)
universe_df['market_currency'] = universe_df.apply(
lambda row: row.symbol.split('_')[0],
axis=1)
# Filter all the exchange pairs to only the ones for a give base currency
universe_df = universe_df[
universe_df['base_currency'] == context.base_currency]
# Filter all the pairs to ensure that pair existed in the current date range
universe_df = universe_df[universe_df.start_date < lookback_date]
universe_df = universe_df[universe_df.end_daily >= current_date]
context.coins = symbols(
*universe_df.symbol) # convert all the pairs to symbols
return universe_df.symbol.tolist()
# Replace all NA, NAN or infinite values with its nearest value
def fill(series):
if isinstance(series, pd.Series):
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
elif isinstance(series, np.ndarray):
return pd.Series(series).replace([np.inf, -np.inf],
np.nan).ffill().bfill().values
else:
return series
if __name__ == '__main__':
start_date = pd.to_datetime('2017-01-08', utc=True)
end_date = pd.to_datetime('2017-11-13', utc=True)
performance = run_algorithm(start=start_date, end=end_date,
capital_base=10000.0,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
data_frequency='minute',
base_currency='btc',
live=False,
live_graph=False,
algo_namespace='simple_universe')
"""
Run in Terminal (inside catalyst environment):
python simple_universe.py
"""
catalyst/support/issue_55.py
-1
View File
@@ -1,4 +1,3 @@
import talib
import pandas as pd
from catalyst import run_algorithm
catalyst/support/issue_57.py
-46
View File
@@ -1,46 +0,0 @@
import talib
import pandas as pd
from catalyst import run_algorithm
from catalyst.api import symbol
def initialize(context):
print('initializing')
context.asset = symbol('btc_usdt')
def handle_data(context, data):
print('handling bar: {}'.format(data.current_dt))
price = data.current(context.asset, 'close')
print('got price {price}'.format(price=price))
try:
prices = data.history(
context.asset,
fields='close',
bar_count=60,
frequency='1D'
)
print('got {} price entries\n'.format(len(prices), prices))
except Exception as e:
print(e)
run_algorithm(
capital_base=1,
start=pd.to_datetime('2016-2-11', utc=True),
end=pd.to_datetime('2017-8-31', utc=True),
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=None,
exchange_name='bittrex',
algo_namespace='issue_57',
base_currency='btc'
<<<<<<< HEAD
)
=======
)
>>>>>>> develop
catalyst/support/issue_74.py
-127
View File
@@ -1,127 +0,0 @@
from __future__ import division
import os
import pytz
import numpy as np
import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import datetime
from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True)
def initialize(context):
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i % context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n + 1, frequency='daily')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n + 1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val / tminus_val - 1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr), xr) / n
# Compute asset correlation matrix (informative only)
corr_m = cov_m / np.dot(np.transpose(stds), stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3 + context.nassets, n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w, np.transpose(m))) * 365
p_std = np.sqrt(
np.dot(np.dot(w, cov_m), np.transpose(w))) * np.sqrt(365)
# store results in results array
results_array[0, p] = p_r
results_array[1, p] = p_std
# store Sharpe Ratio (return / volatility) - risk free rate element
# excluded for simplicity
results_array[2, p] = results_array[0, p] / results_array[1, p]
i = 0
for iw in weights:
results_array[3 + i, p] = weights[i]
i += 1
# convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r', 'stdev',
'sharpe'] + context.assets)
# locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
# locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
# order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
# create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev, results_frame.r,
c=results_frame.sharpe, cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
# plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1], max_sharpe_port[0], marker='o',
color='b', s=200)
# plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr, r=r, m=m, stds=stds, max_sharpe_port=max_sharpe_port,
corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None):
# Form DataFrame with selected data
data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m',
'portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
catalyst/support/rodrigo_1.py
-153
View File
@@ -1,153 +0,0 @@
import pandas as pd
from logbook import Logger, DEBUG
from catalyst import run_algorithm
from catalyst.api import (schedule_function, order_target_percent, symbol,
date_rules, get_open_orders, cancel_order, record,
set_commission, set_slippage)
log = Logger('rodrigo_1', level=DEBUG)
"""
The initialize function sets any data or variables that
you'll use in your algorithm.
It's only called once at the beginning of your algorithm.
"""
def initialize(context):
# Select asset of interest
context.asset = symbol('BTC_USD')
# set_commission(TradingPairFeeSchedule(maker_fee=0.5, taker_fee=0.5))
# set_slippage(TradingPairFixedSlippage(spread=0.5))
# Set up a rebalance method to run every day
schedule_function(rebalance, date_rule=date_rules.every_day())
"""
Rebalance function scheduled to run once per day.
"""
def rebalance(context, data):
# To make market decisions, we're calculating the token's
# moving average for the last 5 days.
# We get the price history for the last 5 days.
price_history = data.history(context.asset, fields='price', bar_count=5,
frequency='1d')
# Then we take an average of those 5 days.
average_price = price_history.mean()
# We also get the coin's current price.
price = data.current(context.asset, 'price')
# Cancel any outstanding orders
orders = get_open_orders(context.asset) or []
for order in orders:
cancel_order(order)
# If our coin is currently listed on a major exchange
if data.can_trade(context.asset):
# If the current price is 1% above the 5-day average price,
# we open a long position. If the current price is below the
# average price, then we want to close our position to 0 shares.
if price > (1.01 * average_price):
# Place the buy order (positive means buy, negative means sell)
order_target_percent(context.asset, .99)
log.info("Buying %s" % (context.asset.symbol))
elif price < average_price:
# Sell all of our shares by setting the target position to zero
order_target_percent(context.asset, 0)
log.info("Selling %s" % (context.asset.symbol))
# Use the record() method to track up to five custom signals.
# Record Apple's current price and the average price over the last
# five days.
cash = context.portfolio.cash
leverage = context.account.leverage
record(price=price, average_price=average_price, cash=cash,
leverage=leverage)
def analyze(context=None, results=None):
import matplotlib.pyplot as plt
# Plot the portfolio and asset data.
ax1 = plt.subplot(511)
results[['portfolio_value']].plot(ax=ax1)
ax1.set_ylabel('Portfolio Value (USD)')
ax2 = plt.subplot(512, sharex=ax1)
ax2.set_ylabel('{asset} (USD)'.format(asset=context.asset))
(results[[
'price',
]]).plot(ax=ax2)
trans = results.ix[[t != [] for t in results.transactions]]
buys = trans.ix[
[t[0]['amount'] > 0 for t in trans.transactions]
]
sells = trans.ix[
[t[0]['amount'] < 0 for t in trans.transactions]
]
ax2.plot(
buys.index,
results.price[buys.index],
'^',
markersize=10,
color='g',
)
ax2.plot(
sells.index,
results.price[sells.index],
'v',
markersize=10,
color='r',
)
ax3 = plt.subplot(513, sharex=ax1)
results[['leverage']].plot(ax=ax3)
ax3.set_ylabel('Leverage ')
ax4 = plt.subplot(514, sharex=ax1)
results[['cash']].plot(ax=ax4)
ax4.set_ylabel('Cash (USD)')
results[[
'algorithm',
'benchmark',
]] = results[[
'algorithm_period_return',
'benchmark_period_return',
]]
ax5 = plt.subplot(515, sharex=ax1)
results[[
'algorithm',
'benchmark',
]].plot(ax=ax5)
ax5.set_ylabel('Percent Change')
plt.legend(loc=3)
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
run_algorithm(
capital_base=100000,
start=pd.to_datetime('2017-1-1', utc=True),
end=pd.to_datetime('2017-10-22', utc=True),
data_frequency='minute',
initialize=initialize,
handle_data=None,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace='rodrigo_1',
base_currency='usd'
)
catalyst/utils/calendars/exchange_calendar_open.py
+2 -1
View File
@@ -31,4 +31,5 @@ class OpenExchangeCalendar(TradingCalendar):
return DateOffset(days=1)
def __init__(self, *args, **kwargs):
super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)
super(OpenExchangeCalendar, self).__init__(
start=Timestamp('2015-3-1', tz='UTC'), **kwargs)
catalyst/utils/cli.py
+3 -1
View File
@@ -9,6 +9,7 @@ DEFAULT_BAR_TEMPLATE = ' [%(bar)s] %(label)s: %(info)s'
DEFAULT_EMPTY_CHAR = ' '
DEFAULT_FILL_CHAR = '='
def item_show_count(total=None):
def maybe_show_total(index):
if total is not None:
@@ -17,12 +18,13 @@ def item_show_count(total=None):
def item_show_func(item, _it=iter(count())):
if item is not None:
starting = False
# starting = False
return maybe_show_total(next(_it))
return 'DONE'
return item_show_func
def maybe_show_progress(it,
show_progress,
empty_char=DEFAULT_EMPTY_CHAR,
catalyst/utils/math_utils.py
+2
View File
@@ -17,9 +17,11 @@ import math
from numpy import isnan
def round_nearest(x, a):
return round(round(x / a) * a, -int(math.floor(math.log10(a))))
def tolerant_equals(a, b, atol=10e-7, rtol=10e-7, equal_nan=False):
"""Check if a and b are equal with some tolerance.
catalyst/utils/paths.py
+1 -1
View File
@@ -126,7 +126,7 @@ def catalyst_root(environ=None):
root = environ.get('ZIPLINE_ROOT', None)
if root is None:
root = os.path.join(expanduser('~'),'.catalyst')
root = os.path.join(expanduser('~'), '.catalyst')
return root
catalyst/utils/run_algo.py
+48 -79
View File
@@ -8,12 +8,11 @@ from time import sleep
import click
import pandas as pd
from logbook import Logger
from catalyst.data.bundles import load
from catalyst.data.data_portal import DataPortal
from catalyst.exchange.bittrex.bittrex import Bittrex
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
from catalyst.exchange.poloniex.poloniex import Poloniex
from catalyst.exchange.factory import get_exchange
try:
from pygments import highlight
@@ -32,19 +31,16 @@ from catalyst.utils.factory import create_simulation_parameters
from catalyst.data.loader import load_crypto_market_data
import catalyst.utils.paths as pth
from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \
ExchangeTradingAlgorithmBacktest
from catalyst.exchange.exchange_algorithm import (
ExchangeTradingAlgorithmLive,
ExchangeTradingAlgorithmBacktest,
)
from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
DataPortalExchangeBacktest
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
from catalyst.exchange.exchange_portfolio import ExchangePortfolio
from catalyst.exchange.exchange_errors import (
ExchangeRequestError, ExchangeAuthEmpty,
ExchangeRequestErrorTooManyAttempts,
BaseCurrencyNotFoundError, ExchangeNotFoundError)
from catalyst.exchange.exchange_utils import get_exchange_auth, \
get_algo_object, get_exchange_folder
from logbook import Logger
ExchangeRequestError, ExchangeRequestErrorTooManyAttempts,
BaseCurrencyNotFoundError, NotEnoughCapitalError)
from catalyst.constants import LOG_LEVEL
@@ -94,7 +90,9 @@ def _run(handle_data,
exchange,
algo_namespace,
base_currency,
live_graph):
live_graph,
simulate_orders,
stats_output):
"""Run a backtest for the given algorithm.
This is shared between the cli and :func:`catalyst.run_algo`.
@@ -143,7 +141,8 @@ def _run(handle_data,
else:
click.echo(algotext)
mode = 'live' if live else 'backtest'
mode = 'paper-trading' if simulate_orders else 'live-trading' \
if live else 'backtest'
log.info('running algo in {mode} mode'.format(mode=mode))
exchange_name = exchange
@@ -154,53 +153,12 @@ def _run(handle_data,
exchanges = dict()
for exchange_name in exchange_list:
# Looking for the portfolio from the cache first
portfolio = get_algo_object(
algo_name=algo_namespace,
key='portfolio_{}'.format(exchange_name),
environ=environ
exchanges[exchange_name] = get_exchange(
exchange_name=exchange_name,
base_currency=base_currency,
must_authenticate=(live and not simulate_orders),
)
if portfolio is None:
portfolio = ExchangePortfolio(
start_date=pd.Timestamp.utcnow()
)
# This corresponds to the json file containing api token info
exchange_auth = get_exchange_auth(exchange_name)
if live and (exchange_auth['key'] == '' \
or exchange_auth['secret'] == ''):
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name, environ), 'auth.json'))
if exchange_name == 'bitfinex':
exchanges[exchange_name] = Bitfinex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'bittrex':
exchanges[exchange_name] = Bittrex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
elif exchange_name == 'poloniex':
exchanges[exchange_name] = Poloniex(
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
else:
raise ExchangeNotFoundError(exchange_name=exchange_name)
open_calendar = get_calendar('OPEN')
env = TradingEnvironment(
@@ -215,7 +173,7 @@ def _run(handle_data,
asset_db_path=None # We don't need an asset db, we have exchanges
)
env.asset_finder = AssetFinderExchange()
choose_loader = None # TODO: use the DataPortal for in the algorithm class for this
choose_loader = None # TODO: use the DataPortal in the algo class for this
if live:
start = pd.Timestamp.utcnow()
@@ -263,35 +221,32 @@ def _run(handle_data,
)
if base_currency in balances:
base_currency_available = balances[base_currency]
base_currency_available = balances[base_currency]['free']
log.info(
'base currency available in the account: {} {}'.format(
base_currency_available, base_currency
)
)
if capital_base is not None \
and capital_base < base_currency_available:
log.info(
'using capital base limit: {} {}'.format(
capital_base, base_currency
)
)
amount = capital_base
else:
amount = base_currency_available
return amount
return base_currency_available
else:
raise BaseCurrencyNotFoundError(
base_currency=base_currency,
exchange=exchange_name
)
combined_capital_base = 0
for exchange_name in exchanges:
exchange = exchanges[exchange_name]
combined_capital_base += fetch_capital_base(exchange)
if not simulate_orders:
for exchange_name in exchanges:
exchange = exchanges[exchange_name]
balance = fetch_capital_base(exchange)
if balance < capital_base:
raise NotEnoughCapitalError(
exchange=exchange_name,
base_currency=base_currency,
balance=balance,
capital_base=capital_base,
)
sim_params = create_simulation_parameters(
start=start,
@@ -308,7 +263,9 @@ def _run(handle_data,
ExchangeTradingAlgorithmLive,
exchanges=exchanges,
algo_namespace=algo_namespace,
live_graph=live_graph
live_graph=live_graph,
simulate_orders=simulate_orders,
stats_output=stats_output,
)
elif exchanges:
# Removed the existing Poloniex fork to keep things simple
@@ -470,6 +427,8 @@ def run_algorithm(initialize,
base_currency=None,
algo_namespace=None,
live_graph=False,
simulate_orders=True,
stats_output=None,
output=os.devnull):
"""Run a trading algorithm.
@@ -544,6 +503,14 @@ def run_algorithm(initialize,
default_extension, extensions, strict_extensions, environ
)
if capital_base is None:
raise ValueError(
'Please specify a `capital_base` parameter which is the maximum '
'amount of base currency available for trading. For example, '
'if the `capital_base` is 5ETH, the '
'`order_target_percent(asset, 1)` command will order 5ETH worth '
'of the specified asset.'
)
# I'm not sure that we need this since the modified DataPortal
# does not require extensions to be explicitly loaded.
@@ -591,5 +558,7 @@ def run_algorithm(initialize,
exchange=exchange_name,
algo_namespace=algo_namespace,
base_currency=base_currency,
live_graph=live_graph
live_graph=live_graph,
simulate_orders=simulate_orders,
stats_output=stats_output
)
docs/source/beginner-tutorial.rst
+15805 -2
View File
File diff suppressed because it is too large Load Diff
docs/source/example-algos.rst
+328
View File
@@ -31,6 +31,18 @@ Overview
`two-part video tutorial <videos.html#backtesting-a-strategy>`_ to show how
to get started in backtesting and live trading with Catalyst.
- :ref:`Simple Universe <simple_universe>`: This code provides the 'universe'
of available trading pairs on a given exchange on any given day. You can use
this code to dynamically select which currency pairs you want to trade each
day of your strategy. This example does not make any trades.
- :ref:`Portfolio Optimization <portfolio_optimization>`: Use this code to
execute a portfolio optimization model. This strategy will select the
portfolio with the maximum Sharpe Ratio. The parameters are set to use 180
days of historical data and rebalance every 30 days. This code was used in
writting the following article:
`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
.. _buy_btc_simple:
@@ -746,4 +758,320 @@ implemented after the video was recorded, which executes the orders at slighlty
different prices, but resulting in significant changes in performance of our
strategy.
.. _simple_universe:
Simple Universe
~~~~~~~~~~~~~~~
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
This example aims to provide an easy way for users to learn how to
collect data from any given exchange and select a subset of the available
currency pairs for trading. You simply need to specify the exchange and
the market (base_currency) that you want to focus on. You will then see
how to create a universe of assets, and filter it based the market you
desire.
The example prints out the closing price of all the pairs for a given
market in a given exchange every 30 minutes. The example also contains
the OHLCV data with minute-resolution for the past seven days which
could be used to create indicators. Use this code as the backbone to
create your own trading strategy.
The lookback_date variable is used to ensure data for a coin existed on
the lookback period specified.
To run, execute the following two commands in a terminal (inside catalyst
environment). The first one retrieves all the pricing data needed for this
script to run (only needs to be run once), and the second one executes this
script with the parameters specified in the run_algorithm() call at the end
of the file:
.. code-block:: bash
catalyst ingest-exchange -x bitfinex -f minute
.. code-block:: bash
python simple_universe.py
Credits: This code was originally submitted by `Abner Ayala-Acevedo
<https://github.com/abnera>`_. Thank you!
.. code-block:: python
from datetime import timedelta
import numpy as np
import pandas as pd
from catalyst import run_algorithm
from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.api import (symbols, )
def initialize(context):
context.i = -1 # minute counter
context.exchange = context.exchanges.values()[0].name.lower()
context.base_currency = context.exchanges.values()[0].base_currency.lower()
def handle_data(context, data):
context.i += 1
lookback_days = 7 # 7 days
# current date & time in each iteration formatted into a string
now = data.current_dt
date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = now - timedelta(days=lookback_days)
# keep only the date as a string, discard the time
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute'
# update universe everyday at midnight
if not context.i % one_day_in_minutes:
context.universe = universe(context, lookback_date, date)
# get data every 30 minutes
minutes = 30
# get lookback_days of history data: that is 'lookback' number of bins
lookback = one_day_in_minutes / minutes * lookback_days
if not context.i % minutes and context.universe:
# we iterate for every pair in the current universe
for coin in context.coins:
pair = str(coin.symbol)
# Get 30 minute interval OHLCV data. This is the standard data
# required for candlestick or indicators/signals. Return Pandas
# DataFrames. 30T means 30-minute re-sampling of one minute data.
# Adjust it to your desired time interval as needed.
opened = fill(data.history(coin, 'open',
bar_count=lookback, frequency='30T')).values
high = fill(data.history(coin, 'high',
bar_count=lookback, frequency='30T')).values
low = fill(data.history(coin, 'low',
bar_count=lookback, frequency='30T')).values
close = fill(data.history(coin, 'price',
bar_count=lookback, frequency='30T')).values
volume = fill(data.history(coin, 'volume',
bar_count=lookback, frequency='30T')).values
# close[-1] is the last value in the set, which is the equivalent
# to current price (as in the most recent value)
# displays the minute price for each pair every 30 minutes
print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format(
now=now,
pair=pair,
o=opened[-1],
h=high[-1],
l=low[-1],
c=close[-1],
v=volume[-1],
))
# -------------------------------------------------------------
# --------------- Insert Your Strategy Here -------------------
# -------------------------------------------------------------
def analyze(context=None, results=None):
pass
# Get the universe for a given exchange and a given base_currency market
# Example: Poloniex BTC Market
def universe(context, lookback_date, current_date):
# get all the pairs for the given exchange
json_symbols = get_exchange_symbols(context.exchange)
# convert into a DataFrame for easier processing
df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1)
df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1)
# Filter all the pairs to get only the ones for a given base_currency
df = df[df['base_currency'] == context.base_currency]
# Filter all the pairs to ensure that pair existed in the current date range
df = df[df.start_date < lookback_date]
df = df[df.end_daily >= current_date]
context.coins = symbols(*df.symbol) # convert all the pairs to symbols
return df.symbol.tolist()
# Replace all NA, NAN or infinite values with its nearest value
def fill(series):
if isinstance(series, pd.Series):
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
elif isinstance(series, np.ndarray):
return pd.Series(series).replace(
[np.inf, -np.inf], np.nan
).ffill().bfill().values
else:
return series
if __name__ == '__main__':
start_date = pd.to_datetime('2017-11-10', utc=True)
end_date = pd.to_datetime('2017-11-13', utc=True)
performance = run_algorithm(start=start_date, end=end_date,
capital_base=100.0, # amount of base_currency
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
data_frequency='minute',
base_currency='btc',
live=False,
live_graph=False,
algo_namespace='simple_universe')
.. _portfolio_optimization:
Portfolio Optimization
~~~~~~~~~~~~~~~~~~~~~~
Use this code to execute a portfolio optimization model. This strategy will
select the portfolio with the maximum Sharpe Ratio. The parameters are set to
use 180 days of historical data and rebalance every 30 days. This code was used
in writting the following article:
`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
.. code-block:: python
'''
You can run this code using the Python interpreter:
$ python portfolio_optimization.py
'''
from __future__ import division
import os
import pytz
import numpy as np
import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import datetime
from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True)
def initialize(context):
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i%context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n+1, frequency='1d')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n+1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val/tminus_val-1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr),xr)/n
# Compute asset correlation matrix (informative only)
corr_m = cov_m/np.dot(np.transpose(stds),stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3+context.nassets,n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w,np.transpose(m)))*365
p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
#store results in results array
results_array[0,p] = p_r
results_array[1,p] = p_std
#store Sharpe Ratio (return / volatility) - risk free rate element
#excluded for simplicity
results_array[2,p] = results_array[0,p] / results_array[1,p]
i = 0
for iw in weights:
results_array[3+i,p] = weights[i]
i += 1
#convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r','stdev','sharpe']+context.assets)
#locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
#locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
#order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
#create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
#plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
#plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None):
# Form DataFrame with selected data
data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
.. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
:align: center
docs/source/naming-convention.rst → docs/source/features.rst
+57 -1
View File
@@ -1,5 +1,61 @@
Features
========
This page describes the features that Catalyst provides in the current version,
and what is planned for future releases.
Current Functionality
~~~~~~~~~~~~~~~~~~~~~
* Backtesting and live-trading modes to run your trading algorithms, with a
seamless transition between the two.
* Paper trading simulates order in live-trading mode.
* Support for 3 exchanges: Bitfinex, Bittrex and Poloniex in both modes
(backtesting and live-trading). Historical data for backtesting is provided
with daily resolution for all three exchanges, and minute resolution for
Bitfinex and Poloniex. No minute-resolution data is currently available for
Bittrex. Refer to
`Catalyst Market Coverage <https://www.enigma.co/catalyst/status>`_ for
details.
* Interface with over 90 exchanges available in live and paper trading modes.
* Granular commission models which closely simulates each exchange fee
structure in backtesting and paper trading.
* Standardized naming convention for all asset pairs trading on any exchange in
the form ``{market_currency}_{base_currency}``. See
:ref:`naming`.
* Output of performance statistics based on Pandas DataFrames to integrate
nicely into the existing PyData ecosystem.
* Support for accessing multiple exchanges per algorithm, which opens the door
to cross-exchange arbitrage opportunities.
* Support for running multiple algorithms on the same exchange independently of
one another. Catalyst performance tracker stores just enough data to allow
algorithms to run independently while still sharing critical data through
exchanges.
* Benchmark defaults to Bitcoin price (btc_usdt in Poloniex exchange) for the
purpose of comparing performance across trading algorithms. A custom benchmark
can be specified through ``set_benchmark()`` (but see
`issue #86 <https://github.com/enigmampc/catalyst/issues/86>`_).
* Support for MacOS, Linux and Windows installations.
* Support for Python2 and Python3.
For additional details on the functionality added on recent releases, see the
:doc:`Release Notes<releases>`.
Upcoming features
~~~~~~~~~~~~~~~~~
* Additional datasets beyond pricing data (Dec. 2017)
* API documentation (Jan. 2017)
* Support for decentralized exchanges (Jan. 2017)
* Support for data ingestion of community-contributed data sets (Jan. 2017)
* Pipeline support (Jan. 2018)
* Web UI (Q2 2018)
.. _naming:
Naming Convention
=================
~~~~~~~~~~~~~~~~~
Catalyst introduces a standardized naming convention for all asset pairs
trading on any exchange in the following form:
docs/source/index.rst
+2 -4
View File
@@ -1,4 +1,4 @@
.. include:: welcome.rst
.. include:: ../../README.rst
|
|
Table of Contents
@@ -9,9 +9,8 @@ Table of Contents
install
beginner-tutorial
jupyter
live-trading
naming-convention
features
example-algos
utilities
videos
@@ -19,7 +18,6 @@ Table of Contents
development-guidelines
releases
.. bundles
.. development-guidelines
.. appendix
.. release-process
docs/source/jupyter.rst
-15794
View File
File diff suppressed because it is too large Load Diff
docs/source/live-trading.rst
+6
View File
@@ -106,6 +106,10 @@ What differs are the arguments provided to the catalyst client or
Here is the breakdown of the new arguments:
- ``live``: Boolean flag which enables live trading.
- ``capital_base``: The amount of base_currency assigned to the strategy.
It has to be lower or equal to the amount of base currency available for
trading on the exchange. For illustration, order_target_percent(asset, 1)
will order the capital_base amount specified here of the specified asset.
- ``exchange_name``: The name of the targeted exchange
(supported values: *bitfinex*, *bittrex*).
- ``algo_namespace``: A arbitrary label assigned to your algorithm for
@@ -113,6 +117,8 @@ Here is the breakdown of the new arguments:
- ``base_currency``: The base currency used to calculate the
statistics of your algorithm. Currently, the base currency of all
trading pairs of your algorithm must match this value.
- ``simulate_orders``: Enables the paper trading mode, in which orders are
simulated in Catalyst instead of processed on the exchange.
Here is a complete algorithm for reference:
`Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_
docs/source/releases.rst
+19
View File
@@ -2,6 +2,15 @@
Release Notes
=============
Version 0.3.10
^^^^^^^^^^^^^
**Release Date**: 2017-12-12
Bug Fixes
~~~~~~~~~
- Fixed issue with fetching assets with daily frequency
Version 0.3.10
^^^^^^^^^^^^^
**Release Date**: 2017-11-28
@@ -10,6 +19,16 @@ Bug Fixes
~~~~~~~~~
- Fixed issue with fetching assets with daily frequency
- Changed Poloniex interface (should solve :issue:`95` and :issue:`94`)
- Solved issue with overriding commission and slippage (:issue:`87`)
- Fixed inefficiency with Bittrex current prices (:issue:`76`)
Build
~~~~~
- Integrated with CCXT
- Added paper trading capability (`simulate_orders=True` param in live mode)
- More granular commissions (:issue:`82`)
- Added market orders in live mode (:issue:`81`)
Version 0.3.9
^^^^^^^^^^^^^
docs/source/videos.rst
+17 -1
View File
@@ -32,7 +32,9 @@ Where things don't:
Backtesting a Strategy
----------------------
This algorithm is based on a simple momentum strategy. When the cryptoasset
This is the first video of a two-part series on using Catalyst for algorithmic
trading. This video implements a simple momentum strategy based on
`mean reversion <example-algos.html#mean-reversion>`_: when the cryptoasset
goes up quickly, we’re going to buy; when it goes down quickly, we’re going to
sell. Hopefully, we’ll ride the waves.
@@ -40,3 +42,17 @@ sell. Hopefully, we’ll ride the waves.
<iframe width="560" height="315" src="https://www.youtube.com/embed/JOBRwst9jUY" frameborder="0" allowfullscreen></iframe>
|
|
Live Trading a Strategy
-----------------------
This is the second part of the two-part series on using Catalyst for algorithmic
trading. Having backtested `our strategy <example-algos.html#mean-reversion>`_
in the previous video, we now take it to trade live against the Bittrex exchange.
.. raw:: html
<iframe width="560" height="315" src="https://www.youtube.com/embed/NupiE-Xuglw" frameborder="0" allowfullscreen></iframe>
|
|
docs/source/welcome.rst
-43
View File
@@ -1,43 +0,0 @@
.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
|
Catalyst is an algorithmic trading library for crypto-assets written in Python.
It allows trading strategies to be easily expressed and backtested against
historical data (with daily and minute resolution), providing analytics and
insights regarding a particular strategy's performance. Catalyst also supports
live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex,
and Poloniex) with more being added over time. Catalyst empowers users to share
and curate data and build profitable, data-driven investment strategies. Please
visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or
refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for
further technical details.
Catalyst builds on top of the well-established
`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to
minimize structural changes to the general API to maximize compatibility with
existing trading algorithms, developer knowledge, and tutorials. Join us on
`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel
for questions around Catalyst, algorithmic trading and technical support.
Features
========
- Ease of use: Catalyst tries to get out of your way so that you can
focus on algorithm development. See
`examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_
provided.
- Support for several of the top crypto-exchanges by trading volume:
`Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
and `Poloniex <https://www.poloniex.com>`_.
- Secure: You and only you have access to each exchange API keys for your accounts.
- Input of historical pricing data of all crypto-assets by exchange,
with daily and minute resolution. See
`Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
- Backtesting and live-trading functionality, with a seamless transition
between the two modes.
- Output of performance statistics are based on Pandas DataFrames to
integrate nicely into the existing PyData eco-system.
- Statistic and machine learning libraries like matplotlib, scipy,
statsmodels, and sklearn support development, analysis, and
visualization of state-of-the-art trading systems.
- Addition of Bitcoin price (btc_usdt) as a benchmark for comparing
performance across trading algorithms.
etc/python2.7-environment.yml
+1
View File
@@ -20,6 +20,7 @@ dependencies:
- bcolz==0.12.1
- bottleneck==1.2.1
- chardet==3.0.4
- ccxt==1.10.319
- click==6.7
- contextlib2==0.5.5
- cycler==0.10.0
etc/requirements.txt
+3
View File
@@ -80,3 +80,6 @@ empyrical==0.2.1
tables==3.3.0
#Catalyst dependencies
ccxt==1.10.283
boto3==1.4.8
tests/exchange/test_bcolz.py
+2 -2
View File
@@ -116,7 +116,7 @@ class TestBcolzWriter(object):
df = self.generate_df(exchange_name, freq, start, end)
print df.index[0],df.index[-1]
print(df.index[0], df.index[-1])
writer = BcolzExchangeBarWriter(
rootdir=self.root_dir,
@@ -140,7 +140,7 @@ class TestBcolzWriter(object):
dx = get_df_from_arrays(arrays, periods)
assert_equals(df.equals(df), True)
assert_equals(df.equals(dx), True)
pass
def test_bcolz_bitfinex_daily_write_read(self):
tests/exchange/test_bitfinex.py
+13 -12
View File
@@ -4,10 +4,12 @@ from base import BaseExchangeTestCase
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
from catalyst.exchange.exchange_utils import get_exchange_auth
from catalyst.finance.execution import (LimitOrder)
from catalyst.utils.deprecate import deprecated
log = Logger('test_bitfinex')
@deprecated
class TestBitfinex(BaseExchangeTestCase):
@classmethod
def setup(self):
@@ -34,7 +36,7 @@ class TestBitfinex(BaseExchangeTestCase):
def test_open_orders(self):
log.info('retrieving open orders')
orders = self.exchange.get_open_orders()
# orders = self.exchange.get_open_orders()
pass
def test_get_order(self):
@@ -47,18 +49,17 @@ class TestBitfinex(BaseExchangeTestCase):
def test_get_candles(self):
log.info('retrieving candles')
ohlcv_neo = self.exchange.get_candles(
freq='1T',
assets=self.exchange.get_asset('neo_btc')
)
# ohlcv_neo = self.exchange.get_candles(
# freq='1T',
# assets=self.exchange.get_asset('neo_btc'))
pass
def test_tickers(self):
log.info('retrieving tickers')
tickers = self.exchange.tickers([
self.exchange.get_asset('eth_btc'),
self.exchange.get_asset('etc_btc')
])
# tickers = self.exchange.tickers([
# self.exchange.get_asset('eth_btc'),
# self.exchange.get_asset('etc_btc')
# ])
pass
def test_get_account(self):
@@ -67,11 +68,11 @@ class TestBitfinex(BaseExchangeTestCase):
def test_get_balances(self):
log.info('testing exchange balances')
balances = self.exchange.get_balances()
# balances = self.exchange.get_balances()
pass
def test_orderbook(self):
log.info('testing order book for bitfinex')
asset = self.exchange.get_asset('eth_btc')
orderbook = self.exchange.get_orderbook(asset)
# asset = self.exchange.get_asset('eth_btc')
# orderbook = self.exchange.get_orderbook(asset)
pass
tests/exchange/test_bittrex.py
+23 -21
View File
@@ -1,13 +1,15 @@
import pandas as pd
# import pandas as pd
from catalyst.exchange.bittrex.bittrex import Bittrex
from catalyst.finance.order import Order
from base import BaseExchangeTestCase
from logbook import Logger
from catalyst.exchange.exchange_utils import get_exchange_auth
from catalyst.utils.deprecate import deprecated
log = Logger('test_bittrex')
@deprecated
class TestBittrex(BaseExchangeTestCase):
@classmethod
def setup(self):
@@ -33,8 +35,8 @@ class TestBittrex(BaseExchangeTestCase):
def test_open_orders(self):
log.info('retrieving open orders')
asset = self.exchange.get_asset('neo_btc')
orders = self.exchange.get_open_orders(asset)
# asset = self.exchange.get_asset('neo_btc')
# orders = self.exchange.get_open_orders(asset)
pass
def test_get_order(self):
@@ -51,21 +53,21 @@ class TestBittrex(BaseExchangeTestCase):
def test_get_candles(self):
log.info('retrieving candles')
ohlcv_neo = self.exchange.get_candles(
freq='5T',
assets=self.exchange.get_asset('neo_btc'),
bar_count=20,
end_dt=pd.to_datetime('2017-10-20', utc=True)
)
ohlcv_neo_ubq = self.exchange.get_candles(
freq='1D',
assets=[
self.exchange.get_asset('neo_btc'),
self.exchange.get_asset('ubq_btc')
],
bar_count=14,
end_dt=pd.to_datetime('2017-10-20', utc=True)
)
# ohlcv_neo = self.exchange.get_candles(
# freq='5T',
# assets=self.exchange.get_asset('neo_btc'),
# bar_count=20,
# end_dt=pd.to_datetime('2017-10-20', utc=True)
# )
# ohlcv_neo_ubq = self.exchange.get_candles(
# freq='1D',
# assets=[
# self.exchange.get_asset('neo_btc'),
# self.exchange.get_asset('ubq_btc')
# ],
# bar_count=14,
# end_dt=pd.to_datetime('2017-10-20', utc=True)
# )
pass
def test_tickers(self):
@@ -79,7 +81,7 @@ class TestBittrex(BaseExchangeTestCase):
def test_get_balances(self):
log.info('testing wallet balances')
balances = self.exchange.get_balances()
# balances = self.exchange.get_balances()
pass
def test_get_account(self):
@@ -88,6 +90,6 @@ class TestBittrex(BaseExchangeTestCase):
def test_orderbook(self):
log.info('testing order book for bittrex')
asset = self.exchange.get_asset('eth_btc')
orderbook = self.exchange.get_orderbook(asset)
# asset = self.exchange.get_asset('eth_btc')
# orderbook = self.exchange.get_orderbook(asset)
pass
tests/exchange/test_bundle.py
+30 -33
View File
@@ -1,11 +1,10 @@
import hashlib
# import hashlib
import os
import tempfile
from logging import getLogger
import pandas as pd
from catalyst import get_calendar
from catalyst.exchange.bundle_utils import get_bcolz_chunk, \
get_start_dt, get_df_from_arrays
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
@@ -22,22 +21,22 @@ log = getLogger('test_exchange_bundle')
class TestExchangeBundle:
def test_spot_value(self):
data_frequency = 'daily'
exchange_name = 'poloniex'
# data_frequency = 'daily'
# exchange_name = 'poloniex'
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
assets = [
exchange.get_asset('btc_usdt')
]
dt = pd.to_datetime('2017-10-14', utc=True)
# exchange = get_exchange(exchange_name)
# exchange_bundle = ExchangeBundle(exchange)
# assets = [
# exchange.get_asset('btc_usdt')
# ]
# dt = pd.to_datetime('2017-10-14', utc=True)
values = exchange_bundle.get_spot_values(
assets=assets,
field='close',
dt=dt,
data_frequency=data_frequency
)
# values = exchange_bundle.get_spot_values(
# assets=assets,
# field='close',
# dt=dt,
# data_frequency=data_frequency
# )
pass
def test_ingest_minute(self):
@@ -215,7 +214,7 @@ class TestExchangeBundle:
# encounter these problems as I have been focusing on minute data.
reader = exchange_bundle.get_reader(data_frequency)
for asset in assets:
# Since this pair was loaded last. It should be there in daily mode.
# Since this pair was loaded last. It should be here in daily mode.
arrays = reader.load_raw_arrays(
sids=[asset.sid],
fields=['close'],
@@ -252,7 +251,6 @@ class TestExchangeBundle:
ensure_directory(path)
exchange_bundle = ExchangeBundle(exchange)
calendar = get_calendar('OPEN')
# We are using a BcolzMinuteBarWriter even though the data is daily
# Each day has a maximum of one bar
@@ -304,26 +302,25 @@ class TestExchangeBundle:
pass
def test_minute_bundle(self):
exchange_name = 'poloniex'
data_frequency = 'minute'
# exchange_name = 'poloniex'
# data_frequency = 'minute'
exchange = get_exchange(exchange_name)
asset = exchange.get_asset('neos_btc')
path = get_bcolz_chunk(
exchange_name=exchange_name,
symbol=asset.symbol,
data_frequency=data_frequency,
period='2017-5',
)
# exchange = get_exchange(exchange_name)
# asset = exchange.get_asset('neos_btc')
# path = get_bcolz_chunk(
# exchange_name=exchange_name,
# symbol=asset.symbol,
# data_frequency=data_frequency,
# period='2017-5',
# )
pass
def test_hash_symbol(self):
symbol = 'etc_btc'
sid = int(
hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
) % 10 ** 6
# symbol = 'etc_btc'
# sid = int(
# hashlib.sha256(symbol.encode('utf-8')).hexdigest(), 16
# ) % 10 ** 6
pass
def test_validate_data(self):
tests/exchange/test_ccxt.py
+93
View File
@@ -0,0 +1,93 @@
import pandas as pd
from logbook import Logger
from base import BaseExchangeTestCase
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.finance.order import Order
from catalyst.exchange.exchange_utils import get_exchange_auth
log = Logger('test_ccxt')
class TestCCXT(BaseExchangeTestCase):
@classmethod
def setup(self):
exchange_name = 'gdax'
auth = get_exchange_auth(exchange_name)
self.exchange = CCXT(
exchange_name=exchange_name,
key=auth['key'],
secret=auth['secret'],
base_currency='eth',
portfolio=None
)
def test_order(self):
log.info('creating order')
asset = self.exchange.get_asset('neo_eth')
order_id = self.exchange.order(
asset=asset,
limit_price=0.07,
amount=1,
)
log.info('order created {}'.format(order_id))
assert order_id is not None
pass
def test_open_orders(self):
# log.info('retrieving open orders')
# asset = self.exchange.get_asset('neo_eth')
# orders = self.exchange.get_open_orders(asset)
pass
def test_get_order(self):
log.info('retrieving order')
order = self.exchange.get_order('2631386', 'neo_eth')
# order = self.exchange.get_order('2631386')
assert isinstance(order, Order)
pass
def test_cancel_order(self, ):
log.info('cancel order')
self.exchange.cancel_order('2631386', 'neo_eth')
pass
def test_get_candles(self):
log.info('retrieving candles')
candles = self.exchange.get_candles(
freq='5T',
assets=[self.exchange.get_asset('eth_btc')],
bar_count=200,
start_dt=pd.to_datetime('2017-01-01', utc=True)
)
for asset in candles:
df = pd.DataFrame(candles[asset])
df.set_index('last_traded', drop=True, inplace=True)
pass
def test_tickers(self):
log.info('retrieving tickers')
tickers = self.exchange.tickers([
self.exchange.get_asset('eth_btc'),
])
assert len(tickers) == 1
pass
def test_get_balances(self):
log.info('testing wallet balances')
# balances = self.exchange.get_balances()
pass
def test_get_account(self):
log.info('testing account data')
pass
def test_orderbook(self):
log.info('testing order book for bittrex')
# asset = self.exchange.get_asset('eth_btc')
# orderbook = self.exchange.get_orderbook(asset, 'all', limit=10)
pass
def test_get_fees(self):
pass
tests/exchange/test_data_portal.py
+25 -23
View File
@@ -3,11 +3,13 @@ from logbook import Logger
from catalyst import get_calendar
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest, \
from catalyst.exchange.exchange_data_portal import (
DataPortalExchangeBacktest,
DataPortalExchangeLive
)
from catalyst.exchange.exchange_utils import get_common_assets
from catalyst.exchange.factory import get_exchange, get_exchanges
from test_utils import rnd_history_date_days, rnd_bar_count, output_df
from catalyst.exchange.factory import get_exchanges
from test_utils import rnd_history_date_days, rnd_bar_count
log = Logger('test_bitfinex')
@@ -35,31 +37,31 @@ class TestExchangeDataPortal:
)
def test_get_history_window_live(self):
asset_finder = self.data_portal_live.asset_finder
# asset_finder = self.data_portal_live.asset_finder
assets = [
asset_finder.lookup_symbol('eth_btc', self.bitfinex),
asset_finder.lookup_symbol('eth_btc', self.bittrex)
]
now = pd.Timestamp.utcnow()
data = self.data_portal_live.get_history_window(
assets,
now,
10,
'1m',
'price')
# assets = [
# asset_finder.lookup_symbol('eth_btc', self.bitfinex),
# asset_finder.lookup_symbol('eth_btc', self.bittrex)
# ]
# now = pd.Timestamp.utcnow()
# data = self.data_portal_live.get_history_window(
# assets,
# now,
# 10,
# '1m',
# 'price')
pass
def test_get_spot_value_live(self):
asset_finder = self.data_portal_live.asset_finder
# asset_finder = self.data_portal_live.asset_finder
assets = [
asset_finder.lookup_symbol('eth_btc', self.bitfinex),
asset_finder.lookup_symbol('eth_btc', self.bittrex)
]
now = pd.Timestamp.utcnow()
value = self.data_portal_live.get_spot_value(
assets, 'price', now, '1m')
# assets = [
# asset_finder.lookup_symbol('eth_btc', self.bitfinex),
# asset_finder.lookup_symbol('eth_btc', self.bittrex)
# ]
# now = pd.Timestamp.utcnow()
# value = self.data_portal_live.get_spot_value(
# assets, 'price', now, '1m')
pass
def test_get_history_window_backtest(self):
tests/exchange/test_poloniex.py
+8 -6
View File
@@ -4,11 +4,14 @@ from base import BaseExchangeTestCase
from logbook import Logger
from catalyst.exchange.exchange_utils import get_exchange_auth
import pandas as pd
from catalyst.utils.deprecate import deprecated
from test_utils import output_df
log = Logger('test_poloniex')
@deprecated
class TestPoloniex(BaseExchangeTestCase):
@classmethod
def setup(self):
@@ -34,8 +37,8 @@ class TestPoloniex(BaseExchangeTestCase):
def test_open_orders(self):
log.info('retrieving open orders')
asset = self.exchange.get_asset('neos_btc')
orders = self.exchange.get_open_orders(asset)
# asset = self.exchange.get_asset('neos_btc')
# orders = self.exchange.get_open_orders(asset)
pass
def test_get_order(self):
@@ -79,7 +82,7 @@ class TestPoloniex(BaseExchangeTestCase):
def test_get_balances(self):
log.info('testing wallet balances')
balances = self.exchange.get_balances()
# balances = self.exchange.get_balances()
pass
def test_get_account(self):
@@ -88,7 +91,6 @@ class TestPoloniex(BaseExchangeTestCase):
def test_orderbook(self):
log.info('testing order book for poloniex')
asset = self.exchange.get_asset('eth_btc')
orderbook = self.exchange.get_orderbook(asset)
# asset = self.exchange.get_asset('eth_btc')
# orderbook = self.exchange.get_orderbook(asset)
pass
tests/exchange/test_server_bundle.py
+8 -13
View File
@@ -1,21 +1,16 @@
import os
import tarfile
import importlib
import pandas as pd
from catalyst import get_calendar
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
from catalyst.data.minute_bars import BcolzMinuteBarMetadata
from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.finance import candlestick2_ohlc
from matplotlib.finance import volume_overlay
# from matplotlib.finance import volume_overlay
import matplotlib.ticker as ticker
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader
from catalyst.exchange.bundle_utils import get_df_from_arrays, get_bcolz_chunk
from catalyst.exchange.factory import get_exchange
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
@@ -51,7 +46,7 @@ class ValidateChunks(object):
if data_frequency == 'daily':
end = end - pd.Timedelta(hours=23, minutes=59)
print start, end, data_frequency
print(start, end, data_frequency)
arrays = reader.load_raw_arrays(self.columns, start, end,
[asset.sid, ])
@@ -85,8 +80,8 @@ class ValidateChunks(object):
matplotlib.transforms.Bbox([[0.125, 0.1], [0.9, 0.26]]))
# Plot the volume overlay
bc = volume_overlay(ax2, df['open'], df['close'], df['volume'],
colorup='g', alpha=0.5, width=1)
# bc = volume_overlay(ax2, df['open'], df['close'], df['volume'],
# colorup='g', alpha=0.5, width=1)
ax.xaxis.set_major_locator(ticker.MaxNLocator(6))
tests/exchange/test_utils.py
+1 -2
View File
@@ -26,8 +26,7 @@ def rnd_history_date_minutes(max_minutes=1440):
def rnd_bar_count(max_bars=21):
now = pd.Timestamp.utcnow()
# now = pd.Timestamp.utcnow()
return randint(0, max_bars)