wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-21 12:30:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: wassname/catalyst:new_exchange_config
Branches Tags
wassname/catalyst:master wassname/catalyst:pandas_0.22.0 wassname/catalyst:pandas0.2 wassname/catalyst:gh-pages wassname/catalyst:develop wassname/catalyst:server-0.3.10 wassname/catalyst:cloud wassname/catalyst:new_exchange_config wassname/catalyst:cloud_conn wassname/catalyst:unit-tests wassname/catalyst:revert-180-jupyter-allow-root-docker wassname/catalyst:abnera-patch-2 wassname/catalyst:damo1884-talib_example wassname/catalyst:concurrent-exchanges wassname/catalyst:live-trading wassname/catalyst:5-minute
wassname/catalyst:0.5.8 wassname/catalyst:0.5.7 wassname/catalyst:0.5.6 wassname/catalyst:0.5.5 wassname/catalyst:0.5.4 wassname/catalyst:0.5.3 wassname/catalyst:0.5.2 wassname/catalyst:0.5.1 wassname/catalyst:0.5.0 wassname/catalyst:0.4.7 wassname/catalyst:0.4.6 wassname/catalyst:0.4.4 wassname/catalyst:0.4.3 wassname/catalyst:0.4.2 wassname/catalyst:0.4.1 wassname/catalyst:0.4.0 wassname/catalyst:0.3.10 wassname/catalyst:0.3.9 wassname/catalyst:0.3.8 wassname/catalyst:0.3.7 wassname/catalyst:0.3.6 wassname/catalyst:0.3.5 wassname/catalyst:0.3.4 wassname/catalyst:0.3.3 wassname/catalyst:0.3.2 wassname/catalyst:0.3.1 wassname/catalyst:0.3 wassname/catalyst:0.2.dev5 wassname/catalyst:0.2.dev4 wassname/catalyst:0.2.dev3 wassname/catalyst:0.2.dev2 wassname/catalyst:0.2.dev1 wassname/catalyst:0.1.dev9 wassname/catalyst:0.1.dev8 wassname/catalyst:0.1.dev7 wassname/catalyst:0.1.dev6 wassname/catalyst:0.1.dev5 wassname/catalyst:0.1.dev4 wassname/catalyst:0.1.dev3
..
compare: wassname/catalyst:revert-180-jupyter-allow-root-docker
Branches Tags
wassname/catalyst:pandas_0.22.0 wassname/catalyst:pandas0.2 wassname/catalyst:gh-pages wassname/catalyst:develop wassname/catalyst:master wassname/catalyst:server-0.3.10 wassname/catalyst:cloud wassname/catalyst:new_exchange_config wassname/catalyst:cloud_conn wassname/catalyst:unit-tests wassname/catalyst:revert-180-jupyter-allow-root-docker wassname/catalyst:abnera-patch-2 wassname/catalyst:damo1884-talib_example wassname/catalyst:concurrent-exchanges wassname/catalyst:live-trading wassname/catalyst:5-minute
wassname/catalyst:0.5.8 wassname/catalyst:0.5.7 wassname/catalyst:0.5.6 wassname/catalyst:0.5.5 wassname/catalyst:0.5.4 wassname/catalyst:0.5.3 wassname/catalyst:0.5.2 wassname/catalyst:0.5.1 wassname/catalyst:0.5.0 wassname/catalyst:0.4.7 wassname/catalyst:0.4.6 wassname/catalyst:0.4.4 wassname/catalyst:0.4.3 wassname/catalyst:0.4.2 wassname/catalyst:0.4.1 wassname/catalyst:0.4.0 wassname/catalyst:0.3.10 wassname/catalyst:0.3.9 wassname/catalyst:0.3.8 wassname/catalyst:0.3.7 wassname/catalyst:0.3.6 wassname/catalyst:0.3.5 wassname/catalyst:0.3.4 wassname/catalyst:0.3.3 wassname/catalyst:0.3.2 wassname/catalyst:0.3.1 wassname/catalyst:0.3 wassname/catalyst:0.2.dev5 wassname/catalyst:0.2.dev4 wassname/catalyst:0.2.dev3 wassname/catalyst:0.2.dev2 wassname/catalyst:0.2.dev1 wassname/catalyst:0.1.dev9 wassname/catalyst:0.1.dev8 wassname/catalyst:0.1.dev7 wassname/catalyst:0.1.dev6 wassname/catalyst:0.1.dev5 wassname/catalyst:0.1.dev4 wassname/catalyst:0.1.dev3
1 Commits
new_exchange_config .. revert-180-jupyter-allow-root-docker
Author SHA1 Message Date
Avishai WeingartenandGitHub 19cdbcaa85 Revert "Allow jupyter to run as root inside Docker image" 2018-01-30 17:25:26 +02:00
68 changed files with 2113 additions and 4902 deletions
Expand all files Collapse all files
README.rst
+4 -2
View File
@@ -1,4 +1,4 @@
.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png
.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
:target: https://enigmampc.github.io/catalyst
:align: center
:alt: Enigma | Catalyst
@@ -17,7 +17,9 @@ insights regarding a particular strategy's performance. Catalyst also supports
live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex,
and Poloniex) with more being added over time. Catalyst empowers users to share
and curate data and build profitable, data-driven investment strategies. Please
visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or
refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for
further technical details.
Catalyst builds on top of the well-established
`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to
catalyst/__main__.py
+2 -130
View File
@@ -6,7 +6,6 @@ import click
import sys
import logbook
import pandas as pd
from catalyst.marketplace.marketplace import Marketplace
from six import text_type
from catalyst.data import bundles as bundles_module
@@ -580,8 +579,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
sys.stdout)
click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout)
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=include_symbols,
@@ -635,8 +633,7 @@ def clean_exchange(ctx, exchange_name, data_frequency):
exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Cleaning exchange bundle {}...'.format(exchange_name),
sys.stdout)
click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout)
exchange_bundle.clean(
data_frequency=data_frequency,
)
@@ -764,130 +761,5 @@ def bundles():
click.echo("%s %s" % (bundle, timestamp), sys.stdout)
@main.group()
@click.pass_context
def marketplace(ctx):
"""Access the Enigma Data Marketplace to:\n
- Register and Publish new datasets (seller-side)\n
- Subscribe and Ingest premium datasets (buyer-side)\n
"""
pass
@marketplace.command()
@click.pass_context
def ls(ctx):
"""List all available datasets.
"""
click.echo('Listing of available data sources on the marketplace:',
sys.stdout)
marketplace = Marketplace()
marketplace.list()
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.pass_context
def subscribe(ctx, dataset):
"""Subscribe to an exisiting dataset.
"""
marketplace = Marketplace()
marketplace.subscribe(dataset)
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.option(
'-f',
'--data-frequency',
type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}),
default='daily',
show_default=True,
help='The data frequency of the desired OHLCV bars.',
)
@click.option(
'-s',
'--start',
default=None,
type=Date(tz='utc', as_timestamp=True),
help='The start date of the data range. (default: one year from end date)',
)
@click.option(
'-e',
'--end',
default=None,
type=Date(tz='utc', as_timestamp=True),
help='The end date of the data range. (default: today)',
)
@click.pass_context
def ingest(ctx, dataset, data_frequency, start, end):
"""Ingest a dataset (requires subscription).
"""
marketplace = Marketplace()
marketplace.ingest(dataset, data_frequency, start, end)
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.pass_context
def clean(ctx, dataset):
"""Clean/Remove local data for a given dataset.
"""
marketplace = Marketplace()
marketplace.clean(dataset)
@marketplace.command()
@click.pass_context
def register(ctx):
"""Register a new dataset.
"""
marketplace = Marketplace()
marketplace.register()
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the Marketplace dataset to publish data for.',
)
@click.option(
'--datadir',
default=None,
help='The folder that contains the CSV data files to publish.',
)
@click.option(
'--watch/--no-watch',
is_flag=True,
default=False,
help='Whether to watch the datadir for live data.',
)
@click.pass_context
def publish(ctx, dataset, datadir, watch):
"""Publish data for a registered dataset.
"""
marketplace = Marketplace()
if dataset is None:
ctx.fail("must specify a dataset to publish data for "
" with '--dataset'\n")
if datadir is None:
ctx.fail("must specify a datadir where to find the files to publish "
" with '--datadir'\n")
marketplace.publish(dataset, datadir, watch)
if __name__ == '__main__':
main()
catalyst/algorithm.py
+5 -5
View File
@@ -939,7 +939,7 @@ class TradingAlgorithm(object):
The field to query. The options have the following meanings:
arena : str
The arena from the simulation parameters. This will normally
be ``backtest`` but some systems may use this distinguish
be ``'backtest'`` but some systems may use this distinguish
live trading from backtesting.
data_frequency : {'daily', 'minute'}
data_frequency tells the algorithm if it is running with
@@ -954,7 +954,7 @@ class TradingAlgorithm(object):
The platform that the code is running on. By default this
will be the string 'catalyst'. This can allow algorithms to
know if they are running on the Quantopian platform instead.
\* : dict[str -> any]
* : dict[str -> any]
Returns all of the fields in a dictionary.
Returns
@@ -1032,7 +1032,7 @@ class TradingAlgorithm(object):
argument is the name of the column in the preprocessed dataframe
containing the symbols. This will be used along with the date
information to map the sids in the asset finder.
\*\*kwargs
**kwargs
Forwarded to :func:`pandas.read_csv`.
Returns
@@ -1156,7 +1156,7 @@ class TradingAlgorithm(object):
Parameters
----------
\*\*kwargs
**kwargs
The names and values to record.
Notes
@@ -1273,7 +1273,7 @@ class TradingAlgorithm(object):
Parameters
----------
\*args : iterable[str]
*args : iterable[str]
The ticker symbols to lookup.
Returns
catalyst/api.pyi
+8 -65
View File
@@ -34,7 +34,6 @@ def attach_pipeline(pipeline, name, chunks=None):
:func:`catalyst.api.pipeline_output`
"""
def batch_market_order(share_counts):
"""Place a batch market order for multiple assets.
@@ -49,7 +48,6 @@ def batch_market_order(share_counts):
Index of ids for newly-created orders.
"""
def cancel_order(order_param):
"""Cancel an open order.
@@ -59,9 +57,7 @@ def cancel_order(order_param):
The order_id or order object to cancel.
"""
def continuous_future(root_symbol_str, offset=0, roll='volume',
adjustment='mul'):
def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'):
"""Create a specifier for a continuous contract.
Parameters
@@ -85,10 +81,7 @@ def continuous_future(root_symbol_str, offset=0, roll='volume',
The continuous future specifier.
"""
def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
date_format=None, timezone='UTC', symbol=None, mask=True,
symbol_column=None, special_params_checker=None, **kwargs):
def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs):
"""Fetch a csv from a remote url and register the data so that it is
queryable from the ``data`` object.
@@ -132,7 +125,6 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
A requests source that will pull data from the url specified.
"""
def future_symbol(symbol):
"""Lookup a futures contract with a given symbol.
@@ -152,7 +144,6 @@ def future_symbol(symbol):
Raised when no contract named 'symbol' is found.
"""
def get_datetime(tz=None):
"""
Returns the current simulation datetime.
@@ -168,7 +159,6 @@ dt : datetime
The current simulation datetime converted to ``tz``.
"""
def get_environment(field='platform'):
"""Query the execution environment.
@@ -208,7 +198,6 @@ def get_environment(field='platform'):
Raised when ``field`` is not a valid option.
"""
def get_order(order_id):
"""Lookup an order based on the order id returned from one of the
order functions.
@@ -224,12 +213,10 @@ def get_order(order_id):
The order object.
"""
def history(bar_count, frequency, field, ffill=True):
"""DEPRECATED: use ``data.history`` instead.
"""
def order(asset, amount, limit_price=None, stop_price=None, style=None):
"""Place an order.
@@ -271,9 +258,7 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_percent`
"""
def order_percent(asset, percent, limit_price=None, stop_price=None,
style=None):
def order_percent(asset, percent, limit_price=None, stop_price=None, style=None):
"""Place an order in the specified asset corresponding to the given
percent of the current portfolio value.
@@ -308,7 +293,6 @@ def order_percent(asset, percent, limit_price=None, stop_price=None,
:func:`catalyst.api.order_value`
"""
def order_target(asset, target, limit_price=None, stop_price=None, style=None):
"""Place an order to adjust a position to a target number of shares. If
the position doesn't already exist, this is equivalent to placing a new
@@ -360,9 +344,7 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_target_value`
"""
def order_target_percent(asset, target, limit_price=None, stop_price=None,
style=None):
def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None):
"""Place an order to adjust a position to a target percent of the
current portfolio value. If the position doesn't already exist, this is
equivalent to placing a new order. If the position does exist, this is
@@ -414,9 +396,7 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None,
:func:`catalyst.api.order_target_value`
"""
def order_target_value(asset, target, limit_price=None, stop_price=None,
style=None):
def order_target_value(asset, target, limit_price=None, stop_price=None, style=None):
"""Place an order to adjust a position to a target value. If
the position doesn't already exist, this is equivalent to placing a new
order. If the position does exist, this is equivalent to placing an
@@ -468,7 +448,6 @@ def order_target_value(asset, target, limit_price=None, stop_price=None,
:func:`catalyst.api.order_target_percent`
"""
def order_value(asset, value, limit_price=None, stop_price=None, style=None):
"""Place an order by desired value rather than desired number of
shares.
@@ -509,7 +488,6 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_percent`
"""
def pipeline_output(name):
"""Get the results of the pipeline that was attached with the name:
``name``.
@@ -536,7 +514,6 @@ def pipeline_output(name):
:meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline`
"""
def record(*args, **kwargs):
"""Track and record values each day.
@@ -552,9 +529,7 @@ def record(*args, **kwargs):
:func:`~catalyst.run_algorithm`.
"""
def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
calendar=None):
def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None):
"""Schedules a function to be called according to some timed rules.
Parameters
@@ -574,7 +549,6 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
:class:`catalyst.api.time_rules`
"""
def set_asset_restrictions(restrictions, on_error='fail'):
"""Set a restriction on which assets can be ordered.
@@ -588,7 +562,6 @@ def set_asset_restrictions(restrictions, on_error='fail'):
catalyst.finance.asset_restrictions.Restrictions
"""
def set_benchmark(benchmark):
"""Set the benchmark asset.
@@ -603,7 +576,6 @@ def set_benchmark(benchmark):
automatically reinvested.
"""
def set_cancel_policy(cancel_policy):
"""Sets the order cancellation policy for the simulation.
@@ -618,7 +590,6 @@ def set_cancel_policy(cancel_policy):
:class:`catalyst.api.NeverCancel`
"""
def set_commission(commission):
"""Sets the commission model for the simulation.
@@ -634,7 +605,6 @@ def set_commission(commission):
:class:`catalyst.finance.commission.PerDollar`
"""
def set_do_not_order_list(restricted_list, on_error='fail'):
"""Set a restriction on which assets can be ordered.
@@ -644,13 +614,11 @@ def set_do_not_order_list(restricted_list, on_error='fail'):
The assets that cannot be ordered.
"""
def set_long_only(on_error='fail'):
"""Set a rule specifying that this algorithm cannot take short
positions.
"""
def set_max_leverage(max_leverage):
"""Set a limit on the maximum leverage of the algorithm.
@@ -661,7 +629,6 @@ def set_max_leverage(max_leverage):
be no maximum.
"""
def set_max_order_count(max_count, on_error='fail'):
"""Set a limit on the number of orders that can be placed in a single
day.
@@ -672,9 +639,7 @@ def set_max_order_count(max_count, on_error='fail'):
The maximum number of orders that can be placed on any single day.
"""
def set_max_order_size(asset=None, max_shares=None, max_notional=None,
on_error='fail'):
def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
"""Set a limit on the number of shares and/or dollar value of any single
order placed for sid. Limits are treated as absolute values and are
enforced at the time that the algo attempts to place an order for sid.
@@ -693,9 +658,7 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None,
The maximum value that can be ordered at one time.
"""
def set_max_position_size(asset=None, max_shares=None, max_notional=None,
on_error='fail'):
def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
"""Set a limit on the number of shares and/or dollar value held for the
given sid. Limits are treated as absolute values and are enforced at
the time that the algo attempts to place an order for sid. This means
@@ -718,7 +681,6 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None,
The maximum value to hold for an asset.
"""
def set_slippage(slippage):
"""Set the slippage model for the simulation.
@@ -732,7 +694,6 @@ def set_slippage(slippage):
:class:`catalyst.finance.slippage.SlippageModel`
"""
def set_symbol_lookup_date(dt):
"""Set the date for which symbols will be resolved to their assets
(symbols may map to different firms or underlying assets at
@@ -744,7 +705,6 @@ def set_symbol_lookup_date(dt):
The new symbol lookup date.
"""
def sid(sid):
"""Lookup an Asset by its unique asset identifier.
@@ -764,7 +724,6 @@ def sid(sid):
When a requested ``sid`` does not map to any asset.
"""
def symbol(symbol_str):
"""Lookup an Equity by its ticker symbol.
@@ -789,7 +748,6 @@ def symbol(symbol_str):
:func:`catalyst.api.set_symbol_lookup_date`
"""
def symbols(*args):
"""Lookup multuple Equities as a list.
@@ -815,18 +773,3 @@ def symbols(*args):
:func:`catalyst.api.set_symbol_lookup_date`
"""
def get_dataset(ds_name, start=None, end=None):
"""
Lookup a data source from the marketplace
Parameters
----------
ds_name: str
start: pd.Timestamp
end: pd.Timestamp
Returns
-------
"""
catalyst/assets/_assets.pyx
+10 -59
View File
@@ -433,7 +433,7 @@ cdef class TradingPair(Asset):
'taker',
'trading_state',
'data_source',
'decimals',
'decimals'
})
def __init__(self,
object symbol,
@@ -455,7 +455,7 @@ cdef class TradingPair(Asset):
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=1,
int trading_state=0,
object data_source='catalyst'):
"""
Replicates the Asset constructor with some built-in conventions
@@ -600,51 +600,14 @@ cdef class TradingPair(Asset):
cpdef to_dict(self):
"""
Convert to a python dict.
Repeat constructor params:
object symbol,
object exchange,
object start_date=None,
object asset_name=None,
int sid=0,
float leverage=1.0,
object end_daily=None,
object end_minute=None,
object end_date=None,
object exchange_symbol=None,
object first_traded=None,
object auto_close_date=None,
object exchange_full=None,
float min_trade_size=0.0001,
float max_trade_size=1000000,
float maker=0.0015,
float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=1,
object data_source='catalyst',
"""
trading_pair_dict = dict(
symbol=self.symbol,
exchange=self.exchange,
start_date=self.start_date,
asset_name=self.asset_name,
leverage=self.leverage,
end_daily=self.end_daily,
end_minute=self.end_minute,
end_date=self.end_date,
exchange_symbol=self.exchange_symbol,
exchange_full=self.exchange_full,
min_trade_size=self.min_trade_size,
max_trade_size=self.max_trade_size,
maker=self.maker,
taker=self.taker,
lot=self.lot,
decimals=self.decimals,
trading_state=self.trading_state,
data_source=self.data_source,
)
return trading_pair_dict
#TODO: missing fields
super_dict = super(TradingPair, self).to_dict()
super_dict['end_daily'] = self.end_daily
super_dict['end_minute'] = self.end_minute
super_dict['leverage'] = self.leverage
super_dict['min_trade_size'] = self.min_trade_size
return super_dict
def is_exchange_open(self, dt_minute):
"""
@@ -660,16 +623,6 @@ cdef class TradingPair(Asset):
#TODO: make more dymanic to catch holds
return True
def set_end_date(self, dt, data_frequency):
if data_frequency == 'minute':
self.end_minute = dt
else:
self.end_daily = dt
def set_start_date(self, dt):
self.start_date = dt
cpdef __reduce__(self):
"""
Function used by pickle to determine how to serialize/deserialize this
@@ -693,9 +646,7 @@ cdef class TradingPair(Asset):
self.lot,
self.decimals,
self.taker,
self.maker,
self.trading_state,
self.data_source))
self.maker))
def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object)
catalyst/constants.py
+1 -32
View File
@@ -11,39 +11,8 @@ LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
'{exchange}/symbols.json'
EXCHANGE_CONFIG_URL = 'https://s3.amazonaws.com/enigmaco/ohlcv/' \
'{exchange}/config.json'
BUNDLE_URL = 'https://s3.amazonaws.com/enigmaco/ohlcv/' \
'{exchange}/{data_frequency}/{name}.tar.gz'
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
DATE_FORMAT = '%Y-%m-%d'
try:
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
except Exception as e:
print('unable to get catalyst path: {}'.format(e))
AUTO_INGEST = False
AUTH_SERVER = 'https://data.enigma.co'
# TODO: switch to mainnet
ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_marketplace_address.txt'
MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_marketplace_abi.json'
# TODO: switch to mainnet
ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
'master/catalyst/marketplace/' \
'contract_enigma_address.txt'
ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_enigma_abi.json'
catalyst/examples/buy_low_sell_high.py
+4 -3
View File
@@ -7,6 +7,7 @@ from catalyst.api import (
order_target_percent,
symbol,
record,
get_open_orders,
)
from catalyst.exchange.utils.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm
@@ -59,7 +60,7 @@ def _handle_data(context, data):
rsi=rsi,
)
orders = context.blotter.open_orders
orders = get_open_orders(context.asset)
if orders:
log.info('skipping bar until all open orders execute')
return
@@ -145,11 +146,11 @@ if __name__ == '__main__':
live = True
if live:
run_algorithm(
capital_base=1000,
capital_base=0.001,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
exchange_name='binance',
live=True,
algo_namespace=algo_namespace,
base_currency='btc',
catalyst/examples/dual_moving_average.py
+15 -16
View File
@@ -4,7 +4,8 @@ import pandas as pd
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import (record, symbol, order_target_percent,)
from catalyst.api import (record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.utils.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
@@ -31,18 +32,16 @@ def handle_data(context, data):
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_data = data.history(context.asset,
short_mavg = data.history(context.asset,
'price',
bar_count=short_window,
frequency="1T",
)
short_mavg = short_data.mean()
long_data = data.history(context.asset,
frequency="1m",
).mean()
long_mavg = data.history(context.asset,
'price',
bar_count=long_window,
frequency="1T",
)
long_mavg = long_data.mean()
frequency="1m",
).mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
@@ -62,7 +61,7 @@ def handle_data(context, data):
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = context.blotter.open_orders
orders = get_open_orders(context.asset)
if len(orders) > 0:
return
@@ -83,6 +82,7 @@ def handle_data(context, data):
def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation
exchange = list(context.exchanges.values())[0]
base_currency = exchange.base_currency.upper()
@@ -93,7 +93,7 @@ def analyze(context, perf):
ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
@@ -104,9 +104,9 @@ def analyze(context, perf):
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.asset.symbol,
base=base_currency
))
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
@@ -136,20 +136,19 @@ def analyze(context, perf):
ax3.legend_.remove()
ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end - start) / 5))
ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end / 5))
ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
catalyst/examples/marketplace/github-research.py
-70
View File
@@ -1,70 +0,0 @@
import pandas as pd
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import symbol, get_dataset
START = '2017-01-01'
END = '2017-12-31'
def initialize(context):
pass
def handle_data(context, data):
context.github = get_dataset('github')
context.github.sort_index(level=0, inplace=True)
context.zec = data.history(symbol('zec_usdt'),
['price', ],
bar_count=365,
frequency="1d")
context.xmr = data.history(symbol('xmr_usdt'),
['price', ],
bar_count=365,
frequency="1d")
def analyze(context=None, results=None):
ax1 = plt.subplot(211)
idx = pd.IndexSlice
df = context.github.loc[START:END].loc[
idx[:, [b'ZEC']], ['commits']].reset_index(
level='symbol', drop=True)
df.plot(ax=ax1, color='blue')
ax1.legend(loc=2)
ax1.set_title('Zcash')
ax2 = ax1.twinx()
context.zec['price'].loc[START:END].plot(ax=ax2, color='green')
ax2.legend(loc=1)
ax3 = plt.subplot(212)
idx = pd.IndexSlice
df = context.github.loc[START:END].loc[
idx[:, [b'XMR']], ['commits']].reset_index(
level='symbol', drop=True)
df.plot(ax=ax3, color='blue')
ax3.legend(loc=2)
ax3.set_title('Monero')
ax4 = ax3.twinx()
context.xmr['price'].loc[START:END].plot(ax=ax4, color='green')
ax4.legend(loc=1)
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace='algo-github',
base_currency='usdt',
live=False,
start=pd.to_datetime(END, utc=True),
end=pd.to_datetime(END, utc=True),
)
catalyst/examples/marketplace/mean_reversion_by_marketcap.py
-237
View File
@@ -1,237 +0,0 @@
# For this example, we're going to write a simple momentum script. When the
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
# going to sell. Hopefully we'll ride the waves.
import os
import tempfile
import time
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_dataset
from catalyst.exchange.utils.stats_utils import set_print_settings, \
get_pretty_stats
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
df = get_dataset('testmarketcap2') # type: pd.DataFrame
# Picking a specific date in our DataFrame
first_dt = df.index.get_level_values(0)[0]
# Since we use a MultiIndex with date / symbol, picking a date will
# result in a new DataFrame for the selected date with a single
# symbol index
df = df.xs(first_dt, level=0)
# Keep only the top coins by market cap
df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))]
set_print_settings()
df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True)
print('the marketplace data:\n{}'.format(df))
# Pick the 5 assets with the lowest market cap for trading
quote_currency = 'eth'
exchange = context.exchanges[next(iter(context.exchanges))]
symbols = [a.symbol for a in exchange.assets
if a.start_date < context.datetime]
context.assets = []
for currency, price in df['market_cap_usd'].iteritems():
if len(context.assets) >= 5:
break
s = '{}_{}'.format(currency.decode('utf-8'), quote_currency)
if s in symbols:
context.assets.append(symbol(s))
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = dict()
context.current_day = today
# Preparing dictionaries for asset-level data points
volumes = dict()
rsis = dict()
price_values = dict()
cash = context.portfolio.cash
for asset in context.assets:
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.assets variable. For this example,
# we're using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this
# convention for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
asset,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare
# to the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(asset, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
# if asset not in context.base_price:
# context.base_price[asset] = price
#
# base_price = context.base_price[asset]
# price_change = (price - base_price) / base_price
# Tracking the relevant data
volumes[asset] = current['volume']
rsis[asset] = rsi[-1]
price_values[asset] = price
# price_changes[asset] = price_change
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if asset in context.traded_today:
continue
# Exit if we cannot trade
if not data.can_trade(asset):
continue
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we
# calculate how long or short our position is at this minute.
pos_amount = context.portfolio.positions[asset].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
target = 1.0 / len(context.assets)
order_target_percent(
asset, target, limit_price=limit_price
)
context.traded_today[asset] = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
asset, 0, limit_price=limit_price
)
context.traded_today[asset] = True
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
current_price=price_values,
volume=volumes,
rsi=rsis,
cash=cash,
)
def analyze(context=None, perf=None):
stats = get_pretty_stats(perf)
print('the algo stats:\n{}'.format(stats))
pass
if __name__ == '__main__':
# The execution mode: backtest or live
live = False
if live:
run_algorithm(
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
live=True,
algo_namespace=NAMESPACE,
base_currency='btc',
live_graph=False,
simulate_orders=False,
stats_output=None,
)
else:
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=100,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace=NAMESPACE,
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-10-15', utc=True),
)
log.info('saved perf stats: {}'.format(out))
catalyst/examples/mean_reversion_simple.py
+1 -1
View File
@@ -248,7 +248,7 @@ if __name__ == '__main__':
if live:
run_algorithm(
capital_base=0.03,
capital_base=0.01,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
catalyst/examples/portfolio_optimization.py
+2 -3
View File
@@ -66,7 +66,7 @@ def handle_data(context, data):
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3 + context.nassets, n_portfolios))
for p in range(n_portfolios):
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
@@ -146,5 +146,4 @@ if __name__ == '__main__':
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000,
base_currency='usdt', )
capital_base=100000, )
catalyst/examples/simple_loop.py
+2 -2
View File
@@ -26,7 +26,7 @@ def handle_data(context, data):
context.asset,
fields='price',
bar_count=20,
frequency='2H'
frequency='30T'
)
last_traded = prices.index[-1]
log.info('last candle date: {}'.format(last_traded))
@@ -114,7 +114,7 @@ def analyze(context, perf):
if __name__ == '__main__':
mode = 'live'
mode = 'backtest'
if mode == 'backtest':
run_algorithm(
catalyst/exchange/ccxt/ccxt_exchange.py
+320 -275
View File
@@ -1,33 +1,34 @@
import json
import os
import re
from collections import defaultdict
import ccxt
import pandas as pd
import six
from catalyst.assets._assets import TradingPair
from redo import retry
from ccxt import InvalidOrder, NetworkError, \
ExchangeError
from logbook import Logger
from six import string_types
from catalyst.algorithm import MarketOrder
from catalyst.assets._assets import TradingPair
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeSymbolsNotFound, ExchangeRequestError, InvalidOrderStyle, \
UnsupportedHistoryFrequencyError, \
ExchangeNotFoundError, CreateOrderError, InvalidHistoryTimeframeError, \
MarketsNotFoundError, InvalidMarketError
UnsupportedHistoryFrequencyError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.utils.ccxt_utils import get_exchange_config
from catalyst.exchange.utils.exchange_utils import mixin_market_params, \
get_exchange_folder, get_catalyst_symbol, \
get_exchange_auth
from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \
get_epoch, \
get_periods_range
from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.finance.transaction import Transaction
from ccxt import InvalidOrder, NetworkError, \
ExchangeError
from logbook import Logger
from six import string_types
log = Logger('CCXT', level=LOG_LEVEL)
@@ -42,8 +43,7 @@ SUPPORTED_EXCHANGES = dict(
class CCXT(Exchange):
def __init__(self, exchange_name, key,
secret, password, base_currency, config=None):
def __init__(self, exchange_name, key, secret, base_currency):
log.debug(
'finding {} in CCXT exchanges:\n{}'.format(
exchange_name, ccxt.exchanges
@@ -60,11 +60,8 @@ class CCXT(Exchange):
self.api = exchange_attr({
'apiKey': key,
'secret': secret,
'password': password,
})
self.api.enableRateLimit = True
self.has = self.api.has
self.fees = self.api.fees
except Exception:
raise ExchangeNotFoundError(exchange_name=exchange_name)
@@ -72,7 +69,6 @@ class CCXT(Exchange):
self._symbol_maps = [None, None]
self.name = exchange_name
self.assets = []
self.base_currency = base_currency
self.transactions = defaultdict(list)
@@ -84,123 +80,97 @@ class CCXT(Exchange):
self._common_symbols = dict()
self.bundle = ExchangeBundle(self.name)
self.markets = None
self._is_init = False
self._config = config
def init(self):
if self._is_init:
return
if self._config is None:
self._config = get_exchange_config(self.name)
log.debug(
'got exchange config {}:\n{}'.format(
self.name, self._config
exchange_folder = get_exchange_folder(self.name)
filename = os.path.join(exchange_folder, 'cctx_markets.json')
if os.path.exists(filename):
timestamp = os.path.getmtime(filename)
dt = pd.to_datetime(timestamp, unit='s', utc=True)
if dt >= pd.Timestamp.utcnow().floor('1D'):
with open(filename) as f:
self.markets = json.load(f)
log.debug('loaded markets for {}'.format(self.name))
if self.markets is None:
try:
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
)
self.markets = self.api.fetch_markets()
with open(filename, 'w+') as f:
json.dump(self.markets, f, indent=4)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch markets {}: {}'.format(
self.name, e
)
)
raise ExchangeRequestError(error=e)
self.load_assets()
self._is_init = True
def load_assets(self):
if self._config is None:
raise ValueError('Exchange config not available.')
@staticmethod
def find_exchanges(features=None, is_authenticated=False):
ccxt_features = []
if features is not None:
for feature in features:
if not feature.endswith('Bundle'):
ccxt_features.append(feature)
self.assets = []
for asset_dict in self._config['assets']:
asset = TradingPair(**asset_dict)
self.assets.append(asset)
exchange_names = []
for exchange_name in ccxt.exchanges:
if is_authenticated:
exchange_auth = get_exchange_auth(exchange_name)
def _fetch_markets(self):
markets_symbols = self.api.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
self.name, markets_symbols
)
)
try:
markets = self.api.fetch_markets()
has_auth = (exchange_auth['key'] != ''
and exchange_auth['secret'] != '')
except NetworkError as e:
raise ExchangeRequestError(error=e)
if not has_auth:
continue
if not markets:
raise MarketsNotFoundError(
exchange=self.name,
)
log.debug('loading exchange: {}'.format(exchange_name))
exchange = getattr(ccxt, exchange_name)()
for market in markets:
if 'id' not in market:
raise InvalidMarketError(
exchange=self.name,
market=market,
)
return markets
if ccxt_features is None:
has_feature = True
def create_exchange_config(self):
config = dict(
name=self.name,
features=[feature for feature in self.has if self.has[feature]]
)
markets = retry(
action=self._fetch_markets,
attempts=5,
sleeptime=5,
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn(
'fetching markets again for {}'.format(self.name)
),
)
else:
try:
has_feature = all(
[exchange.has[feature] for feature in ccxt_features]
)
config['assets'] = []
for market in markets:
asset = self.create_trading_pair(market=market)
config['assets'].append(asset)
except Exception:
has_feature = False
return config
if has_feature:
try:
log.info('initializing {}'.format(exchange_name))
exchange_names.append(exchange_name)
def create_trading_pair(self, market, start_dt=None, end_dt=None,
leverage=1, end_daily=None, end_minute=None):
"""
Creating a TradingPair from market and asset data.
except Exception as e:
log.warn(
'unable to initialize exchange {}: {}'.format(
exchange_name, e
)
)
Parameters
----------
market: dict[str, Object]
start_dt
end_dt
leverage
end_daily
end_minute
Returns
-------
"""
params = dict(
exchange=self.name,
data_source='catalyst',
exchange_symbol=market['id'],
symbol=get_catalyst_symbol(market),
start_date=start_dt,
end_date=end_dt,
leverage=leverage,
asset_name=market['symbol'],
end_daily=end_daily,
end_minute=end_minute,
)
self.apply_conditional_market_params(params, market)
return TradingPair(**params)
def load_assets(self):
if self._config is None or 'error' in self._config:
raise ValueError('Exchange config not available.')
self.assets = []
for asset_dict in self._config['assets']:
asset = TradingPair(**asset_dict)
self.assets.append(asset)
return exchange_names
def account(self):
return None
@@ -232,11 +202,32 @@ class CCXT(Exchange):
return frequencies
def get_market(self, symbol):
"""
The CCXT market.
Parameters
----------
symbol:
The CCXT symbol.
Returns
-------
dict[str, Object]
"""
s = self.get_symbol(symbol)
market = next(
(market for market in self.markets if market['symbol'] == s),
None,
)
return market
def substitute_currency_code(self, currency, source='catalyst'):
if source == 'catalyst':
currency = currency.upper()
key = self.api.common_currency_code(currency).lower()
key = self.api.common_currency_code(currency)
self._common_symbols[key] = currency.lower()
return key
@@ -264,7 +255,13 @@ class CCXT(Exchange):
if source == 'ccxt':
if isinstance(asset_or_symbol, string_types):
parts = asset_or_symbol.split('/')
return '{}_{}'.format(parts[0].lower(), parts[1].lower())
base_currency = self.substitute_currency_code(
parts[0], source
)
quote_currency = self.substitute_currency_code(
parts[1], source
)
return '{}_{}'.format(base_currency, quote_currency)
else:
return asset_or_symbol.symbol
@@ -275,7 +272,13 @@ class CCXT(Exchange):
) else asset_or_symbol.symbol
parts = symbol.split('_')
return '{}/{}'.format(parts[0].upper(), parts[1].upper())
base_currency = self.substitute_currency_code(
parts[0], source
)
quote_currency = self.substitute_currency_code(
parts[1], source
)
return '{}/{}'.format(base_currency, quote_currency)
@staticmethod
def map_frequency(value, source='ccxt', raise_error=True):
@@ -401,7 +404,7 @@ class CCXT(Exchange):
)
def get_candles(self, freq, assets, bar_count=1, start_dt=None,
end_dt=None, floor_dates=True):
end_dt=None):
is_single = (isinstance(assets, TradingPair))
if is_single:
assets = [assets]
@@ -422,19 +425,26 @@ class CCXT(Exchange):
'Please provide either start_dt or end_dt, not both.'
)
if start_dt is None:
if end_dt is None:
end_dt = pd.Timestamp.utcnow()
dt_range = get_periods_range(
end_dt=end_dt,
periods=bar_count,
freq=freq,
elif end_dt is not None:
# Make sure that end_dt really wants data in the past
# if it's close to now, we skip the 'since' parameters to
# lower the probability of error
bars_to_now = pd.date_range(
end_dt, pd.Timestamp.utcnow(), freq=freq
)
start_dt = dt_range[0]
# See: https://github.com/ccxt/ccxt/issues/1360
if len(bars_to_now) > 1 or self.name in ['poloniex']:
dt_range = get_periods_range(
end_dt=end_dt,
periods=bar_count,
freq=freq,
)
start_dt = dt_range[0]
delta = start_dt - get_epoch()
since = int(delta.total_seconds()) * 1000
since = None
if start_dt is not None:
delta = start_dt - get_epoch()
since = int(delta.total_seconds()) * 1000
candles = dict()
for index, asset in enumerate(assets):
@@ -448,20 +458,16 @@ class CCXT(Exchange):
candles[asset] = []
for ohlcv in ohlcvs:
dt = pd.to_datetime(ohlcv[0], unit='ms', utc=True)
if floor_dates:
dt = dt.floor('1T')
candles[asset].append(
dict(
last_traded=dt,
open=ohlcv[1],
high=ohlcv[2],
low=ohlcv[3],
close=ohlcv[4],
volume=ohlcv[5],
)
)
candles[asset].append(dict(
last_traded=pd.to_datetime(
ohlcv[0], unit='ms', utc=True
),
open=ohlcv[1],
high=ohlcv[2],
low=ohlcv[3],
close=ohlcv[4],
volume=ohlcv[5]
))
candles[asset] = sorted(
candles[asset], key=lambda c: c['last_traded']
)
@@ -479,53 +485,144 @@ class CCXT(Exchange):
except ExchangeSymbolsNotFound:
return None
def apply_conditional_market_params(self, params, market):
def get_asset_defs(self, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
The local and Catalyst definitions of the specified market.
Parameters
----------
params: dict[Object]
market: dict[Object]
market: dict[str, Object]
The CCXT market dicts.
Returns
-------
dict[str, Object]
The asset definition.
"""
asset_defs = []
for is_local in (False, True):
asset_def = self.get_asset_def(market, is_local)
asset_defs.append((asset_def, is_local))
return asset_defs
def get_asset_def(self, market, is_local=False):
"""
The asset definition (in symbols.json files) corresponding
to the the specified market.
Parameters
----------
market: dict[str, Object]
The CCXT market dict.
is_local
Whether to search in local or Catalyst asset definitions.
Returns
-------
dict[str, Object]
The asset definition.
"""
exchange_symbol = market['id']
symbol_map = self._fetch_symbol_map(is_local)
if symbol_map is not None:
assets_lower = {k.lower(): v for k, v in symbol_map.items()}
key = exchange_symbol.lower()
asset = assets_lower[key] if key in assets_lower else None
if asset is not None:
return asset
else:
return None
else:
return None
def create_trading_pair(self, market, asset_def=None, is_local=False):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
asset_def: dict[str, Object]
is_local: bool
Returns
-------
"""
# TODO: make this more externalized / configurable
# Consider representing in some type of JSON structure
if 'active' in market:
params['trading_state'] = 1 if market['active'] else 0
data_source = 'local' if is_local else 'catalyst'
params = dict(
exchange=self.name,
data_source=data_source,
exchange_symbol=market['id'],
)
mixin_market_params(self.name, params, market)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
params['start_date'] = asset_def['start_date'] \
if 'start_date' in asset_def else None
params['end_date'] = asset_def['end_date'] \
if 'end_date' in asset_def else None
params['leverage'] = asset_def['leverage'] \
if 'leverage' in asset_def else 1.0
params['asset_name'] = asset_def['asset_name'] \
if 'asset_name' in asset_def else None
params['end_daily'] = asset_def['end_daily'] \
if 'end_daily' in asset_def \
and asset_def['end_daily'] != 'N/A' else None
params['end_minute'] = asset_def['end_minute'] \
if 'end_minute' in asset_def \
and asset_def['end_minute'] != 'N/A' else None
else:
params['trading_state'] = 1
params['symbol'] = get_catalyst_symbol(market)
# TODO: add as an optional column
params['leverage'] = 1.0
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
return TradingPair(**params)
if self.name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
def load_assets(self):
log.debug('loading assets for {}'.format(self.name))
self.assets = []
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None \
and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
for market in self.markets:
if 'id' not in market:
log.warn('invalid market: {}'.format(market))
continue
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
asset_defs = self.get_asset_defs(market)
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
asset = None
for asset_def in asset_defs:
if asset_def[0] is not None or not asset_defs[1]:
try:
asset = self.create_trading_pair(
market=market,
asset_def=asset_def[0],
is_local=asset_def[1]
)
self.assets.append(asset)
if 'lot' not in params:
params['lot'] = params['min_trade_size']
except TypeError as e:
log.warn('unable to add asset: {}'.format(e))
if asset is None:
asset = self.create_trading_pair(market=market)
self.assets.append(asset)
def get_balances(self):
try:
@@ -673,9 +770,14 @@ class CCXT(Exchange):
abs(amount), adj_amount,
)
)
else:
adj_amount = round(abs(amount), asset.decimals)
adj_amount = abs(amount)
if adj_amount == 0:
raise CreateOrderError(
exchange=self.name,
e='order amount lower than the smallest lot: {}'.format(amount)
)
try:
result = self.api.create_order(
@@ -697,22 +799,6 @@ class CCXT(Exchange):
)
raise ExchangeRequestError(error=e)
exchange_amount = None
if 'amount' in result and result['amount'] != adj_amount:
exchange_amount = result['amount']
elif 'info' in result:
if 'origQty' in result['info']:
exchange_amount = float(result['info']['origQty'])
if exchange_amount:
log.info(
'order amount adjusted by {} from {} to {}'.format(
self.name, adj_amount, exchange_amount
)
)
adj_amount = exchange_amount
if 'info' not in result:
raise ValueError('cannot use order without info attribute')
@@ -773,38 +859,31 @@ class CCXT(Exchange):
order.id, order.asset, return_price=True
)
order.status = exc_order.status
order.commission = exc_order.commission
order.filled = exc_order.amount
transactions = []
if exc_order.status == ORDER_STATUS.FILLED:
if order.amount > exc_order.amount:
log.warn(
'executed order amount {} differs '
'from original'.format(
exc_order.amount, order.amount
)
if order.amount != exc_order.amount:
log.warn(
'executed order amount {} differs '
'from original'.format(
exc_order.amount, order.amount
)
order.check_triggers(
price=price,
dt=exc_order.dt,
)
order.amount = exc_order.amount
if order.status == ORDER_STATUS.FILLED:
transaction = Transaction(
asset=order.asset,
amount=order.amount,
dt=pd.Timestamp.utcnow(),
price=price,
order_id=order.id,
commission=order.commission,
commission=order.commission
)
transactions.append(transaction)
return transactions
return [transaction]
def process_order(self, order):
# TODO: move to parent class after tracking features in the parent
if not self.api.has['fetchMyTrades']:
if not self.api.hasFetchMyTrades:
return self._process_order_fallback(order)
try:
@@ -884,8 +963,7 @@ class CCXT(Exchange):
)
raise ExchangeRequestError(error=e)
def cancel_order(self, order_param,
asset_or_symbol=None, params={}):
def cancel_order(self, order_param, asset_or_symbol=None):
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
@@ -897,8 +975,7 @@ class CCXT(Exchange):
try:
symbol = self.get_symbol(asset_or_symbol) \
if asset_or_symbol is not None else None
self.api.cancel_order(id=order_id,
symbol=symbol, params=params)
self.api.cancel_order(id=order_id, symbol=symbol)
except (ExchangeError, NetworkError) as e:
log.warn(
@@ -908,7 +985,7 @@ class CCXT(Exchange):
)
raise ExchangeRequestError(error=e)
def tickers(self, assets, on_ticker_error='raise'):
def tickers(self, assets):
"""
Retrieve current tick data for the given assets
@@ -921,51 +998,27 @@ class CCXT(Exchange):
list[dict[str, float]
"""
if len(assets) == 1:
try:
symbol = self.get_symbol(assets[0])
log.debug('fetching single ticker: {}'.format(symbol))
results = dict()
results[symbol] = self.api.fetch_ticker(symbol=symbol)
except (ExchangeError, NetworkError,) as e:
log.warn(
'unable to fetch ticker {} / {}: {}'.format(
self.name, symbol, e
)
)
raise ExchangeRequestError(error=e)
elif len(assets) > 1:
symbols = self.get_symbols(assets)
try:
log.debug('fetching multiple tickers: {}'.format(symbols))
results = self.api.fetch_tickers(symbols=symbols)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch tickers {} / {}: {}'.format(
self.name, symbols, e
)
)
raise ExchangeRequestError(error=e)
else:
raise ValueError('Cannot request tickers with not assets.')
tickers = dict()
tickers = {}
for asset in assets:
symbol = self.get_symbol(asset)
if symbol not in results:
msg = 'ticker not found {} / {}'.format(
self.name, symbol
)
log.warn(msg)
if on_ticker_error == 'warn':
continue
else:
raise ExchangeRequestError(error=msg)
ticker = results[symbol]
# Test the CCXT throttling further to see if we need this
self.ask_request()
# TODO: use fetch_tickers() for efficiency
# I tried using fetch_tickers() but noticed some
# inconsistencies, see issue:
# https://github.com/ccxt/ccxt/issues/870
try:
ticker = self.api.fetch_ticker(symbol=symbol)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch ticker {} / {}: {}'.format(
self.name, asset.symbol, e
)
)
continue
ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
if 'last_price' not in ticker:
@@ -977,7 +1030,7 @@ class CCXT(Exchange):
ticker['volume'] = ticker['baseVolume']
elif 'info' in ticker and 'bidQty' in ticker['info'] \
and 'askQty' in ticker['info']:
and 'askQty' in ticker['info']:
ticker['volume'] = float(ticker['info']['bidQty']) + \
float(ticker['info']['askQty'])
@@ -1015,28 +1068,20 @@ class CCXT(Exchange):
return result
def get_trades(self, asset, my_trades=True, start_dt=None, limit=100):
def get_trades(self, asset, my_trades=True, start_dt=None, limit=None):
if not my_trades:
raise NotImplemented(
'get_trades only supports "my trades"'
)
# TODO: is it possible to sort this? Limit is useless otherwise.
ccxt_symbol = self.get_symbol(asset)
if start_dt:
delta = start_dt - get_epoch()
since = int(delta.total_seconds()) * 1000
else:
since = None
try:
if my_trades:
trades = self.api.fetch_my_trades(
symbol=ccxt_symbol,
since=since,
limit=limit,
)
else:
trades = self.api.fetch_trades(
symbol=ccxt_symbol,
since=since,
limit=limit,
)
trades = self.api.fetch_my_trades(
symbol=ccxt_symbol,
since=start_dt,
limit=limit,
)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch trades {} / {}: {}'.format(
catalyst/exchange/exchange.py
+63 -65
View File
@@ -5,22 +5,20 @@ from time import sleep
import numpy as np
import pandas as pd
from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
SymbolNotFoundOnExchange, \
PricingDataNotLoadedError, \
NoDataAvailableOnExchange, NoValueForField, \
NoCandlesReceivedFromExchange, \
NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
TickerNotFoundError, NotEnoughCashError
from catalyst.exchange.utils.datetime_utils import get_delta, \
get_periods_range, \
get_periods, get_start_dt, get_frequency
from catalyst.exchange.utils.exchange_utils import \
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
resample_history_df, has_bundle
from logbook import Logger
log = Logger('Exchange', level=LOG_LEVEL)
@@ -180,7 +178,6 @@ class Exchange:
if symbols is None:
# Make a distinct list of all symbols
symbols = list(set([asset.symbol for asset in self.assets]))
symbols.sort()
if quote_currency is not None:
for symbol in symbols[:]:
@@ -257,10 +254,9 @@ class Exchange:
elif data_frequency is not None:
applies = (
(
data_frequency == 'minute' and a.end_minute is not None
) or (
data_frequency == 'daily' and a.end_daily is not None
)
data_frequency == 'minute' and a.end_minute is not None)
or (
data_frequency == 'daily' and a.end_daily is not None)
)
else:
@@ -293,6 +289,16 @@ class Exchange:
log.debug('found asset: {}'.format(asset))
return asset
def fetch_symbol_map(self, is_local=False):
index = 1 if is_local else 0
if self._symbol_maps[index] is not None:
return self._symbol_maps[index]
else:
symbol_map = get_exchange_symbols(self.name, is_local)
self._symbol_maps[index] = symbol_map
return symbol_map
@abstractmethod
def init(self):
"""
@@ -304,13 +310,24 @@ class Exchange:
"""
@abstractmethod
def create_exchange_config(self):
def load_assets(self, is_local=False):
"""
Fetch the exchange market data and generate a config object
Returns
-------
Populate the 'assets' attribute with a dictionary of Assets.
The key of the resulting dictionary is the exchange specific
currency pair symbol. The universal symbol is contained in the
'symbol' attribute of each asset.
Notes
-----
The sid of each asset is calculated based on a numeric hash of the
universal symbol. This simple approach avoids maintaining a mapping
of sids.
This method can be omerridden if an exchange offers equivalent data
via its api.
"""
pass
def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
"""
@@ -484,54 +501,45 @@ class Exchange:
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D', 'H']
frequency, data_frequency
)
# we want to avoid receiving empty candles
# so we request more than needed
# TODO: consider defining a const per asset
# and/or some retry mechanism (in each iteration request more data)
requested_bar_count = bar_count + 30
# The get_history method supports multiple asset
candles = self.get_candles(
freq=freq,
assets=assets,
bar_count=requested_bar_count,
bar_count=bar_count,
end_dt=end_dt if not is_current else None,
)
# candles sanity check - verify no empty candles were received:
series = dict()
for asset in candles:
if not candles[asset]:
raise NoCandlesReceivedFromExchange(
bar_count=requested_bar_count,
end_dt=end_dt,
asset=asset,
exchange=self.name)
first_candle = candles[asset][0]
asset_series = self.get_series_from_candles(
candles=candles[asset],
start_dt=first_candle['last_traded'],
end_dt=end_dt,
data_frequency=frequency,
field=field,
)
series = get_candles_df(candles=candles,
field=field,
freq=frequency,
bar_count=requested_bar_count,
end_dt=end_dt)
# Checking to make sure that the dates match
delta = get_delta(candle_size, data_frequency)
adj_end_dt = end_dt - delta
last_traded = asset_series.index[-1]
# TODO: consider how to approach this edge case
# delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
# delta = get_delta(delta_candle_size, data_frequency)
# adj_end_dt = end_dt - delta
# last_traded = asset_series.index[-1]
# if last_traded < adj_end_dt:
# raise LastCandleTooEarlyError(
# last_traded=last_traded,
# end_dt=adj_end_dt,
# exchange=self.name,
# )
if last_traded < adj_end_dt:
raise LastCandleTooEarlyError(
last_traded=last_traded,
end_dt=adj_end_dt,
exchange=self.name,
)
series[asset] = asset_series
df = pd.DataFrame(series)
df.dropna(inplace=True)
return df.tail(bar_count)
return df
def get_history_window_with_bundle(self,
assets,
@@ -579,8 +587,7 @@ class Exchange:
A dataframe containing the requested data.
"""
# TODO: this function needs some work,
# we're currently using it just for benchmark data
# TODO: this function needs some work, we're currently using it just for benchmark data
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency
)
@@ -647,21 +654,16 @@ class Exchange:
return df
def _check_low_balance(self, currency, balances, amount, open_orders=None):
def _check_low_balance(self, currency, balances, amount):
free = balances[currency]['free'] if currency in balances else 0.0
if open_orders:
# TODO: make sure that this works
free += sum([order.amount for order in open_orders])
if free < amount:
return free, True
else:
return free, False
def sync_positions(self, positions, open_orders=None, cash=None,
check_balances=False):
def sync_positions(self, positions, cash=None, check_balances=False):
"""
Update the portfolio cash and position balances based on the
latest ticker prices.
@@ -690,7 +692,7 @@ class Exchange:
balances=balances,
amount=cash,
)
if is_lower and not open_orders:
if is_lower:
raise NotEnoughCashError(
currency=self.base_currency,
exchange=self.name,
@@ -699,8 +701,8 @@ class Exchange:
)
positions_value = 0.0
if positions:
assets = list(set([position.asset for position in positions]))
if positions is not None:
assets = set([position.asset for position in positions])
tickers = self.tickers(assets)
for position in positions:
@@ -917,8 +919,7 @@ class Exchange:
"""
@abstractmethod
def cancel_order(self, order_param,
symbol_or_asset=None, params={}):
def cancel_order(self, order_param, symbol_or_asset=None):
"""Cancel an open order.
Parameters
@@ -927,7 +928,6 @@ class Exchange:
The order_id or order object to cancel.
symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this
params:
"""
pass
@@ -972,15 +972,13 @@ class Exchange:
pass
@abc.abstractmethod
def tickers(self, assets, on_ticker_error='raise'):
def tickers(self, assets):
"""
Retrieve current tick data for the given assets
Parameters
----------
assets: list[TradingPair]
on_ticker_error: str [raise|warn]
How to handle an error when retrieving a single ticker.
Returns
-------
catalyst/exchange/exchange_algorithm.py
+46 -180
View File
@@ -16,13 +16,11 @@ import signal
import sys
from datetime import timedelta
from os import listdir
from os.path import isfile, join, exists
import logbook
import pandas as pd
from redo import retry
from os.path import isfile, join
import catalyst.protocol as zp
import logbook
import pandas as pd
from catalyst.algorithm import TradingAlgorithm
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange_blotter import ExchangeBlotter
@@ -38,20 +36,18 @@ from catalyst.exchange.utils.exchange_utils import (
get_algo_folder,
get_algo_df,
save_algo_df,
clear_frame_stats_directory,
remove_old_files,
group_assets_by_exchange, )
from catalyst.exchange.utils.stats_utils import \
get_pretty_stats, stats_to_s3, stats_to_algo_folder
from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
stats_to_algo_folder
from catalyst.finance.execution import MarketOrder
from catalyst.finance.performance import PerformanceTracker
from catalyst.finance.performance.period import calc_period_stats
from catalyst.gens.tradesimulation import AlgorithmSimulator
from catalyst.marketplace.marketplace import Marketplace
from catalyst.utils.api_support import api_method
from catalyst.utils.input_validation import error_keywords, ensure_upper_case
from catalyst.utils.math_utils import round_nearest
from catalyst.utils.preprocess import preprocess
from redo import retry
log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
@@ -70,8 +66,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
self.current_day = None
if self.simulate_orders is None and \
self.sim_params.arena == 'backtest':
if self.simulate_orders is None \
and self.sim_params.arena == 'backtest':
self.simulate_orders = True
# Operations with retry features
@@ -96,8 +92,6 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
attempts=self.attempts,
)
self._marketplace = None
@staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style):
"""
@@ -164,25 +158,6 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
style)
return amount, style
def _calculate_order_target_amount(self, asset, target):
"""
removes order amounts so we won't run into issues
when two orders are placed one after the other.
it then proceeds to removing positions amount at TradingAlgorithm
:param asset:
:param target:
:return: target
"""
if asset in self.blotter.open_orders:
for open_order in self.blotter.open_orders[asset]:
current_amount = open_order.amount
target -= current_amount
target = super(ExchangeTradingAlgorithmBase, self). \
_calculate_order_target_amount(asset, target)
return target
def round_order(self, amount, asset):
"""
We need fractions with cryptocurrencies
@@ -192,15 +167,6 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
"""
return round_nearest(amount, asset.min_trade_size)
@api_method
def get_dataset(self, data_source_name, start=None, end=None):
if self._marketplace is None:
self._marketplace = Marketplace()
return self._marketplace.get_dataset(
data_source_name, start, end,
)
@api_method
@preprocess(symbol_str=ensure_upper_case)
def symbol(self, symbol_str, exchange_name=None):
@@ -390,35 +356,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self._clock = None
self.frame_stats = list()
# erase the frame_stats folder to avoid overloading the disk
error = clear_frame_stats_directory(self.algo_namespace)
if error:
log.warning(error)
self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
# in order to save paper & live files separately
self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live'
self.custom_signals_stats = \
get_algo_df(self.algo_namespace, 'custom_signals_stats')
self.pnl_stats = get_algo_df(
self.algo_namespace,
'pnl_stats_{}'.format(self.mode_name),
)
self.custom_signals_stats = get_algo_df(
self.algo_namespace,
'custom_signals_stats_{}'.format(self.mode_name)
)
self.exposure_stats = get_algo_df(
self.algo_namespace,
'exposure_stats_{}'.format(self.mode_name)
)
self.exposure_stats = \
get_algo_df(self.algo_namespace, 'exposure_stats')
self.is_running = True
self.stats_minutes = 1
self._last_orders = []
self._last_open_orders = []
self.trading_client = None
super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
@@ -429,20 +379,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
log.warn("Can't initialize signal handler inside another thread."
"Exit should be handled by the user.")
log.info('initialized trading algorithm in live mode')
def interrupt_algorithm(self):
"""
when algorithm comes to an end this function is called.
extracts the stats and calls analyze.
after finishing, it exits the run.
Parameters
----------
Returns
-------
"""
self.is_running = False
if self._analyze is None:
@@ -452,31 +391,21 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
log.info('Exiting the algorithm. Calling `analyze()` '
'before exiting the algorithm.')
# add the last day stats which is not saved in the directory
current_stats = pd.DataFrame(self.frame_stats)
current_stats.set_index('period_close', drop=False, inplace=True)
# get the location of the directory
algo_folder = get_algo_folder(self.algo_namespace)
folder = join(algo_folder, 'frame_stats')
folder = join(algo_folder, 'daily_performance')
files = [f for f in listdir(folder) if isfile(join(folder, f))]
if exists(folder):
files = [f for f in listdir(folder) if isfile(join(folder, f))]
daily_perf_list = []
for item in files:
filename = join(folder, item)
period_stats_list = []
for item in files:
filename = join(folder, item)
with open(filename, 'rb') as handle:
perf_period = pickle.load(handle)
perf_period_dict = perf_period.to_dict()
daily_perf_list.append(perf_period_dict)
with open(filename, 'rb') as handle:
perf_period = pickle.load(handle)
period_stats_list.extend(perf_period)
stats = pd.DataFrame(period_stats_list)
stats.set_index('period_close', drop=False, inplace=True)
stats = pd.concat([stats, current_stats])
else:
stats = current_stats
stats = pd.DataFrame(daily_perf_list)
stats.set_index('period_close', drop=False, inplace=True)
self.analyze(stats)
@@ -545,7 +474,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
"""
self.state = get_algo_object(
algo_name=self.algo_namespace,
key='context.state_{}'.format(self.mode_name),
key='context.state',
)
if self.state is None:
self.state = {}
@@ -568,7 +497,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# Unpacking the perf_tracker and positions if available
cum_perf = get_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance_{}'.format(self.mode_name),
key='cumulative_performance',
)
if cum_perf is not None:
tracker.cumulative_performance = cum_perf
@@ -579,7 +508,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
todays_perf = get_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
rel_path='daily_performance_{}'.format(self.mode_name),
rel_path='daily_performance',
)
if todays_perf is not None:
# Ensure single common position tracker
@@ -662,6 +591,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
if base_currency is None:
base_currency = exchange.base_currency
# Don't check the cash if there are open orders. This could
# results in false positives.
orders = []
for asset in self.blotter.open_orders:
asset_orders = self.blotter.open_orders[asset]
@@ -671,7 +602,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
required_cash = self.portfolio.cash if not orders else None
cash, positions_value = exchange.sync_positions(
positions=exchange_positions,
open_orders=orders,
check_balances=check_balances,
cash=required_cash,
)
@@ -717,11 +647,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
)
self.pnl_stats = pd.concat([self.pnl_stats, df])
save_algo_df(
self.algo_namespace,
'pnl_stats_{}'.format(self.mode_name),
self.pnl_stats,
)
save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats)
def add_custom_signals_stats(self, period_stats):
"""
@@ -742,11 +668,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
)
self.custom_signals_stats = pd.concat([self.custom_signals_stats, df])
save_algo_df(
self.algo_namespace,
'custom_signals_stats_{}'.format(self.mode_name),
self.custom_signals_stats,
)
save_algo_df(self.algo_namespace, 'custom_signals_stats',
self.custom_signals_stats)
def add_exposure_stats(self, period_stats):
"""
@@ -773,43 +696,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.exposure_stats = pd.concat([self.exposure_stats, df])
save_algo_df(
self.algo_namespace,
'exposure_stats_{}'.format(self.mode_name),
self.exposure_stats
self.algo_namespace, 'exposure_stats', self.exposure_stats
)
def nullify_frame_stats(self, now):
"""
Save all period_stats to local directory
erase old files from the folder and nullify
self.frame_stats
Parameters
----------
now: Timestamp
Returns
-------
"""
save_algo_object(
algo_name=self.algo_namespace,
key=now.floor('1D').strftime('%Y-%m-%d'),
obj=self.frame_stats,
rel_path='frame_stats'
)
error = remove_old_files(
algo_name=self.algo_namespace,
today=now,
rel_path='frame_stats'
)
if error:
log.warning(error)
self.frame_stats = list()
def handle_data(self, data):
"""
Wrapper around the handle_data method of each algo.
@@ -829,20 +718,15 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# Resetting the frame stats every day to minimize memory footprint
today = data.current_dt.floor('1D')
if self.current_day is not None and today > self.current_day:
self.nullify_frame_stats(now=data.current_dt)
self.frame_stats = list()
self.performance_needs_update = False
last_orders_list = list(self.blotter.orders.keys())
open_orders_list = list(self.blotter.open_orders.keys())
if last_orders_list != self._last_orders or \
open_orders_list != self._last_open_orders:
orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
if orders != self._last_orders:
self.performance_needs_update = True
# Saving current order positions
# to detect changes in the next frame
self._last_orders = copy.deepcopy(last_orders_list)
self._last_open_orders = copy.deepcopy(open_orders_list)
# Saving current orders to detect changes in the next frame
self._last_orders = copy.deepcopy(orders)
if self.performance_needs_update:
self.perf_tracker.update_performance()
@@ -884,7 +768,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
log.debug('saving cumulative performance object')
save_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance_{}'.format(self.mode_name),
key='cumulative_performance',
obj=self.perf_tracker.cumulative_performance,
)
log.debug('saving todays performance object')
@@ -892,12 +776,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
obj=self.perf_tracker.todays_performance,
rel_path='daily_performance_{}'.format(self.mode_name)
rel_path='daily_performance'
)
log.debug('saving context.state object')
save_algo_object(
algo_name=self.algo_namespace,
key='context.state_{}'.format(self.mode_name),
key='context.state',
obj=self.state)
def _process_stats(self, data):
@@ -914,8 +798,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# Saving the last hour in memory
self.frame_stats.append(frame_stats)
# creating and saving the pnl_stats into the local
# directory
self.add_pnl_stats(frame_stats)
if self.recorded_vars:
self.add_custom_signals_stats(frame_stats)
@@ -953,7 +835,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
csv_bytes = stats_to_algo_folder(
stats=self.frame_stats,
algo_namespace=self.algo_namespace,
folder_name='stats_{}'.format(self.mode_name),
recorded_cols=recorded_cols,
)
except Exception as e:
@@ -981,13 +862,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
raise NotImplementedError()
def _get_open_orders(self, asset=None):
if self.simulate_orders:
raise ValueError(
'The get_open_orders() method only works in live mode. '
'The purpose is to list open orders on the exchange '
'regardless who placed them. To list the open orders of '
'this algo, use `context.blotter.open_orders`.'
)
if asset:
exchange = self.exchanges[asset.exchange]
return exchange.get_open_orders(asset)
@@ -1021,15 +895,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
If an asset is passed then this will return a list of the open
orders for this asset.
"""
# TODO: should this be a shortcut to the open orders in the blotter?
return retry(
action=self._get_open_orders,
attempts=self.attempts['get_open_orders_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('Fetching open orders again.'),
args=(asset,)
)
args=(asset,))
@api_method
def get_order(self, order_id, exchange_name):
@@ -1058,19 +930,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
args=(order_id,))
@api_method
def cancel_order(self, order_param, exchange_name,
symbol=None, params={}):
def cancel_order(self, order_param, exchange_name):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
exchange_name: name of exchange from
which you want to cancel the order
symbol:
params:
"""
exchange = self.exchanges[exchange_name]
@@ -1084,4 +950,4 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('cancelling order again.'),
args=(order_id, symbol, params))
args=(order_id,))
catalyst/exchange/exchange_blotter.py
+4 -7
View File
@@ -68,7 +68,7 @@ class TradingPairFeeSchedule(CommissionModel):
multiplier = maker \
if ((order.amount > 0 and order.limit < transaction.price)
or (order.amount < 0 and order.limit > transaction.price)) \
and order.limit_reached else taker
and order.limit_reached else taker
fee = cost * multiplier
return fee
@@ -214,7 +214,7 @@ class ExchangeBlotter(Blotter):
# that this is safer until we have a robust way to track
# the trades already processed by the algo. We can't loose
# them if the algo shuts down.
if transactions and order.status == ORDER_STATUS.FILLED:
if transactions and order.open_amount == 0:
avg_price = np.average(
a=[t.price for t in transactions],
weights=[t.amount for t in transactions],
@@ -238,12 +238,9 @@ class ExchangeBlotter(Blotter):
else:
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'{exchange} order {order_id} for {symbol} still open '
'after {delta}'.format(
exchange=exchange.name,
'order {order_id} still open after {delta}'.format(
order_id=order.id,
delta=delta,
symbol=order.asset.symbol,
delta=delta
)
)
catalyst/exchange/exchange_bundle.py
+70 -61
View File
@@ -1,4 +1,3 @@
import copy
import os
import shutil
from datetime import timedelta
@@ -9,12 +8,8 @@ from operator import is_not
import numpy as np
import pandas as pd
import pytz
from catalyst.assets._assets import TradingPair
from logbook import Logger
from pytz import UTC
from six import itervalues
from catalyst import get_calendar
from catalyst.assets._assets import TradingPair
from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
from catalyst.constants import LOG_LEVEL
from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
@@ -27,12 +22,15 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
get_bcolz_chunk, get_df_from_arrays, get_assets
from catalyst.exchange.utils.datetime_utils import get_start_dt, \
get_period_label, get_month_start_end, get_year_start_end, get_period, \
timestr_to_dt
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
get_period_label, get_month_start_end, get_year_start_end
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory
from logbook import Logger
from pytz import UTC
from six import itervalues
log = Logger('exchange_bundle', level=LOG_LEVEL)
@@ -460,7 +458,7 @@ class ExchangeBundle:
last_entry = None
if start is None or \
(earliest_trade is not None and earliest_trade > start):
(earliest_trade is not None and earliest_trade > start):
start = earliest_trade
if last_entry is not None and (end is None or end > last_entry):
@@ -514,8 +512,8 @@ class ExchangeBundle:
continue
dates = pd.date_range(
start=get_period(adj_start, data_frequency),
end=get_period(adj_end, data_frequency),
start=get_period_label(adj_start, data_frequency),
end=get_period_label(adj_end, data_frequency),
freq='MS' if data_frequency == 'minute' else 'AS',
tz=UTC
)
@@ -554,9 +552,7 @@ class ExchangeBundle:
# We sort the chunks by end date to ingest most recent data first
chunks[asset].sort(
key=lambda chunk: timestr_to_dt(
chunk['period'], data_frequency
)
key=lambda chunk: pd.to_datetime(chunk['period'])
)
return chunks
@@ -604,15 +600,14 @@ class ExchangeBundle:
if show_breakdown:
for asset in chunks:
with maybe_show_progress(
chunks[asset],
show_progress,
label='Ingesting {frequency} price data for '
'{symbol} on {exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
symbol=asset.symbol
)
) as it:
chunks[asset],
show_progress,
label='Ingesting {frequency} price data for '
'{symbol} on {exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
symbol=asset.symbol
)) as it:
for chunk in it:
problems += self.ingest_ctable(
asset=chunk['asset'],
@@ -627,19 +622,16 @@ class ExchangeBundle:
# We sort the chunks by end date to ingest most recent data first
all_chunks.sort(
key=lambda chunk: timestr_to_dt(
chunk['period'], data_frequency
)
key=lambda chunk: pd.to_datetime(chunk['period'])
)
with maybe_show_progress(
all_chunks,
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
)
) as it:
all_chunks,
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
)) as it:
for chunk in it:
problems += self.ingest_ctable(
asset=chunk['asset'],
@@ -708,36 +700,42 @@ class ExchangeBundle:
for symbol in symbols:
start_dt = df.index.get_level_values(1).min()
end_dt = df.index.get_level_values(1).max()
end_dt_key = 'end_{}'.format(data_frequency)
try:
asset = self.exchange.get_asset(symbol, is_local=True)
except:
asset = copy.deepcopy(self.exchange.get_asset(symbol))
market = self.exchange.get_market(symbol)
if market is None:
raise ValueError('symbol not available in the exchange.')
if asset.data_source == 'local':
asset.start_date = asset.start_date \
if asset.start_date < start_dt else start_dt
params = dict(
exchange=self.exchange.name,
data_source='local',
exchange_symbol=market['id'],
)
mixin_market_params(self.exchange_name, params, market)
if data_frequency == 'daily':
asset.end_date = asset.end_daily = asset.end_daily \
if asset.end_daily > end_dt else end_dt
asset_def = self.exchange.get_asset_def(market, True)
if asset_def is not None:
params['symbol'] = asset_def['symbol']
else:
asset.end_date = asset.end_minute = asset.end_minute \
if asset.end_minute > end_dt else end_dt
params['start_date'] = asset_def['start_date'] \
if asset_def['start_date'] < start_dt else start_dt
params['end_date'] = asset_def[end_dt_key] \
if asset_def[end_dt_key] > end_dt else end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else asset_def['end_daily']
params['end_minute'] = end_dt \
if data_frequency == 'minute' else asset_def['end_minute']
else:
asset.data_source = 'local'
asset.start_date = start_dt
asset.end_dt = end_dt
params['symbol'] = get_catalyst_symbol(market)
if data_frequency == 'daily':
asset.end_daily = end_dt
asset.end_minute = None
else:
asset.end_daily = None
asset.end_minute = end_dt
params['end_daily'] = end_dt \
if data_frequency == 'daily' else 'N/A'
params['end_minute'] = end_dt \
if data_frequency == 'minute' else 'N/A'
if min_start_dt is None or start_dt < min_start_dt:
min_start_dt = start_dt
@@ -745,9 +743,11 @@ class ExchangeBundle:
if max_end_dt is None or end_dt > max_end_dt:
max_end_dt = end_dt
assets[symbol] = asset
asset = TradingPair(**params)
assets[market['id']] = asset
save_exchange_symbols(self.exchange_name, assets, True)
# TODO: update config.json
writer = self.get_writer(
start_dt=min_start_dt.replace(hour=00, minute=00),
end_dt=max_end_dt.replace(hour=23, minute=59),
@@ -830,6 +830,7 @@ class ExchangeBundle:
field,
data_frequency,
algo_end_dt=None,
trailing_bar_count=None,
force_auto_ingest=False
):
"""
@@ -857,6 +858,7 @@ class ExchangeBundle:
bar_count=bar_count,
field=field,
data_frequency=data_frequency,
trailing_bar_count=trailing_bar_count,
)
return pd.DataFrame(series)
@@ -885,6 +887,7 @@ class ExchangeBundle:
field=field,
data_frequency=data_frequency,
reset_reader=True,
trailing_bar_count=trailing_bar_count,
)
return series
@@ -895,6 +898,7 @@ class ExchangeBundle:
bar_count=bar_count,
field=field,
data_frequency=data_frequency,
trailing_bar_count=trailing_bar_count,
)
return pd.DataFrame(series)
@@ -958,7 +962,12 @@ class ExchangeBundle:
bar_count,
field,
data_frequency,
trailing_bar_count=None,
reset_reader=False):
if trailing_bar_count:
delta = get_delta(trailing_bar_count, data_frequency)
end_dt += delta
start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
start_dt, _ = self.get_adj_dates(
start_dt, end_dt, assets, data_frequency
catalyst/exchange/exchange_data_portal.py
+5 -5
View File
@@ -9,9 +9,8 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
PricingDataNotLoadedError)
from catalyst.exchange.utils.exchange_utils import resample_history_df, \
group_assets_by_exchange
from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt
from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
from catalyst.exchange.utils.datetime_utils import get_frequency
from logbook import Logger
from redo import retry
@@ -299,6 +298,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
frequency, data_frequency
)
adj_bar_count = candle_size * bar_count
trailing_bar_count = candle_size - 1
if data_frequency == 'minute' and adj_data_frequency == 'daily':
end_dt = end_dt.floor('1D')
@@ -310,10 +310,10 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
field=field,
data_frequency=adj_data_frequency,
algo_end_dt=self._last_available_session,
trailing_bar_count=trailing_bar_count,
)
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
df = resample_history_df(pd.DataFrame(series), freq, field)
return df
def get_exchange_spot_value(self,
catalyst/exchange/exchange_errors.py
-21
View File
@@ -322,24 +322,3 @@ class BalanceTooLowError(ZiplineError):
'add positions to hold a free amount greater than {amount}, or clean '
'the state of this algo and restart.'
).strip()
class NoCandlesReceivedFromExchange(ZiplineError):
msg = (
'Although requesting {bar_count} candles until {end_dt} of asset {asset}, '
'an empty list of candles was received for {exchange}.'
).strip()
class MarketsNotFoundError(ZiplineError):
msg = (
'Exchange {exchange} contains no valid market so it is unusable in '
'Catalyst.'
).strip()
class InvalidMarketError(ZiplineError):
msg = (
'Exchange {exchange} contains at least one incorrectly structured '
'market: {market}, so it is unusable in Catalyst.'
).strip()
catalyst/exchange/utils/bundle_utils.py
+5 -7
View File
@@ -5,7 +5,6 @@ from datetime import datetime
import numpy as np
import pandas as pd
from catalyst.constants import BUNDLE_URL
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
import os
@@ -49,11 +48,10 @@ def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
path = os.path.join(root, name)
if not os.path.isdir(path):
url = BUNDLE_URL.format(
url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
'exchange-{exchange}/{name}.tar.gz'.format(
exchange=exchange_name,
data_frequency=data_frequency,
name=name,
)
name=name)
bytes = download_without_progress(url)
with tarfile.open('r', fileobj=bytes) as tar:
@@ -77,14 +75,14 @@ def get_df_from_arrays(arrays, periods):
"""
ohlcv = dict()
for index, field in enumerate(['open', 'high', 'low', 'close', 'volume']):
for index, field in enumerate(
['open', 'high', 'low', 'close', 'volume']):
ohlcv[field] = arrays[index].flatten()
df = pd.DataFrame(
data=ohlcv,
index=periods
)
df.index.name = 'last_traded'
return df
catalyst/exchange/utils/ccxt_utils.py
-307
View File
@@ -1,307 +0,0 @@
import json
import os
import pandas as pd
from six.moves.urllib import request
from catalyst.assets._assets import TradingPair
from ccxt import NetworkError
from catalyst.constants import LOG_LEVEL, EXCHANGE_CONFIG_URL
from catalyst.exchange.exchange_errors import MarketsNotFoundError, \
InvalidMarketError
from catalyst.exchange.utils.exchange_utils import get_catalyst_symbol, \
get_exchange_folder, get_exchange_auth
from catalyst.exchange.utils.serialization_utils import ExchangeJSONDecoder, \
ExchangeJSONEncoder
from logbook import Logger
from redo import retry
from ccxt.base.exchange import Exchange
from catalyst.utils.paths import last_modified_time, data_root, \
ensure_directory
import ccxt
log = Logger('ccxt_utils', level=LOG_LEVEL)
def scan_exchange_configs(features=None, history=None, is_authenticated=False,
path=None):
"""
Finding exchanges from their config files
Parameters
----------
features
is_authenticated
Returns
-------
"""
for exchange_name in ccxt.exchanges:
config = get_exchange_config(exchange_name, path)
if not config or 'error' in config:
log.info(
'skipping invalid exchange {}'.format(exchange_name)
)
# Check if the exchange has an auth.json file
if is_authenticated:
exchange_auth = get_exchange_auth(exchange_name)
has_auth = (exchange_auth['key'] != ''
and exchange_auth['secret'] != '')
if not has_auth:
continue
if features is None:
has_features = True
else:
try:
supported_features = [
feature for feature in features if
feature in config['features']
]
has_features = len(supported_features) > 0
except Exception:
has_features = False
# TODO: filter by history
if has_features:
yield config
def get_exchange_config(exchange_name, path=None, environ=None,
expiry='1H'):
"""
The de-serialized content of the exchange's config.json.
Parameters
----------
exchange_name: str
The exchange name
filename: str
The target file
environ:
Returns
-------
config: dict[srt, Object]
The config dictionary.
"""
try:
if path is None:
root = data_root(environ)
path = os.path.join(root, 'exchanges')
folder = os.path.join(path, exchange_name)
ensure_directory(folder)
filename = os.path.join(folder, 'config.json')
url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
if os.path.isfile(filename):
# If the file exists, only update periodically to avoid
# unnecessary calls
now = pd.Timestamp.utcnow()
limit = pd.Timedelta(expiry)
if pd.Timedelta(now - last_modified_time(filename)) > limit:
try:
request.urlretrieve(url=url, filename=filename)
except Exception as e:
log.warn(
'unable to update config {} => {}: {}'.format(
url, filename, e
)
)
else:
request.urlretrieve(url=url, filename=filename)
with open(filename) as data_file:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
except Exception as e:
log.warn(
'unable to download {} config: {}'.format(
exchange_name, e
)
)
return dict(error=e)
def save_exchange_config(config, filename=None, environ=None):
"""
Save assets into an exchange_config file.
Parameters
----------
exchange_name: str
config
environ
Returns
-------
"""
if filename is None:
name = 'config.json'
exchange_folder = get_exchange_folder(config['id'], environ)
filename = os.path.join(exchange_folder, name)
with open(filename, 'w+') as handle:
json.dump(config, handle, indent=4, cls=ExchangeJSONEncoder)
def fetch_markets(ccxt_exchange):
"""
Fetches CCXT market objects.
Parameters
----------
ccxt_exchange: Exchange
Returns
-------
"""
markets_symbols = ccxt_exchange.load_markets()
log.debug(
'fetching {} markets:\n{}'.format(
ccxt_exchange.name, markets_symbols
)
)
markets = ccxt_exchange.fetch_markets()
if not markets:
raise MarketsNotFoundError(
exchange=ccxt_exchange.name,
)
for market in markets:
if 'id' not in market:
raise InvalidMarketError(
exchange=ccxt_exchange.name,
market=market,
)
return markets
def create_exchange_config(ccxt_exchange):
"""
Creates an exchange config structure.
Parameters
----------
ccxt_exchange: Exchange
Returns
-------
"""
exchange_name = ccxt_exchange.__class__.__name__
config = dict(
id=exchange_name,
name=ccxt_exchange.name,
features=[
feature for feature in ccxt_exchange.has if
ccxt_exchange.has[feature]
]
)
markets = retry(
action=fetch_markets,
attempts=5,
sleeptime=5,
retry_exceptions=(NetworkError,),
cleanup=lambda: log.warn(
'fetching markets again for {}'.format(exchange_name)
),
args=(ccxt_exchange,)
)
config['assets'] = []
for market in markets:
asset = create_trading_pair(exchange_name, market)
config['assets'].append(asset)
return config
def create_trading_pair(exchange_name, market, start_dt=None, end_dt=None,
leverage=1, end_daily=None, end_minute=None):
"""
Creating a TradingPair from market and asset data.
Parameters
----------
market: dict[str, Object]
start_dt
end_dt
leverage
end_daily
end_minute
Returns
-------
"""
params = dict(
exchange=exchange_name,
data_source='catalyst',
exchange_symbol=market['id'],
symbol=get_catalyst_symbol(market),
start_date=start_dt,
end_date=end_dt,
leverage=leverage,
asset_name=market['symbol'],
end_daily=end_daily,
end_minute=end_minute,
)
apply_conditional_market_params(exchange_name, params, market)
return TradingPair(**params)
def apply_conditional_market_params(exchange_name, params, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
params: dict[Object]
market: dict[Object]
Returns
-------
"""
# TODO: make this more externalized / configurable
# Consider representing in some type of JSON structure
if 'active' in market:
params['trading_state'] = 1 if market['active'] else 0
else:
params['trading_state'] = 1
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None \
and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
catalyst/exchange/utils/datetime_utils.py
+10 -37
View File
@@ -92,7 +92,7 @@ def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
adj_periods = periods * unit_periods
# TODO: standardize time aliases to avoid any mapping
unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm'
unit = 'd' if unit == 'D' else 'm'
delta = pd.Timedelta(adj_periods, unit)
if start_dt is not None:
@@ -164,12 +164,6 @@ def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
return start_dt
def timestr_to_dt(timestr, data_frequency):
dt_format = '%Y' if data_frequency == 'daily' else '%Y%m'
dt = pd.to_datetime(timestr, format=dt_format, utc=True)
return dt
def get_period_label(dt, data_frequency):
"""
The period label for the specified date and frequency.
@@ -183,26 +177,6 @@ def get_period_label(dt, data_frequency):
-------
str
"""
if data_frequency == 'minute':
return '{}{:02d}'.format(dt.year, dt.month)
else:
return '{}'.format(dt.year)
def get_period(dt, data_frequency):
"""
The period label for the specified date and frequency.
Parameters
----------
dt: datetime
data_frequency: str
Returns
-------
str
"""
if data_frequency == 'minute':
return '{}-{:02d}'.format(dt.year, dt.month)
@@ -274,7 +248,7 @@ def get_year_start_end(dt, first_day=None, last_day=None):
return year_start, year_end
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
def get_frequency(freq, data_frequency=None):
"""
Get the frequency parameters.
@@ -328,18 +302,17 @@ def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
elif unit.lower() == 'm' or unit == 'T':
unit = 'T'
alias = '{}T'.format(candle_size)
data_frequency = 'minute'
elif unit.lower() == 'h':
if 'H' in supported_freqs:
unit = 'H'
alias = '{}H'.format(candle_size)
else:
candle_size = candle_size * 60
alias = '{}T'.format(candle_size)
if data_frequency == 'daily':
data_frequency = 'minute'
# elif unit.lower() == 'h':
# candle_size = candle_size * 60
#
# alias = '{}T'.format(candle_size)
# if data_frequency == 'daily':
# data_frequency = 'minute'
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
catalyst/exchange/utils/exchange_utils.py
+120 -180
View File
@@ -1,19 +1,19 @@
import hashlib
import os
import shutil
import json
import pandas as pd
import os
import pickle
from catalyst.assets._assets import TradingPair
import shutil
from datetime import date, datetime
import pandas as pd
from catalyst.assets._assets import TradingPair
from six import string_types
from six.moves.urllib import request
from catalyst.constants import EXCHANGE_CONFIG_URL
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
ExchangeJSONDecoder, ConfigJSONEncoder
from catalyst.utils.deprecate import deprecated
ExchangeJSONDecoder
from catalyst.utils.paths import data_root, ensure_directory, \
last_modified_time
@@ -69,7 +69,7 @@ def is_blacklist(exchange_name, environ=None):
return os.path.exists(filename)
def get_exchange_config_filename(exchange_name, environ=None):
def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
"""
The absolute path of the exchange's symbol.json file.
@@ -83,12 +83,12 @@ def get_exchange_config_filename(exchange_name, environ=None):
str
"""
name = 'config.json'
name = 'symbols.json' if not is_local else 'symbols_local.json'
exchange_folder = get_exchange_folder(exchange_name, environ)
return os.path.join(exchange_folder, name)
def download_exchange_config(exchange_name, filename, environ=None):
def download_exchange_symbols(exchange_name, environ=None):
"""
Downloads the exchange's symbols.json from the repository.
@@ -102,14 +102,15 @@ def download_exchange_config(exchange_name, filename, environ=None):
str
"""
url = EXCHANGE_CONFIG_URL.format(exchange=exchange_name)
request.urlretrieve(url=url, filename=filename)
filename = get_exchange_symbols_filename(exchange_name)
url = SYMBOLS_URL.format(exchange=exchange_name)
response = request.urlretrieve(url=url, filename=filename)
return response
@deprecated
def get_exchange_config(exchange_name, filename=None, environ=None):
def get_exchange_symbols(exchange_name, is_local=False, environ=None):
"""
The de-serialized content of the exchange's config.json.
The de-serialized content of the exchange's symbols.json.
Parameters
----------
@@ -122,48 +123,55 @@ def get_exchange_config(exchange_name, filename=None, environ=None):
Object
"""
if filename is None:
filename = get_exchange_config_filename(exchange_name)
filename = get_exchange_symbols_filename(exchange_name, is_local)
if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
pd.Timestamp('now', tz='UTC') - last_modified_time(
filename)).days > 1):
try:
download_exchange_symbols(exchange_name, environ)
except Exception as e:
pass
if os.path.isfile(filename):
now = pd.Timestamp.utcnow()
limit = pd.Timedelta('2H')
if pd.Timedelta(now - last_modified_time(filename)) > limit:
download_exchange_config(exchange_name, filename, environ)
with open(filename) as data_file:
try:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
except ValueError:
return dict()
else:
download_exchange_config(exchange_name, filename, environ)
with open(filename) as data_file:
try:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
except ValueError:
return dict()
raise ExchangeSymbolsNotFound(
exchange=exchange_name,
filename=filename
)
def save_exchange_config(exchange_name, config, filename=None, environ=None):
def save_exchange_symbols(exchange_name, assets, is_local=False, environ=None):
"""
Save assets into an exchange_config file.
Save assets into an exchange_symbols file.
Parameters
----------
exchange_name: str
config
assets: list[dict[str, object]]
is_local: bool
environ
Returns
-------
"""
if filename is None:
name = 'config.json'
exchange_folder = get_exchange_folder(exchange_name, environ)
filename = os.path.join(exchange_folder, name)
asset_dicts = dict()
for symbol in assets:
asset_dicts[symbol] = assets[symbol].to_dict()
with open(filename, 'w+') as handle:
json.dump(config, handle, indent=4, cls=ConfigJSONEncoder)
filename = get_exchange_symbols_filename(
exchange_name, is_local, environ
)
with open(filename, 'wt') as handle:
json.dump(asset_dicts, handle, indent=4, default=symbols_serial)
def get_symbols_string(assets):
@@ -265,7 +273,6 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
key: str
environ:
rel_path: str
how: str
Returns
-------
@@ -309,7 +316,6 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
obj: Object
environ:
rel_path: str
how: str
"""
folder = get_algo_folder(algo_name, environ)
@@ -386,71 +392,6 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
df.to_csv(handle, encoding='UTF_8')
def clear_frame_stats_directory(algo_name):
"""
remove the outdated directory
to avoid overloading the disk
Parameters
----------
algo_name: str
Returns
-------
error: str
"""
error = None
algo_folder = get_algo_folder(algo_name)
folder = os.path.join(algo_folder, 'frame_stats')
if os.path.exists(folder):
try:
shutil.rmtree(folder)
except OSError:
error = 'unable to remove {}, the analyze ' \
'data will be inconsistent'.format(folder)
return error
def remove_old_files(algo_name, today, rel_path, environ=None):
"""
remove old files from a directory
to avoid overloading the disk
Parameters
----------
algo_name: str
today: Timestamp
rel_path: str
environ:
Returns
-------
error: str
"""
error = None
algo_folder = get_algo_folder(algo_name, environ)
folder = os.path.join(algo_folder, rel_path)
ensure_directory(folder)
# run on all files in the folder
for f in os.listdir(folder):
try:
file_path = os.path.join(folder, f)
creation_unix = os.path.getctime(file_path)
creation_time = pd.to_datetime(creation_unix, unit='s', utc=True)
# if the file is older than 30 days erase it
if today - pd.DateOffset(30) > creation_time:
os.unlink(file_path)
except OSError:
error = 'unable to erase files in {}'.format(folder)
return error
def get_exchange_minute_writer_root(exchange_name, environ=None):
"""
The minute writer folder for the exchange.
@@ -504,6 +445,25 @@ def has_bundle(exchange_name, data_frequency, environ=None):
return os.path.isdir(folder)
def symbols_serial(obj):
"""
JSON serializer for objects not serializable by default json code
Parameters
----------
obj: Object
Returns
-------
str
"""
if isinstance(obj, (datetime, date)):
return obj.floor('1D').strftime(DATE_FORMAT)
raise TypeError("Type %s not serializable" % type(obj))
def perf_serial(obj):
"""
JSON serializer for objects not serializable by default json code
@@ -552,7 +512,7 @@ def get_common_assets(exchanges):
return assets
def resample_history_df(df, freq, field, start_dt=None):
def resample_history_df(df, freq, field):
"""
Resample the OHCLV DataFrame using the specified frequency.
@@ -580,25 +540,49 @@ def resample_history_df(df, freq, field, start_dt=None):
else:
raise ValueError('Invalid field.')
resampled_df = df.resample(
freq, closed='left', label='left'
).agg(agg) # type: pd.DataFrame
# Because the samples are closed left, we get one more candle at
# the beginning then the requested number for bars. Removing this
# candle to avoid confusion.
if start_dt and not resampled_df.empty:
resampled_df = resampled_df[resampled_df.index >= start_dt]
resampled_df = df.resample(freq).agg(agg)
return resampled_df
def from_ms_timestamp(ms):
return pd.to_datetime(ms, unit='ms', utc=True)
def mixin_market_params(exchange_name, params, market):
"""
Applies a CCXT market dict to parameters of TradingPair init.
Parameters
----------
params: dict[Object]
market: dict[Object]
def get_epoch():
return pd.to_datetime('1970-1-1', utc=True)
Returns
-------
"""
# TODO: make this more externalized / configurable
if 'lot' in market:
params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex':
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
else:
# TODO: default commission, make configurable
params['maker'] = 0.0015
params['taker'] = 0.0025
info = market['info'] if 'info' in market else None
if info:
if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def group_assets_by_exchange(assets):
@@ -655,69 +639,25 @@ def save_asset_data(folder, df, decimals=8):
)
def forward_fill_df_if_needed(df, periods):
df = df.reindex(periods)
# volume should always be 0 (if there were no trades in this interval)
df['volume'] = df['volume'].fillna(0.0)
# ie pull the last close into this close
df['close'] = df.fillna(method='pad')
# now copy the close that was pulled down from the last timestep
# into this row, across into o/h/l
df['open'] = df['open'].fillna(df['close'])
df['low'] = df['low'].fillna(df['close'])
df['high'] = df['high'].fillna(df['close'])
return df
def transform_candles_to_df(candles):
return pd.DataFrame(candles).set_index('last_traded')
def get_candles_df(candles, field, freq, bar_count, end_dt):
def get_candles_df(candles, field, freq, bar_count, end_dt,
previous_value=None):
all_series = dict()
for asset in candles:
asset_df = transform_candles_to_df(candles[asset])
rounded_end_dt = end_dt.floor(freq)
periods = pd.date_range(end=rounded_end_dt,
periods=bar_count,
freq=freq)
asset_df = forward_fill_df_if_needed(asset_df, periods)
periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
all_series[asset] = pd.Series(asset_df[field])
dates = [candle['last_traded'] for candle in candles[asset]]
values = [candle[field] for candle in candles[asset]]
series = pd.Series(values, index=dates)
series = series.reindex(
periods,
method='ffill',
fill_value=previous_value,
)
series.sort_index(inplace=True)
all_series[asset] = series
df = pd.DataFrame(all_series)
df.dropna(inplace=True)
return df
def get_trades_df(trades):
df = pd.DataFrame(trades)
df.index = pd.to_datetime(df.pop('datetime'))
df.index = df.index.tz_localize('UTC')
return df
def candles_from_trades(trades_df, freq):
"""
Calculate OHLCV from candles.
Parameters
----------
trades_df
freq
Returns
-------
"""
df = trades_df['price'].resample(freq).ohlc() # type: pd.DataFrame
df['volume'] = trades_df['amount'].resample(freq).sum()
df.dropna(axis=0, how='all', inplace=True)
df.sort_index(inplace=True, ascending=False)
return df
catalyst/exchange/utils/factory.py
+23 -35
View File
@@ -4,9 +4,8 @@ from catalyst.constants import LOG_LEVEL
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.utils.ccxt_utils import scan_exchange_configs
from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
get_exchange_folder
get_exchange_folder, is_blacklist
from logbook import Logger
log = Logger('factory', level=LOG_LEVEL)
@@ -14,12 +13,9 @@ exchange_cache = dict()
def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
skip_init=False, auth_alias=None, config=None):
skip_init=False, auth_alias=None):
key = (exchange_name, base_currency)
if key in exchange_cache:
if not skip_init:
exchange_cache[key].init()
return exchange_cache[key]
exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
@@ -37,10 +33,7 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
password=exchange_auth['password'] if 'password'
in exchange_auth.keys() else '',
base_currency=base_currency,
config=config,
)
exchange_cache[key] = exchange
@@ -58,8 +51,8 @@ def get_exchanges(exchange_names):
return exchanges
def find_exchanges(features=None, history=None, skip_blacklist=True, path=None,
is_authenticated=False, base_currency=None):
def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
base_currency=None):
"""
Find exchanges filtered by a list of feature.
@@ -77,33 +70,28 @@ def find_exchanges(features=None, history=None, skip_blacklist=True, path=None,
list[Exchange]
"""
exchange_names = CCXT.find_exchanges(features, is_authenticated)
return list(
scan_exchanges(
features,
history,
skip_blacklist,
path,
is_authenticated,
base_currency
)
)
def scan_exchanges(features=None, history=None, skip_blacklist=True, path=None,
is_authenticated=False, base_currency=None):
for config in scan_exchange_configs(
features=features,
history=history,
is_authenticated=is_authenticated,
path=path,
):
if skip_blacklist and (config is None or 'error' in config):
exchanges = []
for exchange_name in exchange_names:
if skip_blacklist and is_blacklist(exchange_name):
continue
yield get_exchange(
exchange_name=config['id'],
exchange = get_exchange(
exchange_name=exchange_name,
skip_init=True,
base_currency=base_currency,
config=config,
)
if features is not None:
if 'dailyBundle' in features \
and not exchange.has_bundle('daily'):
continue
elif 'minuteBundle' in features \
and not exchange.has_bundle('minute'):
continue
exchanges.append(exchange)
return exchanges
catalyst/exchange/utils/serialization_utils.py
+1 -34
View File
@@ -3,48 +3,15 @@ import re
from json import JSONEncoder
import pandas as pd
from catalyst.constants import DATE_TIME_FORMAT
from six import string_types
from datetime import date, datetime
from catalyst.constants import DATE_TIME_FORMAT, DATE_FORMAT
from catalyst.assets._assets import TradingPair
class ConfigJSONEncoder(json.JSONEncoder):
def default(self, obj):
"""
JSON serializer for objects not serializable by default json code
Parameters
----------
obj: Object
Returns
-------
str
"""
if isinstance(obj, (datetime, date)):
return obj.floor('1D').strftime(DATE_FORMAT)
elif isinstance(obj, TradingPair):
return obj.to_dict()
class ExchangeJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, pd.Timestamp):
return obj.strftime(DATE_TIME_FORMAT)
elif isinstance(obj, TradingPair):
asset = obj.to_dict()
asset['maker'] = round(asset['maker'], asset['decimals'])
asset['taker'] = round(asset['taker'], asset['decimals'])
asset['lot'] = round(asset['lot'], 4)
asset['min_trade_size'] = round(asset['min_trade_size'], 4)
asset['max_trade_size'] = round(asset['max_trade_size'], 4)
return asset
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, obj)
catalyst/exchange/utils/stats_utils.py
+12 -12
View File
@@ -44,7 +44,7 @@ def crossover(source, target):
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
or target is np.nan:
return False
if source[-1] >= target > source[-2]:
@@ -54,7 +54,7 @@ def crossover(source, target):
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] > target[-1] and source[-2] < target[-2]:
@@ -81,7 +81,7 @@ def crossunder(source, target):
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
or target is np.nan:
return False
if source[-1] < target <= source[-2]:
@@ -90,7 +90,7 @@ def crossunder(source, target):
return False
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] < target[-1] and source[-2] >= target[-2]:
@@ -229,10 +229,7 @@ def prepare_stats(stats, recorded_cols=list()):
asset_values)
df = pd.DataFrame(stats)
df['orders'] = df['orders'].apply(lambda orders: len(orders))
df['transactions'] = df['transactions'].apply(
lambda transactions: len(transactions)
)
index_cols = [
'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
@@ -244,6 +241,11 @@ def prepare_stats(stats, recorded_cols=list()):
for column in recorded_cols:
index_cols.append(column)
df['orders'] = df['orders'].apply(lambda orders: len(orders))
df['transactions'] = df['transactions'].apply(
lambda transactions: len(transactions)
)
if asset_cols:
columns = asset_cols
df.set_index(index_cols, drop=True, inplace=True)
@@ -396,8 +398,7 @@ def email_error(algo_name, dt, e, environ=None):
)})
def stats_to_algo_folder(stats, algo_namespace,
folder_name, recorded_cols=None):
def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
"""
Saves the performance stats to the algo local folder.
@@ -405,7 +406,6 @@ def stats_to_algo_folder(stats, algo_namespace,
----------
stats: list[Object]
algo_namespace: str
folder_name: str
recorded_cols: list[str]
Returns
@@ -418,7 +418,7 @@ def stats_to_algo_folder(stats, algo_namespace,
timestr = time.strftime('%Y%m%d')
folder = get_algo_folder(algo_namespace)
stats_folder = os.path.join(folder, folder_name)
stats_folder = os.path.join(folder, 'stats')
ensure_directory(stats_folder)
filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
catalyst/finance/risk/period.py
+2 -4
View File
@@ -29,15 +29,13 @@ from .risk import check_entry
from empyrical import (
alpha_beta_aligned,
annual_volatility,
cum_returns,
downside_risk,
information_ratio,
max_drawdown,
sharpe_ratio,
sortino_ratio
)
from catalyst.patches.stats import (
max_drawdown,
cum_returns,
)
from catalyst.constants import LOG_LEVEL
catalyst/marketplace/__init__.py
View File
catalyst/marketplace/contract_enigma_abi.json
-302
View File
@@ -1,302 +0,0 @@
[
{
"constant": true,
"inputs": [],
"name": "name",
"outputs": [
{
"name": "",
"type": "string"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "approve",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "totalSupply",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_from",
"type": "address"
},
{
"name": "_to",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "transferFrom",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "INITIAL_SUPPLY",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "decimals",
"outputs": [
{
"name": "",
"type": "uint8"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_subtractedValue",
"type": "uint256"
}
],
"name": "decreaseApproval",
"outputs": [
{
"name": "success",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": false,
"inputs": [],
"name": "getAfterApproveTest",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [
{
"name": "_owner",
"type": "address"
}
],
"name": "balanceOf",
"outputs": [
{
"name": "balance",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "symbol",
"outputs": [
{
"name": "",
"type": "string"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_to",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "transfer",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_addedValue",
"type": "uint256"
}
],
"name": "increaseApproval",
"outputs": [
{
"name": "success",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [
{
"name": "_owner",
"type": "address"
},
{
"name": "_spender",
"type": "address"
}
],
"name": "allowance",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"name": "testValue",
"type": "address"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"name": "owner",
"type": "address"
},
{
"indexed": true,
"name": "spender",
"type": "address"
},
{
"indexed": false,
"name": "value",
"type": "uint256"
}
],
"name": "Approval",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"name": "from",
"type": "address"
},
{
"indexed": true,
"name": "to",
"type": "address"
},
{
"indexed": false,
"name": "value",
"type": "uint256"
}
],
"name": "Transfer",
"type": "event"
}
]
catalyst/marketplace/contract_enigma_address.txt
-1
View File
@@ -1 +0,0 @@
0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10
catalyst/marketplace/contract_marketplace_abi.json
-1
View File
File diff suppressed because one or more lines are too long
catalyst/marketplace/contract_marketplace_address.txt
-1
View File
@@ -1 +0,0 @@
0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4
catalyst/marketplace/marketplace.py
-799
View File
@@ -1,799 +0,0 @@
from __future__ import print_function
import glob
import json
import os
import re
import shutil
import sys
import time
import webbrowser
import bcolz
import logbook
import pandas as pd
import requests
from requests_toolbelt import MultipartDecoder
from requests_toolbelt.multipart.decoder import \
NonMultipartContentTypeException
from catalyst.constants import (
LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
from catalyst.exchange.utils.stats_utils import set_print_settings
from catalyst.marketplace.marketplace_errors import (
MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles, MarketplaceRequiresPython3)
from catalyst.marketplace.utils.auth_utils import get_key_secret, \
get_signed_headers
from catalyst.marketplace.utils.bundle_utils import merge_bundles
from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
to_grains
from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
get_data_source_folder, get_marketplace_folder, \
get_user_pubaddr, get_temp_bundles_folder, extract_bundle
from catalyst.utils.paths import ensure_directory
if sys.version_info.major < 3:
import urllib
else:
import urllib.request as urllib
log = logbook.Logger('Marketplace', level=LOG_LEVEL)
class Marketplace:
def __init__(self):
global Web3
try:
from web3 import Web3, HTTPProvider
except ImportError:
raise MarketplaceRequiresPython3()
self.addresses = get_user_pubaddr()
if self.addresses[0]['pubAddr'] == '':
raise MarketplacePubAddressEmpty(
filename=os.path.join(
get_marketplace_folder(), 'addresses.json')
)
self.default_account = self.addresses[0]['pubAddr']
self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE))
contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
self.mkt_contract_address = Web3.toChecksumAddress(
contract_url.readline().decode(
contract_url.info().get_content_charset()).strip())
abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
abi = json.load(abi_url)
self.mkt_contract = self.web3.eth.contract(
self.mkt_contract_address,
abi=abi,
)
contract_url = urllib.urlopen(ENIGMA_CONTRACT)
self.eng_contract_address = Web3.toChecksumAddress(
contract_url.readline().decode(
contract_url.info().get_content_charset()).strip())
abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
abi = json.load(abi_url)
self.eng_contract = self.web3.eth.contract(
self.eng_contract_address,
abi=abi,
)
# def get_data_sources_map(self):
# return [
# dict(
# name='Marketcap',
# desc='The marketcap value in USD.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily'],
# ),
# dict(
# name='GitHub',
# desc='The rate of development activity on GitHub.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily', 'hour'],
# ),
# dict(
# name='Influencers',
# desc='Tweets & related sentiments by selected influencers.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily', 'hour', 'minute'],
# ),
# ]
def to_text(self, hex):
return Web3.toText(hex).rstrip('\0')
def choose_pubaddr(self):
if len(self.addresses) == 1:
address = self.addresses[0]['pubAddr']
address_i = 0
print('Using {} for this transaction.'.format(address))
else:
while True:
for i in range(0, len(self.addresses)):
print('{}\t{}\t{}'.format(
i,
self.addresses[i]['pubAddr'],
self.addresses[i]['desc'])
)
address_i = int(input('Choose your address associated with '
'this transaction: [default: 0] ') or 0)
if not (0 <= address_i < len(self.addresses)):
print('Please choose a number between 0 and {}\n'.format(
len(self.addresses) - 1))
else:
address = Web3.toChecksumAddress(
self.addresses[address_i]['pubAddr'])
break
return address, address_i
def sign_transaction(self, tx):
url = 'https://www.myetherwallet.com/#offline-transaction'
print('\nVisit {url} and enter the following parameters:\n\n'
'From Address:\t\t{_from}\n'
'\n\tClick the "Generate Information" button\n\n'
'To Address:\t\t{to}\n'
'Value / Amount to Send:\t{value}\n'
'Gas Limit:\t\t{gas}\n'
'Gas Price:\t\t[Accept the default value]\n'
'Nonce:\t\t\t{nonce}\n'
'Data:\t\t\t{data}\n'.format(
url=url,
_from=tx['from'],
to=tx['to'],
value=tx['value'],
gas=tx['gas'],
nonce=tx['nonce'],
data=tx['data'], )
)
webbrowser.open_new(url)
signed_tx = input('Copy and Paste the "Signed Transaction" '
'field here:\n')
if signed_tx.startswith('0x'):
signed_tx = signed_tx[2:]
return signed_tx
def check_transaction(self, tx_hash):
if 'ropsten' in ETH_REMOTE_NODE:
etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
tx_hash)
else:
etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
print('\nYou can check the outcome of your transaction here:\n'
'{}\n\n'.format(etherscan))
def _list(self):
data_sources = self.mkt_contract.functions.getAllProviders().call()
data = []
for index, data_source in enumerate(data_sources):
if index > 0:
if 'test' not in Web3.toText(data_source).lower():
data.append(
dict(
dataset=self.to_text(data_source)
)
)
return pd.DataFrame(data)
def list(self):
df = self._list()
set_print_settings()
if df.empty:
print('There are no datasets available yet.')
else:
print(df)
def subscribe(self, dataset=None):
if dataset is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'subscribe to [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
dataset = df_sets.iloc[dataset_num]['dataset']
break
dataset = dataset.lower()
address = self.choose_pubaddr()[0]
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(dataset))
return
grains = provider_info[1]
price = from_grains(grains)
subscribed = self.mkt_contract.functions.checkAddressSubscription(
address, Web3.toHex(dataset)
).call()
if subscribed[5]:
print(
'\nYou are already subscribed to the "{}" dataset.\n'
'Your subscription started on {} UTC, and is valid until '
'{} UTC.'.format(
dataset,
pd.to_datetime(subscribed[3], unit='s', utc=True),
pd.to_datetime(subscribed[4], unit='s', utc=True)
)
)
return
print('\nThe price for a monthly subscription to this dataset is'
' {} ENG'.format(price))
print(
'Checking that the ENG balance in {} is greater than {} '
'ENG... '.format(address, price), end=''
)
wallet_address = address[2:]
balance = self.web3.eth.call({
'from': address,
'to': self.eng_contract_address,
'data': '0x70a08231000000000000000000000000{}'.format(
wallet_address
)
})
try:
balance = Web3.toInt(balance) # web3 >= 4.0.0b7
except TypeError:
balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6
if balance > grains:
print('OK.')
else:
print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower '
'than the price of the dataset that you are trying to\n'
'buy: {} ENG. Get enough ENG to cover the costs of the '
'monthly\nsubscription for what you are trying to buy, '
'and try again.'.format(
address, from_grains(balance), price))
return
while True:
agree_pay = input('Please confirm that you agree to pay {} ENG '
'for a monthly subscription to the dataset "{}" '
'starting today. [default: Y] '.format(
price, dataset)) or 'y'
if agree_pay.lower() not in ('y', 'n'):
print("Please answer Y or N.")
else:
if agree_pay.lower() == 'y':
break
else:
return
print('Ready to subscribe to dataset {}.\n'.format(dataset))
print('In order to execute the subscription, you will need to sign '
'two different transactions:\n'
'1. First transaction is to authorize the Marketplace contract '
'to spend {} ENG on your behalf.\n'
'2. Second transaction is the actual subscription for the '
'desired dataset'.format(price))
tx = self.eng_contract.functions.approve(
self.mkt_contract_address,
grains,
).buildTransaction(
{'from': address,
'nonce': self.web3.eth.getTransactionCount(address)}
)
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
)
print(
'\nThis is the TxHash for this transaction: {}'.format(tx_hash)
)
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the first transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nFirst transaction successful!\n'
'Now processing second transaction.')
tx = self.mkt_contract.functions.subscribe(
Web3.toHex(dataset),
).buildTransaction({
'from': address,
'nonce': self.web3.eth.getTransactionCount(address)})
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(bin_hex(
self.web3.eth.sendRawTransaction(signed_tx)))
print('\nThis is the TxHash for this transaction: '
'{}'.format(tx_hash))
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the second transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nSecond transaction successful!\n'
'You have successfully subscribed to dataset {} with'
'address {}.\n'
'You can now ingest this dataset anytime during the '
'next month by running the following command:\n'
'catalyst marketplace ingest --dataset={}'.format(
dataset, address, dataset))
def process_temp_bundle(self, ds_name, path):
"""
Merge the temp bundle into the main bundle for the specified
data source.
Parameters
----------
ds_name
path
Returns
-------
"""
tmp_bundle = extract_bundle(path)
bundle_folder = get_data_source_folder(ds_name)
ensure_directory(bundle_folder)
if os.listdir(bundle_folder):
zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
merge_bundles(zsource, ztarget)
else:
os.rename(tmp_bundle, bundle_folder)
pass
def ingest(self, ds_name=None, start=None, end=None, force_download=False):
if ds_name is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'ingest [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
ds_name = df_sets.iloc[dataset_num]['dataset']
break
# ds_name = ds_name.lower()
# TODO: catch error conditions
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(ds_name)
).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(ds_name))
return
address, address_i = self.choose_pubaddr()
fns = self.mkt_contract.functions
check_sub = fns.checkAddressSubscription(
address, Web3.toHex(ds_name)
).call()
if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
print('You are not subscribed to dataset "{}" with address {}. '
'Plese subscribe first.'.format(ds_name, address))
return
if not check_sub[5]:
print('Your subscription to dataset "{}" expired on {} UTC.'
'Please renew your subscription by running:\n'
'catalyst marketplace subscribe --dataset={}'.format(
ds_name,
pd.to_datetime(check_sub[4], unit='s', utc=True),
ds_name)
)
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
secret = self.addresses[address_i]['secret']
else:
key, secret = get_key_secret(address)
headers = get_signed_headers(ds_name, key, secret)
log.debug('Starting download of dataset for ingestion...')
r = requests.post(
'{}/marketplace/ingest'.format(AUTH_SERVER),
headers=headers,
stream=True,
)
if r.status_code == 200:
target_path = get_temp_bundles_folder()
try:
decoder = MultipartDecoder.from_response(r)
for part in decoder.parts:
h = part.headers[b'Content-Disposition'].decode('utf-8')
# Extracting the filename from the header
name = re.search(r'filename="(.*)"', h).group(1)
filename = os.path.join(target_path, name)
with open(filename, 'wb') as f:
# for chunk in part.content.iter_content(
# chunk_size=1024):
# if chunk: # filter out keep-alive new chunks
# f.write(chunk)
f.write(part.content)
self.process_temp_bundle(ds_name, filename)
except NonMultipartContentTypeException:
response = r.json()
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=response,
)
else:
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=r.status_code,
)
log.info('{} ingested successfully'.format(ds_name))
def get_dataset(self, ds_name, start=None, end=None):
ds_name = ds_name.lower()
# TODO: filter ctable by start and end date
bundle_folder = get_data_source_folder(ds_name)
z = bcolz.ctable(rootdir=bundle_folder, mode='r')
df = z.todataframe() # type: pd.DataFrame
df.set_index(['date', 'symbol'], drop=True, inplace=True)
# TODO: implement the filter more carefully
# if start and end is None:
# df = df.xs(start, level=0)
return df
def clean(self, ds_name=None, data_frequency=None):
if ds_name is None:
mktplace_root = get_marketplace_folder()
folders = [os.path.basename(f.rstrip('/'))
for f in glob.glob('{}/*/'.format(mktplace_root))
if 'temp_bundles' not in f]
while True:
for idx, f in enumerate(folders):
print('{}\t{}'.format(idx, f))
dataset_num = input('Choose the dataset you want to '
'clean [0..{}]: '.format(
len(folders) - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
len(folders) - 1))
else:
if dataset_num not in range(0, len(folders)):
print('Enter a number between 0 and {}'.format(
len(folders) - 1))
else:
ds_name = folders[dataset_num]
break
ds_name = ds_name.lower()
if data_frequency is None:
folder = get_data_source_folder(ds_name)
else:
folder = get_bundle_folder(ds_name, data_frequency)
shutil.rmtree(folder)
pass
def create_metadata(self, key, secret, ds_name, data_frequency, desc,
has_history=True, has_live=True):
"""
Returns
-------
"""
headers = get_signed_headers(ds_name, key, secret)
r = requests.post(
'{}/marketplace/register'.format(AUTH_SERVER),
json=dict(
ds_name=ds_name,
desc=desc,
data_frequency=data_frequency,
has_history=has_history,
has_live=has_live,
),
headers=headers,
)
if r.status_code != 200:
raise MarketplaceHTTPRequest(
request='register', error=r.status_code
)
if 'error' in r.json():
raise MarketplaceHTTPRequest(
request='upload file', error=r.json()['error']
)
def register(self):
while True:
desc = input('Enter the name of the dataset to register: ')
dataset = desc.lower()
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if provider_info[4]:
print('There is already a dataset registered under '
'the name "{}". Please choose a different '
'name.'.format(dataset))
else:
break
price = int(
input(
'Enter the price for a monthly subscription to '
'this dataset in ENG: '
)
)
while True:
freq = input('Enter the data frequency [daily, hourly, minute]: ')
if freq.lower() not in ('daily', 'hourly', 'minute'):
print('Not a valid frequency.')
else:
break
while True:
reg_pub = input(
'Does it include historical data? [default: Y]: '
) or 'y'
if reg_pub.lower() not in ('y', 'n'):
print('Please answer Y or N.')
else:
if reg_pub.lower() == 'y':
has_history = True
else:
has_history = False
break
while True:
reg_pub = input(
'Doest it include live data? [default: Y]: '
) or 'y'
if reg_pub.lower() not in ('y', 'n'):
print('Please answer Y or N.')
else:
if reg_pub.lower() == 'y':
has_live = True
else:
has_live = False
break
address, address_i = self.choose_pubaddr()
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
secret = self.addresses[address_i]['secret']
else:
key, secret = get_key_secret(address)
grains = to_grains(price)
tx = self.mkt_contract.functions.register(
Web3.toHex(dataset),
grains,
address,
).buildTransaction(
{'from': address,
'nonce': self.web3.eth.getTransactionCount(address)}
)
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
)
print(
'\nThis is the TxHash for this transaction: {}'.format(tx_hash)
)
except Exception as e:
print('Unable to register the requested dataset: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nWarming up the {} dataset'.format(dataset))
self.create_metadata(
key=key,
secret=secret,
ds_name=dataset,
data_frequency=freq,
desc=desc,
has_history=has_history,
has_live=has_live,
)
print('\n{} registered successfully'.format(dataset))
def publish(self, dataset, datadir, watch):
dataset = dataset.lower()
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if not provider_info[4]:
raise MarketplaceDatasetNotFound(dataset=dataset)
match = next(
(l for l in self.addresses if l['pubAddr'] == provider_info[0]),
None
)
if not match:
raise MarketplaceNoAddressMatch(
dataset=dataset,
address=provider_info[0])
print('Using address: {} to publish this dataset.'.format(
provider_info[0]))
if 'key' in match:
key = match['key']
secret = match['secret']
else:
key, secret = get_key_secret(provider_info[0])
headers = get_signed_headers(dataset, key, secret)
filenames = glob.glob(os.path.join(datadir, '*.csv'))
if not filenames:
raise MarketplaceNoCSVFiles(datadir=datadir)
files = []
for file in filenames:
files.append(('file', open(file, 'rb')))
r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
files=files,
headers=headers)
if r.status_code != 200:
raise MarketplaceHTTPRequest(request='upload file',
error=r.status_code)
if 'error' in r.json():
raise MarketplaceHTTPRequest(request='upload file',
error=r.json()['error'])
print('Dataset {} uploaded successfully.'.format(dataset))
catalyst/marketplace/marketplace_errors.py
-97
View File
@@ -1,97 +0,0 @@
import sys
import traceback
from catalyst.errors import ZiplineError
def silent_except_hook(exctype, excvalue, exctraceback):
if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch,
MarketplaceSubscriptionExpired, MarketplaceJSONError,
MarketplaceWalletNotSupported, MarketplaceEmptySignature,
MarketplaceRequiresPython3]:
fn = traceback.extract_tb(exctraceback)[-1][0]
ln = traceback.extract_tb(exctraceback)[-1][1]
print("Error traceback: {1} (line {2})\n"
"{0.__name__}: {3}".format(exctype, fn, ln, excvalue))
else:
sys.__excepthook__(exctype, excvalue, exctraceback)
sys.excepthook = silent_except_hook
class MarketplacePubAddressEmpty(ZiplineError):
msg = (
'Please enter your public address to use in the Data Marketplace '
'in the following file: {filename}'
).strip()
class MarketplaceDatasetNotFound(ZiplineError):
msg = (
'The dataset "{dataset}" is not registered in the Data Marketplace.'
).strip()
class MarketplaceNoAddressMatch(ZiplineError):
msg = (
'The address registered with the dataset {dataset}: {address} '
'does not match any of your addresses.'
).strip()
class MarketplaceHTTPRequest(ZiplineError):
msg = (
'Request to remote server to {request} failed: {error}'
).strip()
class MarketplaceNoCSVFiles(ZiplineError):
msg = (
'No CSV files found on {datadir} to upload.'
)
class MarketplaceContractDataNoMatch(ZiplineError):
msg = (
'The information found on the contract does not match the '
'requested data:\n{params}.'
)
class MarketplaceSubscriptionExpired(ZiplineError):
msg = (
'Your subscription to dataset "{dataset}" expired on {date} '
'and is no longer active. You have to subscribe again running the '
'following command:\n'
'catalyst marketplace subscribe --dataset={dataset}'
)
class MarketplaceWalletNotSupported(ZiplineError):
msg = (
'Wallet {wallet} is not supported.'
)
class MarketplaceEmptySignature(ZiplineError):
msg = (
'Signature cannot be empty.'
)
class MarketplaceJSONError(ZiplineError):
msg = (
'The configuration file {file} is malformed. Please correct '
'the following error:\n{error}'
)
class MarketplaceRequiresPython3(ZiplineError):
msg = (
'\nCatalyst requires Python3 to access the Enigma Data Marketplace.\n'
'If you want to use the Data Marketplace, you need to reinstall '
'Catalyst\nwith Python3. See the documentation website for additional '
'information.')
catalyst/marketplace/utils/__init__.py
View File
catalyst/marketplace/utils/auth_utils.py
-131
View File
@@ -1,131 +0,0 @@
import hashlib
import hmac
import requests
import time
from catalyst.marketplace.marketplace_errors import (
MarketplaceHTTPRequest, MarketplaceWalletNotSupported,
MarketplaceEmptySignature)
from catalyst.marketplace.utils.path_utils import (
get_user_pubaddr, save_user_pubaddr)
from catalyst.constants import AUTH_SERVER
def get_key_secret(pubAddr, wallet='mew'):
"""
Obtain a new key/secret pair from authentication server
Parameters
----------
pubAddr: str
dataset: str
Returns
-------
key: str
secret: str
"""
session = requests.Session()
response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
headers={
'Authorization': 'Digest username="{0}"'.format(
pubAddr)})
if response.status_code != 401:
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error='Unexpected response code: '
'{}'.format(response.status_code))
header = response.headers.get('WWW-Authenticate')
auth_type, auth_info = header.split(None, 1)
d = requests.utils.parse_dict_header(auth_info)
nonce = '0x{}'.format(d['nonce'])
if wallet == 'mew':
print('\nObtaining a key/secret pair to streamline all future '
'requests with the authentication server.\n'
'Visit https://www.myetherwallet.com/signmsg.html and sign the '
'following message:\n{}'.format(nonce))
signature = input('Copy and Paste the "sig" field from '
'the signature here (without the double quotes, '
'only the HEX value):\n')
else:
raise MarketplaceWalletNotSupported(wallet=wallet)
if signature is None:
raise MarketplaceEmptySignature()
signature = signature[2:]
r = int(signature[0:64], base=16)
s = int(signature[64:128], base=16)
v = int(signature[128:130], base=16)
vrs = [v, r, s]
response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
headers={
'Authorization': 'Digest username="{0}",realm="{1}",'
'nonce="{2}",uri="/marketplace/getkeysecret",response="{3}",'
'opaque="{4}"'.format(pubAddr,
d['realm'],
d['nonce'],
','.join(str(e) for e in vrs+[wallet]),
d['opaque'])})
if response.status_code == 200:
if 'error' in response.json():
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error=str(response.json()['error']))
else:
addresses = get_user_pubaddr()
match = next((l for l in addresses if
l['pubAddr'] == pubAddr), None)
match['key'] = response.json()['key']
match['secret'] = response.json()['secret']
addresses[addresses.index(match)] = match
save_user_pubaddr(addresses)
print('Key/secret pair retrieved successfully from server.')
return match['key'], match['secret']
else:
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error=response.status_code)
def get_signed_headers(ds_name, key, secret):
"""
Return a new request header including the key / secret signature
Parameters
----------
ds_name
key
secret
Returns
-------
"""
nonce = str(int(time.time()))
signature = hmac.new(
secret.encode('utf-8'),
'{}{}'.format(ds_name, nonce).encode('utf-8'),
hashlib.sha512
).hexdigest()
headers = {
'Sign': signature,
'Key': key,
'Nonce': nonce,
'Dataset': ds_name,
}
return headers
catalyst/marketplace/utils/bundle_utils.py
-94
View File
@@ -1,94 +0,0 @@
import os
import random
import re
import shutil
import bcolz
import numpy as np
import pandas as pd
from six import string_types
def merge_bundles(zsource, ztarget):
"""
Merge
Parameters
----------
zsource
ztarget
Returns
-------
"""
# TODO: find a way to do this iteratively instead of in-memory
df_source = zsource.todataframe()
df_target = ztarget.todataframe()
df = pd.concat(
[df_source, df_target], ignore_index=True
) # type: pd.DataFrame
df.drop_duplicates(inplace=True)
df.set_index(['date', 'symbol'], drop=False, inplace=True)
sanitize_df(df)
dirname = os.path.basename(ztarget.rootdir)
bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
shutil.move(ztarget.rootdir, bak_dir)
z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
shutil.rmtree(bak_dir)
return z
def sanitize_df(df):
# Using a sampling method to identify dates for efficiency with
# large datasets
if len(df) > 100:
indexes = random.sample(range(0, len(df) - 1), 100)
elif len(df) > 1:
indexes = range(0, len(df) - 1)
else:
indexes = [0, ]
for column in df.columns:
is_date = False
for index in indexes:
value = df[column].iloc[index]
if not isinstance(value, string_types):
continue
# TODO: assuming that the date is at least daily
exp = re.compile(r'^\d{4}-\d{2}-\d{2}.*$')
matches = exp.findall(value)
if matches:
is_date = True
break
if is_date:
df[column] = pd.to_datetime(df[column])
else:
try:
ser = safely_reduce_dtype(df[column])
df[column] = ser
except Exception:
pass
return df
def safely_reduce_dtype(ser): # pandas.Series or numpy.array
orig_dtype = "".join(
[x for x in ser.dtype.name if x.isalpha()]) # float/int
mx = 1
for val in ser.values:
new_itemsize = np.min_scalar_type(val).itemsize
if mx < new_itemsize:
mx = new_itemsize
if orig_dtype == 'int':
mx = max(mx, 4)
new_dtype = orig_dtype + str(mx * 8)
return ser.astype(new_dtype)
catalyst/marketplace/utils/eth_utils.py
-82
View File
@@ -1,82 +0,0 @@
import binascii
# def bytes32(string):
# """
# Convert string to bytes32 data type for smart contract
# Parameters
# ----------
# string: str
# Returns
# -------
# list
# """
# return binascii.hexlify(string.encode('utf-8'))
# def b32_str(bytes32):
# """
# Convert bytes32 to string
# Parameters
# ----------
# input: bytes object
# Returns
# -------
# str
# """
# return binascii.unhexlify(
# bytes32.decode('utf-8').rstrip('\0')).decode('ascii')
def bin_hex(binary):
"""
Convert bytes32 to string
Parameters
----------
input: bytes object
Returns
-------
str
"""
return binascii.hexlify(binary).decode('utf-8')
def from_grains(amount):
"""
Convert from grains to cryptocurrency
Parameters
----------
input: amount
Returns
-------
int
"""
return amount // 10 ** 8
def to_grains(amount):
"""
Convert from cryptocurrency to grains
Parameters
----------
input: amount
Returns
-------
int
"""
return amount * 10 ** 8
catalyst/marketplace/utils/path_utils.py
-166
View File
@@ -1,166 +0,0 @@
import os
import json
import tarfile
from catalyst.utils.deprecate import deprecated
from catalyst.utils.paths import data_root, ensure_directory
from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
def get_marketplace_folder(environ=None):
"""
The root path of the marketplace folder.
Parameters
----------
environ:
Returns
-------
str
"""
if not environ:
environ = os.environ
root = data_root(environ)
marketplace_folder = os.path.join(root, 'marketplace')
ensure_directory(marketplace_folder)
return marketplace_folder
def get_data_source_folder(data_source_name, environ=None):
"""
The root path of an data_source folder.
Parameters
----------
data_source_name: str
environ:
Returns
-------
str
"""
if not environ:
environ = os.environ
root = data_root(environ)
data_source_folder = os.path.join(root, 'marketplace', data_source_name)
ensure_directory(data_source_folder)
return data_source_folder
@deprecated
def get_bundle_folder(data_source_name, data_frequency, environ=None):
data_source_folder = get_data_source_folder(data_source_name, environ)
bundle_folder = os.path.join(data_source_folder, data_frequency)
ensure_directory(bundle_folder)
return bundle_folder
def get_temp_bundles_folder(environ=None):
"""
The temp folder for bundle downloads by algo name.
Parameters
----------
ds_name: str
environ:
Returns
-------
str
"""
root = data_root(environ)
folder = os.path.join(root, 'marketplace', 'temp_bundles')
ensure_directory(folder)
return folder
def extract_bundle(tar_filename):
"""
Extract a bcolz bundle.
Parameters
----------
ds_name
Returns
-------
str
"""
target_path = tar_filename.replace('.tar.gz', '')
with tarfile.open(tar_filename, 'r') as tar:
tar.extractall(target_path)
return target_path
def get_user_pubaddr(environ=None):
"""
The de-serialized contend of the user's addresses.json file.
Parameters
----------
environ:
Returns
-------
Object
"""
marketplace_folder = get_marketplace_folder(environ)
filename = os.path.join(marketplace_folder, 'addresses.json')
if os.path.isfile(filename):
with open(filename) as data_file:
try:
data = json.load(data_file)
except json.decoder.JSONDecodeError as e:
raise MarketplaceJSONError(file=filename, error=e)
try:
d = data[0]['pubAddr']
except Exception as e:
return [data, ]
return data
else:
data = []
data.append(dict(pubAddr='', desc=''))
with open(filename, 'w') as f:
json.dump(data, f, sort_keys=False, indent=2,
separators=(',', ':'))
return data
def save_user_pubaddr(data, environ=None):
"""
Saves the user's public addresses and their related metadata in
the corresponding addresses.json file.
Parameters
----------
data: dict
Returns
-------
True
"""
marketplace_folder = get_marketplace_folder(environ)
filename = os.path.join(marketplace_folder, 'addresses.json')
with open(filename, 'w') as f:
json.dump(data, f, sort_keys=False, indent=2,
separators=(',', ':'))
return True
catalyst/support/issue_216.py
-376
View File
@@ -1,376 +0,0 @@
# -*- coding: utf-8 -*-
# !/usr/bin/env python2
import sys
import os
import pandas as pd
import signal
# import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import (
symbol,
record,
order,
order_target,
order_target_percent,
get_open_orders
)
from catalyst.finance import commission
# from base.telegrambot import TelegramBot
class GracefulKiller:
# Source: https://stackoverflow.com/a/31464349
def __init__(self, context):
self.kill_now = False
self.signal = 0
self.context = context
signal.signal(signal.SIGINT, self.exit_gracefully)
def exit_gracefully(self, signum, frame):
self.kill_now = True
self.signal = signum
if hasattr(self.context,
'telegram_bot') and self.context.telegram_bot is not None:
self.context.telegram_bot.updater.stop()
sys.exit(0)
def exit(self):
return self.kill_now
class SimulationParameters:
MODE = 'paper'
CAPITAL_BASE = 1000
"""
Capital base used on this simulation
"""
DATA_FREQUECY = 'minute'
EXCHANGE_NAME = 'bitfinex'
# EXCHANGE_NAME = 'binance'
"""
Exchange used on this simulation
"""
DATA_DIR = '/home/av/Dropbox/simulations/data'
ALGO_NAMESPACE = os.path.basename(__file__).split('.')[0]
ALGO_NAMESPACE_IMAGE = '{}/{}/{}.png'.format(DATA_DIR, 'images',
ALGO_NAMESPACE)
ALGO_NAMESPACE_RESULTS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, 'tables',
ALGO_NAMESPACE + '_results')
ALGO_NAMESPACE_TRANSACTIONS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR,
'tables',
ALGO_NAMESPACE + '_transactions')
BASE_CURRENCY = 'usd'
# BASE_CURRENCY = 'usdt'
# SHORT PERIOD
START_DATE = '2017-09-07'
"""
Start date used on this simulation
"""
END_DATE = '2017-12-12'
"""
End date used on this simulation
"""
SKIP_FIRST_CANDLES = 0
# CANDLES_SAMPLE_RATE = 60
# CANDLES_SAMPLE_RATE = 30
CANDLES_SAMPLE_RATE = 1
"""
Candle interval used on this simulation (in minutes)
"""
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
# 30 minute interval ohlcv data (the standard data required for candlestick or
# indicators/signals)
# 30T means 30 minutes re-sampling of one minute data.
# CANDLES_FREQUENCY = '60T'
# CANDLES_FREQUENCY = '30T'
CANDLES_FREQUENCY = '1T'
CANDLES_BUFFER_SIZE = 48
COIN_PAIR = 'btc_usd'
# COIN_PAIR = 'btc_usdt'
"""
Coin pair used on this simulation
"""
# TRANSACTIONS
COMMISSION_FEE = 0.0030
BUY_MIN_AMOUNT = 5 # i.e: USD
SELL_MIN_AMOUNT = 0.001 # i.e: USD
BUY_SELL_PERCENTAGE = 1 # 0.50
BUY_PERCENTAGE = BUY_SELL_PERCENTAGE
SELL_PERCENTAGE = BUY_SELL_PERCENTAGE
BASE_PRICE = 'close'
"""
Base price used (close / Heiken Ashi)
"""
log = None
parameters = None
def print_facts(context):
context.log.info("""
Index: {}
Date: {}
Candle:
O: {}
H: {}
L: {}
C: {}
V: {}
Metrics:
...
Portfolio:
Base price: {}
Base coin (coin2/usd): {}
Amount (coin1/btc): {}
""".format(
# Facts
context.i,
context.curr_minute,
context.candles_open[-1],
context.candles_high[-1],
context.candles_low[-1],
context.candles_close[-1],
context.candles_volume[-1],
# Metrics
# ...
# Portfolio
context.curr_base_price,
context.portfolio.cash,
context.portfolio.positions[context.coin_pair].amount,
))
def print_facts_telegram(context):
price = context.curr_base_price
amount = context.portfolio.positions[context.coin_pair].amount
pnl = context.portfolio.pnl
capital_used = context.portfolio.capital_used
portfolio_value = context.portfolio.portfolio_value
portfolio_returns = context.portfolio.returns
starting_cash = context.portfolio.starting_cash
cash = context.portfolio.cash
msg = """
Status...
Price: {}
Starting cash: {}
Cash: {}
Capital used: {}
Amount: {}
Portfolio value: {}
Returns: {}
PnL: {}
""".format(
price,
starting_cash,
cash,
capital_used,
amount,
portfolio_value,
portfolio_returns,
pnl,
)
if hasattr(context, 'telegram_bot') and context.telegram_bot is not None:
context.telegram_bot.msg(msg)
def default_initialize(context):
# FIXME: set_benchmark
# set_benchmark(symbol(context.parameters.COIN_PAIR))
context.coin_pair = symbol(context.parameters.COIN_PAIR)
context.base_price = None
context.current_day = None
context.counter = -1
context.i = 0
context.candles_sample_rate = context.parameters.CANDLES_SAMPLE_RATE
context.candles_frequency = context.parameters.CANDLES_FREQUENCY
context.candles_buffer_size = context.parameters.CANDLES_BUFFER_SIZE
context.set_commission(
commission.PerShare(cost=context.parameters.COMMISSION_FEE))
def default_handle_data(context, data):
context.curr_minute = data.current_dt
context.counter += 1
if context.candles_sample_rate == 1:
context.i += 1
elif context.counter % context.candles_sample_rate != 0:
context.i += 1
return
if context.i < context.parameters.SKIP_FIRST_CANDLES:
return
context.candles_open = data.history(
context.coin_pair,
'open',
bar_count=context.candles_buffer_size,
frequency=context.candles_frequency)
context.candles_high = data.history(
context.coin_pair,
'high',
bar_count=context.candles_buffer_size,
frequency=context.candles_frequency)
context.candles_low = data.history(
context.coin_pair,
'low',
bar_count=context.candles_buffer_size,
frequency=context.candles_frequency)
context.candles_close = data.history(
context.coin_pair,
'price',
bar_count=context.candles_buffer_size,
frequency=context.candles_frequency)
context.candles_volume = data.history(
context.coin_pair,
'volume',
bar_count=context.candles_buffer_size,
frequency=context.candles_frequency)
# FIXME: Here is the error!
# The candles_close frame shows more or less always a value of 94, while
# bitcoin price is very different from that
print(context.candles_close)
context.base_prices = context.candles_close
cash = context.portfolio.cash
amount = context.portfolio.positions[context.coin_pair].amount
price = data.current(context.coin_pair, 'price')
order_id = None
context.last_base_price = context.base_prices[-2]
context.curr_base_price = context.base_prices[-1]
# TA calculations
# ...
# Sanity checks
# assert cash >= 0
if cash < 0:
import ipdb;
ipdb.set_trace() # BREAKPOINT
print_facts(context)
print_facts_telegram(context)
# Order management
net_shares = 0
if context.counter == 2:
brute_shares = (cash / price) * context.parameters.BUY_PERCENTAGE
share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE
net_shares = brute_shares - share_commission_fee
buy_order_id = order(context.coin_pair, net_shares)
if context.counter == 3:
brute_shares = amount * context.parameters.SELL_PERCENTAGE
share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE
net_shares = -(brute_shares - share_commission_fee)
sell_order_id = order(context.coin_pair, net_shares)
# Record
record(
price=price,
foo='bar',
# volume=current['volume'],
# price_change=price_change,
# Metrics
cash=cash,
# buy=context.buy,
# sell=context.sell
)
def default_analyze(context=None, perf=None):
pass
def initialize(context):
global log
context.parameters = parameters
context.log = Logger(context.parameters.ALGO_NAMESPACE)
log = context.log
default_initialize(context)
context.killer = GracefulKiller(context)
context.telegram_bot = None
# TELEGRAM_TOKEN='token'
# context.telegram_bot = TelegramBot()
# context.telegram_bot.initialize(TELEGRAM_TOKEN, context)
if __name__ == '__main__':
# Parameters:
parameters = SimulationParameters()
start_date = pd.to_datetime(parameters.START_DATE, utc=True)
end_date = pd.to_datetime(parameters.END_DATE, utc=True)
if parameters.MODE == 'backtest':
results = run_algorithm(
capital_base=parameters.CAPITAL_BASE,
data_frequency=parameters.DATA_FREQUECY,
initialize=initialize,
handle_data=default_handle_data,
analyze=default_analyze,
exchange_name=parameters.EXCHANGE_NAME,
algo_namespace=parameters.ALGO_NAMESPACE,
base_currency=parameters.BASE_CURRENCY,
start=start_date,
end=end_date,
live=False,
live_graph=False
)
returns_daily = results
results.to_csv('{}'.format(parameters.ALGO_NAMESPACE_RESULTS_TABLE))
# returns_daily = returns_minutely.add(1).groupby(pd.TimeGrouper('24H')).prod().add(-1)
# FIXME: pyfolio integration
# pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results)
# pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results[:'2017-01-01'])
# pyfolio.create_full_tear_sheet(*pf_data)
elif parameters.MODE == 'paper':
results = run_algorithm(
capital_base=parameters.CAPITAL_BASE,
data_frequency=parameters.DATA_FREQUECY,
initialize=initialize,
handle_data=default_handle_data,
analyze=default_analyze,
exchange_name=parameters.EXCHANGE_NAME,
algo_namespace=parameters.ALGO_NAMESPACE,
base_currency=parameters.BASE_CURRENCY,
live=True,
simulate_orders=True,
live_graph=False
)
elif parameters.MODE == 'live':
results = run_algorithm(
initialize=initialize,
handle_data=default_handle_data,
analyze=default_analyze,
exchange_name=parameters.EXCHANGE_NAME,
algo_namespace=parameters.ALGO_NAMESPACE,
base_currency=parameters.BASE_CURRENCY,
live=True,
live_graph=True
)
catalyst/support/issue_236.py
-34
View File
@@ -1,34 +0,0 @@
from catalyst.api import symbol
from catalyst.utils.run_algo import run_algorithm
coins = ['dash', 'btc', 'dash', 'etc', 'eth', 'ltc', 'nxt', 'rep', 'str',
'xmr', 'xrp', 'zec']
symbols = None
def initialize(context):
pass
def _handle_data(context, data):
global symbols
if symbols is None: symbols = [symbol(c + '_usdt') for c in coins]
print('getting history for: %s' % [s.symbol for s in symbols])
history = data.history(symbols,
['close', 'volume'],
bar_count=1, # EXCEPTION, Change to 2
frequency='5T')
# print 'history: %s' % history.shape
run_algorithm(initialize=initialize,
handle_data=_handle_data,
analyze=lambda _, results: True,
exchange_name='poloniex',
base_currency='usdt',
algo_namespace='issue-236',
live=True,
data_frequency='minute',
capital_base=3000,
simulate_orders=True)
catalyst/utils/run_algo.py
+9 -18
View File
@@ -10,7 +10,6 @@ import click
import pandas as pd
from six import string_types
import catalyst
from catalyst.data.bundles import load
from catalyst.data.data_portal import DataPortal
from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
@@ -24,7 +23,7 @@ try:
from pygments.formatters import TerminalFormatter
PYGMENTS = True
except ImportError:
except:
PYGMENTS = False
from toolz import valfilter, concatv
from functools import partial
@@ -56,7 +55,6 @@ class _RunAlgoError(click.ClickException, ValueError):
----------
pyfunc_msg : str
The message that will be shown when called as a python function.
cmdline_msg : str
The message that will be shown on the command line.
"""
@@ -152,7 +150,6 @@ def _run(handle_data,
'We encourage you to report any issue on GitHub: '
'https://github.com/enigmampc/catalyst/issues'
)
log.info('Catalyst version {}'.format(catalyst.__version__))
sleep(3)
if live:
@@ -263,15 +260,6 @@ def _run(handle_data,
# We still need to support bundles for other misc data, but we
# can handle this later.
if start != pd.tslib.normalize_date(start) or \
end != pd.tslib.normalize_date(end):
# todo: add to Sim_Params the option to start & end at specific times
log.warn(
"Catalyst currently starts and ends on the start and "
"end of the dates specified, respectively. We hope to "
"Modify this and support specific times in a future release."
)
data = DataPortalExchangeBacktest(
exchange_names=[exchange_name for exchange_name in exchanges],
asset_finder=None,
@@ -428,8 +416,7 @@ def run_algorithm(initialize,
auth_aliases=None,
stats_output=None,
output=os.devnull):
"""
Run a trading algorithm.
"""Run a trading algorithm.
Parameters
----------
@@ -471,7 +458,7 @@ def run_algorithm(initialize,
This argument is mutually exclusive with ``data``.
default_extension : bool, optional
Should the default catalyst extension be loaded. This is found at
``$CATALYST_ROOT/extension.py``
``$ZIPLINE_ROOT/extension.py``
extensions : iterable[str], optional
The names of any other extensions to load. Each element may either be
a dotted module path like ``a.b.c`` or a path to a python file ending
@@ -482,8 +469,12 @@ def run_algorithm(initialize,
environ : mapping[str -> str], optional
The os environment to use. Many extensions use this to get parameters.
This defaults to ``os.environ``.
live : bool, optional
Execute algorithm in live trading mode.
live: execute live trading
exchange_conn: The exchange connection parameters
Supported Exchanges
-------------------
bitfinex
Returns
-------
docs/source/appendix.rst
+179 -173
View File
@@ -4,7 +4,7 @@ API Reference
Running a Backtest
~~~~~~~~~~~~~~~~~~
.. autofunction:: catalyst.run_algorithm(...)
.. autofunction:: zipline.run_algorithm(...)
Algorithm API
~~~~~~~~~~~~~
@@ -18,335 +18,341 @@ currently-executing :class:`~zipline.algorithm.TradingAlgorithm` instance.
Data Object
```````````
.. autoclass:: catalyst.protocol.BarData
.. autoclass:: zipline.protocol.BarData
:members:
Scheduling Functions
````````````````````
.. autofunction:: catalyst.api.schedule_function
.. autofunction:: zipline.api.schedule_function
.. autoclass:: catalyst.api.date_rules
.. autoclass:: zipline.api.date_rules
:members:
:undoc-members:
.. autoclass:: catalyst.api.time_rules
.. autoclass:: zipline.api.time_rules
:members:
Orders
``````
.. autofunction:: catalyst.api.order
.. autofunction:: zipline.api.order
.. autofunction:: catalyst.api.order_value
.. autofunction:: zipline.api.order_value
.. autofunction:: catalyst.api.order_percent
.. autofunction:: zipline.api.order_percent
.. autofunction:: catalyst.api.order_target
.. autofunction:: zipline.api.order_target
.. autofunction:: catalyst.api.order_target_value
.. autofunction:: zipline.api.order_target_value
.. autofunction:: catalyst.api.order_target_percent
.. autofunction:: zipline.api.order_target_percent
.. autoclass:: catalyst.finance.execution.ExecutionStyle
.. autoclass:: zipline.finance.execution.ExecutionStyle
:members:
.. autoclass:: catalyst.finance.execution.MarketOrder
.. autoclass:: zipline.finance.execution.MarketOrder
.. autoclass:: catalyst.finance.execution.LimitOrder
.. autoclass:: zipline.finance.execution.LimitOrder
.. autoclass:: catalyst.finance.execution.StopOrder
.. autoclass:: zipline.finance.execution.StopOrder
.. autoclass:: catalyst.finance.execution.StopLimitOrder
.. autoclass:: zipline.finance.execution.StopLimitOrder
.. autofunction:: catalyst.api.get_order
.. autofunction:: zipline.api.get_order
.. autofunction:: catalyst.api.get_open_orders
.. autofunction:: zipline.api.get_open_orders
.. autofunction:: catalyst.api.cancel_order
.. autofunction:: zipline.api.cancel_order
Order Cancellation Policies
'''''''''''''''''''''''''''
.. autofunction:: catalyst.api.set_cancel_policy
.. autofunction:: zipline.api.set_cancel_policy
.. autoclass:: catalyst.finance.cancel_policy.CancelPolicy
.. autoclass:: zipline.finance.cancel_policy.CancelPolicy
:members:
.. autofunction:: catalyst.api.EODCancel
.. autofunction:: zipline.api.EODCancel
.. autofunction:: catalyst.api.NeverCancel
.. autofunction:: zipline.api.NeverCancel
Assets
``````
.. autofunction:: catalyst.api.symbol
.. autofunction:: zipline.api.symbol
.. autofunction:: catalyst.api.symbols
.. autofunction:: zipline.api.symbols
.. autofunction:: catalyst.api.set_symbol_lookup_date
.. autofunction:: zipline.api.future_symbol
.. autofunction:: catalyst.api.sid
.. autofunction:: zipline.api.set_symbol_lookup_date
.. autofunction:: zipline.api.sid
Trading Controls
````````````````
zipline provides trading controls to help ensure that the algorithm is
Zipline provides trading controls to help ensure that the algorithm is
performing as expected. The functions help protect the algorithm from certian
bugs that could cause undesirable behavior when trading with real money.
.. autofunction:: catalyst.api.set_do_not_order_list
.. autofunction:: zipline.api.set_do_not_order_list
.. autofunction:: catalyst.api.set_long_only
.. autofunction:: zipline.api.set_long_only
.. autofunction:: catalyst.api.set_max_leverage
.. autofunction:: zipline.api.set_max_leverage
.. autofunction:: catalyst.api.set_max_order_count
.. autofunction:: zipline.api.set_max_order_count
.. autofunction:: catalyst.api.set_max_order_size
.. autofunction:: zipline.api.set_max_order_size
.. autofunction:: catalyst.api.set_max_position_size
.. autofunction:: zipline.api.set_max_position_size
Simulation Parameters
`````````````````````
.. autofunction:: catalyst.api.set_benchmark
.. autofunction:: zipline.api.set_benchmark
Commission Models
'''''''''''''''''
.. autofunction:: catalyst.api.set_commission
.. autofunction:: zipline.api.set_commission
.. autoclass:: catalyst.finance.commission.CommissionModel
.. autoclass:: zipline.finance.commission.CommissionModel
:members:
.. autoclass:: catalyst.finance.commission.PerShare
.. autoclass:: zipline.finance.commission.PerShare
.. autoclass:: catalyst.finance.commission.PerTrade
.. autoclass:: zipline.finance.commission.PerTrade
.. autoclass:: catalyst.finance.commission.PerDollar
.. autoclass:: zipline.finance.commission.PerDollar
Slippage Models
'''''''''''''''
.. autofunction:: catalyst.api.set_slippage
.. autofunction:: zipline.api.set_slippage
.. autoclass:: catalyst.finance.slippage.SlippageModel
.. autoclass:: zipline.finance.slippage.SlippageModel
:members:
.. autoclass:: catalyst.finance.slippage.FixedSlippage
.. autoclass:: zipline.finance.slippage.FixedSlippage
.. autoclass:: catalyst.finance.slippage.VolumeShareSlippage
.. autoclass:: zipline.finance.slippage.VolumeShareSlippage
Pipeline
````````
Not supported yet.
For more information, see :ref:`pipeline-api`
.. For more information, see :ref:`pipeline-api`
.. autofunction:: zipline.api.attach_pipeline
.. .. autofunction:: catalyst.api.attach_pipeline
.. .. autofunction:: catalyst.api.pipeline_output
.. autofunction:: zipline.api.pipeline_output
Miscellaneous
`````````````
.. autofunction:: catalyst.api.record
.. autofunction:: zipline.api.record
.. autofunction:: catalyst.api.get_environment
.. autofunction:: zipline.api.get_environment
.. autofunction:: catalyst.api.fetch_csv
.. autofunction:: zipline.api.fetch_csv
.. _pipeline-api:
.. Pipeline API
.. ~~~~~~~~~~~~
Pipeline API
~~~~~~~~~~~~
.. .. autoclass:: zipline.pipeline.Pipeline
.. :members:
.. :member-order: groupwise
.. autoclass:: zipline.pipeline.Pipeline
:members:
:member-order: groupwise
.. .. autoclass:: zipline.pipeline.CustomFactor
.. :members:
.. :member-order: groupwise
.. autoclass:: zipline.pipeline.CustomFactor
:members:
:member-order: groupwise
.. .. autoclass:: zipline.pipeline.filters.Filter
.. :members: __and__, __or__
.. :exclude-members: dtype
.. autoclass:: zipline.pipeline.filters.Filter
:members: __and__, __or__
:exclude-members: dtype
.. .. autoclass:: zipline.pipeline.factors.Factor
.. :members: bottom, deciles, demean, linear_regression, pearsonr,
.. percentile_between, quantiles, quartiles, quintiles, rank,
.. spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq,
.. \__add__, \__sub__, \__mul__, \__div__, \__mod__, \__pow__,
.. \__lt__, \__le__, \__ne__, \__ge__, \__gt__
.. :exclude-members: dtype
.. :member-order: bysource
.. autoclass:: zipline.pipeline.factors.Factor
:members: bottom, deciles, demean, linear_regression, pearsonr,
percentile_between, quantiles, quartiles, quintiles, rank,
spearmanr, top, winsorize, zscore, isnan, notnan, isfinite, eq,
__add__, __sub__, __mul__, __div__, __mod__, __pow__, __lt__,
__le__, __ne__, __ge__, __gt__
:exclude-members: dtype
:member-order: bysource
.. .. autoclass:: zipline.pipeline.term.Term
.. :members:
.. :exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed
.. autoclass:: zipline.pipeline.term.Term
:members:
:exclude-members: compute_extra_rows, dependencies, inputs, mask, windowed
.. .. autoclass:: zipline.pipeline.data.USEquityPricing
.. :members: open, high, low, close, volume
.. :undoc-members:
.. autoclass:: zipline.pipeline.data.USEquityPricing
:members: open, high, low, close, volume
:undoc-members:
.. Built-in Factors
.. ````````````````
Built-in Factors
````````````````
.. .. autoclass:: zipline.pipeline.factors.AverageDollarVolume
.. :members:
.. autoclass:: zipline.pipeline.factors.AverageDollarVolume
:members:
.. .. autoclass:: zipline.pipeline.factors.BollingerBands
.. :members:
.. autoclass:: zipline.pipeline.factors.BollingerBands
:members:
.. .. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent
.. :members:
.. autoclass:: zipline.pipeline.factors.BusinessDaysSincePreviousEvent
:members:
.. .. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent
.. :members:
.. autoclass:: zipline.pipeline.factors.BusinessDaysUntilNextEvent
:members:
.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage
.. :members:
.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingAverage
:members:
.. .. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev
.. :members:
.. autoclass:: zipline.pipeline.factors.ExponentialWeightedMovingStdDev
:members:
.. .. autoclass:: zipline.pipeline.factors.Latest
.. :members:
.. autoclass:: zipline.pipeline.factors.Latest
:members:
.. .. autoclass:: zipline.pipeline.factors.MaxDrawdown
.. :members:
.. autoclass:: zipline.pipeline.factors.MaxDrawdown
:members:
.. .. autoclass:: zipline.pipeline.factors.Returns
.. :members:
.. autoclass:: zipline.pipeline.factors.Returns
:members:
.. .. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns
.. :members:
.. autoclass:: zipline.pipeline.factors.RollingLinearRegressionOfReturns
:members:
.. .. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns
.. :members:
.. autoclass:: zipline.pipeline.factors.RollingPearsonOfReturns
:members:
.. .. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns
.. :members:
.. autoclass:: zipline.pipeline.factors.RollingSpearmanOfReturns
:members:
.. .. autoclass:: zipline.pipeline.factors.RSI
.. :members:
.. autoclass:: zipline.pipeline.factors.RSI
:members:
.. .. autoclass:: zipline.pipeline.factors.SimpleMovingAverage
.. :members:
.. autoclass:: zipline.pipeline.factors.SimpleMovingAverage
:members:
.. .. autoclass:: zipline.pipeline.factors.VWAP
.. :members:
.. autoclass:: zipline.pipeline.factors.VWAP
:members:
.. .. autoclass:: zipline.pipeline.factors.WeightedAverageValue
.. :members:
.. autoclass:: zipline.pipeline.factors.WeightedAverageValue
:members:
.. Pipeline Engine
.. ```````````````
Pipeline Engine
```````````````
.. .. autoclass:: zipline.pipeline.engine.PipelineEngine
.. :members: run_pipeline, run_chunked_pipeline
.. :member-order: bysource
.. autoclass:: zipline.pipeline.engine.PipelineEngine
:members: run_pipeline, run_chunked_pipeline
:member-order: bysource
.. .. autoclass:: zipline.pipeline.engine.SimplePipelineEngine
.. :members: __init__, run_pipeline, run_chunked_pipeline
.. :member-order: bysource
.. autoclass:: zipline.pipeline.engine.SimplePipelineEngine
:members: __init__, run_pipeline, run_chunked_pipeline
:member-order: bysource
.. .. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace
.. autofunction:: zipline.pipeline.engine.default_populate_initial_workspace
.. Data Loaders
.. ````````````
Data Loaders
````````````
.. .. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader
.. :members: __init__, from_files, load_adjusted_array
.. :member-order: bysource
.. autoclass:: zipline.pipeline.loaders.equity_pricing_loader.USEquityPricingLoader
:members: __init__, from_files, load_adjusted_array
:member-order: bysource
Asset Metadata
~~~~~~~~~~~~~~
.. autoclass:: catalyst.assets.Asset
.. autoclass:: zipline.assets.Asset
:members:
.. autoclass:: catalyst.assets.AssetConvertible
.. autoclass:: zipline.assets.Equity
:members:
.. autoclass:: zipline.assets.Future
:members:
.. autoclass:: zipline.assets.AssetConvertible
:members:
Trading Calendar API
~~~~~~~~~~~~~~~~~~~~
.. autofunction:: catalyst.utils.calendars.get_calendar
.. autofunction:: zipline.utils.calendars.get_calendar
.. autoclass:: catalyst.utils.calendars.TradingCalendar
.. autoclass:: zipline.utils.calendars.TradingCalendar
:members:
.. autofunction:: catalyst.utils.calendars.register_calendar
.. autofunction:: zipline.utils.calendars.register_calendar
.. autofunction:: catalyst.utils.calendars.register_calendar_type
.. autofunction:: zipline.utils.calendars.register_calendar_type
.. autofunction:: catalyst.utils.calendars.deregister_calendar
.. autofunction:: zipline.utils.calendars.deregister_calendar
.. autofunction:: catalyst.utils.calendars.clear_calendars
.. autofunction:: zipline.utils.calendars.clear_calendars
Data API
~~~~~~~~
.. Writers
.. ```````
.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter
.. :members:
Writers
```````
.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarWriter
:members:
.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter
.. :members:
.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarWriter
:members:
.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter
.. :members:
.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentWriter
:members:
.. .. autoclass:: zipline.assets.AssetDBWriter
.. :members:
.. autoclass:: zipline.assets.AssetDBWriter
:members:
.. Readers
.. ```````
.. .. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader
.. :members:
Readers
```````
.. autoclass:: zipline.data.minute_bars.BcolzMinuteBarReader
:members:
.. .. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader
.. :members:
.. autoclass:: zipline.data.us_equity_pricing.BcolzDailyBarReader
:members:
.. .. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader
.. :members:
.. autoclass:: zipline.data.us_equity_pricing.SQLiteAdjustmentReader
:members:
.. .. autoclass:: zipline.assets.AssetFinder
.. :members:
.. autoclass:: zipline.assets.AssetFinder
:members:
.. .. autoclass:: zipline.data.data_portal.DataPortal
.. :members:
.. autoclass:: zipline.data.data_portal.DataPortal
:members:
.. Bundles
.. ```````
.. .. autofunction:: zipline.data.bundles.register
Bundles
```````
.. autofunction:: zipline.data.bundles.register
.. .. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True)
.. autofunction:: zipline.data.bundles.ingest(name, environ=os.environ, date=None, show_progress=True)
.. .. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None)
.. autofunction:: zipline.data.bundles.load(name, environ=os.environ, date=None)
.. .. autofunction:: zipline.data.bundles.unregister
.. autofunction:: zipline.data.bundles.unregister
.. .. data:: zipline.data.bundles.bundles
.. data:: zipline.data.bundles.bundles
.. The bundles that have been registered as a mapping from bundle name to bundle
.. data. This mapping is immutable and should only be updated through
.. :func:`~zipline.data.bundles.register` or
.. :func:`~zipline.data.bundles.unregister`.
The bundles that have been registered as a mapping from bundle name to bundle
data. This mapping is immutable and should only be updated through
:func:`~zipline.data.bundles.register` or
:func:`~zipline.data.bundles.unregister`.
.. .. autofunction:: zipline.data.bundles.yahoo_equities
.. autofunction:: zipline.data.bundles.yahoo_equities
@@ -356,16 +362,16 @@ Utilities
Caching
```````
.. autoclass:: catalyst.utils.cache.CachedObject
.. autoclass:: zipline.utils.cache.CachedObject
.. autoclass:: catalyst.utils.cache.ExpiringCache
.. autoclass:: zipline.utils.cache.ExpiringCache
.. autoclass:: catalyst.utils.cache.dataframe_cache
.. autoclass:: zipline.utils.cache.dataframe_cache
.. autoclass:: catalyst.utils.cache.working_file
.. autoclass:: zipline.utils.cache.working_file
.. autoclass:: catalyst.utils.cache.working_dir
.. autoclass:: zipline.utils.cache.working_dir
Command Line
````````````
.. autofunction:: catalyst.utils.cli.maybe_show_progress
.. autofunction:: zipline.utils.cli.maybe_show_progress
docs/source/beginner-tutorial.rst
+161 -51
View File
@@ -168,7 +168,7 @@ We'll start with the CLI, and introduce the ``run_algorithm()`` in the last
example of this tutorial. Some of the :doc:`example algorithms <example-algos>`
provide instructions on how to run them both from the CLI, and using the
:func:`~catalyst.run_algorithm` function. For the third method, refer to the
corresponding section on :ref:`Catalyst & Jupyter Notebook <jupyter>` after you
corresponding section on :doc:`Catalyst & Jupyter Notebook <jupyter>` after you
have assimilated the contents of this tutorial.
Command line interface
@@ -473,7 +473,6 @@ Which we execute by running:
</div>
|
There is a row for each trading day, starting on the first day of our
simulation Jan 1st, 2016. In the columns you can find various
information about the state of your algorithm. The column
@@ -519,7 +518,7 @@ alongside enigma-catalyst (with the exception of the ``Conda`` install, where it
was included by default inside the conda environment we created). If for any
reason you don't have it installed, you can add it by running:
.. code-block:: bash
.. code-block:: python
(catalyst)$ pip install matplotlib
@@ -580,8 +579,162 @@ which you can skim through for now. A copy of this algorithm is available in
the ``examples`` directory:
`dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_.
.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
:language: python
.. code-block:: python
import numpy as np
import pandas as pd
from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order, record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.utils.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
def initialize(context):
context.i = 0
context.asset = symbol('ltc_usd')
context.base_price = None
def handle_data(context, data):
# define the windows for the moving averages
short_window = 50
long_window = 200
# Skip as many bars as long_window to properly compute the average
context.i += 1
if context.i < long_window:
return
# Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_mavg = data.history(context.asset, 'price',
bar_count=short_window, frequency="1m").mean()
long_mavg = data.history(context.asset, 'price',
bar_count=long_window, frequency="1m").mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
# Save values for later inspection
record(price=price,
cash=context.portfolio.cash,
price_change=price_change,
short_mavg=short_mavg,
long_mavg=long_mavg)
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.asset):
return
# We check what's our position on our portfolio and trade accordingly
pos_amount = context.portfolio.positions[context.asset].amount
# Trading logic
if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation
exchange = list(context.exchanges.values())[0]
base_currency = exchange.base_currency.upper()
# First chart: Plot portfolio value using base_currency
ax1 = plt.subplot(411)
perf.loc[:, ['portfolio_value']].plot(ax=ax1)
ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format(
asset = context.asset.symbol,
base = base_currency
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index, 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index, 'price'],
marker='v',
s=100,
c='red',
label=''
)
# Third chart: Compare percentage change between our portfolio
# and the price of the asset
ax3 = plt.subplot(413, sharex=ax1)
perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
ax3.legend_.remove()
ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
In order to run the code above, you have to ingest the needed data first:
@@ -653,7 +806,6 @@ the ``scikit-learn`` functions require ``numpy.ndarray``\ s rather than
``pandas.DataFrame``\ s, so you can simply pass the underlying
``ndarray`` of a ``DataFrame`` via ``.values``).
.. _jupyter:
Jupyter Notebook
~~~~~~~~~~~~~~~~
@@ -674,13 +826,13 @@ In order to use Jupyter Notebook, you first have to install it inside your
environment. It's available as ``pip`` package, so regardless of how you
installed Catalyst, go inside your catalyst environemnt and run:
.. code-block:: bash
.. code:: bash
(catalyst)$ pip install jupyter
Once you have Jupyter Notebook installed, every time you want to use it run:
.. code-block:: bash
.. code:: bash
(catalyst)$ jupyter notebook
@@ -694,7 +846,7 @@ Before running your algorithms inside the Jupyter Notebook, remember to ingest
the data from the command line interface (CLI). In the example below, you would
need to run first:
.. code-block:: bash
.. code:: bash
catalyst ingest-exchange -x bitfinex -i btc_usd
@@ -16455,49 +16607,7 @@ NaN
</div>
PyCharm IDE
~~~~~~~~~~~
PyCharm is an Integrated Development Environment (IDE) used in computer
programming, specifically for the Python language. It streamlines the continuos
development of Python code, and among other things includes a debugger that
comes in handy to see the inner workings of Catalyst, and your trading
algorithms.
Install
^^^^^^^
Install PyCharm from their `Website <https://www.jetbrains.com/pycharm/download/>`__.
There is a free and open-source **Community** version.
Setup
^^^^^
1. When creating a new project in PyCharm, right under you specify the Location,
click on **Project Interpreter** to display a drop down menu
2. Select **Existing interpreter**, click the gear box right next to it and
select 'add local'. Depending on your installation, select either
"*Virtual Environemnt*" or "*Conda Environment" and click the '...' button to
navigate to your catalyst env and select the Python binary file:
``bin/python`` for Linux/MacOS installations or 'python.exe' for Windows
installs (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe').
Select OK. You may want to click on *Make available to all projects* for your
future reference. Click OK again, and create your new environment using the
set up of your virtual environment.
Alternatively, if you already have your project created, in Windows do:
1. File -> Default Settings -> Project Interpreter. Click the gear box next to
the project interpreter and select ‘add local’, and follow the steps from the
second step above.
On MacOS:
1. PyCharm -> Preferences -> Settings -> Project:’NAME_OF_PROJECT’ ->
Project Interpreter. Click the gear box next to the project interpreter
and select ‘add local’, and follow the steps from the second step above.
You should now be able to run your project/scripts in PyCharm.
Next steps
~~~~~~~~~~
docs/source/conf.py
+2 -5
View File
@@ -27,8 +27,8 @@ extlinks = {
# -- Docstrings ---------------------------------------------------------------
extensions += ['numpydoc']
numpydoc_show_class_members = False
#extensions += ['numpydoc']
#numpydoc_show_class_members = False
# Add any paths that contain templates here, relative to this directory.
templates_path = ['.templates']
@@ -97,6 +97,3 @@ intersphinx_mapping = {
doctest_global_setup = "import catalyst"
todo_include_todos = True
suppress_warnings = ['image.nonlocal_uri']
docs/source/development-guidelines.rst
+17 -7
View File
@@ -36,15 +36,25 @@ Finally, you can build the C extensions by running:
$ python setup.py build_ext --inplace
Development with Docker
-----------------------
.. To finish, make sure `tests`__ pass.
If you want to work with zipline using a `Docker`__ container, you'll need to
build the ``Dockerfile`` in the Zipline root directory, and then build
``Dockerfile-dev``. Instructions for building both containers can be found in
``Dockerfile`` and ``Dockerfile-dev``, respectively.
.. __ #style-guide-running-tests
__ https://docs.docker.com/get-started/
.. If you get an error running nosetests after setting up a fresh virtualenv, please try running
.. code-block
.. # where zipline is the name of your virtualenv
.. $ deactivate zipline
.. $ workon zipline
.. Development with Docker
.. -----------------------
..If you want to work with zipline using a `Docker`__ container, you'll need to build the ``Dockerfile`` in the Zipline root directory, and then build ``Dockerfile-dev``. Instructions for building both containers can be found in ``Dockerfile`` and ``Dockerfile-dev``, respectively.
.. __ https://docs.docker.com/get-started/
Git Branching Structure
-----------------------
docs/source/example-algos.rst
+881 -18
View File
@@ -1,5 +1,4 @@
|
Example Algorithms
==================
@@ -52,8 +51,35 @@ Buy BTC Simple Algorithm
Source code: `examples/buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_
.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py
:language: python
.. code-block:: python
'''
Run this example, by executing the following from your terminal:
catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
If you want to run this code using another exchange, make sure that
the asset is available on that exchange. For example, if you were to run
it for exchange Poloniex, you would need to edit the following line:
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
and specify exchange poloniex as follows:
catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
To see which assets are available on each exchange, visit:
https://www.enigma.co/catalyst/status
'''
from catalyst.api import order, record, symbol
def initialize(context):
context.asset = symbol('btc_usd')
def handle_data(context, data):
order(context.asset, 1)
record(btc = data.current(context.asset, 'price'))
This simple algorithm does not produce any output nor displays any chart.
@@ -63,6 +89,8 @@ This simple algorithm does not produce any output nor displays any chart.
Buy and Hodl Algorithm
~~~~~~~~~~~~~~~~~~~~~~
Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
First ingest the historical pricing data needed to run this algorithm:
.. code-block:: bash
@@ -90,10 +118,157 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an
earlier date, you'll get an error), and the most recent date you can choose is
one day prior to the current date.
Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py
:language: python
.. code-block:: python
#!/usr/bin/env python
#
# Copyright 2017 Enigma MPC, Inc.
# Copyright 2015 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pandas as pd
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order_target_value, symbol, record,
cancel_order, get_open_orders, )
def initialize(context):
context.ASSET_NAME = 'btc_usd'
context.TARGET_HODL_RATIO = 0.8
context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
context.is_buying = True
context.asset = symbol(context.ASSET_NAME)
context.i = 0
def handle_data(context, data):
context.i += 1
starting_cash = context.portfolio.starting_cash
target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
reserve_value = context.RESERVE_RATIO * starting_cash
# Cancel any outstanding orders
orders = get_open_orders(context.asset) or []
for order in orders:
cancel_order(order)
# Stop buying after passing the reserve threshold
cash = context.portfolio.cash
if cash <= reserve_value:
context.is_buying = False
# Retrieve current asset price from pricing data
price = data.current(context.asset, 'price')
# Check if still buying and could (approximately) afford another purchase
if context.is_buying and cash > price:
print('buying')
# Place order to make position in asset equal to target_hodl_value
order_target_value(
context.asset,
target_hodl_value,
limit_price=price * 1.1,
)
record(
price=price,
volume=data.current(context.asset, 'volume'),
cash=cash,
starting_cash=context.portfolio.starting_cash,
leverage=context.account.leverage,
)
def analyze(context=None, results=None):
# Plot the portfolio and asset data.
ax1 = plt.subplot(611)
results[['portfolio_value']].plot(ax=ax1)
ax1.set_ylabel('Portfolio Value (USD)')
ax2 = plt.subplot(612, sharex=ax1)
ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
results[['price']].plot(ax=ax2)
trans = results.ix[[t != [] for t in results.transactions]]
buys = trans.ix[
[t[0]['amount'] > 0 for t in trans.transactions]
]
ax2.scatter(
buys.index.to_pydatetime(),
results.price[buys.index],
marker='^',
s=100,
c='g',
label=''
)
ax3 = plt.subplot(613, sharex=ax1)
results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
ax3.set_ylabel('Leverage ')
ax4 = plt.subplot(614, sharex=ax1)
results[['starting_cash', 'cash']].plot(ax=ax4)
ax4.set_ylabel('Cash (USD)')
results[[
'treasury',
'algorithm',
'benchmark',
]] = results[[
'treasury_period_return',
'algorithm_period_return',
'benchmark_period_return',
]]
ax5 = plt.subplot(615, sharex=ax1)
results[[
'treasury',
'algorithm',
'benchmark',
]].plot(ax=ax5)
ax5.set_ylabel('Percent Change')
ax6 = plt.subplot(616, sharex=ax1)
results[['volume']].plot(ax=ax6)
ax6.set_ylabel('Volume (mCoins/5min)')
plt.legend(loc=3)
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=10000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace='buy_and_hodl',
base_currency='usd',
start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True),
)
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png
@@ -102,13 +277,166 @@ Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/bl
Dual Moving Average Crossover
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
This strategy is covered in detail in the last part of
`this tutorial <beginner-tutorial.html#history>`_.
Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
.. code-block:: python
.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
:language: python
import numpy as np
import pandas as pd
from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order, record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
def initialize(context):
context.i = 0
context.asset = symbol('ltc_usd')
context.base_price = None
def handle_data(context, data):
# define the windows for the moving averages
short_window = 50
long_window = 200
# Skip as many bars as long_window to properly compute the average
context.i += 1
if context.i < long_window:
return
# Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_mavg = data.history(context.asset, 'price',
bar_count=short_window, frequency="1m").mean()
long_mavg = data.history(context.asset, 'price',
bar_count=long_window, frequency="1m").mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
# Save values for later inspection
record(price=price,
cash=context.portfolio.cash,
price_change=price_change,
short_mavg=short_mavg,
long_mavg=long_mavg)
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.asset):
return
# We check what's our position on our portfolio and trade accordingly
pos_amount = context.portfolio.positions[context.asset].amount
# Trading logic
if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation
base_currency = context.exchanges.values()[0].base_currency.upper()
# First chart: Plot portfolio value using base_currency
ax1 = plt.subplot(411)
perf.loc[:, ['portfolio_value']].plot(ax=ax1)
ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format(
asset = context.asset.symbol,
base = base_currency
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index, 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index, 'price'],
marker='v',
s=100,
c='red',
label=''
)
# Third chart: Compare percentage change between our portfolio
# and the price of the asset
ax3 = plt.subplot(413, sharex=ax1)
perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
ax3.legend_.remove()
ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
@@ -118,6 +446,8 @@ Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/cata
Mean Reversion Algorithm
~~~~~~~~~~~~~~~~~~~~~~~~
Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
This algorithm is based on a simple momentum strategy. When the cryptoasset goes
up quickly, we're going to buy; when it goes down quickly, we're going to sell.
Hopefully, we'll ride the waves.
@@ -138,10 +468,284 @@ lines 218-245, so in order to run the algorithm we just type:
python mean_reversion_simple.py
Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
.. code-block:: python
.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py
:language: python
import os
import tempfile
import time
import numpy as np
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in USD.
context.neo_eth = symbol('neo_usd')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 30
context.RSI_OVERBOUGHT = 80
context.CANDLE_SIZE = '15T'
context.start_time = time.time()
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = False
context.current_day = today
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.neo_eth variable. For this example, we're
# using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention
# for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
context.neo_eth,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(context.neo_eth, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
cash = context.portfolio.cash
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
price=price,
volume=current['volume'],
price_change=price_change,
rsi=rsi[-1],
cash=cash
)
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if context.traded_today:
return
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.neo_eth)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.neo_eth):
return
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.neo_eth].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
order_target_percent(
context.neo_eth, 1, limit_price=limit_price
)
context.traded_today = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
context.neo_eth, 0, limit_price=limit_price
)
context.traded_today = True
def analyze(context=None, perf=None):
end = time.time()
log.info('elapsed time: {}'.format(end - context.start_time))
import matplotlib.pyplot as plt
# The base currency of the algo exchange
base_currency = context.exchanges.values()[0].base_currency.upper()
# Plot the portfolio value over time.
ax1 = plt.subplot(611)
perf.loc[:, 'portfolio_value'].plot(ax=ax1)
ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
# Plot the price increase or decrease over time.
ax2 = plt.subplot(612, sharex=ax1)
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.neo_eth.symbol, base=base_currency
))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'price'],
marker='v',
s=100,
c='red',
label=''
)
ax4 = plt.subplot(613, sharex=ax1)
perf.loc[:, 'cash'].plot(
ax=ax4, label='Base Currency ({})'.format(base_currency)
)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
ax5 = plt.subplot(614, sharex=ax1)
perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
ax5.set_ylabel('Percent\nChange')
ax6 = plt.subplot(615, sharex=ax1)
perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
ax6.set_ylabel('RSI')
ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
if not transaction_df.empty:
ax6.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'rsi'],
marker='^',
s=100,
c='green',
label=''
)
ax6.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'rsi'],
marker='v',
s=100,
c='red',
label=''
)
plt.legend(loc=3)
start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end/5))
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
pass
if __name__ == '__main__':
# The execution mode: backtest or live
MODE = 'backtest'
if MODE == 'backtest':
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=10000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
)
log.info('saved perf stats: {}'.format(out))
elif MODE == 'live':
run_algorithm(
capital_base=0.5,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
live=True,
algo_namespace=NAMESPACE,
base_currency='usd',
live_graph=False
)
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png
@@ -158,6 +762,8 @@ strategy.
Simple Universe
~~~~~~~~~~~~~~~
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
This example aims to provide an easy way for users to learn how to
collect data from any given exchange and select a subset of the available
currency pairs for trading. You simply need to specify the exchange and
@@ -184,10 +790,142 @@ of the file:
catalyst ingest-exchange -x bitfinex -f minute
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
.. code-block:: bash
python simple_universe.py
Credits: This code was originally submitted by `Abner Ayala-Acevedo
<https://github.com/abnera>`_. Thank you!
.. code-block:: python
from datetime import timedelta
import numpy as np
import pandas as pd
from catalyst import run_algorithm
from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.api import (symbols, )
def initialize(context):
context.i = -1 # minute counter
context.exchange = context.exchanges.values()[0].name.lower()
context.base_currency = context.exchanges.values()[0].base_currency.lower()
def handle_data(context, data):
context.i += 1
lookback_days = 7 # 7 days
# current date & time in each iteration formatted into a string
now = data.current_dt
date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = now - timedelta(days=lookback_days)
# keep only the date as a string, discard the time
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute'
# update universe everyday at midnight
if not context.i % one_day_in_minutes:
context.universe = universe(context, lookback_date, date)
# get data every 30 minutes
minutes = 30
# get lookback_days of history data: that is 'lookback' number of bins
lookback = one_day_in_minutes / minutes * lookback_days
if not context.i % minutes and context.universe:
# we iterate for every pair in the current universe
for coin in context.coins:
pair = str(coin.symbol)
# Get 30 minute interval OHLCV data. This is the standard data
# required for candlestick or indicators/signals. Return Pandas
# DataFrames. 30T means 30-minute re-sampling of one minute data.
# Adjust it to your desired time interval as needed.
opened = fill(data.history(coin, 'open',
bar_count=lookback, frequency='30T')).values
high = fill(data.history(coin, 'high',
bar_count=lookback, frequency='30T')).values
low = fill(data.history(coin, 'low',
bar_count=lookback, frequency='30T')).values
close = fill(data.history(coin, 'price',
bar_count=lookback, frequency='30T')).values
volume = fill(data.history(coin, 'volume',
bar_count=lookback, frequency='30T')).values
# close[-1] is the last value in the set, which is the equivalent
# to current price (as in the most recent value)
# displays the minute price for each pair every 30 minutes
print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format(
now=now,
pair=pair,
o=opened[-1],
h=high[-1],
l=low[-1],
c=close[-1],
v=volume[-1],
))
# -------------------------------------------------------------
# --------------- Insert Your Strategy Here -------------------
# -------------------------------------------------------------
def analyze(context=None, results=None):
pass
# Get the universe for a given exchange and a given base_currency market
# Example: Poloniex BTC Market
def universe(context, lookback_date, current_date):
# get all the pairs for the given exchange
json_symbols = get_exchange_symbols(context.exchange)
# convert into a DataFrame for easier processing
df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1)
df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1)
# Filter all the pairs to get only the ones for a given base_currency
df = df[df['base_currency'] == context.base_currency]
# Filter all the pairs to ensure that pair existed in the current date range
df = df[df.start_date < lookback_date]
df = df[df.end_daily >= current_date]
context.coins = symbols(*df.symbol) # convert all the pairs to symbols
return df.symbol.tolist()
# Replace all NA, NAN or infinite values with its nearest value
def fill(series):
if isinstance(series, pd.Series):
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
elif isinstance(series, np.ndarray):
return pd.Series(series).replace(
[np.inf, -np.inf], np.nan
).ffill().bfill().values
else:
return series
if __name__ == '__main__':
start_date = pd.to_datetime('2017-11-10', utc=True)
end_date = pd.to_datetime('2017-11-13', utc=True)
performance = run_algorithm(start=start_date, end=end_date,
capital_base=100.0, # amount of base_currency
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
data_frequency='minute',
base_currency='btc',
live=False,
live_graph=False,
algo_namespace='simple_universe')
.. literalinclude:: ../../catalyst/examples/simple_universe.py
:language: python
.. _portfolio_optimization:
@@ -201,10 +939,135 @@ use 180 days of historical data and rebalance every 30 days. This code was used
in writting the following article:
`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/portfolio_optimization.py>`_
.. code-block:: python
.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py
:language: python
'''
You can run this code using the Python interpreter:
$ python portfolio_optimization.py
'''
from __future__ import division
import os
import pytz
import numpy as np
import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import datetime
from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True)
def initialize(context):
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i%context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n+1, frequency='1d')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n+1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val/tminus_val-1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr),xr)/n
# Compute asset correlation matrix (informative only)
corr_m = cov_m/np.dot(np.transpose(stds),stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3+context.nassets,n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w,np.transpose(m)))*365
p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
#store results in results array
results_array[0,p] = p_r
results_array[1,p] = p_std
#store Sharpe Ratio (return / volatility) - risk free rate element
#excluded for simplicity
results_array[2,p] = results_array[0,p] / results_array[1,p]
i = 0
for iw in weights:
results_array[3+i,p] = weights[i]
i += 1
#convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r','stdev','sharpe']+context.assets)
#locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
#locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
#order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
#create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
#plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
#plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None):
# Form DataFrame with selected data
data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
.. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
:align: center
docs/source/index.rst
-2
View File
@@ -1,8 +1,6 @@
.. include:: ../../README.rst
|
|
Table of Contents
-----------------
docs/source/install.rst
+12 -42
View File
@@ -47,10 +47,8 @@ you can install MiniConda, which is a smaller footprint (fewer packages and
smaller size) than its big brother Anaconda, but it still contains all the
main packages needed. To install MiniConda, you can follow these steps:
1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select either
Python 3.6 (recommended) or Python 2.7 for your Operating System. The
`Enigma Data Marketplace <https://enigmampc.github.io/marketplace/>`_ will
require Python3, that's why we are recommending to opt for the newer version.
1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7
for your Operating System.
2. Install MiniConda. See the `Installation Instructions
<https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
3. Ensure the correct installation by running ``conda list`` in a Terminal
@@ -66,27 +64,18 @@ main packages needed. To install MiniConda, you can follow these steps:
Once either Conda or MiniConda has been set up you can install Catalyst:
1. Download the file `python3.6-environment.yml
<https://github.com/enigmampc/catalyst/blob/master/etc/python3.6-environment.yml>`_
(recommended) or `python2.7-environment.yml
<https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_
matching your Conda installation from step #1 above.
1. Download the file `python2.7-environment.yml
<https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
To download, simply click on the 'Raw' button and save the file locally
to a folder you can remember. Make sure that the file gets saved with the
``.yml`` extension, and nothing like a ``.txt`` file or anything else.
2. Open a Terminal window and enter [``cd/dir``] into the directory where you
saved the above ``.yml`` file.
saved the above ``python2.7-environment.yml`` file.
3. Install using this file. This step can take about 5-10 minutes to install.
.. code-block:: bash
conda env create -f python3.6-environment.yml
or
.. code-block:: bash
conda env create -f python2.7-environment.yml
@@ -133,14 +122,6 @@ with the following steps:
2. Create the environment:
for python 2.7:
.. code-block:: bash
conda create --name catalyst python=2.7 scipy zlib
or for python 3.6:
.. code-block:: bash
conda create --name catalyst python=2.7 scipy zlib
@@ -317,7 +298,7 @@ Troubleshooting ``pip`` Install
.. _pipenv:
Installing with ``pipenv``
--------------------------
-------------------------
Installing Catalyst via ``pipenv`` is perhaps easier that installing it via
``pip`` itself but you need to install ``pipenv`` first via ``pip``.
@@ -462,22 +443,12 @@ about matplotlib backends, please refer to the
Windows Requirements
--------------------
In Windows, you will first need to install the Microsoft Visual C++ Compiler,
which is different depending on the version of Python that you plan to use:
* Python 3.5, 3.6: `Visual C++ 2015 Build Tools
<http://landinghub.visualstudio.com/visual-cpp-build-tools>`_,
which installs Visual C++ version 14.0. **This is the recommended version**
* Python 2.7: `Microsoft Visual C++ Compiler for Python 2.7
<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_, which
installs version Visual C++ version 9.0
This package contains the compiler and the set of system headers necessary for
producing binary wheels for Python packages. If it's not already in your
system, download it and install it before proceeding to the next step. If you
need additional help, or are looking for other versions of Visual C++ for
Windows (only advanced users), follow `this link <https://wiki.python.org/moin/WindowsCompilers>`_.
In Windows, you will first need to install the `Microsoft Visual C++ Compiler
for Python 2.7
<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This
package contains the compiler and the set of system headers necessary for
producing binary wheels for Python 2.7 packages. If it's not already in your
system, download it and install it before proceeding to the next step.
Once you have the above compiler installed, the easiest and best supported way
to install Catalyst in Windows is to use :ref:`Conda <conda>`. If you didn't
@@ -505,7 +476,6 @@ mentioned above are as follows:
default you get 0 as the Value Data)
|
- **The installer has encountered an unexpected error installing this package.
This may indicate a problem with this package. The error code is 2503.**
docs/source/live-trading.rst
+11 -29
View File
@@ -30,24 +30,22 @@ Paper Trading vs Live Trading modes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Catalyst currently supports three different modes in which you can execute your
trading algorithm. The first is **backtesting**, which is covered extensively in
the tutorial, and uses historical data to run your algorithm. There is no
trading algorithm. The first is backtesting, which is covered extensively in the
tutorial, and uses historical data to run your algorithm. There is no
interaction with the exchange in backtesting mode, and this is the first mode
that you should test any new algorithm.
Once you are confident with the simulations that you have obtained with your
algorithm in backtesting, you may switch to live trading, where you have two
different modes:
* **Paper Trading**: The simulated algorithm runs in real time, and fetches
pricing data in real time from the exchange, but the orders never reach the
exchange, and are instead kept within Catalyst and simulated. No real currency
is bought or sold. Think of it as a `backtesting happening in real time`.
* **Live Trading**: This is the proper live trading mode in which an algorithm
runs in real time, fetching pricing data from live exchanges and placing
orders against the exchange. Real currency is transacted on the exchange
driven by the algorithm.
* *Paper Trading*: The simulated algorithm runs in real time, and fetches
pricing data in real time from the exchange, but the orders never reach the
exchange, and are instead kept within Catalyst and simulated. No real currency
is bought or sold. Think of it as a `backtesting happening in real time`.
* *Live Trading*: This is the proper live trading mode in which an algorithm
runs in real time, fetching pricing data from live exchanges and placing orders
against the exchange. Real currency is transacted on the exchange driven by the
algorithm.
These three modes are controlled by the following variables:
@@ -115,7 +113,7 @@ Currency symbols (e.g. btc, eth, ltc) follow the Bittrex convention.
Here are some examples:
.. code:: python
.. code-block:: json
# With Bitfinex
bitcoin_usd_asset = symbol('btc_usd')
@@ -176,22 +174,6 @@ Here is the breakdown of the new arguments:
- ``simulate_orders``: Enables the paper trading mode, in which orders are
simulated in Catalyst instead of processed on the exchange. It defaults to
``True``.
- ``end_date``: When setting the end_date to a time in the **future**,
it will schedule the live algo to finish gracefully at the specified date.
- ``start_date``: (**Will be implemented in the future**)
The live algo starts by default in the present, as mentioned above.
by setting the start_date to a time in the future, the algorithm would
essentially sleep and when the predefined time comes, it would start executing.
The `catalyst live` command offers additional parameters.
You can learn more by running the following from the command line:
.. code-block:: bash
catalyst live --help
Here is a complete algorithm for reference:
`Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_
docs/source/releases.rst
-32
View File
@@ -2,38 +2,6 @@
Release Notes
=============
Version 0.5.3
^^^^^^^^^^^^^
**Release Date**: 2018-02-09
Bug Fixes
~~~~~~~~~
- Fixed an issue with last candle in backtesting :issue:`219`
Version 0.5.2
^^^^^^^^^^^^^
**Release Date**: 2018-02-08
Bug Fixes
~~~~~~~~~
- Fixed an issue with live candle values :issue:`216` and :issue:`199`
Version 0.5.1
^^^^^^^^^^^^^
**Release Date**: 2018-02-07
Bug Fixes
~~~~~~~~~
- Fixed an issue with orders that stay open :issue:`211`
- Fixed Jupyter issues :issue:`179`
- Fetching multiple tickers in one call to minimize rate limit risks :issue:`174`
- Improved live state presentation :issue:`171`
Build
~~~~~
- Introducing the Enigma Marketplace
Version 0.4.7
^^^^^^^^^^^^^
**Release Date**: 2018-01-19
docs/source/videos.rst
+1 -6
View File
@@ -11,7 +11,6 @@ Installation: MacOS
|
|
Installation: Windows
---------------------
@@ -22,7 +21,6 @@ Where things go smoothly:
<iframe width="560" height="315" src="https://www.youtube.com/embed/H8HqcEbZmkk" frameborder="0" allowfullscreen></iframe>
|
Where things don't:
.. raw:: html
@@ -31,7 +29,6 @@ Where things don't:
|
|
Backtesting a Strategy
----------------------
@@ -47,7 +44,6 @@ sell. Hopefully, we’ll ride the waves.
|
|
Live Trading a Strategy
-----------------------
@@ -58,6 +54,5 @@ in the previous video, we now take it to trade live against the Bittrex exchange
.. raw:: html
<iframe width="560" height="315" src="https://www.youtube.com/embed/NupiE-Xuglw" frameborder="0" allowfullscreen></iframe>
|
|
|
etc/docker_cmd.sh
+1 -1
View File
@@ -16,4 +16,4 @@ fi
jupyter notebook -y --no-browser --notebook-dir=${PROJECT_DIR} \
--certfile=${SSL_CERT_PEM} --keyfile=${SSL_CERT_KEY} --ip='*' \
--config=${CONFIG_PATH} --allow-root
--config=${CONFIG_PATH}
etc/python2.7-environment.yml
+2 -8
View File
@@ -1,11 +1,9 @@
name: catalyst
channels:
- defaults
- conda-forge
dependencies:
- certifi=2016.2.28=py27_0
- mkl=2017.0.3
- matplotlib=2.1.2=py36_0
- numpy=1.13.1=py27_0
- openssl=1.0.2l
- pip=9.0.1=py27_1
@@ -22,11 +20,7 @@ dependencies:
- bcolz==0.12.1
- bottleneck==1.2.1
- chardet==3.0.4
- ccxt==1.11.22
# The Enigma Data Marketplace requires Python3 because it depends on
# web3, which requires Python3, as building its dependencies breaks in Python2
# - web3==4.0.0b7
- requests-toolbelt==0.8.0
- ccxt==1.10.774
- click==6.7
- contextlib2==0.5.5
- cycler==0.10.0
@@ -63,4 +57,4 @@ dependencies:
- tables==3.4.2
- toolz==0.8.2
- urllib3==1.22
- enigma-catalyst>=0.5
- enigma-catalyst>=0.3
etc/python3.6-environment.yml
-90
View File
@@ -1,90 +0,0 @@
name: catalyst
channels:
- defaults
- conda-forge
dependencies:
- ca-certificates=2017.08.26
- certifi=2018.1.18
- intel-openmp=2018.0.0
- mkl=2018.0.1
- numpy=1.14.0
- openssl=1.0.2n
- matplotlib=2.1.2=py36_0
- pip=9.0.1
- python=3.6.4
- scipy=1.0.0
- setuptools=38.4.0=py36_0
- sqlite=3.22.0
- tk=8.6.7
- wheel=0.30.0
- xz=5.2.3
- zlib=1.2.11
- pip:
- aiodns==1.1.1
- aiohttp==3.0.1
- alembic==0.9.7
- async-timeout==2.0.0
- attrdict==2.0.0
- attrs==17.4.0
- bcolz==0.12.1
- boto3==1.5.27
- botocore==1.8.41
- bottleneck==1.2.1
- cchardet==2.1.1
- ccxt==1.11.22
- chardet==3.0.4
- click==6.7
- contextlib2==0.5.5
- cyordereddict==1.0.0
- cython==0.27.3
- cytoolz==0.9.0
- decorator==4.2.1
- docutils==0.14
- empyrical==0.2.1
- enigma-catalyst>=0.5.3
- eth-abi==1.0.0b0
- eth-account==0.1.0a2
- eth-keyfile==0.5.1
- eth-keys==0.2.0b1
- eth-rlp==0.1.0a2
- eth-utils==1.0.0b1
- hexbytes==0.1.0b0
- idna==2.6
- idna-ssl==1.0.0
- intervaltree==2.1.0
- jmespath==0.9.3
- logbook==1.2.1
- lru-dict==1.1.6
- lxml==4.1.1
- mako==1.0.7
- markupsafe==1.0
- multidict==4.1.0
- multipledispatch==0.4.9
- networkx==2.1
- numexpr==2.6.4
- pandas==0.19.2
- pandas-datareader==0.6.0
- patsy==0.5.0
- pycares==2.3.0
- pycryptodome==3.4.11
- pysha3==1.0.2
- python-dateutil==2.6.1
- python-editor==1.0.3
- pytz==2018.3
- redo==1.6
- requests==2.18.4
- requests-file==1.4.3
- requests-ftp==0.3.1
- requests-toolbelt==0.8.0
- rlp==0.6.0
- s3transfer==0.1.12
- six==1.11.0
- sortedcontainers==1.5.9
- sqlalchemy==1.2.2
- statsmodels==0.8.0
- tables==3.4.2
- toolz==0.9.0
- urllib3==1.22
- web3==4.0.0b9
- wrapt==1.10.11
- yarl==1.1.0
etc/requirements.txt
+1 -3
View File
@@ -81,8 +81,6 @@ empyrical==0.2.1
tables==3.3.0
#Catalyst dependencies
ccxt==1.11.22
ccxt==1.10.774
boto3==1.4.8
redo==1.6
web3==4.0.0b11; python_version > '3.4'
requests-toolbelt==0.8.0
etc/requirements_dev.txt
+1 -1
View File
@@ -16,7 +16,7 @@ babel==1.3
docutils==0.12
snowballstemmer==1.2.0
sphinx-rtd-theme==0.1.8
sphinx==1.6.7
sphinx==1.3.4
pbr==1.10.0
mock==2.0.0
etc/requirements_docs.txt
+1 -1
View File
@@ -1,4 +1,4 @@
Sphinx==1.6.7
Sphinx>=1.3.2
numpydoc>=0.5.0
sphinx-autobuild==0.6.0
docutils==0.12
tests/exchange/test_bundle.py
+17 -35
View File
@@ -11,11 +11,10 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle, \
BUNDLE_NAME_TEMPLATE
from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
get_df_from_arrays
from catalyst.exchange.utils.datetime_utils import get_start_dt
from exchange.utils.datetime_utils import get_start_dt
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import get_exchange
from catalyst.exchange.utils.stats_utils import df_to_string, \
set_print_settings
from catalyst.exchange.utils.stats_utils import df_to_string
from catalyst.utils.paths import ensure_directory
log = getLogger('test_exchange_bundle')
@@ -43,16 +42,16 @@ class TestExchangeBundle:
def test_ingest_minute(self):
data_frequency = 'minute'
exchange_name = 'binance'
exchange_name = 'poloniex'
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
assets = [
exchange.get_asset('bch_eth')
exchange.get_asset('eth_btc')
]
start = pd.to_datetime('2018-03-01', utc=True)
end = pd.to_datetime('2018-03-8', utc=True)
start = pd.to_datetime('2016-03-01', utc=True)
end = pd.to_datetime('2017-11-1', utc=True)
log.info('ingesting exchange bundle {}'.format(exchange_name))
exchange_bundle.ingest(
@@ -62,8 +61,7 @@ class TestExchangeBundle:
exclude_symbols=None,
start=start,
end=end,
show_progress=False,
show_breakdown=False
show_progress=True
)
reader = exchange_bundle.get_reader(data_frequency)
@@ -74,15 +72,9 @@ class TestExchangeBundle:
start_dt=start,
end_dt=end
)
periods = exchange_bundle.get_calendar_periods_range(
start, end, data_frequency
print('found {} rows for {} ingestion\n{}'.format(
len(arrays[0]), asset.symbol, arrays[0])
)
dx = get_df_from_arrays(arrays[0], periods)
set_print_settings()
print('found {} rows for last ingestion:\n{}\n{}'.format(
len(dx), dx.head(10), dx.tail(10)
))
pass
def test_ingest_minute_all(self):
@@ -109,7 +101,7 @@ class TestExchangeBundle:
# data_frequency = 'daily'
# include_symbols = 'neo_btc,bch_btc,eth_btc'
exchange_name = 'binance'
exchange_name = 'bitfinex'
data_frequency = 'minute'
exchange = get_exchange(exchange_name)
@@ -230,14 +222,9 @@ class TestExchangeBundle:
start_dt=start,
end_dt=end
)
periods = exchange_bundle.get_calendar_periods_range(
start, end, data_frequency
print('found {} rows for {} ingestion\n{}'.format(
len(arrays[0]), asset.symbol, arrays[0])
)
dx = get_df_from_arrays(arrays, periods)
print('found {} rows for last ingestion'.format(
len(dx)
))
pass
def test_daily_data_to_minute_table(self):
@@ -303,22 +290,17 @@ class TestExchangeBundle:
for asset in assets:
sid = asset.sid
arrays = reader.load_raw_arrays(
daily_values = reader.load_raw_arrays(
fields=['open', 'high', 'low', 'close', 'volume'],
start_dt=start,
end_dt=end,
sids=[sid],
)
periods = exchange_bundle.get_calendar_periods_range(
start, end, data_frequency
)
dx = get_df_from_arrays(arrays, periods)
print('found {} rows for last ingestion'.format(
len(dx)
))
pass
len(daily_values[0]))
)
pass
def test_minute_bundle(self):
# exchange_name = 'poloniex'
tests/exchange/test_ccxt.py
+13 -51
View File
@@ -1,12 +1,12 @@
import pandas as pd
from logbook import Logger
from catalyst.exchange.utils.stats_utils import set_print_settings
from catalyst.testing import ZiplineTestCase
from catalyst.testing.fixtures import WithLogger
from .base import BaseExchangeTestCase
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
get_trades_df, candles_from_trades
from catalyst.exchange.utils.exchange_utils import get_exchange_auth
from catalyst.finance.order import Order
log = Logger('test_ccxt')
@@ -15,24 +15,23 @@ log = Logger('test_ccxt')
class TestCCXT(BaseExchangeTestCase):
@classmethod
def setup(self):
exchange_name = 'binance'
exchange_name = 'bitfinex'
auth = get_exchange_auth(exchange_name)
self.exchange = CCXT(
exchange_name=exchange_name,
key=auth['key'],
secret=auth['secret'],
password=None,
base_currency='usdt',
base_currency='bnb',
)
self.exchange.init()
def test_order(self):
log.info('creating order')
asset = self.exchange.get_asset('eth_usdt')
asset = self.exchange.get_asset('neo_bnb')
order_id = self.exchange.order(
asset=asset,
style=ExchangeLimitOrder(limit_price=1000),
amount=1.01,
style=ExchangeLimitOrder(limit_price=10),
amount=1,
)
log.info('order created {}'.format(order_id))
assert order_id is not None
@@ -59,30 +58,24 @@ class TestCCXT(BaseExchangeTestCase):
def test_get_candles(self):
log.info('retrieving candles')
candles = self.exchange.get_candles(
freq='1T',
assets=[self.exchange.get_asset('eng_eth')],
freq='30T',
assets=[self.exchange.get_asset('eth_btc')],
bar_count=200,
start_dt=pd.to_datetime('2017-09-01', utc=True),
start_dt=pd.to_datetime('2017-09-01', utc=True)
)
for asset in candles:
df = pd.DataFrame(candles[asset])
df.set_index('last_traded', drop=True, inplace=True)
set_print_settings()
print('got {} candles'.format(len(df)))
print(df.head(10))
print(df.tail(10))
pass
def test_tickers(self):
log.info('retrieving tickers')
assets = [
self.exchange.get_asset('ada_eth'),
self.exchange.get_asset('zrx_eth'),
self.exchange.get_asset('iot_usd'),
]
tickers = self.exchange.tickers(assets)
assert len(tickers) == 2
assert len(tickers) == 1
pass
def test_my_trades(self):
@@ -92,37 +85,6 @@ class TestCCXT(BaseExchangeTestCase):
assert trades
pass
def test_validate_volume(self):
asset = self.exchange.get_asset('eng_eth')
candles = self.exchange.get_candles(
freq='1T',
assets=[asset],
bar_count=10,
)
df = pd.DataFrame(candles[asset])
df.set_index('last_traded', drop=True, inplace=True)
df.drop_duplicates()
df.sort_index(inplace=True, ascending=False)
assert candles
start_dt = df.index[-1]
trades = self.exchange.get_trades(
asset, start_dt=start_dt, my_trades=False
)
assert trades
trades_df = get_trades_df(trades)
df2 = candles_from_trades(trades_df, '1T')
set_print_settings()
log.info(
'comparing candles / resampled trades:\n{}\n{}'.format(
df, df2
)
)
pass
def test_get_executed_order(self):
log.info('retrieving executed order')
asset = self.exchange.get_asset('eng_eth')
tests/exchange/test_config.py
-8
View File
@@ -1,8 +0,0 @@
from catalyst.exchange.utils.factory import get_exchange
class TestConfig:
def test_create_config(self):
exchange = get_exchange('binance', skip_init=True)
config = exchange.create_exchange_config()
pass
tests/exchange/test_data_portal.py
+1 -1
View File
@@ -9,7 +9,7 @@ from catalyst.exchange.exchange_data_portal import (
)
from catalyst.exchange.utils.exchange_utils import get_common_assets
from catalyst.exchange.utils.factory import get_exchanges
from .test_utils import rnd_history_date_days, rnd_bar_count
from test_utils import rnd_history_date_days, rnd_bar_count
log = Logger('test_bitfinex')
tests/exchange/test_exchange_utils.py
-175
View File
@@ -1,175 +0,0 @@
from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, \
forward_fill_df_if_needed, get_candles_df
from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
from datetime import timedelta
from pandas import Timestamp, DataFrame, concat
import numpy as np
class TestExchangeUtils(WithLogger, ZiplineTestCase):
@classmethod
def get_specific_field_from_df(cls, df, field, asset):
new_df = DataFrame(df[field])
new_df.columns = [asset]
new_df.index.name = None
return new_df
@classmethod
def verify_forward_fill_df_if_needed(cls, candles, periods, expected_df):
observed_df = forward_fill_df_if_needed(
transform_candles_to_df(candles),
periods)
assert (expected_df.equals(observed_df))
@classmethod
def verify_get_candles_df(cls, assets, candles, end_fixed_dt,
expected_df, check_next_candle=False):
# run on all the fields
for field in ['volume', 'open', 'close', 'high', 'low']:
field_dt = cls.get_specific_field_from_df(expected_df,
field,
assets[0])
# run on several timestamps
for delta in range(5):
end_dt = end_fixed_dt + timedelta(minutes=delta)
assert (field_dt.equals(get_candles_df({assets[0]: candles},
field, '5T', 3,
end_dt=end_dt)))
field_dt_a1 = cls.get_specific_field_from_df(expected_df,
field,
assets[0])
field_dt_a2 = cls.get_specific_field_from_df(expected_df,
field,
assets[1])
observed_df = get_candles_df({assets[0]: candles,
assets[1]: candles},
field, '5T', 3,
end_dt=end_dt)
assert (observed_df.equals(concat([field_dt_a1, field_dt_a2],
axis=1)))
if check_next_candle:
# one candle forward
end_dt = end_fixed_dt + timedelta(minutes=6)
observed_df = get_candles_df({assets[0]: candles,
assets[1]: candles},
field, '5T', 3,
end_dt=end_dt)
assert (not observed_df.equals(concat([field_dt_a1,
field_dt_a2],
axis=1)))
assert (concat([field_dt_a1, field_dt_a2],
axis=1)[1:].equals(observed_df[:-1]))
def test_get_candles_df(self):
assets = ['btc_usdt', 'eth_usdt']
# test forward fill in the end
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
}]
expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 593.0, 'volume': 0.0, 'low': 593.0,
'close': 593.0, 'open': 593.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
periods = [Timestamp('2018-03-01 09:45:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:50:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:55:00+0000', tz='UTC')]
expected_df = transform_candles_to_df(expected)
self.verify_forward_fill_df_if_needed(candles, periods,
expected_df)
self.verify_get_candles_df(assets, candles, periods[2],
expected_df, True)
# test forward fill in the middle
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 595.0, 'volume': 0.0, 'low': 595.0,
'close': 595.0, 'open': 595.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected_df = transform_candles_to_df(expected)
self.verify_forward_fill_df_if_needed(candles, periods, expected_df)
self.verify_get_candles_df(assets, candles, periods[2], expected_df)
# test "forward fill" at the beginning
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected = [{'high': np.NaN, 'volume': 0.0, 'low': np.NaN,
'close': np.NaN, 'open': np.NaN,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected_df = transform_candles_to_df(expected)
self.verify_forward_fill_df_if_needed(candles, periods, expected_df)
# Not the same due to dropna - commenting out for now
# self.verify_get_candles_df(assets, candles, periods[2], expected_df)
tests/exchange/test_suites/test_suite_bundle.py
+17 -148
View File
@@ -2,7 +2,6 @@ import random
import os
import pandas as pd
from datetime import timedelta
from logbook import TestHandler
from pandas.util.testing import assert_frame_equal
@@ -13,7 +12,6 @@ from catalyst.exchange.utils.exchange_utils import get_candles_df
from catalyst.exchange.utils.factory import get_exchange
from catalyst.exchange.utils.test_utils import output_df, \
select_random_assets
from catalyst.exchange.utils.stats_utils import set_print_settings
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 8)
@@ -37,7 +35,7 @@ class TestSuiteBundle:
return data_portal
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
freq, data_frequency, data_portal, field):
freq, data_frequency, data_portal):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
@@ -60,26 +58,14 @@ class TestSuiteBundle:
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing {} for {}/{} with {} timeframe until {}'.format(
field, exchange.name, symbols, freq, end_dt
)
)
data['bundle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field=field,
field='close',
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle data:\n{}'.format(
data['bundle']
)
)
candles = exchange.get_candles(
end_dt=end_dt,
freq=freq,
@@ -88,16 +74,11 @@ class TestSuiteBundle:
)
data['exchange'] = get_candles_df(
candles=candles,
field=field,
field='close',
freq=freq,
bar_count=bar_count,
end_dt=end_dt,
)
print(
'the exchange data:\n{}'.format(
data['exchange']
)
)
for source in data:
df = data[source]
path, folder = output_df(
@@ -107,146 +88,29 @@ class TestSuiteBundle:
print('saved {} test results: {}'.format(end_dt, folder))
assert_frame_equal(
right=data['bundle'][:-1],
left=data['exchange'][:-1],
right=data['bundle'],
left=data['exchange'],
check_less_precise=1,
)
try:
assert_frame_equal(
right=data['bundle'][:-1],
left=data['exchange'][:-1],
right=data['bundle'],
left=data['exchange'],
check_less_precise=min([a.decimals for a in assets]),
)
except Exception as e:
print(
'Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e)
)
print('Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e))
with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
handle.write(e.args[0])
pass
def compare_current_with_last_candle(self, exchange, assets, end_dt,
freq, data_frequency, data_portal):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
freq
data_frequency
data_portal
Returns
-------
"""
data = dict()
assets = sorted(assets, key=lambda a: a.symbol)
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing data for {}/{} with {} timeframe on {}'.format(
exchange.name, symbols, freq, end_dt
)
)
data['candle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=1,
frequency=freq,
field='close',
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle first / last row:\n{}'.format(
data['candle'].iloc[[-1]]
)
)
current = data_portal.get_spot_value(
assets=assets,
field='close',
dt=end_dt,
data_frequency=data_frequency,
)
data['current'] = pd.Series(data=current, index=assets)
print(
'the current price:\n{}'.format(
data['current']
)
)
pass
def test_validate_bundles(self):
# exchange_population = 3
asset_population = 3
data_frequency = random.choice(['minute'])
# bundle = 'dailyBundle' if data_frequency
# == 'daily' else 'minuteBundle'
# exchanges = select_random_exchanges(
# population=exchange_population,
# features=[bundle],
# ) # Type: list[Exchange]
# TODO: currently focusing on Binance, try other exchanges
exchanges = [get_exchange('poloniex', skip_init=True)]
data_portal = TestSuiteBundle.get_data_portal(exchanges)
for exchange in exchanges:
exchange.init()
frequencies = exchange.get_candle_frequencies(data_frequency)
# freq = random.sample(frequencies, 1)[0]
freq = '5T'
rnd = random.SystemRandom()
# field = rnd.choice(['open', 'high', 'low', 'close', 'volume'])
field = rnd.choice(['close'])
# bar_count = random.randint(3, 6)
bar_count = 5
# assets = select_random_assets(
# exchange.assets, asset_population
# )
assets = [exchange.get_asset('bch_eth')]
end_dt = pd.to_datetime('2018-03-01', utc=True)
for asset in assets:
attribute = 'end_{}'.format(data_frequency)
asset_end_dt = getattr(asset, attribute)
if end_dt is None or asset_end_dt < end_dt:
end_dt = asset_end_dt
end_dt = end_dt + timedelta(minutes=3)
dt_range = pd.date_range(
end=end_dt, periods=bar_count, freq=freq
)
self.compare_bundle_with_exchange(
exchange=exchange,
assets=assets,
end_dt=dt_range[-1],
bar_count=bar_count,
freq=freq,
data_frequency=data_frequency,
data_portal=data_portal,
field=field,
)
pass
def test_validate_last_candle(self):
# exchange_population = 3
asset_population = 3
data_frequency = random.choice(['minute'])
# bundle = 'dailyBundle' if data_frequency
# == 'daily' else 'minuteBundle'
# exchanges = select_random_exchanges(
@@ -262,6 +126,8 @@ class TestSuiteBundle:
frequencies = exchange.get_candle_frequencies(data_frequency)
freq = random.sample(frequencies, 1)[0]
bar_count = random.randint(1, 10)
assets = select_random_assets(
exchange.assets, asset_population
)
@@ -273,11 +139,14 @@ class TestSuiteBundle:
if end_dt is None or asset_end_dt < end_dt:
end_dt = asset_end_dt
end_dt = end_dt + timedelta(minutes=3)
self.compare_current_with_last_candle(
dt_range = pd.date_range(
end=end_dt, periods=bar_count, freq=freq
)
self.compare_bundle_with_exchange(
exchange=exchange,
assets=assets,
end_dt=end_dt,
end_dt=dt_range[-1],
bar_count=bar_count,
freq=freq,
data_frequency=data_frequency,
data_portal=data_portal,
tests/exchange/test_suites/test_suite_exchange.py
+51 -23
View File
@@ -5,28 +5,63 @@ from logging import Logger, WARNING
from time import sleep
import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import TestHandler
from catalyst.assets._assets import TradingPair
from catalyst.exchange.exchange_errors import ExchangeRequestError
from catalyst.exchange.exchange_execution import ExchangeLimitOrder
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import get_exchanges, get_exchange
from catalyst.exchange.utils.test_utils import select_random_exchanges, \
select_random_assets
handle_exchange_error, select_random_assets
from catalyst.testing import ZiplineTestCase
from catalyst.testing.fixtures import WithLogger
from exchange.utils.factory import get_exchanges
log = Logger('TestSuiteExchange')
class TestSuiteExchange(WithLogger, ZiplineTestCase):
def _test_markets_exchange(self, exchange, attempts=0):
assets = None
try:
exchange.init()
# Verify that the assets and markets are populated
if not exchange.markets:
raise ValueError(
'no markets found'
)
if not exchange.assets:
raise ValueError(
'no assets derived from markets'
)
assets = exchange.assets
except ExchangeRequestError as e:
sleep(5)
if attempts > 5:
handle_exchange_error(exchange, e)
else:
print(
're-trying an exchange request {} {}'.format(
exchange.name, attempts
)
)
self._test_markets_exchange(exchange, attempts + 1)
except Exception as e:
handle_exchange_error(exchange, e)
return assets
def test_markets(self):
population = 3
results = dict()
exchanges = select_random_exchanges(population) # Type: list[Exchange]
for exchange in exchanges:
exchange.init()
assets = self._test_markets_exchange(exchange)
if assets is not None:
@@ -55,7 +90,7 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
# exchange_population,
# features=['fetchTickers'],
# ) # Type: list[Exchange]
exchanges = list(get_exchanges(['binance']).values())
exchanges = list(get_exchanges(['bitfinex']).values())
for exchange in exchanges:
exchange.init()
@@ -78,11 +113,10 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
exchange_population = 3
asset_population = 3
# exchanges = select_random_exchanges(
# population=exchange_population,
# features=['fetchOHLCV'],
# ) # Type: list[Exchange]
exchanges = list(get_exchanges(['binance']).values())
exchanges = select_random_exchanges(
population=exchange_population,
features=['fetchOHLCV'],
) # Type: list[Exchange]
for exchange in exchanges:
exchange.init()
@@ -104,6 +138,7 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
assets=assets,
bar_count=bar_count,
start_dt=dt_range[0],
end_dt=dt_range[-1],
)
assert len(candles) == asset_population
@@ -120,20 +155,13 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
quote_currency = 'eth'
order_amount = 0.1
# exchanges = select_random_exchanges(
# population=population,
# features=['fetchOrder'],
# is_authenticated=True,
# base_currency=quote_currency,
# ) # Type: list[Exchange]
exchanges = select_random_exchanges(
population=population,
features=['fetchOrder'],
is_authenticated=True,
base_currency=quote_currency,
) # Type: list[Exchange]
exchanges = [
get_exchange(
'binance',
base_currency=quote_currency,
must_authenticate=True,
)
]
log_catcher = TestHandler()
with log_catcher:
for exchange in exchanges:
tests/marketplace/__init__.py
View File
tests/marketplace/test_marketplace.py
-35
View File
@@ -1,35 +0,0 @@
from catalyst.marketplace.marketplace import Marketplace
from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
class TestMarketplace(WithLogger, ZiplineTestCase):
def test_list(self):
marketplace = Marketplace()
marketplace.list()
pass
def test_register(self):
marketplace = Marketplace()
marketplace.register()
pass
def test_subscribe(self):
marketplace = Marketplace()
marketplace.subscribe('marketcap')
pass
def test_ingest(self):
marketplace = Marketplace()
ds_def = marketplace.ingest('marketcap')
pass
def test_publish(self):
marketplace = Marketplace()
datadir = '/Users/fredfortier/Downloads/marketcap_test_single'
marketplace.publish('marketcap1234', datadir, False)
pass
def test_clean(self):
marketplace = Marketplace()
marketplace.clean('marketcap')
pass