wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-06-27 17:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Frederic Fortier
2018-02-08 00:33:55 -05:00
parent 881e3e5953 e248831719
commit c0b9939580
33 changed files with 2190 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.rst
+1 -1
View File
@@ -1,4 +1,4 @@
.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png
:target: https://enigmampc.github.io/catalyst
:align: center
:alt: Enigma | Catalyst
catalyst/__main__.py
+129 -2
View File
@@ -6,6 +6,7 @@ import click
import sys
import logbook
import pandas as pd
from catalyst.marketplace.marketplace import Marketplace
from six import text_type
from catalyst.data import bundles as bundles_module
@@ -579,7 +580,8 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout)
click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
sys.stdout)
exchange_bundle.ingest(
data_frequency=data_frequency,
include_symbols=include_symbols,
@@ -633,7 +635,8 @@ def clean_exchange(ctx, exchange_name, data_frequency):
exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout)
click.echo('Cleaning exchange bundle {}...'.format(exchange_name),
sys.stdout)
exchange_bundle.clean(
data_frequency=data_frequency,
)
@@ -761,5 +764,129 @@ def bundles():
click.echo("%s %s" % (bundle, timestamp), sys.stdout)
@main.group()
@click.pass_context
def marketplace(ctx):
pass
@marketplace.command()
@click.pass_context
def ls(ctx):
click.echo('Listing of available data sources on the marketplace:',
sys.stdout)
marketplace = Marketplace()
marketplace.list()
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.pass_context
def subscribe(ctx, dataset):
if dataset is None:
ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
marketplace = Marketplace()
marketplace.subscribe(dataset)
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.option(
'-f',
'--data-frequency',
type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}),
default='daily',
show_default=True,
help='The data frequency of the desired OHLCV bars.',
)
@click.option(
'-s',
'--start',
default=None,
type=Date(tz='utc', as_timestamp=True),
help='The start date of the data range. (default: one year from end date)',
)
@click.option(
'-e',
'--end',
default=None,
type=Date(tz='utc', as_timestamp=True),
help='The end date of the data range. (default: today)',
)
@click.pass_context
def ingest(ctx, dataset, data_frequency, start, end):
if dataset is None:
ctx.fail("must specify a dataset to clean with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
marketplace = Marketplace()
marketplace.ingest(dataset, data_frequency, start, end)
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.pass_context
def clean(ctx, dataset):
if dataset is None:
ctx.fail("must specify a dataset to ingest with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
marketplace = Marketplace()
marketplace.clean(dataset)
click.echo('Done', sys.stdout)
@marketplace.command()
@click.pass_context
def register(ctx):
marketplace = Marketplace()
marketplace.register()
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the Marketplace dataset to publish data for.',
)
@click.option(
'--datadir',
default=None,
help='The folder that contains the CSV data files to publish.',
)
@click.option(
'--watch/--no-watch',
is_flag=True,
default=False,
help='Whether to watch the datadir for live data.',
)
@click.pass_context
def publish(ctx, dataset, datadir, watch):
marketplace = Marketplace()
if dataset is None:
ctx.fail("must specify a dataset to publish data for "
" with '--dataset'\n")
if datadir is None:
ctx.fail("must specify a datadir where to find the files to publish "
" with '--datadir'\n")
marketplace.publish(dataset, datadir, watch)
if __name__ == '__main__':
main()
catalyst/api.pyi
+65 -8
View File
@@ -34,6 +34,7 @@ def attach_pipeline(pipeline, name, chunks=None):
:func:`catalyst.api.pipeline_output`
"""
def batch_market_order(share_counts):
"""Place a batch market order for multiple assets.
@@ -48,6 +49,7 @@ def batch_market_order(share_counts):
Index of ids for newly-created orders.
"""
def cancel_order(order_param):
"""Cancel an open order.
@@ -57,7 +59,9 @@ def cancel_order(order_param):
The order_id or order object to cancel.
"""
def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'):
def continuous_future(root_symbol_str, offset=0, roll='volume',
adjustment='mul'):
"""Create a specifier for a continuous contract.
Parameters
@@ -81,7 +85,10 @@ def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'
The continuous future specifier.
"""
def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs):
def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
date_format=None, timezone='UTC', symbol=None, mask=True,
symbol_column=None, special_params_checker=None, **kwargs):
"""Fetch a csv from a remote url and register the data so that it is
queryable from the ``data`` object.
@@ -125,6 +132,7 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_forma
A requests source that will pull data from the url specified.
"""
def future_symbol(symbol):
"""Lookup a futures contract with a given symbol.
@@ -144,6 +152,7 @@ def future_symbol(symbol):
Raised when no contract named 'symbol' is found.
"""
def get_datetime(tz=None):
"""
Returns the current simulation datetime.
@@ -159,6 +168,7 @@ dt : datetime
The current simulation datetime converted to ``tz``.
"""
def get_environment(field='platform'):
"""Query the execution environment.
@@ -198,6 +208,7 @@ def get_environment(field='platform'):
Raised when ``field`` is not a valid option.
"""
def get_order(order_id):
"""Lookup an order based on the order id returned from one of the
order functions.
@@ -213,10 +224,12 @@ def get_order(order_id):
The order object.
"""
def history(bar_count, frequency, field, ffill=True):
"""DEPRECATED: use ``data.history`` instead.
"""
def order(asset, amount, limit_price=None, stop_price=None, style=None):
"""Place an order.
@@ -258,7 +271,9 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_percent`
"""
def order_percent(asset, percent, limit_price=None, stop_price=None, style=None):
def order_percent(asset, percent, limit_price=None, stop_price=None,
style=None):
"""Place an order in the specified asset corresponding to the given
percent of the current portfolio value.
@@ -293,6 +308,7 @@ def order_percent(asset, percent, limit_price=None, stop_price=None, style=None)
:func:`catalyst.api.order_value`
"""
def order_target(asset, target, limit_price=None, stop_price=None, style=None):
"""Place an order to adjust a position to a target number of shares. If
the position doesn't already exist, this is equivalent to placing a new
@@ -344,7 +360,9 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_target_value`
"""
def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None):
def order_target_percent(asset, target, limit_price=None, stop_price=None,
style=None):
"""Place an order to adjust a position to a target percent of the
current portfolio value. If the position doesn't already exist, this is
equivalent to placing a new order. If the position does exist, this is
@@ -396,7 +414,9 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None, style
:func:`catalyst.api.order_target_value`
"""
def order_target_value(asset, target, limit_price=None, stop_price=None, style=None):
def order_target_value(asset, target, limit_price=None, stop_price=None,
style=None):
"""Place an order to adjust a position to a target value. If
the position doesn't already exist, this is equivalent to placing a new
order. If the position does exist, this is equivalent to placing an
@@ -448,6 +468,7 @@ def order_target_value(asset, target, limit_price=None, stop_price=None, style=N
:func:`catalyst.api.order_target_percent`
"""
def order_value(asset, value, limit_price=None, stop_price=None, style=None):
"""Place an order by desired value rather than desired number of
shares.
@@ -488,6 +509,7 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_percent`
"""
def pipeline_output(name):
"""Get the results of the pipeline that was attached with the name:
``name``.
@@ -514,6 +536,7 @@ def pipeline_output(name):
:meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline`
"""
def record(*args, **kwargs):
"""Track and record values each day.
@@ -529,7 +552,9 @@ def record(*args, **kwargs):
:func:`~catalyst.run_algorithm`.
"""
def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None):
def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
calendar=None):
"""Schedules a function to be called according to some timed rules.
Parameters
@@ -549,6 +574,7 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True, cale
:class:`catalyst.api.time_rules`
"""
def set_asset_restrictions(restrictions, on_error='fail'):
"""Set a restriction on which assets can be ordered.
@@ -562,6 +588,7 @@ def set_asset_restrictions(restrictions, on_error='fail'):
catalyst.finance.asset_restrictions.Restrictions
"""
def set_benchmark(benchmark):
"""Set the benchmark asset.
@@ -576,6 +603,7 @@ def set_benchmark(benchmark):
automatically reinvested.
"""
def set_cancel_policy(cancel_policy):
"""Sets the order cancellation policy for the simulation.
@@ -590,6 +618,7 @@ def set_cancel_policy(cancel_policy):
:class:`catalyst.api.NeverCancel`
"""
def set_commission(commission):
"""Sets the commission model for the simulation.
@@ -605,6 +634,7 @@ def set_commission(commission):
:class:`catalyst.finance.commission.PerDollar`
"""
def set_do_not_order_list(restricted_list, on_error='fail'):
"""Set a restriction on which assets can be ordered.
@@ -614,11 +644,13 @@ def set_do_not_order_list(restricted_list, on_error='fail'):
The assets that cannot be ordered.
"""
def set_long_only(on_error='fail'):
"""Set a rule specifying that this algorithm cannot take short
positions.
"""
def set_max_leverage(max_leverage):
"""Set a limit on the maximum leverage of the algorithm.
@@ -629,6 +661,7 @@ def set_max_leverage(max_leverage):
be no maximum.
"""
def set_max_order_count(max_count, on_error='fail'):
"""Set a limit on the number of orders that can be placed in a single
day.
@@ -639,7 +672,9 @@ def set_max_order_count(max_count, on_error='fail'):
The maximum number of orders that can be placed on any single day.
"""
def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
def set_max_order_size(asset=None, max_shares=None, max_notional=None,
on_error='fail'):
"""Set a limit on the number of shares and/or dollar value of any single
order placed for sid. Limits are treated as absolute values and are
enforced at the time that the algo attempts to place an order for sid.
@@ -658,7 +693,9 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error=
The maximum value that can be ordered at one time.
"""
def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
def set_max_position_size(asset=None, max_shares=None, max_notional=None,
on_error='fail'):
"""Set a limit on the number of shares and/or dollar value held for the
given sid. Limits are treated as absolute values and are enforced at
the time that the algo attempts to place an order for sid. This means
@@ -681,6 +718,7 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_err
The maximum value to hold for an asset.
"""
def set_slippage(slippage):
"""Set the slippage model for the simulation.
@@ -694,6 +732,7 @@ def set_slippage(slippage):
:class:`catalyst.finance.slippage.SlippageModel`
"""
def set_symbol_lookup_date(dt):
"""Set the date for which symbols will be resolved to their assets
(symbols may map to different firms or underlying assets at
@@ -705,6 +744,7 @@ def set_symbol_lookup_date(dt):
The new symbol lookup date.
"""
def sid(sid):
"""Lookup an Asset by its unique asset identifier.
@@ -724,6 +764,7 @@ def sid(sid):
When a requested ``sid`` does not map to any asset.
"""
def symbol(symbol_str):
"""Lookup an Equity by its ticker symbol.
@@ -748,6 +789,7 @@ def symbol(symbol_str):
:func:`catalyst.api.set_symbol_lookup_date`
"""
def symbols(*args):
"""Lookup multuple Equities as a list.
@@ -773,3 +815,18 @@ def symbols(*args):
:func:`catalyst.api.set_symbol_lookup_date`
"""
def get_dataset(ds_name, start=None, end=None):
"""
Lookup a data source from the marketplace
Parameters
----------
ds_name: str
start: pd.Timestamp
end: pd.Timestamp
Returns
-------
"""
catalyst/constants.py
+27
View File
@@ -15,5 +15,32 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
DATE_FORMAT = '%Y-%m-%d'
try:
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
except Exception as e:
print('unable to get catalyst path: {}'.format(e))
AUTO_INGEST = False
AUTH_SERVER = 'https://data.enigma.co'
# TODO: switch to mainnet
ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
# TODO: move to MASTER branch on github
MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/develop/catalyst/marketplace/' \
'contract_marketplace_address.txt'
MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/develop/catalyst/marketplace/' \
'contract_marketplace_abi.json'
# TODO: switch to mainnet
ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
'develop/catalyst/marketplace/' \
'contract_enigma_address.txt'
ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/develop/catalyst/marketplace/' \
'contract_enigma_abi.json'
catalyst/examples/mean_reversion_by_marketcap.py
+237
View File
@@ -0,0 +1,237 @@
# For this example, we're going to write a simple momentum script. When the
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
# going to sell. Hopefully we'll ride the waves.
import os
import tempfile
import time
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_dataset
from catalyst.exchange.utils.stats_utils import set_print_settings, \
get_pretty_stats
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
df = get_dataset('testmarketcap2') # type: pd.DataFrame
# Picking a specific date in our DataFrame
first_dt = df.index.get_level_values(0)[0]
# Since we use a MultiIndex with date / symbol, picking a date will
# result in a new DataFrame for the selected date with a single
# symbol index
df = df.xs(first_dt, level=0)
# Keep only the top coins by market cap
df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))]
set_print_settings()
df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True)
print('the marketplace data:\n{}'.format(df))
# Pick the 5 assets with the lowest market cap for trading
quote_currency = 'eth'
exchange = context.exchanges[next(iter(context.exchanges))]
symbols = [a.symbol for a in exchange.assets
if a.start_date < context.datetime]
context.assets = []
for currency, price in df['market_cap_usd'].iteritems():
if len(context.assets) >= 5:
break
s = '{}_{}'.format(currency.decode('utf-8'), quote_currency)
if s in symbols:
context.assets.append(symbol(s))
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = dict()
context.current_day = today
# Preparing dictionaries for asset-level data points
volumes = dict()
rsis = dict()
price_values = dict()
cash = context.portfolio.cash
for asset in context.assets:
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.assets variable. For this example,
# we're using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this
# convention for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
asset,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare
# to the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(asset, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
# if asset not in context.base_price:
# context.base_price[asset] = price
#
# base_price = context.base_price[asset]
# price_change = (price - base_price) / base_price
# Tracking the relevant data
volumes[asset] = current['volume']
rsis[asset] = rsi[-1]
price_values[asset] = price
# price_changes[asset] = price_change
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if asset in context.traded_today:
continue
# Exit if we cannot trade
if not data.can_trade(asset):
continue
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we
# calculate how long or short our position is at this minute.
pos_amount = context.portfolio.positions[asset].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
target = 1.0 / len(context.assets)
order_target_percent(
asset, target, limit_price=limit_price
)
context.traded_today[asset] = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
asset, 0, limit_price=limit_price
)
context.traded_today[asset] = True
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
current_price=price_values,
volume=volumes,
rsi=rsis,
cash=cash,
)
def analyze(context=None, perf=None):
stats = get_pretty_stats(perf)
print('the algo stats:\n{}'.format(stats))
pass
if __name__ == '__main__':
# The execution mode: backtest or live
live = False
if live:
run_algorithm(
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
live=True,
algo_namespace=NAMESPACE,
base_currency='btc',
live_graph=False,
simulate_orders=False,
stats_output=None,
)
else:
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=100,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace=NAMESPACE,
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-10-15', utc=True),
)
log.info('saved perf stats: {}'.format(out))
catalyst/examples/mean_reversion_simple.py
+6 -6
View File
@@ -33,11 +33,11 @@ def initialize(context):
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
context.market = symbol('eth_btc')
context.market = symbol('bnb_eth')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERSOLD = 40
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '15T'
@@ -248,14 +248,14 @@ if __name__ == '__main__':
if live:
run_algorithm(
capital_base=0.01,
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
exchange_name='binance',
live=True,
algo_namespace=NAMESPACE,
base_currency='btc',
base_currency='eth',
live_graph=False,
simulate_orders=False,
stats_output=None,
@@ -274,7 +274,7 @@ if __name__ == '__main__':
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=0.1,
capital_base=0.035,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
catalyst/exchange/ccxt/ccxt_exchange.py
+92 -46
View File
@@ -760,24 +760,23 @@ class CCXT(Exchange):
side = 'buy' if amount > 0 else 'sell'
if hasattr(self.api, 'amount_to_lots'):
adj_amount = self.api.amount_to_lots(
symbol=symbol,
amount=abs(amount),
)
if adj_amount != abs(amount):
log.info(
'adjusted order amount {} to {} based on lot size'.format(
abs(amount), adj_amount,
# TODO: is this right?
if self.api.markets is None:
self.api.load_markets()
# https://github.com/ccxt/ccxt/issues/1483
adj_amount = round(abs(amount), asset.decimals)
market = self.api.markets[symbol]
if 'lots' in market and market['lots'] > amount:
raise CreateOrderError(
exchange=self.name,
e='order amount lower than the smallest lot: {}'.format(
amount
)
)
else:
adj_amount = abs(amount)
if adj_amount == 0:
raise CreateOrderError(
exchange=self.name,
e='order amount lower than the smallest lot: {}'.format(amount)
)
else:
adj_amount = round(abs(amount), asset.decimals)
try:
result = self.api.create_order(
@@ -799,6 +798,22 @@ class CCXT(Exchange):
)
raise ExchangeRequestError(error=e)
exchange_amount = None
if 'amount' in result and result['amount'] != adj_amount:
exchange_amount = result['amount']
elif 'info' in result:
if 'origQty' in result['info']:
exchange_amount = float(result['info']['origQty'])
if exchange_amount:
log.info(
'order amount adjusted by {} from {} to {}'.format(
self.name, adj_amount, exchange_amount
)
)
adj_amount = exchange_amount
if 'info' not in result:
raise ValueError('cannot use order without info attribute')
@@ -859,31 +874,38 @@ class CCXT(Exchange):
order.id, order.asset, return_price=True
)
order.status = exc_order.status
order.commission = exc_order.commission
if order.amount != exc_order.amount:
log.warn(
'executed order amount {} differs '
'from original'.format(
exc_order.amount, order.amount
)
)
order.amount = exc_order.amount
order.filled = exc_order.amount
if order.status == ORDER_STATUS.FILLED:
transactions = []
if exc_order.status == ORDER_STATUS.FILLED:
if order.amount > exc_order.amount:
log.warn(
'executed order amount {} differs '
'from original'.format(
exc_order.amount, order.amount
)
)
order.check_triggers(
price=price,
dt=exc_order.dt,
)
transaction = Transaction(
asset=order.asset,
amount=order.amount,
dt=pd.Timestamp.utcnow(),
price=price,
order_id=order.id,
commission=order.commission
commission=order.commission,
)
return [transaction]
transactions.append(transaction)
return transactions
def process_order(self, order):
# TODO: move to parent class after tracking features in the parent
if not self.api.hasFetchMyTrades:
if not self.api.has['fetchMyTrades']:
return self._process_order_fallback(order)
try:
@@ -985,7 +1007,7 @@ class CCXT(Exchange):
)
raise ExchangeRequestError(error=e)
def tickers(self, assets):
def tickers(self, assets, on_ticker_error='raise'):
"""
Retrieve current tick data for the given assets
@@ -998,27 +1020,51 @@ class CCXT(Exchange):
list[dict[str, float]
"""
tickers = {}
for asset in assets:
symbol = self.get_symbol(asset)
# Test the CCXT throttling further to see if we need this
self.ask_request()
# TODO: use fetch_tickers() for efficiency
# I tried using fetch_tickers() but noticed some
# inconsistencies, see issue:
# https://github.com/ccxt/ccxt/issues/870
if len(assets) == 1:
try:
ticker = self.api.fetch_ticker(symbol=symbol)
except (ExchangeError, NetworkError) as e:
symbol = self.get_symbol(assets[0])
log.debug('fetching single ticker: {}'.format(symbol))
results = dict()
results[symbol] = self.api.fetch_ticker(symbol=symbol)
except (ExchangeError, NetworkError,) as e:
log.warn(
'unable to fetch ticker {} / {}: {}'.format(
self.name, asset.symbol, e
self.name, symbol, e
)
)
continue
raise ExchangeRequestError(error=e)
elif len(assets) > 1:
symbols = self.get_symbols(assets)
try:
log.debug('fetching multiple tickers: {}'.format(symbols))
results = self.api.fetch_tickers(symbols=symbols)
except (ExchangeError, NetworkError) as e:
log.warn(
'unable to fetch tickers {} / {}: {}'.format(
self.name, symbols, e
)
)
raise ExchangeRequestError(error=e)
else:
raise ValueError('Cannot request tickers with not assets.')
tickers = dict()
for asset in assets:
symbol = self.get_symbol(asset)
if symbol not in results:
msg = 'ticker not found {} / {}'.format(
self.name, symbol
)
log.warn(msg)
if on_ticker_error == 'warn':
continue
else:
raise ExchangeRequestError(error=msg)
ticker = results[symbol]
ticker['last_traded'] = from_ms_timestamp(ticker['timestamp'])
if 'last_price' not in ticker:
@@ -1030,7 +1076,7 @@ class CCXT(Exchange):
ticker['volume'] = ticker['baseVolume']
elif 'info' in ticker and 'bidQty' in ticker['info'] \
and 'askQty' in ticker['info']:
and 'askQty' in ticker['info']:
ticker['volume'] = float(ticker['info']['bidQty']) + \
float(ticker['info']['askQty'])
@@ -1068,7 +1114,7 @@ class CCXT(Exchange):
return result
def get_trades(self, asset, my_trades=True, start_dt=None, limit=None):
def get_trades(self, asset, my_trades=True, start_dt=None, limit=100):
if not my_trades:
raise NotImplemented(
'get_trades only supports "my trades"'
catalyst/exchange/exchange.py
+6 -3
View File
@@ -178,6 +178,7 @@ class Exchange:
if symbols is None:
# Make a distinct list of all symbols
symbols = list(set([asset.symbol for asset in self.assets]))
symbols.sort()
if quote_currency is not None:
for symbol in symbols[:]:
@@ -701,8 +702,8 @@ class Exchange:
)
positions_value = 0.0
if positions is not None:
assets = set([position.asset for position in positions])
if positions:
assets = list(set([position.asset for position in positions]))
tickers = self.tickers(assets)
for position in positions:
@@ -972,13 +973,15 @@ class Exchange:
pass
@abc.abstractmethod
def tickers(self, assets):
def tickers(self, assets, on_ticker_error='raise'):
"""
Retrieve current tick data for the given assets
Parameters
----------
assets: list[TradingPair]
on_ticker_error: str [raise|warn]
How to handle an error when retrieving a single ticker.
Returns
-------
catalyst/exchange/exchange_algorithm.py
+16 -3
View File
@@ -43,6 +43,7 @@ from catalyst.finance.execution import MarketOrder
from catalyst.finance.performance import PerformanceTracker
from catalyst.finance.performance.period import calc_period_stats
from catalyst.gens.tradesimulation import AlgorithmSimulator
from catalyst.marketplace.marketplace import Marketplace
from catalyst.utils.api_support import api_method
from catalyst.utils.input_validation import error_keywords, ensure_upper_case
from catalyst.utils.math_utils import round_nearest
@@ -67,7 +68,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
self.current_day = None
if self.simulate_orders is None \
and self.sim_params.arena == 'backtest':
and self.sim_params.arena == 'backtest':
self.simulate_orders = True
# Operations with retry features
@@ -92,6 +93,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
attempts=self.attempts,
)
self._marketplace = None
@staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style):
"""
@@ -115,7 +118,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
# be in-line with CXXT and many exchanges. We'll consider
# adding more order types in the future.
if not isinstance(style, ExchangeLimitOrder) or \
not isinstance(style, MarketOrder):
not isinstance(style, MarketOrder):
raise OrderTypeNotSupported(
order_type=style.__class__.__name__
)
@@ -167,6 +170,15 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
"""
return round_nearest(amount, asset.min_trade_size)
@api_method
def get_dataset(self, data_source_name, start=None, end=None):
if self._marketplace is None:
self._marketplace = Marketplace()
return self._marketplace.get_dataset(
data_source_name, start, end,
)
@api_method
@preprocess(symbol_str=ensure_upper_case)
def symbol(self, symbol_str, exchange_name=None):
@@ -901,7 +913,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('Fetching open orders again.'),
args=(asset,))
args=(asset,)
)
@api_method
def get_order(self, order_id, exchange_name):
catalyst/exchange/exchange_blotter.py
+1 -1
View File
@@ -214,7 +214,7 @@ class ExchangeBlotter(Blotter):
# that this is safer until we have a robust way to track
# the trades already processed by the algo. We can't loose
# them if the algo shuts down.
if transactions and order.open_amount == 0:
if transactions and order.status == ORDER_STATUS.FILLED:
avg_price = np.average(
a=[t.price for t in transactions],
weights=[t.amount for t in transactions],
catalyst/exchange/utils/stats_utils.py
+8 -10
View File
@@ -44,7 +44,7 @@ def crossover(source, target):
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
or target is np.nan:
return False
if source[-1] >= target > source[-2]:
@@ -54,7 +54,7 @@ def crossover(source, target):
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] > target[-1] and source[-2] < target[-2]:
@@ -81,7 +81,7 @@ def crossunder(source, target):
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
or target is np.nan:
return False
if source[-1] < target <= source[-2]:
@@ -90,7 +90,7 @@ def crossunder(source, target):
return False
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] < target[-1] and source[-2] >= target[-2]:
@@ -229,7 +229,10 @@ def prepare_stats(stats, recorded_cols=list()):
asset_values)
df = pd.DataFrame(stats)
df['orders'] = df['orders'].apply(lambda orders: len(orders))
df['transactions'] = df['transactions'].apply(
lambda transactions: len(transactions)
)
index_cols = [
'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
@@ -241,11 +244,6 @@ def prepare_stats(stats, recorded_cols=list()):
for column in recorded_cols:
index_cols.append(column)
df['orders'] = df['orders'].apply(lambda orders: len(orders))
df['transactions'] = df['transactions'].apply(
lambda transactions: len(transactions)
)
if asset_cols:
columns = asset_cols
df.set_index(index_cols, drop=True, inplace=True)
catalyst/finance/risk/period.py
+4 -2
View File
@@ -29,13 +29,15 @@ from .risk import check_entry
from empyrical import (
alpha_beta_aligned,
annual_volatility,
cum_returns,
downside_risk,
information_ratio,
max_drawdown,
sharpe_ratio,
sortino_ratio
)
from catalyst.patches.stats import (
max_drawdown,
cum_returns,
)
from catalyst.constants import LOG_LEVEL
catalyst/marketplace/__init__.py
View File
catalyst/marketplace/contract_enigma_abi.json
+302
View File
@@ -0,0 +1,302 @@
[
{
"constant": true,
"inputs": [],
"name": "name",
"outputs": [
{
"name": "",
"type": "string"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "approve",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "totalSupply",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_from",
"type": "address"
},
{
"name": "_to",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "transferFrom",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "INITIAL_SUPPLY",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "decimals",
"outputs": [
{
"name": "",
"type": "uint8"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_subtractedValue",
"type": "uint256"
}
],
"name": "decreaseApproval",
"outputs": [
{
"name": "success",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": false,
"inputs": [],
"name": "getAfterApproveTest",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [
{
"name": "_owner",
"type": "address"
}
],
"name": "balanceOf",
"outputs": [
{
"name": "balance",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "symbol",
"outputs": [
{
"name": "",
"type": "string"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_to",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "transfer",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_addedValue",
"type": "uint256"
}
],
"name": "increaseApproval",
"outputs": [
{
"name": "success",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [
{
"name": "_owner",
"type": "address"
},
{
"name": "_spender",
"type": "address"
}
],
"name": "allowance",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"name": "testValue",
"type": "address"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"name": "owner",
"type": "address"
},
{
"indexed": true,
"name": "spender",
"type": "address"
},
{
"indexed": false,
"name": "value",
"type": "uint256"
}
],
"name": "Approval",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"name": "from",
"type": "address"
},
{
"indexed": true,
"name": "to",
"type": "address"
},
{
"indexed": false,
"name": "value",
"type": "uint256"
}
],
"name": "Transfer",
"type": "event"
}
]
catalyst/marketplace/contract_enigma_address.txt
+1
View File
@@ -0,0 +1 @@
0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10
catalyst/marketplace/contract_marketplace_abi.json
+1
View File
File diff suppressed because one or more lines are too long
catalyst/marketplace/contract_marketplace_address.txt
+1
View File
@@ -0,0 +1 @@
0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4
catalyst/marketplace/marketplace.py
+705
View File
@@ -0,0 +1,705 @@
from __future__ import print_function
import glob
import json
import os
import re
import shutil
import sys
import time
import bcolz
import logbook
import pandas as pd
import requests
from requests_toolbelt import MultipartDecoder
from requests_toolbelt.multipart.decoder import \
NonMultipartContentTypeException
from catalyst.constants import (
LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
from catalyst.exchange.utils.stats_utils import set_print_settings
from catalyst.marketplace.marketplace_errors import (
MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles)
from catalyst.marketplace.utils.auth_utils import get_key_secret, \
get_signed_headers
from catalyst.marketplace.utils.bundle_utils import merge_bundles
from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
to_grains
from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
get_data_source_folder, get_marketplace_folder, \
get_user_pubaddr, get_temp_bundles_folder, extract_bundle
if sys.version_info.major < 3:
import urllib
else:
import urllib.request as urllib
log = logbook.Logger('Marketplace', level=LOG_LEVEL)
class Marketplace:
def __init__(self):
global Web3
from web3 import Web3, HTTPProvider
self.addresses = get_user_pubaddr()
if self.addresses[0]['pubAddr'] == '':
raise MarketplacePubAddressEmpty(
filename=os.path.join(
get_marketplace_folder(), 'addresses.json')
)
self.default_account = self.addresses[0]['pubAddr']
self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE))
contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
self.mkt_contract_address = Web3.toChecksumAddress(
contract_url.readline().strip())
abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
abi = json.load(abi_url)
self.mkt_contract = self.web3.eth.contract(
self.mkt_contract_address,
abi=abi,
)
contract_url = urllib.urlopen(ENIGMA_CONTRACT)
self.eng_contract_address = Web3.toChecksumAddress(
contract_url.readline().strip())
abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
abi = json.load(abi_url)
self.eng_contract = self.web3.eth.contract(
self.eng_contract_address,
abi=abi,
)
# def get_data_sources_map(self):
# return [
# dict(
# name='Marketcap',
# desc='The marketcap value in USD.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily'],
# ),
# dict(
# name='GitHub',
# desc='The rate of development activity on GitHub.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily', 'hour'],
# ),
# dict(
# name='Influencers',
# desc='Tweets & related sentiments by selected influencers.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily', 'hour', 'minute'],
# ),
# ]
def to_text(self, hex):
return Web3.toText(hex).rstrip('\0')
def choose_pubaddr(self):
if len(self.addresses) == 1:
address = self.addresses[0]['pubAddr']
address_i = 0
print('Using {} for this transaction.'.format(address))
else:
while True:
for i in range(0, len(self.addresses)):
print('{}\t{}\t{}'.format(
i,
self.addresses[i]['pubAddr'],
self.addresses[i]['desc'])
)
address_i = int(input('Choose your address associated with '
'this transaction: [default: 0] ') or 0)
if not (0 <= address_i < len(self.addresses)):
print('Please choose a number between 0 and {}\n'.format(
len(self.addresses) - 1))
else:
address = Web3.toChecksumAddress(
self.addresses[address_i]['pubAddr'])
break
return address, address_i
def sign_transaction(self, from_address, tx):
print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
'enter the following parameters:\n\n'
'From Address:\t\t{_from}\n'
'\n\tClick the "Generate Information" button\n\n'
'To Address:\t\t{to}\n'
'Value / Amount to Send:\t{value}\n'
'Gas Limit:\t\t{gas}\n'
'Gas Price:\t\t[Accept the default value]\n'
'Nonce:\t\t\t{nonce}\n'
'Data:\t\t\t{data}\n'.format(
_from=from_address,
to=tx['to'],
value=tx['value'],
gas=tx['gas'],
nonce=tx['nonce'],
data=tx['data'], )
)
signed_tx = input('Copy and Paste the "Signed Transaction" '
'field here:\n')
if signed_tx.startswith('0x'):
signed_tx = signed_tx[2:]
return signed_tx
def check_transaction(self, tx_hash):
if 'ropsten' in ETH_REMOTE_NODE:
etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
tx_hash)
else:
etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
print('\nYou can check the outcome of your transaction here:\n'
'{}\n\n'.format(etherscan))
def list(self):
data_sources = self.mkt_contract.functions.getAllProviders().call()
data = []
for index, data_source in enumerate(data_sources):
if index > 0:
if 'test' not in Web3.toText(data_source).lower():
data.append(
dict(
dataset=self.to_text(data_source)
)
)
df = pd.DataFrame(data)
set_print_settings()
if df.empty:
print('There are no datasets available yet.')
else:
print(df)
def subscribe(self, dataset):
dataset = dataset.lower()
address = self.choose_pubaddr()[0]
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(dataset))
return
grains = provider_info[1]
price = from_grains(grains)
subscribed = self.mkt_contract.functions.checkAddressSubscription(
address, Web3.toHex(dataset)
).call()
if subscribed[5]:
print(
'\nYou are already subscribed to the "{}" dataset.\n'
'Your subscription started on {} UTC, and is valid until '
'{} UTC.'.format(
dataset,
pd.to_datetime(subscribed[3], unit='s', utc=True),
pd.to_datetime(subscribed[4], unit='s', utc=True)
)
)
return
print('\nThe price for a monthly subscription to this dataset is'
' {} ENG'.format(price))
print(
'Checking that the ENG balance in {} is greater than {} '
'ENG... '.format(address, price), end=''
)
wallet_address = address[2:]
balance = self.web3.eth.call({
'from': address,
'to': self.eng_contract_address,
'data': '0x70a08231000000000000000000000000{}'.format(
wallet_address
)
})
try:
balance = Web3.toInt(balance) # web3 >= 4.0.0b7
except TypeError:
balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6
if balance > grains:
print('OK.')
else:
print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower '
'than the price of the dataset that you are trying to\n'
'buy: {} ENG. Get enough ENG to cover the costs of the '
'monthly\nsubscription for what you are trying to buy, '
'and try again.'.format(
address, from_grains(balance), price))
return
while True:
agree_pay = input('Please confirm that you agree to pay {} ENG '
'for a monthly subscription to the dataset "{}" '
'starting today. [default: Y] '.format(
price, dataset)) or 'y'
if agree_pay.lower() not in ('y', 'n'):
print("Please answer Y or N.")
else:
if agree_pay.lower() == 'y':
break
else:
return
print('Ready to subscribe to dataset {}.\n'.format(dataset))
print('In order to execute the subscription, you will need to sign '
'two different transactions:\n'
'1. First transaction is to authorize the Marketplace contract '
'to spend {} ENG on your behalf.\n'
'2. Second transaction is the actual subscription for the '
'desired dataset'.format(price))
tx = self.eng_contract.functions.approve(
self.mkt_contract_address,
grains,
).buildTransaction(
{'nonce': self.web3.eth.getTransactionCount(address)}
)
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(address, tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
)
print(
'\nThis is the TxHash for this transaction: {}'.format(tx_hash)
)
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the first transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nFirst transaction successful!\n'
'Now processing second transaction.')
tx = self.mkt_contract.functions.subscribe(
Web3.toHex(dataset),
).buildTransaction(
{'nonce': self.web3.eth.getTransactionCount(address)})
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(address, tx)
try:
tx_hash = '0x{}'.format(bin_hex(
self.web3.eth.sendRawTransaction(signed_tx)))
print('\nThis is the TxHash for this transaction: '
'{}'.format(tx_hash))
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the second transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nSecond transaction successful!\n'
'You have successfully subscribed to dataset {} with'
'address {}.\n'
'You can now ingest this dataset anytime during the '
'next month by running the following command:\n'
'catalyst marketplace ingest --dataset={}'.format(
dataset, address, dataset))
def process_temp_bundle(self, ds_name, path):
"""
Merge the temp bundle into the main bundle for the specified
data source.
Parameters
----------
ds_name
path
Returns
-------
"""
tmp_bundle = extract_bundle(path)
bundle_folder = get_data_source_folder(ds_name)
if os.listdir(bundle_folder):
zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
merge_bundles(zsource, ztarget)
else:
os.rename(tmp_bundle, bundle_folder)
pass
def ingest(self, ds_name, start=None, end=None, force_download=False):
# ds_name = ds_name.lower()
# TODO: catch error conditions
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(ds_name)
).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(ds_name))
return
address, address_i = self.choose_pubaddr()
fns = self.mkt_contract.functions
check_sub = fns.checkAddressSubscription(
address, Web3.toHex(ds_name)
).call()
if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
print('You are not subscribed to dataset "{}" with address {}. '
'Plese subscribe first.'.format(ds_name, address))
return
if not check_sub[5]:
print('Your subscription to dataset "{}" expired on {} UTC.'
'Please renew your subscription by running:\n'
'catalyst marketplace subscribe --dataset={}'.format(
ds_name,
pd.to_datetime(check_sub[4], unit='s', utc=True),
ds_name)
)
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
secret = self.addresses[address_i]['secret']
else:
key, secret = get_key_secret(address)
headers = get_signed_headers(ds_name, key, secret)
log.debug('Starting download of dataset for ingestion...')
r = requests.post(
'{}/marketplace/ingest'.format(AUTH_SERVER),
headers=headers,
stream=True,
)
if r.status_code == 200:
target_path = get_temp_bundles_folder()
try:
decoder = MultipartDecoder.from_response(r)
for part in decoder.parts:
h = part.headers[b'Content-Disposition'].decode('utf-8')
# Extracting the filename from the header
name = re.search(r'filename="(.*)"', h).group(1)
filename = os.path.join(target_path, name)
with open(filename, 'wb') as f:
# for chunk in part.content.iter_content(
# chunk_size=1024):
# if chunk: # filter out keep-alive new chunks
# f.write(chunk)
f.write(part.content)
self.process_temp_bundle(ds_name, filename)
except NonMultipartContentTypeException:
response = r.json()
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=response,
)
else:
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=r.status_code,
)
log.info('{} ingested successfully'.format(ds_name))
def get_dataset(self, ds_name, start=None, end=None):
ds_name = ds_name.lower()
# TODO: filter ctable by start and end date
bundle_folder = get_data_source_folder(ds_name)
z = bcolz.ctable(rootdir=bundle_folder, mode='r')
df = z.todataframe() # type: pd.DataFrame
df.set_index(['date', 'symbol'], drop=True, inplace=True)
# TODO: implement the filter more carefully
# if start and end is None:
# df = df.xs(start, level=0)
return df
def clean(self, data_source_name, data_frequency=None):
data_source_name = data_source_name.lower()
if data_frequency is None:
folder = get_data_source_folder(data_source_name)
else:
folder = get_bundle_folder(data_source_name, data_frequency)
shutil.rmtree(folder)
pass
def create_metadata(self, key, secret, ds_name, data_frequency, desc,
has_history=True, has_live=True):
"""
Returns
-------
"""
headers = get_signed_headers(ds_name, key, secret)
r = requests.post(
'{}/marketplace/register'.format(AUTH_SERVER),
json=dict(
ds_name=ds_name,
desc=desc,
data_frequency=data_frequency,
has_history=has_history,
has_live=has_live,
),
headers=headers,
)
if r.status_code != 200:
raise MarketplaceHTTPRequest(
request='register', error=r.status_code
)
if 'error' in r.json():
raise MarketplaceHTTPRequest(
request='upload file', error=r.json()['error']
)
def register(self):
while True:
desc = input('Enter the name of the dataset to register: ')
dataset = desc.lower()
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if provider_info[4]:
print('There is already a dataset registered under '
'the name "{}". Please choose a different '
'name.'.format(dataset))
else:
break
price = int(
input(
'Enter the price for a monthly subscription to '
'this dataset in ENG: '
)
)
while True:
freq = input('Enter the data frequency [daily, hourly, minute]: ')
if freq.lower() not in ('daily', 'hourly', 'minute'):
print('Not a valid frequency.')
else:
break
while True:
reg_pub = input(
'Does it include historical data? [default: Y]: '
) or 'y'
if reg_pub.lower() not in ('y', 'n'):
print('Please answer Y or N.')
else:
if reg_pub.lower() == 'y':
has_history = True
else:
has_history = False
break
while True:
reg_pub = input(
'Doest it include live data? [default: Y]: '
) or 'y'
if reg_pub.lower() not in ('y', 'n'):
print('Please answer Y or N.')
else:
if reg_pub.lower() == 'y':
has_live = True
else:
has_live = False
break
address, address_i = self.choose_pubaddr()
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
secret = self.addresses[address_i]['secret']
else:
key, secret = get_key_secret(address)
grains = to_grains(price)
tx = self.mkt_contract.functions.register(
Web3.toHex(dataset),
grains,
address,
).buildTransaction(
{'nonce': self.web3.eth.getTransactionCount(address)}
)
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(address, tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
)
print(
'\nThis is the TxHash for this transaction: {}'.format(tx_hash)
)
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nWarming up the {} dataset'.format(dataset))
self.create_metadata(
key=key,
secret=secret,
ds_name=dataset,
data_frequency=freq,
desc=desc,
has_history=has_history,
has_live=has_live,
)
print('\n{} registered successfully'.format(dataset))
def publish(self, dataset, datadir, watch):
dataset = dataset.lower()
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if not provider_info[4]:
raise MarketplaceDatasetNotFound(dataset=dataset)
match = next(
(l for l in self.addresses if l['pubAddr'] == provider_info[0]),
None
)
if not match:
raise MarketplaceNoAddressMatch(
dataset=dataset,
address=provider_info[0])
print('Using address: {} to publish this dataset.'.format(
provider_info[0]))
if 'key' in match:
key = match['key']
secret = match['secret']
else:
key, secret = get_key_secret(provider_info[0])
headers = get_signed_headers(dataset, key, secret)
filenames = glob.glob(os.path.join(datadir, '*.csv'))
if not filenames:
raise MarketplaceNoCSVFiles(datadir=datadir)
files = []
for file in filenames:
files.append(('file', open(file, 'rb')))
r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
files=files,
headers=headers)
if r.status_code != 200:
raise MarketplaceHTTPRequest(request='upload file',
error=r.status_code)
if 'error' in r.json():
raise MarketplaceHTTPRequest(request='upload file',
error=r.json()['error'])
print('Dataset {} uploaded successfully.'.format(dataset))
catalyst/marketplace/marketplace_errors.py
+88
View File
@@ -0,0 +1,88 @@
import sys
import traceback
from catalyst.errors import ZiplineError
def silent_except_hook(exctype, excvalue, exctraceback):
if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch,
MarketplaceSubscriptionExpired, MarketplaceJSONError,
MarketplaceWalletNotSupported, MarketplaceEmptySignature]:
fn = traceback.extract_tb(exctraceback)[-1][0]
ln = traceback.extract_tb(exctraceback)[-1][1]
print("Error traceback: {1} (line {2})\n"
"{0.__name__}: {3}".format(exctype, fn, ln, excvalue))
else:
sys.__excepthook__(exctype, excvalue, exctraceback)
sys.excepthook = silent_except_hook
class MarketplacePubAddressEmpty(ZiplineError):
msg = (
'Please enter your public address to use in the Data Marketplace '
'in the following file: {filename}'
).strip()
class MarketplaceDatasetNotFound(ZiplineError):
msg = (
'The dataset "{dataset}" is not registered in the Data Marketplace.'
).strip()
class MarketplaceNoAddressMatch(ZiplineError):
msg = (
'The address registered with the dataset {dataset}: {address} '
'does not match any of your addresses.'
).strip()
class MarketplaceHTTPRequest(ZiplineError):
msg = (
'Request to remote server to {request} failed: {error}'
).strip()
class MarketplaceNoCSVFiles(ZiplineError):
msg = (
'No CSV files found on {datadir} to upload.'
)
class MarketplaceContractDataNoMatch(ZiplineError):
msg = (
'The information found on the contract does not match the '
'requested data:\n{params}.'
)
class MarketplaceSubscriptionExpired(ZiplineError):
msg = (
'Your subscription to dataset "{dataset}" expired on {date} '
'and is no longer active. You have to subscribe again running the '
'following command:\n'
'catalyst marketplace subscribe --dataset={dataset}'
)
class MarketplaceWalletNotSupported(ZiplineError):
msg = (
'Wallet {wallet} is not supported.'
)
class MarketplaceEmptySignature(ZiplineError):
msg = (
'Signature cannot be empty.'
)
class MarketplaceJSONError(ZiplineError):
msg = (
'The configuration file {file} is malformed. Please correct '
'the following error:\n{error}'
)
catalyst/marketplace/utils/__init__.py
View File
catalyst/marketplace/utils/auth_utils.py
+131
View File
@@ -0,0 +1,131 @@
import hashlib
import hmac
import requests
import time
from catalyst.marketplace.marketplace_errors import (
MarketplaceHTTPRequest, MarketplaceWalletNotSupported,
MarketplaceEmptySignature)
from catalyst.marketplace.utils.path_utils import (
get_user_pubaddr, save_user_pubaddr)
from catalyst.constants import AUTH_SERVER
def get_key_secret(pubAddr, wallet='mew'):
"""
Obtain a new key/secret pair from authentication server
Parameters
----------
pubAddr: str
dataset: str
Returns
-------
key: str
secret: str
"""
session = requests.Session()
response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
headers={
'Authorization': 'Digest username="{0}"'.format(
pubAddr)})
if response.status_code != 401:
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error='Unexpected response code: '
'{}'.format(response.status_code))
header = response.headers.get('WWW-Authenticate')
auth_type, auth_info = header.split(None, 1)
d = requests.utils.parse_dict_header(auth_info)
nonce = '0x{}'.format(d['nonce'])
if wallet == 'mew':
print('\nObtaining a key/secret pair to streamline all future '
'requests with the authentication server.\n'
'Visit https://www.myetherwallet.com/signmsg.html and sign the'
'following message:\n{}'.format(nonce))
signature = input('Copy and Paste the "sig" field from '
'the signature here (without the double quotes, '
'only the HEX value:\n')
else:
raise MarketplaceWalletNotSupported(wallet=wallet)
if signature is None:
raise MarketplaceEmptySignature()
signature = signature[2:]
r = int(signature[0:64], base=16)
s = int(signature[64:128], base=16)
v = int(signature[128:130], base=16)
vrs = [v, r, s]
response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
headers={
'Authorization': 'Digest username="{0}",realm="{1}",'
'nonce="{2}",uri="/marketplace/getkeysecret",response="{3}",'
'opaque="{4}"'.format(pubAddr,
d['realm'],
d['nonce'],
','.join(str(e) for e in vrs+[wallet]),
d['opaque'])})
if response.status_code == 200:
if 'error' in response.json():
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error=str(response.json()['error']))
else:
addresses = get_user_pubaddr()
match = next((l for l in addresses if
l['pubAddr'] == pubAddr), None)
match['key'] = response.json()['key']
match['secret'] = response.json()['secret']
addresses[addresses.index(match)] = match
save_user_pubaddr(addresses)
print('Key/secret pair retrieved successfully from server.')
return match['key'], match['secret']
else:
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error=response.status_code)
def get_signed_headers(ds_name, key, secret):
"""
Return a new request header including the key / secret signature
Parameters
----------
ds_name
key
secret
Returns
-------
"""
nonce = str(int(time.time()))
signature = hmac.new(
secret.encode('utf-8'),
'{}{}'.format(ds_name, nonce).encode('utf-8'),
hashlib.sha512
).hexdigest()
headers = {
'Sign': signature,
'Key': key,
'Nonce': nonce,
'Dataset': ds_name,
}
return headers
catalyst/marketplace/utils/bundle_utils.py
+36
View File
@@ -0,0 +1,36 @@
import os
import shutil
import bcolz
def merge_bundles(zsource, ztarget):
"""
Merge
Parameters
----------
zsource
ztarget
Returns
-------
"""
# TODO: find a way to do this iteratively instead of in-memory
df_source = zsource.todataframe()
df_source.set_index('date', drop=False, inplace=True)
df_target = ztarget.todataframe()
df_target.set_index('date', drop=False, inplace=True)
df = df_target.merge(
right=df_source,
how='right',
) # type: pd.DataFrame
dirname = os.path.basename(ztarget.rootdir)
bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
os.rename(ztarget.rootdir, bak_dir)
z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
shutil.rmtree(bak_dir)
return z
catalyst/marketplace/utils/eth_utils.py
+82
View File
@@ -0,0 +1,82 @@
import binascii
# def bytes32(string):
# """
# Convert string to bytes32 data type for smart contract
# Parameters
# ----------
# string: str
# Returns
# -------
# list
# """
# return binascii.hexlify(string.encode('utf-8'))
# def b32_str(bytes32):
# """
# Convert bytes32 to string
# Parameters
# ----------
# input: bytes object
# Returns
# -------
# str
# """
# return binascii.unhexlify(
# bytes32.decode('utf-8').rstrip('\0')).decode('ascii')
def bin_hex(binary):
"""
Convert bytes32 to string
Parameters
----------
input: bytes object
Returns
-------
str
"""
return binascii.hexlify(binary).decode('utf-8')
def from_grains(amount):
"""
Convert from grains to cryptocurrency
Parameters
----------
input: amount
Returns
-------
int
"""
return amount // 10 ** 8
def to_grains(amount):
"""
Convert from cryptocurrency to grains
Parameters
----------
input: amount
Returns
-------
int
"""
return amount * 10 ** 8
catalyst/marketplace/utils/path_utils.py
+166
View File
@@ -0,0 +1,166 @@
import os
import json
import tarfile
from catalyst.utils.deprecate import deprecated
from catalyst.utils.paths import data_root, ensure_directory
from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
def get_marketplace_folder(environ=None):
"""
The root path of the marketplace folder.
Parameters
----------
environ:
Returns
-------
str
"""
if not environ:
environ = os.environ
root = data_root(environ)
marketplace_folder = os.path.join(root, 'marketplace')
ensure_directory(marketplace_folder)
return marketplace_folder
def get_data_source_folder(data_source_name, environ=None):
"""
The root path of an data_source folder.
Parameters
----------
data_source_name: str
environ:
Returns
-------
str
"""
if not environ:
environ = os.environ
root = data_root(environ)
data_source_folder = os.path.join(root, 'marketplace', data_source_name)
ensure_directory(data_source_folder)
return data_source_folder
@deprecated
def get_bundle_folder(data_source_name, data_frequency, environ=None):
data_source_folder = get_data_source_folder(data_source_name, environ)
bundle_folder = os.path.join(data_source_folder, data_frequency)
ensure_directory(bundle_folder)
return bundle_folder
def get_temp_bundles_folder(environ=None):
"""
The temp folder for bundle downloads by algo name.
Parameters
----------
ds_name: str
environ:
Returns
-------
str
"""
root = data_root(environ)
folder = os.path.join(root, 'marketplace', 'temp_bundles')
ensure_directory(folder)
return folder
def extract_bundle(tar_filename):
"""
Extract a bcolz bundle.
Parameters
----------
ds_name
Returns
-------
str
"""
target_path = tar_filename.replace('.tar.gz', '')
with tarfile.open(tar_filename, 'r') as tar:
tar.extractall(target_path)
return target_path
def get_user_pubaddr(environ=None):
"""
The de-serialized contend of the user's addresses.json file.
Parameters
----------
environ:
Returns
-------
Object
"""
marketplace_folder = get_marketplace_folder(environ)
filename = os.path.join(marketplace_folder, 'addresses.json')
if os.path.isfile(filename):
with open(filename) as data_file:
try:
data = json.load(data_file)
except json.decoder.JSONDecodeError as e:
raise MarketplaceJSONError(file=filename, error=e)
try:
d = data[0]['pubAddr']
except Exception as e:
return [data, ]
return data
else:
data = []
data.append(dict(pubAddr='', desc=''))
with open(filename, 'w') as f:
json.dump(data, f, sort_keys=False, indent=2,
separators=(',', ':'))
return data
def save_user_pubaddr(data, environ=None):
"""
Saves the user's public addresses and their related metadata in
the corresponding addresses.json file.
Parameters
----------
data: dict
Returns
-------
True
"""
marketplace_folder = get_marketplace_folder(environ)
filename = os.path.join(marketplace_folder, 'addresses.json')
with open(filename, 'w') as f:
json.dump(data, f, sort_keys=False, indent=2,
separators=(',', ':'))
return True
docs/source/example-algos.rst
+1 -1
View File
@@ -805,7 +805,7 @@ Credits: This code was originally submitted by `Abner Ayala-Acevedo
import pandas as pd
from catalyst import run_algorithm
from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
from catalyst.api import (symbols, )
docs/source/releases.rst
+16
View File
@@ -2,6 +2,22 @@
Release Notes
=============
Version 0.5.0
^^^^^^^^^^^^^
**Release Date**: 2018-02-07
Bug Fixes
~~~~~~~~~
- Fixed an issue with orders that stay open :issue:`211`
- Fixed Jupyter issues :issue:`179`
- Fetching multiple tickers in one call to minimize rate limit risks :issue:`174`
- Improved live state presentation :issue:`171`
Build
~~~~~
- Introducing the Enigma Marketplace
Version 0.4.7
^^^^^^^^^^^^^
**Release Date**: 2018-01-19
etc/python2.7-environment.yml
+1 -1
View File
@@ -20,7 +20,7 @@ dependencies:
- bcolz==0.12.1
- bottleneck==1.2.1
- chardet==3.0.4
- ccxt==1.10.774
- ccxt==1.10.1049
- click==6.7
- contextlib2==0.5.5
- cycler==0.10.0
etc/requirements.txt
+1 -1
View File
@@ -81,6 +81,6 @@ empyrical==0.2.1
tables==3.3.0
#Catalyst dependencies
ccxt==1.10.774
ccxt==1.10.1049
boto3==1.4.8
redo==1.6
etc/requirements_marketplace.txt
+2
View File
@@ -0,0 +1,2 @@
web3==4.0.0b7
requests-toolbelt==0.8.0
tests/exchange/test_ccxt.py
+8 -7
View File
@@ -15,23 +15,23 @@ log = Logger('test_ccxt')
class TestCCXT(BaseExchangeTestCase):
@classmethod
def setup(self):
exchange_name = 'bitfinex'
exchange_name = 'binance'
auth = get_exchange_auth(exchange_name)
self.exchange = CCXT(
exchange_name=exchange_name,
key=auth['key'],
secret=auth['secret'],
base_currency='bnb',
base_currency='usdt',
)
self.exchange.init()
def test_order(self):
log.info('creating order')
asset = self.exchange.get_asset('neo_bnb')
asset = self.exchange.get_asset('eth_usdt')
order_id = self.exchange.order(
asset=asset,
style=ExchangeLimitOrder(limit_price=10),
amount=1,
style=ExchangeLimitOrder(limit_price=1000),
amount=1.01,
)
log.info('order created {}'.format(order_id))
assert order_id is not None
@@ -72,10 +72,11 @@ class TestCCXT(BaseExchangeTestCase):
def test_tickers(self):
log.info('retrieving tickers')
assets = [
self.exchange.get_asset('iot_usd'),
self.exchange.get_asset('ada_eth'),
self.exchange.get_asset('zrx_eth'),
]
tickers = self.exchange.tickers(assets)
assert len(tickers) == 1
assert len(tickers) == 2
pass
def test_my_trades(self):
tests/exchange/test_suites/test_suite_exchange.py
+20 -13
View File
@@ -15,7 +15,7 @@ from catalyst.exchange.utils.test_utils import select_random_exchanges, \
handle_exchange_error, select_random_assets
from catalyst.testing import ZiplineTestCase
from catalyst.testing.fixtures import WithLogger
from exchange.utils.factory import get_exchanges
from catalyst.exchange.utils.factory import get_exchanges, get_exchange
log = Logger('TestSuiteExchange')
@@ -90,7 +90,7 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
# exchange_population,
# features=['fetchTickers'],
# ) # Type: list[Exchange]
exchanges = list(get_exchanges(['bitfinex']).values())
exchanges = list(get_exchanges(['binance']).values())
for exchange in exchanges:
exchange.init()
@@ -113,10 +113,11 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
exchange_population = 3
asset_population = 3
exchanges = select_random_exchanges(
population=exchange_population,
features=['fetchOHLCV'],
) # Type: list[Exchange]
# exchanges = select_random_exchanges(
# population=exchange_population,
# features=['fetchOHLCV'],
# ) # Type: list[Exchange]
exchanges = list(get_exchanges(['binance']).values())
for exchange in exchanges:
exchange.init()
@@ -138,7 +139,6 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
assets=assets,
bar_count=bar_count,
start_dt=dt_range[0],
end_dt=dt_range[-1],
)
assert len(candles) == asset_population
@@ -155,13 +155,20 @@ class TestSuiteExchange(WithLogger, ZiplineTestCase):
quote_currency = 'eth'
order_amount = 0.1
exchanges = select_random_exchanges(
population=population,
features=['fetchOrder'],
is_authenticated=True,
base_currency=quote_currency,
) # Type: list[Exchange]
# exchanges = select_random_exchanges(
# population=population,
# features=['fetchOrder'],
# is_authenticated=True,
# base_currency=quote_currency,
# ) # Type: list[Exchange]
exchanges = [
get_exchange(
'binance',
base_currency=quote_currency,
must_authenticate=True,
)
]
log_catcher = TestHandler()
with log_catcher:
for exchange in exchanges:
tests/marketplace/__init__.py
View File
tests/marketplace/test_marketplace.py
+36
View File
@@ -0,0 +1,36 @@
from catalyst.marketplace.marketplace import Marketplace
from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
import pandas as pd
class TestMarketplace(WithLogger, ZiplineTestCase):
def test_list(self):
marketplace = Marketplace()
marketplace.list()
pass
def test_register(self):
marketplace = Marketplace()
marketplace.register()
pass
def test_subscribe(self):
marketplace = Marketplace()
marketplace.subscribe('marketcap2222')
pass
def test_ingest(self):
marketplace = Marketplace()
ds_def = marketplace.ingest('marketcap1234')
pass
def test_publish(self):
marketplace = Marketplace()
datadir = '/Users/fredfortier/Downloads/marketcap_test_single'
marketplace.publish('marketcap1234', datadir, False)
pass
def test_clean(self):
marketplace = Marketplace()
marketplace.clean('marketcap')
pass