wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-21 12:30:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop'

Browse Source
This commit is contained in:
Victor Grau Serrat
2018-03-14 00:52:33 -06:00
parent 25e9f0f58f 5bb7eed072
commit 296dbf4ada
38 changed files with 1080 additions and 1309 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/__main__.py
+16 -15
View File
@@ -767,12 +767,18 @@ def bundles():
@main.group()
@click.pass_context
def marketplace(ctx):
"""Access the Enigma Data Marketplace to:\n
- Register and Publish new datasets (seller-side)\n
- Subscribe and Ingest premium datasets (buyer-side)\n
"""
pass
@marketplace.command()
@click.pass_context
def ls(ctx):
"""List all available datasets.
"""
click.echo('Listing of available data sources on the marketplace:',
sys.stdout)
marketplace = Marketplace()
@@ -787,10 +793,8 @@ def ls(ctx):
)
@click.pass_context
def subscribe(ctx, dataset):
if dataset is None:
ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
"""Subscribe to an existing dataset.
"""
marketplace = Marketplace()
marketplace.subscribe(dataset)
@@ -825,11 +829,8 @@ def subscribe(ctx, dataset):
)
@click.pass_context
def ingest(ctx, dataset, data_frequency, start, end):
if dataset is None:
ctx.fail("must specify a dataset to clean with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
"""Ingest a dataset (requires subscription).
"""
marketplace = Marketplace()
marketplace.ingest(dataset, data_frequency, start, end)
@@ -842,19 +843,17 @@ def ingest(ctx, dataset, data_frequency, start, end):
)
@click.pass_context
def clean(ctx, dataset):
if dataset is None:
ctx.fail("must specify a dataset to ingest with '--dataset'\n"
"List available dataset on the marketplace with "
"'catalyst marketplace ls'")
click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
"""Clean/Remove local data for a given dataset.
"""
marketplace = Marketplace()
marketplace.clean(dataset)
click.echo('Done', sys.stdout)
@marketplace.command()
@click.pass_context
def register(ctx):
"""Register a new dataset.
"""
marketplace = Marketplace()
marketplace.register()
@@ -878,6 +877,8 @@ def register(ctx):
)
@click.pass_context
def publish(ctx, dataset, datadir, watch):
"""Publish data for a registered dataset.
"""
marketplace = Marketplace()
if dataset is None:
ctx.fail("must specify a dataset to publish data for "
catalyst/constants.py
+3 -4
View File
@@ -25,8 +25,7 @@ AUTO_INGEST = False
AUTH_SERVER = 'https://data.enigma.co'
# TODO: switch to mainnet
ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
ETH_REMOTE_NODE = 'https://rinkeby.infura.io/'
MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
@@ -37,8 +36,8 @@ MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
'contract_marketplace_abi.json'
# TODO: switch to mainnet
ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
'master/catalyst/marketplace/' \
ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_enigma_address.txt'
ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
catalyst/examples/buy_low_sell_high.py
-1
View File
@@ -7,7 +7,6 @@ from catalyst.api import (
order_target_percent,
symbol,
record,
get_open_orders,
)
from catalyst.exchange.utils.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm
catalyst/examples/dual_moving_average.py
+2 -3
View File
@@ -4,8 +4,7 @@ import pandas as pd
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import (record, symbol, order_target_percent,
get_open_orders)
from catalyst.api import (record, symbol, order_target_percent,)
from catalyst.exchange.utils.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
@@ -63,7 +62,7 @@ def handle_data(context, data):
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
orders = context.blotter.open_orders
if len(orders) > 0:
return
catalyst/examples/mean_reversion_simple.py
+2 -2
View File
@@ -37,8 +37,8 @@ def initialize(context):
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 40
context.RSI_OVERBOUGHT = 60
context.RSI_OVERSOLD = 60
context.RSI_OVERBOUGHT = 70
context.CANDLE_SIZE = '15T'
context.start_time = time.time()
catalyst/examples/portfolio_optimization.py
+3 -2
View File
@@ -66,7 +66,7 @@ def handle_data(context, data):
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3 + context.nassets, n_portfolios))
for p in xrange(n_portfolios):
for p in range(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
@@ -146,4 +146,5 @@ if __name__ == '__main__':
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
capital_base=100000,
base_currency='usdt', )
catalyst/examples/simple_loop.py
+1 -1
View File
@@ -114,7 +114,7 @@ def analyze(context, perf):
if __name__ == '__main__':
mode = 'backtest'
mode = 'live'
if mode == 'backtest':
run_algorithm(
catalyst/exchange/ccxt/ccxt_exchange.py
+16 -19
View File
@@ -43,7 +43,8 @@ SUPPORTED_EXCHANGES = dict(
class CCXT(Exchange):
def __init__(self, exchange_name, key, secret, base_currency):
def __init__(self, exchange_name, key,
secret, password, base_currency):
log.debug(
'finding {} in CCXT exchanges:\n{}'.format(
exchange_name, ccxt.exchanges
@@ -60,6 +61,7 @@ class CCXT(Exchange):
self.api = exchange_attr({
'apiKey': key,
'secret': secret,
'password': password,
})
self.api.enableRateLimit = True
@@ -426,25 +428,18 @@ class CCXT(Exchange):
)
if start_dt is None:
# TODO: determine why binance is failing
if end_dt is None and self.name not in ['binance']:
if end_dt is None:
end_dt = pd.Timestamp.utcnow()
if end_dt is not None:
dt_range = get_periods_range(
end_dt=end_dt,
periods=bar_count,
freq=freq,
)
start_dt = dt_range[0]
dt_range = get_periods_range(
end_dt=end_dt,
periods=bar_count,
freq=freq,
)
start_dt = dt_range[0]
if start_dt is not None:
# Convert out start date to a UNIX timestamp, then translate to
# milliseconds
delta = start_dt - get_epoch()
since = int(delta.total_seconds()) * 1000
else:
since = None
delta = start_dt - get_epoch()
since = int(delta.total_seconds()) * 1000
candles = dict()
for index, asset in enumerate(assets):
@@ -985,7 +980,8 @@ class CCXT(Exchange):
)
raise ExchangeRequestError(error=e)
def cancel_order(self, order_param, asset_or_symbol=None):
def cancel_order(self, order_param,
asset_or_symbol=None, params={}):
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
@@ -997,7 +993,8 @@ class CCXT(Exchange):
try:
symbol = self.get_symbol(asset_or_symbol) \
if asset_or_symbol is not None else None
self.api.cancel_order(id=order_id, symbol=symbol)
self.api.cancel_order(id=order_id,
symbol=symbol, params= params)
except (ExchangeError, NetworkError) as e:
log.warn(
catalyst/exchange/exchange.py
+60 -31
View File
@@ -11,13 +11,16 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
SymbolNotFoundOnExchange, \
PricingDataNotLoadedError, \
NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
NoDataAvailableOnExchange, NoValueForField, \
NoCandlesReceivedFromExchange, \
InvalidHistoryFrequencyAlias, \
TickerNotFoundError, NotEnoughCashError
from catalyst.exchange.utils.datetime_utils import get_delta, \
get_periods_range, \
get_periods, get_start_dt, get_frequency
get_periods, get_start_dt, get_frequency, \
get_candles_number_from_minutes
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
resample_history_df, has_bundle
resample_history_df, has_bundle, get_candles_df
from logbook import Logger
log = Logger('Exchange', level=LOG_LEVEL)
@@ -255,7 +258,8 @@ class Exchange:
elif data_frequency is not None:
applies = (
(
data_frequency == 'minute' and a.end_minute is not None)
data_frequency == 'minute' and
a.end_minute is not None)
or (
data_frequency == 'daily' and a.end_daily is not None)
)
@@ -502,45 +506,66 @@ class Exchange:
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency
frequency, data_frequency, supported_freqs=['T', 'D', 'H']
)
if unit == 'H':
raise InvalidHistoryFrequencyAlias(
freq=frequency)
# we want to avoid receiving empty candles
# so we request more than needed
# TODO: consider defining a const per asset
# and/or some retry mechanism (in each iteration request more data)
kExtra_minutes_candles = 150
requested_bar_count = bar_count + \
get_candles_number_from_minutes(unit,
candle_size,
kExtra_minutes_candles)
# The get_history method supports multiple asset
candles = self.get_candles(
freq=freq,
assets=assets,
bar_count=bar_count,
bar_count=requested_bar_count,
end_dt=end_dt if not is_current else None,
)
series = dict()
# candles sanity check - verify no empty candles were received:
for asset in candles:
first_candle = candles[asset][0]
asset_series = self.get_series_from_candles(
candles=candles[asset],
start_dt=first_candle['last_traded'],
end_dt=end_dt,
data_frequency=frequency,
field=field,
)
if not candles[asset]:
raise NoCandlesReceivedFromExchange(
bar_count=requested_bar_count,
end_dt=end_dt,
asset=asset,
exchange=self.name)
# Checking to make sure that the dates match
delta = get_delta(candle_size, data_frequency)
adj_end_dt = end_dt - delta
last_traded = asset_series.index[-1]
# for avoiding unnecessary forward fill end_dt is taken back one second
forward_fill_till_dt = end_dt - timedelta(seconds=1)
if last_traded < adj_end_dt:
raise LastCandleTooEarlyError(
last_traded=last_traded,
end_dt=adj_end_dt,
exchange=self.name,
)
series = get_candles_df(candles=candles,
field=field,
freq=frequency,
bar_count=requested_bar_count,
end_dt=forward_fill_till_dt)
series[asset] = asset_series
# TODO: consider how to approach this edge case
# delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
# delta = get_delta(delta_candle_size, data_frequency)
# adj_end_dt = end_dt - delta
# last_traded = asset_series.index[-1]
# if last_traded < adj_end_dt:
# raise LastCandleTooEarlyError(
# last_traded=last_traded,
# end_dt=adj_end_dt,
# exchange=self.name,
# )
df = pd.DataFrame(series)
df.dropna(inplace=True)
return df
return df.tail(bar_count)
def get_history_window_with_bundle(self,
assets,
@@ -588,7 +613,8 @@ class Exchange:
A dataframe containing the requested data.
"""
# TODO: this function needs some work, we're currently using it just for benchmark data
# TODO: this function needs some work,
# we're currently using it just for benchmark data
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency
)
@@ -614,7 +640,7 @@ class Exchange:
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
trailing_dt = \
series[asset].index[-1] + get_delta(1, data_frequency) \
if asset in series else start_dt
if asset in series else start_dt
# The get_history method supports multiple asset
# Use the original frequency to let each api optimize
@@ -664,7 +690,8 @@ class Exchange:
else:
return free, False
def sync_positions(self, positions, cash=None, check_balances=False):
def sync_positions(self, positions, cash=None,
check_balances=False):
"""
Update the portfolio cash and position balances based on the
latest ticker prices.
@@ -920,7 +947,8 @@ class Exchange:
"""
@abstractmethod
def cancel_order(self, order_param, symbol_or_asset=None):
def cancel_order(self, order_param,
symbol_or_asset=None, params={}):
"""Cancel an open order.
Parameters
@@ -929,6 +957,7 @@ class Exchange:
The order_id or order object to cancel.
symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this
params:
"""
pass
catalyst/exchange/exchange_algorithm.py
+154 -41
View File
@@ -16,7 +16,7 @@ import signal
import sys
from datetime import timedelta
from os import listdir
from os.path import isfile, join
from os.path import isfile, join, exists
import catalyst.protocol as zp
import logbook
@@ -36,9 +36,11 @@ from catalyst.exchange.utils.exchange_utils import (
get_algo_folder,
get_algo_df,
save_algo_df,
clear_frame_stats_directory,
remove_old_files,
group_assets_by_exchange, )
from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \
stats_to_algo_folder
from catalyst.exchange.utils.stats_utils import \
get_pretty_stats, stats_to_s3, stats_to_algo_folder
from catalyst.finance.execution import MarketOrder
from catalyst.finance.performance import PerformanceTracker
from catalyst.finance.performance.period import calc_period_stats
@@ -67,8 +69,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
self.current_day = None
if self.simulate_orders is None \
and self.sim_params.arena == 'backtest':
if self.simulate_orders is None and \
self.sim_params.arena == 'backtest':
self.simulate_orders = True
# Operations with retry features
@@ -118,7 +120,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
# be in-line with CXXT and many exchanges. We'll consider
# adding more order types in the future.
if not isinstance(style, ExchangeLimitOrder) or \
not isinstance(style, MarketOrder):
not isinstance(style, MarketOrder):
raise OrderTypeNotSupported(
order_type=style.__class__.__name__
)
@@ -161,6 +163,25 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
style)
return amount, style
def _calculate_order_target_amount(self, asset, target):
"""
removes order amounts so we won't run into issues
when two orders are placed one after the other.
it then proceeds to removing positions amount at TradingAlgorithm
:param asset:
:param target:
:return: target
"""
if asset in self.blotter.open_orders:
for open_order in self.blotter.open_orders[asset]:
current_amount = open_order.amount
target -= current_amount
target = super(ExchangeTradingAlgorithmBase, self). \
_calculate_order_target_amount(asset, target)
return target
def round_order(self, amount, asset):
"""
We need fractions with cryptocurrencies
@@ -368,19 +389,35 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self._clock = None
self.frame_stats = list()
self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats')
# erase the frame_stats folder to avoid overloading the disk
error = clear_frame_stats_directory(self.algo_namespace)
if error:
log.warning(error)
self.custom_signals_stats = \
get_algo_df(self.algo_namespace, 'custom_signals_stats')
# in order to save paper & live files separately
self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live'
self.exposure_stats = \
get_algo_df(self.algo_namespace, 'exposure_stats')
self.pnl_stats = get_algo_df(
self.algo_namespace,
'pnl_stats_{}'.format(self.mode_name),
)
self.custom_signals_stats = get_algo_df(
self.algo_namespace,
'custom_signals_stats_{}'.format(self.mode_name)
)
self.exposure_stats = get_algo_df(
self.algo_namespace,
'exposure_stats_{}'.format(self.mode_name)
)
self.is_running = True
self.stats_minutes = 1
self._last_orders = []
self._last_open_orders = []
self.trading_client = None
super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs)
@@ -392,6 +429,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
"Exit should be handled by the user.")
def interrupt_algorithm(self):
"""
when algorithm comes to an end this function is called.
extracts the stats and calls analyze.
after finishing, it exits the run.
Parameters
----------
Returns
-------
"""
self.is_running = False
if self._analyze is None:
@@ -401,21 +451,31 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
log.info('Exiting the algorithm. Calling `analyze()` '
'before exiting the algorithm.')
# add the last day stats which is not saved in the directory
current_stats = pd.DataFrame(self.frame_stats)
current_stats.set_index('period_close', drop=False, inplace=True)
# get the location of the directory
algo_folder = get_algo_folder(self.algo_namespace)
folder = join(algo_folder, 'daily_performance')
files = [f for f in listdir(folder) if isfile(join(folder, f))]
folder = join(algo_folder, 'frame_stats')
daily_perf_list = []
for item in files:
filename = join(folder, item)
if exists(folder):
files = [f for f in listdir(folder) if isfile(join(folder, f))]
with open(filename, 'rb') as handle:
perf_period = pickle.load(handle)
perf_period_dict = perf_period.to_dict()
daily_perf_list.append(perf_period_dict)
period_stats_list = []
for item in files:
filename = join(folder, item)
stats = pd.DataFrame(daily_perf_list)
stats.set_index('period_close', drop=False, inplace=True)
with open(filename, 'rb') as handle:
perf_period = pickle.load(handle)
period_stats_list.extend(perf_period)
stats = pd.DataFrame(period_stats_list)
stats.set_index('period_close', drop=False, inplace=True)
stats = pd.concat([stats, current_stats])
else:
stats = current_stats
self.analyze(stats)
@@ -484,7 +544,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
"""
self.state = get_algo_object(
algo_name=self.algo_namespace,
key='context.state',
key='context.state_{}'.format(self.mode_name),
)
if self.state is None:
self.state = {}
@@ -507,7 +567,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# Unpacking the perf_tracker and positions if available
cum_perf = get_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance',
key='cumulative_performance_{}'.format(self.mode_name),
)
if cum_perf is not None:
tracker.cumulative_performance = cum_perf
@@ -518,7 +578,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
todays_perf = get_algo_object(
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
rel_path='daily_performance',
rel_path='daily_performance_{}'.format(self.mode_name),
)
if todays_perf is not None:
# Ensure single common position tracker
@@ -601,8 +661,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
if base_currency is None:
base_currency = exchange.base_currency
# Don't check the cash if there are open orders. This could
# results in false positives.
orders = []
for asset in self.blotter.open_orders:
asset_orders = self.blotter.open_orders[asset]
@@ -657,7 +715,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
)
self.pnl_stats = pd.concat([self.pnl_stats, df])
save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats)
save_algo_df(
self.algo_namespace,
'pnl_stats_{}'.format(self.mode_name),
self.pnl_stats,
)
def add_custom_signals_stats(self, period_stats):
"""
@@ -678,8 +740,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
)
self.custom_signals_stats = pd.concat([self.custom_signals_stats, df])
save_algo_df(self.algo_namespace, 'custom_signals_stats',
self.custom_signals_stats)
save_algo_df(
self.algo_namespace,
'custom_signals_stats_{}'.format(self.mode_name),
self.custom_signals_stats,
)
def add_exposure_stats(self, period_stats):
"""
@@ -706,9 +771,43 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
self.exposure_stats = pd.concat([self.exposure_stats, df])
save_algo_df(
self.algo_namespace, 'exposure_stats', self.exposure_stats
self.algo_namespace,
'exposure_stats_{}'.format(self.mode_name),
self.exposure_stats
)
def nullify_frame_stats(self, now):
"""
Save all period_stats to local directory
erase old files from the folder and nullify
self.frame_stats
Parameters
----------
now: Timestamp
Returns
-------
"""
save_algo_object(
algo_name=self.algo_namespace,
key=now.floor('1D').strftime('%Y-%m-%d'),
obj=self.frame_stats,
rel_path='frame_stats'
)
error = remove_old_files(
algo_name=self.algo_namespace,
today=now,
rel_path='frame_stats'
)
if error:
log.warning(error)
self.frame_stats = list()
def handle_data(self, data):
"""
Wrapper around the handle_data method of each algo.
@@ -728,15 +827,20 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# Resetting the frame stats every day to minimize memory footprint
today = data.current_dt.floor('1D')
if self.current_day is not None and today > self.current_day:
self.frame_stats = list()
self.nullify_frame_stats(now=data.current_dt)
self.performance_needs_update = False
orders = list(self.perf_tracker.todays_performance.orders_by_id.keys())
if orders != self._last_orders:
last_orders_list = list(self.blotter.orders.keys())
open_orders_list = list(self.blotter.open_orders.keys())
if last_orders_list != self._last_orders or \
open_orders_list != self._last_open_orders:
self.performance_needs_update = True
# Saving current orders to detect changes in the next frame
self._last_orders = copy.deepcopy(orders)
# Saving current order positions
# to detect changes in the next frame
self._last_orders = copy.deepcopy(last_orders_list)
self._last_open_orders = copy.deepcopy(open_orders_list)
if self.performance_needs_update:
self.perf_tracker.update_performance()
@@ -778,7 +882,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
log.debug('saving cumulative performance object')
save_algo_object(
algo_name=self.algo_namespace,
key='cumulative_performance',
key='cumulative_performance_{}'.format(self.mode_name),
obj=self.perf_tracker.cumulative_performance,
)
log.debug('saving todays performance object')
@@ -786,12 +890,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
algo_name=self.algo_namespace,
key=today.strftime('%Y-%m-%d'),
obj=self.perf_tracker.todays_performance,
rel_path='daily_performance'
rel_path='daily_performance_{}'.format(self.mode_name)
)
log.debug('saving context.state object')
save_algo_object(
algo_name=self.algo_namespace,
key='context.state',
key='context.state_{}'.format(self.mode_name),
obj=self.state)
def _process_stats(self, data):
@@ -808,6 +912,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
# Saving the last hour in memory
self.frame_stats.append(frame_stats)
# creating and saving the pnl_stats into the local
# directory
self.add_pnl_stats(frame_stats)
if self.recorded_vars:
self.add_custom_signals_stats(frame_stats)
@@ -845,6 +951,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
csv_bytes = stats_to_algo_folder(
stats=self.frame_stats,
algo_namespace=self.algo_namespace,
folder_name='stats_{}'.format(self.mode_name),
recorded_cols=recorded_cols,
)
except Exception as e:
@@ -949,13 +1056,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
args=(order_id,))
@api_method
def cancel_order(self, order_param, exchange_name):
def cancel_order(self, order_param, exchange_name,
symbol=None, params={}):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
exchange_name: name of exchange from
which you want to cancel the order
symbol:
params:
"""
exchange = self.exchanges[exchange_name]
@@ -969,4 +1082,4 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('cancelling order again.'),
args=(order_id,))
args=(order_id, symbol, params))
catalyst/exchange/exchange_blotter.py
+6 -3
View File
@@ -68,7 +68,7 @@ class TradingPairFeeSchedule(CommissionModel):
multiplier = maker \
if ((order.amount > 0 and order.limit < transaction.price)
or (order.amount < 0 and order.limit > transaction.price)) \
and order.limit_reached else taker
and order.limit_reached else taker
fee = cost * multiplier
return fee
@@ -238,9 +238,12 @@ class ExchangeBlotter(Blotter):
else:
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'order {order_id} still open after {delta}'.format(
'{exchange} order {order_id} for {symbol} still open '
'after {delta}'.format(
exchange=exchange.name,
order_id=order.id,
delta=delta
delta=delta,
symbol=order.asset.symbol,
)
)
catalyst/exchange/exchange_data_portal.py
+5 -3
View File
@@ -9,8 +9,9 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
PricingDataNotLoadedError)
from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
from catalyst.exchange.utils.datetime_utils import get_frequency
from catalyst.exchange.utils.exchange_utils import resample_history_df, \
group_assets_by_exchange
from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt
from logbook import Logger
from redo import retry
@@ -311,7 +312,8 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
algo_end_dt=self._last_available_session,
)
df = resample_history_df(pd.DataFrame(series), freq, field)
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
return df
def get_exchange_spot_value(self,
catalyst/exchange/exchange_errors.py
+7
View File
@@ -322,3 +322,10 @@ class BalanceTooLowError(ZiplineError):
'add positions to hold a free amount greater than {amount}, or clean '
'the state of this algo and restart.'
).strip()
class NoCandlesReceivedFromExchange(ZiplineError):
msg = (
'Although requesting {bar_count} candles until {end_dt} of asset {asset}, '
'an empty list of candles was received for {exchange}.'
).strip()
catalyst/exchange/utils/datetime_utils.py
+42 -10
View File
@@ -1,4 +1,5 @@
import calendar
import math
import re
from datetime import datetime, timedelta, date
@@ -92,7 +93,7 @@ def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
adj_periods = periods * unit_periods
# TODO: standardize time aliases to avoid any mapping
unit = 'd' if unit == 'D' else 'm'
unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm'
delta = pd.Timedelta(adj_periods, unit)
if start_dt is not None:
@@ -248,7 +249,7 @@ def get_year_start_end(dt, first_day=None, last_day=None):
return year_start, year_end
def get_frequency(freq, data_frequency=None):
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
"""
Get the frequency parameters.
@@ -302,17 +303,18 @@ def get_frequency(freq, data_frequency=None):
elif unit.lower() == 'm' or unit == 'T':
unit = 'T'
alias = '{}T'.format(candle_size)
data_frequency = 'minute'
if data_frequency == 'daily':
elif unit.lower() == 'h':
if 'H' in supported_freqs:
unit = 'H'
alias = '{}H'.format(candle_size)
else:
candle_size = candle_size * 60
alias = '{}T'.format(candle_size)
data_frequency = 'minute'
# elif unit.lower() == 'h':
# candle_size = candle_size * 60
#
# alias = '{}T'.format(candle_size)
# if data_frequency == 'daily':
# data_frequency = 'minute'
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
@@ -325,3 +327,33 @@ def from_ms_timestamp(ms):
def get_epoch():
return pd.to_datetime('1970-1-1', utc=True)
def get_candles_number_from_minutes(unit, candle_size, minutes):
"""
Get the number of bars needed for the given time interval
in minutes.
Notes
-----
Supports only "T", "D" and "H" units
Parameters
----------
unit: str
candle_size : int
minutes: int
Returns
-------
int
"""
if unit == "T":
res = (float(minutes) / candle_size)
elif unit == "H":
res = (minutes / 60.0) / candle_size
else: # unit == "D"
res = (minutes / 1440.0) / candle_size
return int(math.ceil(res))
catalyst/exchange/utils/exchange_utils.py
+111 -21
View File
@@ -126,11 +126,11 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
filename = get_exchange_symbols_filename(exchange_name, is_local)
if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
pd.Timestamp('now', tz='UTC') - last_modified_time(
filename)).days > 1):
pd.Timestamp('now', tz='UTC') - last_modified_time(
filename)).days > 1):
try:
download_exchange_symbols(exchange_name, environ)
except Exception as e:
except Exception:
pass
if os.path.isfile(filename):
@@ -273,6 +273,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
key: str
environ:
rel_path: str
how: str
Returns
-------
@@ -316,6 +317,7 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
obj: Object
environ:
rel_path: str
how: str
"""
folder = get_algo_folder(algo_name, environ)
@@ -392,6 +394,71 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
df.to_csv(handle, encoding='UTF_8')
def clear_frame_stats_directory(algo_name):
"""
remove the outdated directory
to avoid overloading the disk
Parameters
----------
algo_name: str
Returns
-------
error: str
"""
error = None
algo_folder = get_algo_folder(algo_name)
folder = os.path.join(algo_folder, 'frame_stats')
if os.path.exists(folder):
try:
shutil.rmtree(folder)
except OSError:
error = 'unable to remove {}, the analyze ' \
'data will be inconsistent'.format(folder)
return error
def remove_old_files(algo_name, today, rel_path, environ=None):
"""
remove old files from a directory
to avoid overloading the disk
Parameters
----------
algo_name: str
today: Timestamp
rel_path: str
environ:
Returns
-------
error: str
"""
error = None
algo_folder = get_algo_folder(algo_name, environ)
folder = os.path.join(algo_folder, rel_path)
ensure_directory(folder)
# run on all files in the folder
for f in os.listdir(folder):
try:
file_path = os.path.join(folder, f)
creation_unix = os.path.getctime(file_path)
creation_time = pd.to_datetime(creation_unix, unit='s', utc=True)
# if the file is older than 30 days erase it
if today - pd.DateOffset(30) > creation_time:
os.unlink(file_path)
except OSError:
error = 'unable to erase files in {}'.format(folder)
return error
def get_exchange_minute_writer_root(exchange_name, environ=None):
"""
The minute writer folder for the exchange.
@@ -512,7 +579,7 @@ def get_common_assets(exchanges):
return assets
def resample_history_df(df, freq, field):
def resample_history_df(df, freq, field, start_dt=None):
"""
Resample the OHCLV DataFrame using the specified frequency.
@@ -540,7 +607,16 @@ def resample_history_df(df, freq, field):
else:
raise ValueError('Invalid field.')
resampled_df = df.resample(freq, closed='left', label='left').agg(agg)
resampled_df = df.resample(
freq, closed='left', label='left'
).agg(agg) # type: pd.DataFrame
# Because the samples are closed left, we get one more candle at
# the beginning then the requested number for bars. Removing this
# candle to avoid confusion.
if start_dt and not resampled_df.empty:
resampled_df = resampled_df[resampled_df.index >= start_dt]
return resampled_df
@@ -566,8 +642,9 @@ def mixin_market_params(exchange_name, params, market):
params['maker'] = 0.001
params['taker'] = 0.002
elif 'maker' in market and 'taker' in market \
and market['maker'] is not None and market['taker'] is not None:
elif 'maker' in market and 'taker' in market and \
market['maker'] is not None and market['taker'] is not None:
params['maker'] = market['maker']
params['taker'] = market['taker']
@@ -639,23 +716,36 @@ def save_asset_data(folder, df, decimals=8):
)
def get_candles_df(candles, field, freq, bar_count, end_dt,
previous_value=None):
def forward_fill_df_if_needed(df, periods):
df = df.reindex(periods)
# volume should always be 0 (if there were no trades in this interval)
df['volume'] = df['volume'].fillna(0.0)
# ie pull the last close into this close
df['close'] = df.fillna(method='pad')
# now copy the close that was pulled down from the last timestep
# into this row, across into o/h/l
df['open'] = df['open'].fillna(df['close'])
df['low'] = df['low'].fillna(df['close'])
df['high'] = df['high'].fillna(df['close'])
return df
def transform_candles_to_df(candles):
return pd.DataFrame(candles).set_index('last_traded')
def get_candles_df(candles, field, freq, bar_count, end_dt):
all_series = dict()
for asset in candles:
periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
asset_df = transform_candles_to_df(candles[asset])
rounded_end_dt = end_dt.floor(freq)
periods = pd.date_range(end=rounded_end_dt,
periods=bar_count,
freq=freq)
asset_df = forward_fill_df_if_needed(asset_df, periods)
dates = [candle['last_traded'] for candle in candles[asset]]
values = [candle[field] for candle in candles[asset]]
series = pd.Series(values, index=dates)
series = series.reindex(
periods,
method='ffill',
fill_value=previous_value,
)
series.sort_index(inplace=True)
all_series[asset] = series
all_series[asset] = pd.Series(asset_df[field])
df = pd.DataFrame(all_series)
df.dropna(inplace=True)
catalyst/exchange/utils/factory.py
+2
View File
@@ -33,6 +33,8 @@ def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
password=exchange_auth['password'] if 'password'
in exchange_auth.keys() else '',
base_currency=base_currency,
)
exchange_cache[key] = exchange
catalyst/exchange/utils/stats_utils.py
+4 -2
View File
@@ -396,7 +396,8 @@ def email_error(algo_name, dt, e, environ=None):
)})
def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
def stats_to_algo_folder(stats, algo_namespace,
folder_name, recorded_cols=None):
"""
Saves the performance stats to the algo local folder.
@@ -404,6 +405,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
----------
stats: list[Object]
algo_namespace: str
folder_name: str
recorded_cols: list[str]
Returns
@@ -416,7 +418,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
timestr = time.strftime('%Y%m%d')
folder = get_algo_folder(algo_namespace)
stats_folder = os.path.join(folder, 'stats')
stats_folder = os.path.join(folder, folder_name)
ensure_directory(stats_folder)
filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
catalyst/marketplace/contract_enigma_address.txt
+1 -1
View File
@@ -1 +1 @@
0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10
0x39a54f480d922a58c963de8091a6c9afc69db2cf
catalyst/marketplace/contract_marketplace_abi.json
+1 -1
View File
File diff suppressed because one or more lines are too long
catalyst/marketplace/contract_marketplace_address.txt
+1 -1
View File
@@ -1 +1 @@
0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4
0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc
catalyst/marketplace/marketplace.py
+137 -50
View File
@@ -7,6 +7,7 @@ import re
import shutil
import sys
import time
import webbrowser
import bcolz
import logbook
@@ -23,7 +24,7 @@ from catalyst.exchange.utils.stats_utils import set_print_settings
from catalyst.marketplace.marketplace_errors import (
MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles)
MarketplaceNoCSVFiles, MarketplaceRequiresPython3)
from catalyst.marketplace.utils.auth_utils import get_key_secret, \
get_signed_headers
from catalyst.marketplace.utils.bundle_utils import merge_bundles
@@ -32,6 +33,7 @@ from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
get_data_source_folder, get_marketplace_folder, \
get_user_pubaddr, get_temp_bundles_folder, extract_bundle
from catalyst.utils.paths import ensure_directory
if sys.version_info.major < 3:
import urllib
@@ -44,7 +46,10 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL)
class Marketplace:
def __init__(self):
global Web3
from web3 import Web3, HTTPProvider
try:
from web3 import Web3, HTTPProvider
except ImportError:
raise MarketplaceRequiresPython3()
self.addresses = get_user_pubaddr()
@@ -60,7 +65,8 @@ class Marketplace:
contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
self.mkt_contract_address = Web3.toChecksumAddress(
contract_url.readline().strip())
contract_url.readline().decode(
contract_url.info().get_content_charset()).strip())
abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
abi = json.load(abi_url)
@@ -73,7 +79,8 @@ class Marketplace:
contract_url = urllib.urlopen(ENIGMA_CONTRACT)
self.eng_contract_address = Web3.toChecksumAddress(
contract_url.readline().strip())
contract_url.readline().decode(
contract_url.info().get_content_charset()).strip())
abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
abi = json.load(abi_url)
@@ -136,10 +143,10 @@ class Marketplace:
return address, address_i
def sign_transaction(self, from_address, tx):
def sign_transaction(self, tx):
print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
'enter the following parameters:\n\n'
url = 'https://www.myetherwallet.com/#offline-transaction'
print('\nVisit {url} and enter the following parameters:\n\n'
'From Address:\t\t{_from}\n'
'\n\tClick the "Generate Information" button\n\n'
'To Address:\t\t{to}\n'
@@ -148,13 +155,16 @@ class Marketplace:
'Gas Price:\t\t[Accept the default value]\n'
'Nonce:\t\t\t{nonce}\n'
'Data:\t\t\t{data}\n'.format(
_from=from_address,
to=tx['to'],
value=tx['value'],
gas=tx['gas'],
nonce=tx['nonce'],
data=tx['data'], )
)
url=url,
_from=tx['from'],
to=tx['to'],
value=tx['value'],
gas=tx['gas'],
nonce=tx['nonce'],
data=tx['data'], )
)
webbrowser.open_new(url)
signed_tx = input('Copy and Paste the "Signed Transaction" '
'field here:\n')
@@ -167,16 +177,17 @@ class Marketplace:
def check_transaction(self, tx_hash):
if 'ropsten' in ETH_REMOTE_NODE:
etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
tx_hash)
etherscan = 'https://ropsten.etherscan.io/tx/'
elif 'rinkeby' in ETH_REMOTE_NODE:
etherscan = 'https://rinkeby.etherscan.io/tx/'
else:
etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
etherscan = 'https://etherscan.io/tx/'
etherscan = '{}{}'.format(etherscan, tx_hash)
print('\nYou can check the outcome of your transaction here:\n'
'{}\n\n'.format(etherscan))
def list(self):
def _list(self):
data_sources = self.mkt_contract.functions.getAllProviders().call()
data = []
@@ -188,15 +199,44 @@ class Marketplace:
dataset=self.to_text(data_source)
)
)
return pd.DataFrame(data)
def list(self):
df = self._list()
df = pd.DataFrame(data)
set_print_settings()
if df.empty:
print('There are no datasets available yet.')
else:
print(df)
def subscribe(self, dataset):
def subscribe(self, dataset=None):
if dataset is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'subscribe to [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
dataset = df_sets.iloc[dataset_num]['dataset']
break
dataset = dataset.lower()
@@ -259,14 +299,14 @@ class Marketplace:
'buy: {} ENG. Get enough ENG to cover the costs of the '
'monthly\nsubscription for what you are trying to buy, '
'and try again.'.format(
address, from_grains(balance), price))
address, from_grains(balance), price))
return
while True:
agree_pay = input('Please confirm that you agree to pay {} ENG '
'for a monthly subscription to the dataset "{}" '
'starting today. [default: Y] '.format(
price, dataset)) or 'y'
price, dataset)) or 'y'
if agree_pay.lower() not in ('y', 'n'):
print("Please answer Y or N.")
else:
@@ -287,13 +327,11 @@ class Marketplace:
self.mkt_contract_address,
grains,
).buildTransaction(
{'nonce': self.web3.eth.getTransactionCount(address)}
{'from': address,
'nonce': self.web3.eth.getTransactionCount(address)}
)
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(address, tx)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
@@ -328,13 +366,11 @@ class Marketplace:
tx = self.mkt_contract.functions.subscribe(
Web3.toHex(dataset),
).buildTransaction(
{'nonce': self.web3.eth.getTransactionCount(address)})
).buildTransaction({
'from': address,
'nonce': self.web3.eth.getTransactionCount(address)})
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(address, tx)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(bin_hex(
@@ -369,7 +405,7 @@ class Marketplace:
'You can now ingest this dataset anytime during the '
'next month by running the following command:\n'
'catalyst marketplace ingest --dataset={}'.format(
dataset, address, dataset))
dataset, address, dataset))
def process_temp_bundle(self, ds_name, path):
"""
@@ -387,6 +423,7 @@ class Marketplace:
"""
tmp_bundle = extract_bundle(path)
bundle_folder = get_data_source_folder(ds_name)
ensure_directory(bundle_folder)
if os.listdir(bundle_folder):
zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
@@ -397,7 +434,33 @@ class Marketplace:
pass
def ingest(self, ds_name, start=None, end=None, force_download=False):
def ingest(self, ds_name=None, start=None, end=None, force_download=False):
if ds_name is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'ingest [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
ds_name = df_sets.iloc[dataset_num]['dataset']
break
# ds_name = ds_name.lower()
@@ -426,10 +489,10 @@ class Marketplace:
print('Your subscription to dataset "{}" expired on {} UTC.'
'Please renew your subscription by running:\n'
'catalyst marketplace subscribe --dataset={}'.format(
ds_name,
pd.to_datetime(check_sub[4], unit='s', utc=True),
ds_name)
)
ds_name,
pd.to_datetime(check_sub[4], unit='s', utc=True),
ds_name)
)
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
@@ -493,14 +556,40 @@ class Marketplace:
return df
def clean(self, data_source_name, data_frequency=None):
data_source_name = data_source_name.lower()
def clean(self, ds_name=None, data_frequency=None):
if ds_name is None:
mktplace_root = get_marketplace_folder()
folders = [os.path.basename(f.rstrip('/'))
for f in glob.glob('{}/*/'.format(mktplace_root))
if 'temp_bundles' not in f]
while True:
for idx, f in enumerate(folders):
print('{}\t{}'.format(idx, f))
dataset_num = input('Choose the dataset you want to '
'clean [0..{}]: '.format(
len(folders) - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
len(folders) - 1))
else:
if dataset_num not in range(0, len(folders)):
print('Enter a number between 0 and {}'.format(
len(folders) - 1))
else:
ds_name = folders[dataset_num]
break
ds_name = ds_name.lower()
if data_frequency is None:
folder = get_data_source_folder(data_source_name)
folder = get_data_source_folder(ds_name)
else:
folder = get_bundle_folder(data_source_name, data_frequency)
folder = get_bundle_folder(ds_name, data_frequency)
shutil.rmtree(folder)
pass
@@ -604,13 +693,11 @@ class Marketplace:
grains,
address,
).buildTransaction(
{'nonce': self.web3.eth.getTransactionCount(address)}
{'from': address,
'nonce': self.web3.eth.getTransactionCount(address)}
)
if 'ropsten' in ETH_REMOTE_NODE:
tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
signed_tx = self.sign_transaction(address, tx)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
@@ -621,7 +708,7 @@ class Marketplace:
)
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
print('Unable to register the requested dataset: {}'.format(e))
return
self.check_transaction(tx_hash)
catalyst/marketplace/marketplace_errors.py
+10 -1
View File
@@ -9,7 +9,8 @@ def silent_except_hook(exctype, excvalue, exctraceback):
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch,
MarketplaceSubscriptionExpired, MarketplaceJSONError,
MarketplaceWalletNotSupported, MarketplaceEmptySignature]:
MarketplaceWalletNotSupported, MarketplaceEmptySignature,
MarketplaceRequiresPython3]:
fn = traceback.extract_tb(exctraceback)[-1][0]
ln = traceback.extract_tb(exctraceback)[-1][1]
print("Error traceback: {1} (line {2})\n"
@@ -86,3 +87,11 @@ class MarketplaceJSONError(ZiplineError):
'The configuration file {file} is malformed. Please correct '
'the following error:\n{error}'
)
class MarketplaceRequiresPython3(ZiplineError):
msg = (
'\nCatalyst requires Python3 to access the Enigma Data Marketplace.\n'
'If you want to use the Data Marketplace, you need to reinstall '
'Catalyst\nwith Python3. See the documentation website for additional '
'information.')
catalyst/marketplace/utils/auth_utils.py
+2 -2
View File
@@ -47,11 +47,11 @@ def get_key_secret(pubAddr, wallet='mew'):
if wallet == 'mew':
print('\nObtaining a key/secret pair to streamline all future '
'requests with the authentication server.\n'
'Visit https://www.myetherwallet.com/signmsg.html and sign the'
'Visit https://www.myetherwallet.com/signmsg.html and sign the '
'following message:\n{}'.format(nonce))
signature = input('Copy and Paste the "sig" field from '
'the signature here (without the double quotes, '
'only the HEX value:\n')
'only the HEX value):\n')
else:
raise MarketplaceWalletNotSupported(wallet=wallet)
catalyst/marketplace/utils/bundle_utils.py
+64 -6
View File
@@ -1,7 +1,12 @@
import os
import random
import re
import shutil
import bcolz
import numpy as np
import pandas as pd
from six import string_types
def merge_bundles(zsource, ztarget):
@@ -18,19 +23,72 @@ def merge_bundles(zsource, ztarget):
"""
# TODO: find a way to do this iteratively instead of in-memory
df_source = zsource.todataframe()
df_source.set_index('date', drop=False, inplace=True)
df_target = ztarget.todataframe()
df_target.set_index('date', drop=False, inplace=True)
df = df_target.merge(
right=df_source,
how='right',
df = pd.concat(
[df_source, df_target], ignore_index=True
) # type: pd.DataFrame
df.drop_duplicates(inplace=True)
df.set_index(['date', 'symbol'], drop=False, inplace=True)
sanitize_df(df)
dirname = os.path.basename(ztarget.rootdir)
bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
os.rename(ztarget.rootdir, bak_dir)
shutil.move(ztarget.rootdir, bak_dir)
z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
shutil.rmtree(bak_dir)
return z
def sanitize_df(df):
# Using a sampling method to identify dates for efficiency with
# large datasets
if len(df) > 100:
indexes = random.sample(range(0, len(df) - 1), 100)
elif len(df) > 1:
indexes = range(0, len(df) - 1)
else:
indexes = [0, ]
for column in df.columns:
is_date = False
for index in indexes:
value = df[column].iloc[index]
if not isinstance(value, string_types):
continue
# TODO: assuming that the date is at least daily
exp = re.compile(r'^\d{4}-\d{2}-\d{2}.*$')
matches = exp.findall(value)
if matches:
is_date = True
break
if is_date:
df[column] = pd.to_datetime(df[column])
else:
try:
ser = safely_reduce_dtype(df[column])
df[column] = ser
except Exception:
pass
return df
def safely_reduce_dtype(ser): # pandas.Series or numpy.array
orig_dtype = "".join(
[x for x in ser.dtype.name if x.isalpha()]) # float/int
mx = 1
for val in ser.values:
new_itemsize = np.min_scalar_type(val).itemsize
if mx < new_itemsize:
mx = new_itemsize
if orig_dtype == 'int':
mx = max(mx, 4)
new_dtype = orig_dtype + str(mx * 8)
return ser.astype(new_dtype)
catalyst/support/issue_236.py
+32
View File
@@ -0,0 +1,32 @@
from catalyst.api import symbol
from catalyst.utils.run_algo import run_algorithm
coins = ['dash', 'btc', 'dash', 'etc', 'eth', 'ltc', 'nxt', 'rep', 'str', 'xmr', 'xrp', 'zec']
symbols = None
def initialize(context):
pass
def _handle_data(context, data):
global symbols
if symbols is None: symbols = [symbol(c + '_usdt') for c in coins]
print'getting history for: %s' % [s.symbol for s in symbols]
history = data.history(symbols,
['close', 'volume'],
bar_count=1, # EXCEPTION, Change to 2
frequency='5T')
#print 'history: %s' % history.shape
run_algorithm(initialize=initialize,
handle_data=_handle_data,
analyze=lambda _, results: True,
exchange_name='poloniex',
base_currency='usdt',
algo_namespace='issue-236',
live=True,
data_frequency='minute',
capital_base=3000,
simulate_orders=True)
catalyst/utils/run_algo.py
+12 -1
View File
@@ -10,6 +10,7 @@ import click
import pandas as pd
from six import string_types
import catalyst
from catalyst.data.bundles import load
from catalyst.data.data_portal import DataPortal
from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
@@ -23,7 +24,7 @@ try:
from pygments.formatters import TerminalFormatter
PYGMENTS = True
except:
except ImportError:
PYGMENTS = False
from toolz import valfilter, concatv
from functools import partial
@@ -151,6 +152,7 @@ def _run(handle_data,
'We encourage you to report any issue on GitHub: '
'https://github.com/enigmampc/catalyst/issues'
)
log.info('Catalyst version {}'.format(catalyst.__version__))
sleep(3)
if live:
@@ -261,6 +263,15 @@ def _run(handle_data,
# We still need to support bundles for other misc data, but we
# can handle this later.
if start != pd.tslib.normalize_date(start) or \
end != pd.tslib.normalize_date(end):
# todo: add to Sim_Params the option to start & end at specific times
log.warn(
"Catalyst currently starts and ends on the start and "
"end of the dates specified, respectively. We hope to "
"Modify this and support specific times in a future release."
)
data = DataPortalExchangeBacktest(
exchange_names=[exchange_name for exchange_name in exchanges],
asset_finder=None,
docs/source/beginner-tutorial.rst
+44 -156
View File
@@ -580,162 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in
the ``examples`` directory:
`dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_.
.. code-block:: python
import numpy as np
import pandas as pd
from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order, record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.utils.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
def initialize(context):
context.i = 0
context.asset = symbol('ltc_usd')
context.base_price = None
def handle_data(context, data):
# define the windows for the moving averages
short_window = 50
long_window = 200
# Skip as many bars as long_window to properly compute the average
context.i += 1
if context.i < long_window:
return
# Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_mavg = data.history(context.asset, 'price',
bar_count=short_window, frequency="1m").mean()
long_mavg = data.history(context.asset, 'price',
bar_count=long_window, frequency="1m").mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
# Save values for later inspection
record(price=price,
cash=context.portfolio.cash,
price_change=price_change,
short_mavg=short_mavg,
long_mavg=long_mavg)
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.asset):
return
# We check what's our position on our portfolio and trade accordingly
pos_amount = context.portfolio.positions[context.asset].amount
# Trading logic
if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation
exchange = list(context.exchanges.values())[0]
base_currency = exchange.base_currency.upper()
# First chart: Plot portfolio value using base_currency
ax1 = plt.subplot(411)
perf.loc[:, ['portfolio_value']].plot(ax=ax1)
ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format(
asset = context.asset.symbol,
base = base_currency
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index, 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index, 'price'],
marker='v',
s=100,
c='red',
label=''
)
# Third chart: Compare percentage change between our portfolio
# and the price of the asset
ax3 = plt.subplot(413, sharex=ax1)
perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
ax3.legend_.remove()
ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
:language: python
In order to run the code above, you have to ingest the needed data first:
@@ -16609,7 +16455,49 @@ NaN
</div>
PyCharm IDE
~~~~~~~~~~~
PyCharm is an Integrated Development Environment (IDE) used in computer
programming, specifically for the Python language. It streamlines the continuos
development of Python code, and among other things includes a debugger that
comes in handy to see the inner workings of Catalyst, and your trading
algorithms.
Install
^^^^^^^
Install PyCharm from their `Website <https://www.jetbrains.com/pycharm/download/>`__.
There is a free and open-source **Community** version.
Setup
^^^^^
1. When creating a new project in PyCharm, right under you specify the Location,
click on **Project Interpreter** to display a drop down menu
2. Select **Existing interpreter**, click the gear box right next to it and
select 'add local'. Depending on your installation, select either
"*Virtual Environemnt*" or "*Conda Environment" and click the '...' button to
navigate to your catalyst env and select the Python binary file:
``bin/python`` for Linux/MacOS installations or 'python.exe' for Windows
installs (for example: 'C:\\Users\\user\\Anaconda2\\envs\\catalyst\\python.exe').
Select OK. You may want to click on *Make available to all projects* for your
future reference. Click OK again, and create your new environment using the
set up of your virtual environment.
Alternatively, if you already have your project created, in Windows do:
1. File -> Default Settings -> Project Interpreter. Click the gear box next to
the project interpreter and select ‘add local’, and follow the steps from the
second step above.
On MacOS:
1. PyCharm -> Preferences -> Settings -> Project:’NAME_OF_PROJECT’ ->
Project Interpreter. Click the gear box next to the project interpreter
and select ‘add local’, and follow the steps from the second step above.
You should now be able to run your project/scripts in PyCharm.
Next steps
~~~~~~~~~~
docs/source/example-algos.rst
+17 -881
View File
@@ -52,35 +52,8 @@ Buy BTC Simple Algorithm
Source code: `examples/buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_
.. code-block:: python
'''
Run this example, by executing the following from your terminal:
catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
If you want to run this code using another exchange, make sure that
the asset is available on that exchange. For example, if you were to run
it for exchange Poloniex, you would need to edit the following line:
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
and specify exchange poloniex as follows:
catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
To see which assets are available on each exchange, visit:
https://www.enigma.co/catalyst/status
'''
from catalyst.api import order, record, symbol
def initialize(context):
context.asset = symbol('btc_usd')
def handle_data(context, data):
order(context.asset, 1)
record(btc = data.current(context.asset, 'price'))
.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py
:language: python
This simple algorithm does not produce any output nor displays any chart.
@@ -90,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart.
Buy and Hodl Algorithm
~~~~~~~~~~~~~~~~~~~~~~
Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
First ingest the historical pricing data needed to run this algorithm:
.. code-block:: bash
@@ -119,157 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an
earlier date, you'll get an error), and the most recent date you can choose is
one day prior to the current date.
Source code: `examples/buy_and_hodl.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_and_hodl.py>`_
.. code-block:: python
#!/usr/bin/env python
#
# Copyright 2017 Enigma MPC, Inc.
# Copyright 2015 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pandas as pd
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order_target_value, symbol, record,
cancel_order, get_open_orders, )
def initialize(context):
context.ASSET_NAME = 'btc_usd'
context.TARGET_HODL_RATIO = 0.8
context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
context.is_buying = True
context.asset = symbol(context.ASSET_NAME)
context.i = 0
def handle_data(context, data):
context.i += 1
starting_cash = context.portfolio.starting_cash
target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
reserve_value = context.RESERVE_RATIO * starting_cash
# Cancel any outstanding orders
orders = get_open_orders(context.asset) or []
for order in orders:
cancel_order(order)
# Stop buying after passing the reserve threshold
cash = context.portfolio.cash
if cash <= reserve_value:
context.is_buying = False
# Retrieve current asset price from pricing data
price = data.current(context.asset, 'price')
# Check if still buying and could (approximately) afford another purchase
if context.is_buying and cash > price:
print('buying')
# Place order to make position in asset equal to target_hodl_value
order_target_value(
context.asset,
target_hodl_value,
limit_price=price * 1.1,
)
record(
price=price,
volume=data.current(context.asset, 'volume'),
cash=cash,
starting_cash=context.portfolio.starting_cash,
leverage=context.account.leverage,
)
def analyze(context=None, results=None):
# Plot the portfolio and asset data.
ax1 = plt.subplot(611)
results[['portfolio_value']].plot(ax=ax1)
ax1.set_ylabel('Portfolio Value (USD)')
ax2 = plt.subplot(612, sharex=ax1)
ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
results[['price']].plot(ax=ax2)
trans = results.ix[[t != [] for t in results.transactions]]
buys = trans.ix[
[t[0]['amount'] > 0 for t in trans.transactions]
]
ax2.scatter(
buys.index.to_pydatetime(),
results.price[buys.index],
marker='^',
s=100,
c='g',
label=''
)
ax3 = plt.subplot(613, sharex=ax1)
results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
ax3.set_ylabel('Leverage ')
ax4 = plt.subplot(614, sharex=ax1)
results[['starting_cash', 'cash']].plot(ax=ax4)
ax4.set_ylabel('Cash (USD)')
results[[
'treasury',
'algorithm',
'benchmark',
]] = results[[
'treasury_period_return',
'algorithm_period_return',
'benchmark_period_return',
]]
ax5 = plt.subplot(615, sharex=ax1)
results[[
'treasury',
'algorithm',
'benchmark',
]].plot(ax=ax5)
ax5.set_ylabel('Percent Change')
ax6 = plt.subplot(616, sharex=ax1)
results[['volume']].plot(ax=ax6)
ax6.set_ylabel('Volume (mCoins/5min)')
plt.legend(loc=3)
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=10000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace='buy_and_hodl',
base_currency='usd',
start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True),
)
.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py
:language: python
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png
@@ -278,166 +102,13 @@ one day prior to the current date.
Dual Moving Average Crossover
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
This strategy is covered in detail in the last part of
`this tutorial <beginner-tutorial.html#history>`_.
.. code-block:: python
Source Code: `examples/dual_moving_average.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/dual_moving_average.py>`_
import numpy as np
import pandas as pd
from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import (order, record, symbol, order_target_percent,
get_open_orders)
from catalyst.exchange.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE)
def initialize(context):
context.i = 0
context.asset = symbol('ltc_usd')
context.base_price = None
def handle_data(context, data):
# define the windows for the moving averages
short_window = 50
long_window = 200
# Skip as many bars as long_window to properly compute the average
context.i += 1
if context.i < long_window:
return
# Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe.
short_mavg = data.history(context.asset, 'price',
bar_count=short_window, frequency="1m").mean()
long_mavg = data.history(context.asset, 'price',
bar_count=long_window, frequency="1m").mean()
# Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price')
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
# Save values for later inspection
record(price=price,
cash=context.portfolio.cash,
price_change=price_change,
short_mavg=short_mavg,
long_mavg=long_mavg)
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.asset)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.asset):
return
# We check what's our position on our portfolio and trade accordingly
pos_amount = context.portfolio.positions[context.asset].amount
# Trading logic
if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset
order_target_percent(context.asset, 0)
def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation
base_currency = context.exchanges.values()[0].base_currency.upper()
# First chart: Plot portfolio value using base_currency
ax1 = plt.subplot(411)
perf.loc[:, ['portfolio_value']].plot(ax=ax1)
ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format(
asset = context.asset.symbol,
base = base_currency
))
start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index, 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index, 'price'],
marker='v',
s=100,
c='red',
label=''
)
# Third chart: Compare percentage change between our portfolio
# and the price of the asset
ax3 = plt.subplot(413, sharex=ax1)
perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3)
ax3.legend_.remove()
ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True),
)
.. literalinclude:: ../../catalyst/examples/dual_moving_average.py
:language: python
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png
@@ -447,8 +118,6 @@ This strategy is covered in detail in the last part of
Mean Reversion Algorithm
~~~~~~~~~~~~~~~~~~~~~~~~
Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
This algorithm is based on a simple momentum strategy. When the cryptoasset goes
up quickly, we're going to buy; when it goes down quickly, we're going to sell.
Hopefully, we'll ride the waves.
@@ -469,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type:
python mean_reversion_simple.py
.. code-block:: python
Source code: `examples/mean_reversion_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/mean_reversion_simple.py>`_
import os
import tempfile
import time
import numpy as np
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in USD.
context.neo_eth = symbol('neo_usd')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 30
context.RSI_OVERBOUGHT = 80
context.CANDLE_SIZE = '15T'
context.start_time = time.time()
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = False
context.current_day = today
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.neo_eth variable. For this example, we're
# using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention
# for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
context.neo_eth,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(context.neo_eth, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
cash = context.portfolio.cash
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
price=price,
volume=current['volume'],
price_change=price_change,
rsi=rsi[-1],
cash=cash
)
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if context.traded_today:
return
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.neo_eth)
if len(orders) > 0:
return
# Exit if we cannot trade
if not data.can_trade(context.neo_eth):
return
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.neo_eth].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
order_target_percent(
context.neo_eth, 1, limit_price=limit_price
)
context.traded_today = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
context.neo_eth, 0, limit_price=limit_price
)
context.traded_today = True
def analyze(context=None, perf=None):
end = time.time()
log.info('elapsed time: {}'.format(end - context.start_time))
import matplotlib.pyplot as plt
# The base currency of the algo exchange
base_currency = context.exchanges.values()[0].base_currency.upper()
# Plot the portfolio value over time.
ax1 = plt.subplot(611)
perf.loc[:, 'portfolio_value'].plot(ax=ax1)
ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
# Plot the price increase or decrease over time.
ax2 = plt.subplot(612, sharex=ax1)
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.neo_eth.symbol, base=base_currency
))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'price'],
marker='v',
s=100,
c='red',
label=''
)
ax4 = plt.subplot(613, sharex=ax1)
perf.loc[:, 'cash'].plot(
ax=ax4, label='Base Currency ({})'.format(base_currency)
)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
ax5 = plt.subplot(614, sharex=ax1)
perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
ax5.set_ylabel('Percent\nChange')
ax6 = plt.subplot(615, sharex=ax1)
perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
ax6.set_ylabel('RSI')
ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
if not transaction_df.empty:
ax6.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'rsi'],
marker='^',
s=100,
c='green',
label=''
)
ax6.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'rsi'],
marker='v',
s=100,
c='red',
label=''
)
plt.legend(loc=3)
start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end/5))
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
pass
if __name__ == '__main__':
# The execution mode: backtest or live
MODE = 'backtest'
if MODE == 'backtest':
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=10000,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='usd',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
)
log.info('saved perf stats: {}'.format(out))
elif MODE == 'live':
run_algorithm(
capital_base=0.5,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
live=True,
algo_namespace=NAMESPACE,
base_currency='usd',
live_graph=False
)
.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py
:language: python
.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png
@@ -763,8 +158,6 @@ strategy.
Simple Universe
~~~~~~~~~~~~~~~
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
This example aims to provide an easy way for users to learn how to
collect data from any given exchange and select a subset of the available
currency pairs for trading. You simply need to specify the exchange and
@@ -791,142 +184,10 @@ of the file:
catalyst ingest-exchange -x bitfinex -f minute
.. code-block:: bash
python simple_universe.py
Credits: This code was originally submitted by `Abner Ayala-Acevedo
<https://github.com/abnera>`_. Thank you!
.. code-block:: python
from datetime import timedelta
import numpy as np
import pandas as pd
from catalyst import run_algorithm
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
from catalyst.api import (symbols, )
def initialize(context):
context.i = -1 # minute counter
context.exchange = context.exchanges.values()[0].name.lower()
context.base_currency = context.exchanges.values()[0].base_currency.lower()
def handle_data(context, data):
context.i += 1
lookback_days = 7 # 7 days
# current date & time in each iteration formatted into a string
now = data.current_dt
date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = now - timedelta(days=lookback_days)
# keep only the date as a string, discard the time
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute'
# update universe everyday at midnight
if not context.i % one_day_in_minutes:
context.universe = universe(context, lookback_date, date)
# get data every 30 minutes
minutes = 30
# get lookback_days of history data: that is 'lookback' number of bins
lookback = one_day_in_minutes / minutes * lookback_days
if not context.i % minutes and context.universe:
# we iterate for every pair in the current universe
for coin in context.coins:
pair = str(coin.symbol)
# Get 30 minute interval OHLCV data. This is the standard data
# required for candlestick or indicators/signals. Return Pandas
# DataFrames. 30T means 30-minute re-sampling of one minute data.
# Adjust it to your desired time interval as needed.
opened = fill(data.history(coin, 'open',
bar_count=lookback, frequency='30T')).values
high = fill(data.history(coin, 'high',
bar_count=lookback, frequency='30T')).values
low = fill(data.history(coin, 'low',
bar_count=lookback, frequency='30T')).values
close = fill(data.history(coin, 'price',
bar_count=lookback, frequency='30T')).values
volume = fill(data.history(coin, 'volume',
bar_count=lookback, frequency='30T')).values
# close[-1] is the last value in the set, which is the equivalent
# to current price (as in the most recent value)
# displays the minute price for each pair every 30 minutes
print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format(
now=now,
pair=pair,
o=opened[-1],
h=high[-1],
l=low[-1],
c=close[-1],
v=volume[-1],
))
# -------------------------------------------------------------
# --------------- Insert Your Strategy Here -------------------
# -------------------------------------------------------------
def analyze(context=None, results=None):
pass
# Get the universe for a given exchange and a given base_currency market
# Example: Poloniex BTC Market
def universe(context, lookback_date, current_date):
# get all the pairs for the given exchange
json_symbols = get_exchange_symbols(context.exchange)
# convert into a DataFrame for easier processing
df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1)
df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1)
# Filter all the pairs to get only the ones for a given base_currency
df = df[df['base_currency'] == context.base_currency]
# Filter all the pairs to ensure that pair existed in the current date range
df = df[df.start_date < lookback_date]
df = df[df.end_daily >= current_date]
context.coins = symbols(*df.symbol) # convert all the pairs to symbols
return df.symbol.tolist()
# Replace all NA, NAN or infinite values with its nearest value
def fill(series):
if isinstance(series, pd.Series):
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
elif isinstance(series, np.ndarray):
return pd.Series(series).replace(
[np.inf, -np.inf], np.nan
).ffill().bfill().values
else:
return series
if __name__ == '__main__':
start_date = pd.to_datetime('2017-11-10', utc=True)
end_date = pd.to_datetime('2017-11-13', utc=True)
performance = run_algorithm(start=start_date, end=end_date,
capital_base=100.0, # amount of base_currency
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
data_frequency='minute',
base_currency='btc',
live=False,
live_graph=False,
algo_namespace='simple_universe')
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/simple_universe.py>`_
.. literalinclude:: ../../catalyst/examples/simple_universe.py
:language: python
.. _portfolio_optimization:
@@ -940,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used
in writting the following article:
`Markowitz Portfolio Optimization for Cryptocurrencies <https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556>`_.
.. code-block:: python
Source code: `examples/simple_universe.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/portfolio_optimization.py>`_
'''
You can run this code using the Python interpreter:
$ python portfolio_optimization.py
'''
from __future__ import division
import os
import pytz
import numpy as np
import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from datetime import datetime
from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True)
def initialize(context):
# Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt')
context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return
# and asset correlations
context.window = 180
# Set the number of days between each portfolio rebalancing
context.rebalance_period = 30
context.i = 0
def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period
if context.i == 0 or context.i%context.rebalance_period == 0:
n = context.window
prices = data.history(context.assets, fields='price',
bar_count=n+1, frequency='1d')
pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n+1]
t_val = t_prices.values
tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values
# Compute daily returns (r)
r = np.asmatrix(t_val/tminus_val-1)
# Compute the expected returns of each asset with the average
# daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0))
# ###
stds = np.std(r, axis=0)
# Compute excess returns matrix (xr)
xr = r - m
# Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr),xr)/n
# Compute asset correlation matrix (informative only)
corr_m = cov_m/np.dot(np.transpose(stds),stds)
# Define portfolio optimization parameters
n_portfolios = 50000
results_array = np.zeros((3+context.nassets,n_portfolios))
for p in xrange(n_portfolios):
weights = np.random.random(context.nassets)
weights /= np.sum(weights)
w = np.asmatrix(weights)
p_r = np.sum(np.dot(w,np.transpose(m)))*365
p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
#store results in results array
results_array[0,p] = p_r
results_array[1,p] = p_std
#store Sharpe Ratio (return / volatility) - risk free rate element
#excluded for simplicity
results_array[2,p] = results_array[0,p] / results_array[1,p]
i = 0
for iw in weights:
results_array[3+i,p] = weights[i]
i += 1
#convert results array to Pandas DataFrame
results_frame = pd.DataFrame(np.transpose(results_array),
columns=['r','stdev','sharpe']+context.assets)
#locate position of portfolio with highest Sharpe Ratio
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
#locate positon of portfolio with minimum standard deviation
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
#order optimal weights for each asset
for asset in context.assets:
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset])
#create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
plt.xlabel('Volatility')
plt.ylabel('Returns')
plt.colorbar()
#plot red star to highlight position of portfolio with highest Sharpe Ratio
plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
#plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None):
# Form DataFrame with selected data
data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
# Save results in CSV file
filename = os.path.splitext(os.path.basename(__file__))[0]
data.to_csv(filename + '.csv')
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
results = run_algorithm(initialize=initialize,
handle_data=handle_data,
analyze=analyze,
start=start,
end=end,
exchange_name='poloniex',
capital_base=100000, )
.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py
:language: python
.. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ.
:align: center
docs/source/install.rst
+50 -11
View File
@@ -47,8 +47,10 @@ you can install MiniConda, which is a smaller footprint (fewer packages and
smaller size) than its big brother Anaconda, but it still contains all the
main packages needed. To install MiniConda, you can follow these steps:
1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select Python 2.7
for your Operating System.
1. Download `MiniConda <https://conda.io/miniconda.html>`_. Select either
Python 3.6 (recommended) or Python 2.7 for your Operating System. The
`Enigma Data Marketplace <https://enigmampc.github.io/marketplace/>`_ will
require Python3, that's why we are recommending to opt for the newer version.
2. Install MiniConda. See the `Installation Instructions
<https://conda.io/docs/user-guide/install/index.html>`_ if you need help.
3. Ensure the correct installation by running ``conda list`` in a Terminal
@@ -64,18 +66,27 @@ main packages needed. To install MiniConda, you can follow these steps:
Once either Conda or MiniConda has been set up you can install Catalyst:
1. Download the file `python2.7-environment.yml
<https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_.
1. Download the file `python3.6-environment.yml
<https://github.com/enigmampc/catalyst/blob/master/etc/python3.6-environment.yml>`_
(recommended) or `python2.7-environment.yml
<https://github.com/enigmampc/catalyst/blob/master/etc/python2.7-environment.yml>`_
matching your Conda installation from step #1 above.
To download, simply click on the 'Raw' button and save the file locally
to a folder you can remember. Make sure that the file gets saved with the
``.yml`` extension, and nothing like a ``.txt`` file or anything else.
2. Open a Terminal window and enter [``cd/dir``] into the directory where you
saved the above ``python2.7-environment.yml`` file.
saved the above ``.yml`` file.
3. Install using this file. This step can take about 5-10 minutes to install.
.. code-block:: bash
conda env create -f python3.6-environment.yml
or
.. code-block:: bash
conda env create -f python2.7-environment.yml
@@ -122,6 +133,14 @@ with the following steps:
2. Create the environment:
for python 2.7:
.. code-block:: bash
conda create --name catalyst python=2.7 scipy zlib
or for python 3.6:
.. code-block:: bash
conda create --name catalyst python=2.7 scipy zlib
@@ -295,6 +314,16 @@ Troubleshooting ``pip`` Install
$ sudo apt-get install python-dev
----
**Issue**:
Missing TA_Lib
**Solution**:
Follow `these instructions
<https://mrjbq7.github.io/ta-lib/install.html>`_ to install the TA_Lib Python wrapper
(and if needed, its underlying C library as well).
.. _pipenv:
Installing with ``pipenv``
@@ -443,12 +472,22 @@ about matplotlib backends, please refer to the
Windows Requirements
--------------------
In Windows, you will first need to install the `Microsoft Visual C++ Compiler
for Python 2.7
<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_. This
package contains the compiler and the set of system headers necessary for
producing binary wheels for Python 2.7 packages. If it's not already in your
system, download it and install it before proceeding to the next step.
In Windows, you will first need to install the Microsoft Visual C++ Compiler,
which is different depending on the version of Python that you plan to use:
* Python 3.5, 3.6: `Visual C++ 2015 Build Tools
<http://landinghub.visualstudio.com/visual-cpp-build-tools>`_,
which installs Visual C++ version 14.0. **This is the recommended version**
* Python 2.7: `Microsoft Visual C++ Compiler for Python 2.7
<https://www.microsoft.com/en-us/download/details.aspx?id=44266>`_, which
installs version Visual C++ version 9.0
This package contains the compiler and the set of system headers necessary for
producing binary wheels for Python packages. If it's not already in your
system, download it and install it before proceeding to the next step. If you
need additional help, or are looking for other versions of Visual C++ for
Windows (only advanced users), follow `this link <https://wiki.python.org/moin/WindowsCompilers>`_.
Once you have the above compiler installed, the easiest and best supported way
to install Catalyst in Windows is to use :ref:`Conda <conda>`. If you didn't
docs/source/live-trading.rst
+28 -10
View File
@@ -30,22 +30,24 @@ Paper Trading vs Live Trading modes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Catalyst currently supports three different modes in which you can execute your
trading algorithm. The first is backtesting, which is covered extensively in the
tutorial, and uses historical data to run your algorithm. There is no
trading algorithm. The first is **backtesting**, which is covered extensively in
the tutorial, and uses historical data to run your algorithm. There is no
interaction with the exchange in backtesting mode, and this is the first mode
that you should test any new algorithm.
Once you are confident with the simulations that you have obtained with your
algorithm in backtesting, you may switch to live trading, where you have two
different modes:
* *Paper Trading*: The simulated algorithm runs in real time, and fetches
pricing data in real time from the exchange, but the orders never reach the
exchange, and are instead kept within Catalyst and simulated. No real currency
is bought or sold. Think of it as a `backtesting happening in real time`.
* *Live Trading*: This is the proper live trading mode in which an algorithm
runs in real time, fetching pricing data from live exchanges and placing orders
against the exchange. Real currency is transacted on the exchange driven by the
algorithm.
* **Paper Trading**: The simulated algorithm runs in real time, and fetches
pricing data in real time from the exchange, but the orders never reach the
exchange, and are instead kept within Catalyst and simulated. No real currency
is bought or sold. Think of it as a `backtesting happening in real time`.
* **Live Trading**: This is the proper live trading mode in which an algorithm
runs in real time, fetching pricing data from live exchanges and placing
orders against the exchange. Real currency is transacted on the exchange
driven by the algorithm.
These three modes are controlled by the following variables:
@@ -174,6 +176,22 @@ Here is the breakdown of the new arguments:
- ``simulate_orders``: Enables the paper trading mode, in which orders are
simulated in Catalyst instead of processed on the exchange. It defaults to
``True``.
- ``end_date``: When setting the end_date to a time in the **future**,
it will schedule the live algo to finish gracefully at the specified date.
- ``start_date``: (**Will be implemented in the future**)
The live algo starts by default in the present, as mentioned above.
by setting the start_date to a time in the future, the algorithm would
essentially sleep and when the predefined time comes, it would start executing.
The `catalyst live` command offers additional parameters.
You can learn more by running the following from the command line:
.. code-block:: bash
catalyst live --help
Here is a complete algorithm for reference:
`Buy Low and Sell High <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_low_sell_high_live.py>`_
docs/source/releases.rst
+37
View File
@@ -2,6 +2,43 @@
Release Notes
=============
Version 0.5.4
^^^^^^^^^^^^^
**Release Date**: 2018-03-14
Build
~~~~~
- Switched Data Marketplace from Ropstein testnet to Rinkeby testnet after
incorporating changes resulting from the marketplace contract audit
- Several usability improvements of the Data Marketplace that make the
`--dataset` parameter optional. If it is not included in the command line,
will list available datasets, and let you choose interactively.
Bug Fixes
~~~~~~~~~
- Fix Binance requirement of symbol to be included in the cancelled order
:issue:`204`
- Fix `notenoughcasherror` when an open order is filled minutes later
:issue:`237`
- Properly handle of empty candles received from exchanges :issue:`236`
- Added a function to reduce open orders amount from calculated target/amount
for target orders :issue:`243`
- Fix missing file in live trading mode on date change :issue:`252`,
:issue:`253`
- Upgraded Data Marketplace to Web3==4.0.0b11, which was breaking some
functionality from prior version 4.0.0b7 :issue:`257`
- Always request more data to avoid empty bars and always give the exact bar
number :issue:`260`
Documentation
~~~~~~~~~~~~~
- PyCharm documentation :issue:`195`
- Added TA-Lib troubleshooting instructions
- Added instructions on how to create a Conda environment for Python 3.6, and
updated Visual C++ instructions for Windows and Python 3
- Linking example algorithms in the documentation to their sources
Version 0.5.3
^^^^^^^^^^^^^
**Release Date**: 2018-02-09
etc/python2.7-environment.yml
+4 -2
View File
@@ -22,8 +22,10 @@ dependencies:
- bcolz==0.12.1
- bottleneck==1.2.1
- chardet==3.0.4
- ccxt==1.10.1049
- web3==4.0.0b7
- ccxt==1.10.1094
# The Enigma Data Marketplace requires Python3 because it depends on
# web3, which requires Python3, as building its dependencies breaks in Python2
# - web3==4.0.0b7
- requests-toolbelt==0.8.0
- click==6.7
- contextlib2==0.5.5
etc/requirements.txt
+2 -2
View File
@@ -81,8 +81,8 @@ empyrical==0.2.1
tables==3.3.0
#Catalyst dependencies
ccxt==1.10.1049
ccxt==1.10.1094
boto3==1.4.8
redo==1.6
web3==4.0.0b7
web3==4.0.0b11; python_version > '3.4'
requests-toolbelt==0.8.0
etc/requirements_marketplace.txt
-2
View File
@@ -1,2 +0,0 @@
web3==4.0.0b7
requests-toolbelt==0.8.0
tests/exchange/test_bundle.py
+5 -5
View File
@@ -11,7 +11,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle, \
BUNDLE_NAME_TEMPLATE
from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
get_df_from_arrays
from exchange.utils.datetime_utils import get_start_dt
from catalyst.exchange.utils.datetime_utils import get_start_dt
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import get_exchange
from catalyst.exchange.utils.stats_utils import df_to_string
@@ -42,7 +42,7 @@ class TestExchangeBundle:
def test_ingest_minute(self):
data_frequency = 'minute'
exchange_name = 'poloniex'
exchange_name = 'binance'
exchange = get_exchange(exchange_name)
exchange_bundle = ExchangeBundle(exchange)
@@ -50,8 +50,8 @@ class TestExchangeBundle:
exchange.get_asset('eth_btc')
]
start = pd.to_datetime('2016-03-01', utc=True)
end = pd.to_datetime('2017-11-1', utc=True)
start = pd.to_datetime('2018-03-01', utc=True)
end = pd.to_datetime('2018-03-8', utc=True)
log.info('ingesting exchange bundle {}'.format(exchange_name))
exchange_bundle.ingest(
@@ -101,7 +101,7 @@ class TestExchangeBundle:
# data_frequency = 'daily'
# include_symbols = 'neo_btc,bch_btc,eth_btc'
exchange_name = 'bitfinex'
exchange_name = 'binance'
data_frequency = 'minute'
exchange = get_exchange(exchange_name)
tests/exchange/test_exchange_utils.py
+175
View File
@@ -0,0 +1,175 @@
from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, \
forward_fill_df_if_needed, get_candles_df
from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
from datetime import timedelta
from pandas import Timestamp, DataFrame, concat
import numpy as np
class TestExchangeUtils(WithLogger, ZiplineTestCase):
@classmethod
def get_specific_field_from_df(cls, df, field, asset):
new_df = DataFrame(df[field])
new_df.columns = [asset]
new_df.index.name = None
return new_df
@classmethod
def verify_forward_fill_df_if_needed(cls, candles, periods, expected_df):
observed_df = forward_fill_df_if_needed(
transform_candles_to_df(candles),
periods)
assert (expected_df.equals(observed_df))
@classmethod
def verify_get_candles_df(cls, assets, candles, end_fixed_dt,
expected_df, check_next_candle=False):
# run on all the fields
for field in ['volume', 'open', 'close', 'high', 'low']:
field_dt = cls.get_specific_field_from_df(expected_df,
field,
assets[0])
# run on several timestamps
for delta in range(5):
end_dt = end_fixed_dt + timedelta(minutes=delta)
assert (field_dt.equals(get_candles_df({assets[0]: candles},
field, '5T', 3,
end_dt=end_dt)))
field_dt_a1 = cls.get_specific_field_from_df(expected_df,
field,
assets[0])
field_dt_a2 = cls.get_specific_field_from_df(expected_df,
field,
assets[1])
observed_df = get_candles_df({assets[0]: candles,
assets[1]: candles},
field, '5T', 3,
end_dt=end_dt)
assert (observed_df.equals(concat([field_dt_a1, field_dt_a2],
axis=1)))
if check_next_candle:
# one candle forward
end_dt = end_fixed_dt + timedelta(minutes=6)
observed_df = get_candles_df({assets[0]: candles,
assets[1]: candles},
field, '5T', 3,
end_dt=end_dt)
assert (not observed_df.equals(concat([field_dt_a1,
field_dt_a2],
axis=1)))
assert (concat([field_dt_a1, field_dt_a2],
axis=1)[1:].equals(observed_df[:-1]))
def test_get_candles_df(self):
assets = ['btc_usdt', 'eth_usdt']
# test forward fill in the end
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
}]
expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 593.0, 'volume': 0.0, 'low': 593.0,
'close': 593.0, 'open': 593.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
periods = [Timestamp('2018-03-01 09:45:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:50:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:55:00+0000', tz='UTC')]
expected_df = transform_candles_to_df(expected)
self.verify_forward_fill_df_if_needed(candles, periods,
expected_df)
self.verify_get_candles_df(assets, candles, periods[2],
expected_df, True)
# test forward fill in the middle
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 595.0, 'volume': 0.0, 'low': 595.0,
'close': 595.0, 'open': 595.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected_df = transform_candles_to_df(expected)
self.verify_forward_fill_df_if_needed(candles, periods, expected_df)
self.verify_get_candles_df(assets, candles, periods[2], expected_df)
# test "forward fill" at the beginning
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected = [{'high': np.NaN, 'volume': 0.0, 'low': np.NaN,
'close': np.NaN, 'open': np.NaN,
'last_traded': Timestamp('2018-03-01 09:45:00+0000',
tz='UTC')
},
{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000',
tz='UTC')
},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000',
tz='UTC')
}]
expected_df = transform_candles_to_df(expected)
self.verify_forward_fill_df_if_needed(candles, periods, expected_df)
# Not the same due to dropna - commenting out for now
# self.verify_get_candles_df(assets, candles, periods[2], expected_df)
tests/exchange/test_suites/test_suite_bundle.py
+22 -16
View File
@@ -37,7 +37,7 @@ class TestSuiteBundle:
return data_portal
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
freq, data_frequency, data_portal):
freq, data_frequency, data_portal, field):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
@@ -62,8 +62,8 @@ class TestSuiteBundle:
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing data for {}/{} with {} timeframe until {}'.format(
exchange.name, symbols, freq, end_dt
'comparing {} for {}/{} with {} timeframe until {}'.format(
field, exchange.name, symbols, freq, end_dt
)
)
data['bundle'] = data_portal.get_history_window(
@@ -71,13 +71,13 @@ class TestSuiteBundle:
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field='close',
field=field,
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle first / last row:\n{}'.format(
data['bundle'].iloc[[-1, 0]]
'the bundle data:\n{}'.format(
data['bundle']
)
)
candles = exchange.get_candles(
@@ -88,14 +88,14 @@ class TestSuiteBundle:
)
data['exchange'] = get_candles_df(
candles=candles,
field='close',
field=field,
freq=freq,
bar_count=bar_count,
end_dt=end_dt,
)
print(
'the exchange first / last row:\n{}'.format(
data['exchange'].iloc[[-1, 0]]
'the exchange data:\n{}'.format(
data['exchange']
)
)
for source in data:
@@ -107,19 +107,21 @@ class TestSuiteBundle:
print('saved {} test results: {}'.format(end_dt, folder))
assert_frame_equal(
right=data['bundle'],
left=data['exchange'],
right=data['bundle'][:-1],
left=data['exchange'][:-1],
check_less_precise=1,
)
try:
assert_frame_equal(
right=data['bundle'],
left=data['exchange'],
right=data['bundle'][:-1],
left=data['exchange'][:-1],
check_less_precise=min([a.decimals for a in assets]),
)
except Exception as e:
print('Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e))
print(
'Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e)
)
with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
handle.write(e.args[0])
@@ -203,8 +205,11 @@ class TestSuiteBundle:
frequencies = exchange.get_candle_frequencies(data_frequency)
freq = random.sample(frequencies, 1)[0]
rnd = random.SystemRandom()
# field = rnd.choice(['open', 'high', 'low', 'close', 'volume'])
field = rnd.choice(['volume'])
bar_count = random.randint(1, 10)
bar_count = random.randint(3, 6)
assets = select_random_assets(
exchange.assets, asset_population
@@ -229,6 +234,7 @@ class TestSuiteBundle:
freq=freq,
data_frequency=data_frequency,
data_portal=data_portal,
field=field,
)
pass
tests/marketplace/test_marketplace.py
+2 -3
View File
@@ -1,6 +1,5 @@
from catalyst.marketplace.marketplace import Marketplace
from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
import pandas as pd
class TestMarketplace(WithLogger, ZiplineTestCase):
@@ -16,12 +15,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
def test_subscribe(self):
marketplace = Marketplace()
marketplace.subscribe('marketcap2222')
marketplace.subscribe('marketcap')
pass
def test_ingest(self):
marketplace = Marketplace()
ds_def = marketplace.ingest('marketcap1234')
ds_def = marketplace.ingest('marketcap')
pass
def test_publish(self):