mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-21 12:30:16 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Victor Grau Serrat
|
3399e22ea2 | ||
|
Victor Grau Serrat
|
79e5dec813 |
@@ -40,7 +40,6 @@ develop-eggs
|
||||
coverage.xml
|
||||
htmlcov
|
||||
nosetests.xml
|
||||
.python-version
|
||||
|
||||
# C Extensions
|
||||
*.o
|
||||
|
||||
+3
-3
@@ -11,13 +11,13 @@
|
||||
#
|
||||
# https://127.0.0.1
|
||||
#
|
||||
# Default password is 'jupyter'. To provide another, see:
|
||||
# default password is jupyter. to provide another, see:
|
||||
# http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
|
||||
#
|
||||
# Once generated, you can pass the new value via `docker run --env` the first time
|
||||
# once generated, you can pass the new value via `docker run --env` the first time
|
||||
# you start the container.
|
||||
#
|
||||
# You can also run an algo using the docker exec command. For example:
|
||||
# You can also run an algo using the docker exec command. For example:
|
||||
#
|
||||
# docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
|
||||
#
|
||||
|
||||
+4
-4
@@ -5,7 +5,7 @@
|
||||
#
|
||||
# Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
|
||||
#
|
||||
# docker build -t quantopian/catalyst -f Dockerfile .
|
||||
# docker build -t quantopian/catalyst -f Dockerfile
|
||||
#
|
||||
# To run the container:
|
||||
#
|
||||
@@ -15,13 +15,13 @@
|
||||
#
|
||||
# https://127.0.0.1
|
||||
#
|
||||
# Default password is 'jupyter'. To provide another, see:
|
||||
# default password is jupyter. to provide another, see:
|
||||
# http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
|
||||
#
|
||||
# Once generated, you can pass the new value via `docker run --env` the first time
|
||||
# once generated, you can pass the new value via `docker run --env` the first time
|
||||
# you start the container.
|
||||
#
|
||||
# You can also run an algo using the docker exec command. For example:
|
||||
# You can also run an algo using the docker exec command. For example:
|
||||
#
|
||||
# docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
|
||||
#
|
||||
|
||||
+3
-72
@@ -1,72 +1,3 @@
|
||||
.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.jpg
|
||||
:target: https://enigmampc.github.io/catalyst
|
||||
:align: center
|
||||
:alt: Enigma | Catalyst
|
||||
|
||||
|version tag|
|
||||
|version status|
|
||||
|discord|
|
||||
|twitter|
|
||||
|
||||
|
|
||||
|
||||
Catalyst is an algorithmic trading library for crypto-assets written in Python.
|
||||
It allows trading strategies to be easily expressed and backtested against
|
||||
historical data (with daily and minute resolution), providing analytics and
|
||||
insights regarding a particular strategy's performance. Catalyst also supports
|
||||
live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex,
|
||||
and Poloniex) with more being added over time. Catalyst empowers users to share
|
||||
and curate data and build profitable, data-driven investment strategies. Please
|
||||
visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst, or
|
||||
refer to the `whitepaper <https://www.enigma.co/enigma_catalyst.pdf>`_ for
|
||||
further technical details.
|
||||
|
||||
Catalyst builds on top of the well-established
|
||||
`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to
|
||||
minimize structural changes to the general API to maximize compatibility with
|
||||
existing trading algorithms, developer knowledge, and tutorials. Join us on
|
||||
`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel
|
||||
for questions around Catalyst, algorithmic trading and technical support.
|
||||
|
||||
Overview
|
||||
========
|
||||
|
||||
- Ease of use: Catalyst tries to get out of your way so that you can
|
||||
focus on algorithm development. See
|
||||
`examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_
|
||||
provided.
|
||||
- Support for several of the top crypto-exchanges by trading volume:
|
||||
`Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
|
||||
and `Poloniex <https://www.poloniex.com>`_.
|
||||
- Secure: You and only you have access to each exchange API keys for your accounts.
|
||||
- Input of historical pricing data of all crypto-assets by exchange,
|
||||
with daily and minute resolution. See
|
||||
`Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
|
||||
- Backtesting and live-trading functionality, with a seamless transition
|
||||
between the two modes.
|
||||
- Output of performance statistics are based on Pandas DataFrames to
|
||||
integrate nicely into the existing PyData eco-system.
|
||||
- Statistic and machine learning libraries like matplotlib, scipy,
|
||||
statsmodels, and sklearn support development, analysis, and
|
||||
visualization of state-of-the-art trading systems.
|
||||
- Addition of Bitcoin price (btc_usdt) as a benchmark for comparing
|
||||
performance across trading algorithms.
|
||||
|
||||
Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
|
||||
|
||||
|
||||
|
||||
|
||||
.. |version tag| image:: https://img.shields.io/pypi/v/enigma-catalyst.svg
|
||||
:target: https://pypi.python.org/pypi/enigma-catalyst
|
||||
|
||||
.. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
|
||||
:target: https://pypi.python.org/pypi/enigma-catalyst
|
||||
|
||||
.. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
|
||||
:target: https://discordapp.com/invite/SJK32GY
|
||||
|
||||
.. |twitter| image:: https://img.shields.io/twitter/follow/enigmampc.svg?style=social&label=Follow&style=flat-square
|
||||
:target: https://twitter.com/enigmampc
|
||||
|
||||
|
||||
All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_
|
||||
can be found in the
|
||||
`documentation website <https://enigmampc.github.io/catalyst>`_.
|
||||
+10
-4
@@ -29,14 +29,11 @@ from ._version import get_versions
|
||||
from . algorithm import TradingAlgorithm
|
||||
from . import api
|
||||
|
||||
from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
|
||||
|
||||
__version__ = get_versions()['version']
|
||||
del get_versions
|
||||
|
||||
# PERF: Fire a warning if calendars were instantiated during catalyst import.
|
||||
# Having calendars doesn't break anything per-se, but it makes catalyst imports
|
||||
# noticeably slower, which becomes particularly noticeable in the Zipline CLI.
|
||||
from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
|
||||
if global_calendar_dispatcher._calendars:
|
||||
import warnings
|
||||
warnings.warn(
|
||||
@@ -47,6 +44,10 @@ if global_calendar_dispatcher._calendars:
|
||||
del global_calendar_dispatcher
|
||||
|
||||
|
||||
__version__ = get_versions()['version']
|
||||
del get_versions
|
||||
|
||||
|
||||
def load_ipython_extension(ipython):
|
||||
from .__main__ import catalyst_magic
|
||||
ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst')
|
||||
@@ -68,6 +69,7 @@ if os.name == 'nt':
|
||||
_()
|
||||
del _
|
||||
|
||||
|
||||
__all__ = [
|
||||
'TradingAlgorithm',
|
||||
'api',
|
||||
@@ -78,3 +80,7 @@ __all__ = [
|
||||
'run_algorithm',
|
||||
'utils',
|
||||
]
|
||||
|
||||
from ._version import get_versions
|
||||
__version__ = get_versions()['version']
|
||||
del get_versions
|
||||
|
||||
+33
-77
@@ -3,14 +3,14 @@ import os
|
||||
from functools import wraps
|
||||
|
||||
import click
|
||||
import sys
|
||||
import logbook
|
||||
import pandas as pd
|
||||
from six import text_type
|
||||
|
||||
from catalyst.data import bundles as bundles_module
|
||||
from catalyst.exchange.exchange_bundle import ExchangeBundle
|
||||
from catalyst.exchange.utils.exchange_utils import delete_algo_folder
|
||||
from catalyst.exchange.exchange_utils import delete_algo_folder
|
||||
from catalyst.exchange.factory import get_exchange
|
||||
from catalyst.utils.cli import Date, Timestamp
|
||||
from catalyst.utils.run_algo import _run, load_extensions
|
||||
|
||||
@@ -194,7 +194,9 @@ def ipython_only(option):
|
||||
@click.option(
|
||||
'-x',
|
||||
'--exchange-name',
|
||||
help='The name of the targeted exchange.',
|
||||
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
|
||||
help='The name of the targeted exchange (supported: bitfinex,'
|
||||
' bittrex, poloniex).',
|
||||
)
|
||||
@click.option(
|
||||
'-n',
|
||||
@@ -256,9 +258,8 @@ def run(ctx,
|
||||
ctx.fail("must specify a base currency with '-c' in backtest mode")
|
||||
|
||||
if capital_base is None:
|
||||
ctx.fail("must specify a capital base with '--capital-base'")
|
||||
|
||||
click.echo('Running in backtesting mode.', sys.stdout)
|
||||
ctx.fail("must specify a capital base with '--capital-base'"
|
||||
" in backtest mode")
|
||||
|
||||
perf = _run(
|
||||
initialize=None,
|
||||
@@ -283,15 +284,11 @@ def run(ctx,
|
||||
exchange=exchange_name,
|
||||
algo_namespace=algo_namespace,
|
||||
base_currency=base_currency,
|
||||
analyze_live=None,
|
||||
live_graph=False,
|
||||
simulate_orders=True,
|
||||
auth_aliases=None,
|
||||
stats_output=None,
|
||||
live_graph=False
|
||||
)
|
||||
|
||||
if output == '-':
|
||||
click.echo(str(perf), sys.stdout)
|
||||
click.echo(str(perf))
|
||||
elif output != os.devnull: # make the catalyst magic not write any data
|
||||
perf.to_pickle(output)
|
||||
|
||||
@@ -339,12 +336,6 @@ def catalyst_magic(line, cell=None):
|
||||
type=click.File('r'),
|
||||
help='The file that contains the algorithm to run.',
|
||||
)
|
||||
@click.option(
|
||||
'--capital-base',
|
||||
type=float,
|
||||
show_default=True,
|
||||
help='The amount of capital (in base_currency) allocated to trading.',
|
||||
)
|
||||
@click.option(
|
||||
'-t',
|
||||
'--algotext',
|
||||
@@ -383,7 +374,9 @@ def catalyst_magic(line, cell=None):
|
||||
@click.option(
|
||||
'-x',
|
||||
'--exchange-name',
|
||||
help='The name of the targeted exchange.',
|
||||
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
|
||||
help='The name of the targeted exchange (supported: bitfinex,'
|
||||
' bittrex, poloniex).',
|
||||
)
|
||||
@click.option(
|
||||
'-n',
|
||||
@@ -396,38 +389,15 @@ def catalyst_magic(line, cell=None):
|
||||
help='The base currency used to calculate statistics '
|
||||
'(e.g. usd, btc, eth).',
|
||||
)
|
||||
@click.option(
|
||||
'-e',
|
||||
'--end',
|
||||
type=Date(tz='utc', as_timestamp=True),
|
||||
help='An optional end date at which to stop the execution.',
|
||||
)
|
||||
@click.option(
|
||||
'--live-graph/--no-live-graph',
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help='Display live graph.',
|
||||
)
|
||||
@click.option(
|
||||
'--simulate-orders/--no-simulate-orders',
|
||||
is_flag=True,
|
||||
default=True,
|
||||
help='Simulating orders enable the paper trading mode. No orders will be '
|
||||
'sent to the exchange unless set to false.',
|
||||
)
|
||||
@click.option(
|
||||
'--auth-aliases',
|
||||
default=None,
|
||||
help='Authentication file aliases for the specified exchanges. By default,'
|
||||
'each exchange uses the "auth.json" file in the exchange folder. '
|
||||
'Specifying an "auth2" alias would use "auth2.json". It should be '
|
||||
'specified like this: "[exchange_name],[alias],..." For example, '
|
||||
'"binance,auth2" or "binance,auth2,bittrex,auth2".',
|
||||
)
|
||||
@click.pass_context
|
||||
def live(ctx,
|
||||
algofile,
|
||||
capital_base,
|
||||
algotext,
|
||||
define,
|
||||
output,
|
||||
@@ -436,10 +406,7 @@ def live(ctx,
|
||||
exchange_name,
|
||||
algo_namespace,
|
||||
base_currency,
|
||||
end,
|
||||
live_graph,
|
||||
auth_aliases,
|
||||
simulate_orders):
|
||||
live_graph):
|
||||
"""Trade live with the given algorithm.
|
||||
"""
|
||||
if (algotext is not None) == (algofile is not None):
|
||||
@@ -450,22 +417,11 @@ def live(ctx,
|
||||
|
||||
if exchange_name is None:
|
||||
ctx.fail("must specify an exchange name '-x'")
|
||||
|
||||
if algo_namespace is None:
|
||||
ctx.fail("must specify an algorithm name '-n' in live execution mode")
|
||||
|
||||
if base_currency is None:
|
||||
ctx.fail("must specify a base currency '-c' in live execution mode")
|
||||
|
||||
if capital_base is None:
|
||||
ctx.fail("must specify a capital base with '--capital-base'")
|
||||
|
||||
if simulate_orders:
|
||||
click.echo('Running in paper trading mode.', sys.stdout)
|
||||
|
||||
else:
|
||||
click.echo('Running in live trading mode.', sys.stdout)
|
||||
|
||||
perf = _run(
|
||||
initialize=None,
|
||||
handle_data=None,
|
||||
@@ -475,12 +431,12 @@ def live(ctx,
|
||||
algotext=algotext,
|
||||
defines=define,
|
||||
data_frequency=None,
|
||||
capital_base=capital_base,
|
||||
capital_base=None,
|
||||
data=None,
|
||||
bundle=None,
|
||||
bundle_timestamp=None,
|
||||
start=None,
|
||||
end=end,
|
||||
end=None,
|
||||
output=output,
|
||||
print_algo=print_algo,
|
||||
local_namespace=local_namespace,
|
||||
@@ -489,15 +445,11 @@ def live(ctx,
|
||||
exchange=exchange_name,
|
||||
algo_namespace=algo_namespace,
|
||||
base_currency=base_currency,
|
||||
live_graph=live_graph,
|
||||
analyze_live=None,
|
||||
simulate_orders=simulate_orders,
|
||||
auth_aliases=auth_aliases,
|
||||
stats_output=None,
|
||||
live_graph=live_graph
|
||||
)
|
||||
|
||||
if output == '-':
|
||||
click.echo(str(perf), sys.stdout)
|
||||
click.echo(str(perf))
|
||||
elif output != os.devnull: # make the catalyst magic not write any data
|
||||
perf.to_pickle(output)
|
||||
|
||||
@@ -508,7 +460,9 @@ def live(ctx,
|
||||
@click.option(
|
||||
'-x',
|
||||
'--exchange-name',
|
||||
help='The name of the exchange bundle to ingest.',
|
||||
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
|
||||
help='The name of the exchange bundle to ingest (supported: bitfinex,'
|
||||
' bittrex, poloniex).',
|
||||
)
|
||||
@click.option(
|
||||
'-f',
|
||||
@@ -566,8 +520,7 @@ def live(ctx,
|
||||
default=False,
|
||||
help='Report potential anomalies found in data bundles.'
|
||||
)
|
||||
@click.pass_context
|
||||
def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
|
||||
def ingest_exchange(exchange_name, data_frequency, start, end,
|
||||
include_symbols, exclude_symbols, csv, show_progress,
|
||||
verbose, validate):
|
||||
"""
|
||||
@@ -579,7 +532,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
|
||||
|
||||
exchange_bundle = ExchangeBundle(exchange_name)
|
||||
|
||||
click.echo('Ingesting exchange bundle {}...'.format(exchange_name), sys.stdout)
|
||||
click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
|
||||
exchange_bundle.ingest(
|
||||
data_frequency=data_frequency,
|
||||
include_symbols=include_symbols,
|
||||
@@ -602,18 +555,19 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
|
||||
@click.pass_context
|
||||
def clean_algo(ctx, algo_namespace):
|
||||
click.echo(
|
||||
'Cleaning algo state: {}'.format(algo_namespace),
|
||||
sys.stdout
|
||||
'Cleaning algo state: {}'.format(algo_namespace)
|
||||
)
|
||||
delete_algo_folder(algo_namespace)
|
||||
click.echo('Done', sys.stdout)
|
||||
click.echo('Done')
|
||||
|
||||
|
||||
@main.command(name='clean-exchange')
|
||||
@click.option(
|
||||
'-x',
|
||||
'--exchange-name',
|
||||
help='The name of the exchange bundle to ingest.',
|
||||
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
|
||||
help='The name of the exchange bundle to ingest (supported: bitfinex,'
|
||||
' bittrex, poloniex).',
|
||||
)
|
||||
@click.option(
|
||||
'-f',
|
||||
@@ -633,11 +587,11 @@ def clean_exchange(ctx, exchange_name, data_frequency):
|
||||
|
||||
exchange_bundle = ExchangeBundle(exchange_name)
|
||||
|
||||
click.echo('Cleaning exchange bundle {}...'.format(exchange_name), sys.stdout)
|
||||
click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
|
||||
exchange_bundle.clean(
|
||||
data_frequency=data_frequency,
|
||||
)
|
||||
click.echo('Done', sys.stdout)
|
||||
click.echo('Done')
|
||||
|
||||
|
||||
@main.command()
|
||||
@@ -652,7 +606,9 @@ def clean_exchange(ctx, exchange_name, data_frequency):
|
||||
@click.option(
|
||||
'-x',
|
||||
'--exchange-name',
|
||||
help='The name of the exchange bundle to ingest.',
|
||||
type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
|
||||
help='The name of the exchange bundle to ingest (supported: bitfinex,'
|
||||
' bittrex, poloniex).',
|
||||
)
|
||||
@click.option(
|
||||
'-c',
|
||||
@@ -758,7 +714,7 @@ def bundles():
|
||||
# because there were no entries, print a single message indicating that
|
||||
# no ingestions have yet been made.
|
||||
for timestamp in ingestions or ["<no ingestions>"]:
|
||||
click.echo("%s %s" % (bundle, timestamp), sys.stdout)
|
||||
click.echo("%s %s" % (bundle, timestamp))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
@@ -124,6 +124,7 @@ from catalyst.utils.events import (
|
||||
from catalyst.utils.factory import create_simulation_parameters
|
||||
from catalyst.utils.math_utils import (
|
||||
tolerant_equals,
|
||||
round_if_near_integer,
|
||||
round_nearest
|
||||
)
|
||||
from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
|
||||
@@ -1484,6 +1485,7 @@ class TradingAlgorithm(object):
|
||||
"""
|
||||
Converts the number of shares to the smallest tradable lot size for
|
||||
the asset being ordered.
|
||||
|
||||
"""
|
||||
return round_nearest(amount, asset.min_trade_size)
|
||||
|
||||
@@ -1521,7 +1523,6 @@ class TradingAlgorithm(object):
|
||||
self.updated_portfolio(),
|
||||
self.get_datetime(),
|
||||
self.trading_client.current_data)
|
||||
|
||||
@staticmethod
|
||||
def __convert_order_params_for_blotter(limit_price, stop_price, style):
|
||||
"""
|
||||
|
||||
+15
-63
@@ -17,7 +17,6 @@
|
||||
"""
|
||||
Cythonized Asset object.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
|
||||
cimport cython
|
||||
@@ -39,7 +38,7 @@ from numpy cimport int64_t
|
||||
import warnings
|
||||
cimport numpy as np
|
||||
|
||||
from catalyst.exchange.utils.exchange_utils import get_sid
|
||||
from catalyst.exchange.exchange_utils import get_sid
|
||||
from catalyst.utils.calendars import get_calendar
|
||||
from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
|
||||
|
||||
@@ -397,18 +396,11 @@ cdef class Future(Asset):
|
||||
|
||||
cdef class TradingPair(Asset):
|
||||
cdef readonly float leverage
|
||||
cdef readonly object quote_currency
|
||||
cdef readonly object market_currency
|
||||
cdef readonly object base_currency
|
||||
cdef readonly object end_daily
|
||||
cdef readonly object end_minute
|
||||
cdef readonly object exchange_symbol
|
||||
cdef readonly float maker
|
||||
cdef readonly float taker
|
||||
cdef readonly int trading_state
|
||||
cdef readonly object data_source
|
||||
cdef readonly float max_trade_size
|
||||
cdef readonly float lot
|
||||
cdef readonly int decimals
|
||||
|
||||
_kwargnames = frozenset({
|
||||
'sid',
|
||||
@@ -421,19 +413,12 @@ cdef class TradingPair(Asset):
|
||||
'exchange',
|
||||
'exchange_full',
|
||||
'leverage',
|
||||
'quote_currency',
|
||||
'market_currency',
|
||||
'base_currency',
|
||||
'end_daily',
|
||||
'end_minute',
|
||||
'exchange_symbol',
|
||||
'min_trade_size',
|
||||
'max_trade_size',
|
||||
'lot',
|
||||
'maker',
|
||||
'taker',
|
||||
'trading_state',
|
||||
'data_source',
|
||||
'decimals'
|
||||
'min_trade_size'
|
||||
})
|
||||
def __init__(self,
|
||||
object symbol,
|
||||
@@ -449,17 +434,10 @@ cdef class TradingPair(Asset):
|
||||
object first_traded=None,
|
||||
object auto_close_date=None,
|
||||
object exchange_full=None,
|
||||
float min_trade_size=0.0001,
|
||||
float max_trade_size=1000000,
|
||||
float maker=0.0015,
|
||||
float taker=0.0025,
|
||||
float lot=0,
|
||||
int decimals = 8,
|
||||
int trading_state=0,
|
||||
object data_source='catalyst'):
|
||||
object min_trade_size=None):
|
||||
"""
|
||||
Replicates the Asset constructor with some built-in conventions
|
||||
and adds properties for leverage and fees.
|
||||
and a new 'leverage' attribute.
|
||||
|
||||
Symbol
|
||||
------
|
||||
@@ -491,6 +469,8 @@ cdef class TradingPair(Asset):
|
||||
highest volume and market cap generally benefit from high leverage.
|
||||
New currencies from ICO generally cannot be leveraged.
|
||||
|
||||
The leverage value is either None or and integer.
|
||||
|
||||
Leverage allows you to open a larger position with a smaller amount
|
||||
of funds. For example, if you open a $5,000 position in BTC/USD
|
||||
with 5:1 leverage, only one-fifth of this amount, or $1000, will be
|
||||
@@ -500,11 +480,6 @@ cdef class TradingPair(Asset):
|
||||
the position. If you open with 1:1 leverage, $5,000 of your balance
|
||||
will be tied to the position.
|
||||
|
||||
Fees
|
||||
----
|
||||
Exchanges generally charge a taker (taking from the order book) or
|
||||
maker (adding to the order book) fee.
|
||||
|
||||
:param symbol:
|
||||
:param exchange:
|
||||
:param start_date:
|
||||
@@ -519,17 +494,11 @@ cdef class TradingPair(Asset):
|
||||
:param auto_close_date:
|
||||
:param exchange_full:
|
||||
:param min_trade_size:
|
||||
:param max_trade_size:
|
||||
:param maker:
|
||||
:param taker:
|
||||
:param data_source
|
||||
:param decimals
|
||||
:param lot
|
||||
"""
|
||||
|
||||
symbol = symbol.lower()
|
||||
try:
|
||||
self.base_currency, self.quote_currency = symbol.split('_')
|
||||
self.market_currency, self.base_currency = symbol.split('_')
|
||||
except Exception as e:
|
||||
raise InvalidSymbolError(symbol=symbol, error=e)
|
||||
|
||||
@@ -543,14 +512,11 @@ cdef class TradingPair(Asset):
|
||||
asset_name = ' / '.join(symbol.split('_')).upper()
|
||||
|
||||
if start_date is None:
|
||||
start_date = pd.to_datetime('2009-1-1', utc=True)
|
||||
start_date = pd.Timestamp.utcnow()
|
||||
|
||||
if end_date is None:
|
||||
end_date = pd.Timestamp.utcnow() + timedelta(days=365)
|
||||
|
||||
if lot == 0 and min_trade_size > 0:
|
||||
lot = min_trade_size
|
||||
|
||||
super().__init__(
|
||||
sid,
|
||||
exchange,
|
||||
@@ -561,26 +527,19 @@ cdef class TradingPair(Asset):
|
||||
first_traded=first_traded,
|
||||
auto_close_date=auto_close_date,
|
||||
exchange_full=exchange_full,
|
||||
min_trade_size=min_trade_size,
|
||||
min_trade_size=min_trade_size
|
||||
)
|
||||
|
||||
self.maker = maker
|
||||
self.taker = taker
|
||||
self.leverage = leverage
|
||||
self.end_daily = end_daily
|
||||
self.end_minute = end_minute
|
||||
self.exchange_symbol = exchange_symbol
|
||||
self.trading_state = trading_state
|
||||
self.data_source = data_source
|
||||
self.max_trade_size = max_trade_size
|
||||
self.lot = lot
|
||||
self.decimals = decimals
|
||||
|
||||
def __repr__(self):
|
||||
return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
|
||||
'Introduced On: {start_date}, ' \
|
||||
'Market Currency: {market_currency}, ' \
|
||||
'Base Currency: {base_currency}, ' \
|
||||
'Quote Currency: {quote_currency}, ' \
|
||||
'Exchange Leverage: {leverage}, ' \
|
||||
'Minimum Trade Size: {min_trade_size} ' \
|
||||
'Last daily ingestion: {end_daily} ' \
|
||||
@@ -589,7 +548,7 @@ cdef class TradingPair(Asset):
|
||||
sid=self.sid,
|
||||
exchange=self.exchange,
|
||||
start_date=self.start_date,
|
||||
quote_currency=self.quote_currency,
|
||||
market_currency=self.market_currency,
|
||||
base_currency=self.base_currency,
|
||||
leverage=self.leverage,
|
||||
min_trade_size=self.min_trade_size,
|
||||
@@ -601,7 +560,6 @@ cdef class TradingPair(Asset):
|
||||
"""
|
||||
Convert to a python dict.
|
||||
"""
|
||||
#TODO: missing fields
|
||||
super_dict = super(TradingPair, self).to_dict()
|
||||
super_dict['end_daily'] = self.end_daily
|
||||
super_dict['end_minute'] = self.end_minute
|
||||
@@ -620,7 +578,7 @@ cdef class TradingPair(Asset):
|
||||
-------
|
||||
boolean: whether the asset's exchange is open at the given minute.
|
||||
"""
|
||||
#TODO: make more dymanic to catch holds
|
||||
#TODO: consider implementing to spot holds
|
||||
return True
|
||||
|
||||
cpdef __reduce__(self):
|
||||
@@ -630,7 +588,6 @@ cdef class TradingPair(Asset):
|
||||
and whose second element is a tuple of all the attributes that should
|
||||
be serialized/deserialized during pickling.
|
||||
"""
|
||||
#TODO: make sure that all fields set there
|
||||
return (self.__class__, (self.symbol,
|
||||
self.exchange,
|
||||
self.start_date,
|
||||
@@ -641,12 +598,7 @@ cdef class TradingPair(Asset):
|
||||
self.first_traded,
|
||||
self.auto_close_date,
|
||||
self.exchange_full,
|
||||
self.min_trade_size,
|
||||
self.max_trade_size,
|
||||
self.lot,
|
||||
self.decimals,
|
||||
self.taker,
|
||||
self.maker))
|
||||
self.min_trade_size))
|
||||
|
||||
def make_asset_array(int size, Asset asset):
|
||||
cdef np.ndarray out = np.empty([size], dtype=object)
|
||||
|
||||
@@ -15,4 +15,4 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
|
||||
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
|
||||
DATE_FORMAT = '%Y-%m-%d'
|
||||
|
||||
AUTO_INGEST = False
|
||||
AUTO_INGEST = False
|
||||
+135
-140
@@ -1,33 +1,25 @@
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import json, time, csv
|
||||
from datetime import datetime
|
||||
|
||||
import logbook
|
||||
import pandas as pd
|
||||
import requests
|
||||
import os, time, shutil, requests, logbook
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
|
||||
|
||||
from catalyst.exchange.utils.exchange_utils import \
|
||||
get_exchange_symbols_filename
|
||||
|
||||
DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
|
||||
DT_END = pd.to_datetime('today').value // 10 ** 9
|
||||
CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
|
||||
CONN_RETRIES = 2
|
||||
DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
|
||||
DT_END = pd.to_datetime('today').value // 10 ** 9
|
||||
CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
|
||||
CONN_RETRIES = 2
|
||||
|
||||
logbook.StderrHandler().push_application()
|
||||
log = logbook.Logger(__name__)
|
||||
|
||||
|
||||
class PoloniexCurator(object):
|
||||
'''
|
||||
OHLCV data feed generator for crypto data. Based on Poloniex market data
|
||||
'''
|
||||
|
||||
_api_path = 'https://poloniex.com/public?'
|
||||
currency_pairs = []
|
||||
_api_path = 'https://poloniex.com/public?'
|
||||
currency_pairs = []
|
||||
|
||||
def __init__(self):
|
||||
if not os.path.exists(CSV_OUT_FOLDER):
|
||||
@@ -38,9 +30,10 @@ class PoloniexCurator(object):
|
||||
CSV_OUT_FOLDER))
|
||||
log.exception(e)
|
||||
|
||||
|
||||
def get_currency_pairs(self):
|
||||
'''
|
||||
Retrieves and returns all currency pairs from the exchange
|
||||
Retrieves and returns all currency pairs from the exchange
|
||||
'''
|
||||
url = self._api_path + 'command=returnTicker'
|
||||
|
||||
@@ -52,7 +45,7 @@ class PoloniexCurator(object):
|
||||
return None
|
||||
|
||||
data = response.json()
|
||||
self.currency_pairs = []
|
||||
self.currency_pairs = []
|
||||
for ticker in data:
|
||||
self.currency_pairs.append(ticker)
|
||||
self.currency_pairs.sort()
|
||||
@@ -61,60 +54,54 @@ class PoloniexCurator(object):
|
||||
len(self.currency_pairs)
|
||||
))
|
||||
|
||||
|
||||
|
||||
def _retrieve_tradeID_date(self, row):
|
||||
'''
|
||||
Helper function that reads tradeID and date fields from CSV readline
|
||||
'''
|
||||
tId = int(row.split(',')[0])
|
||||
d = pd.to_datetime(row.split(',')[1],
|
||||
infer_datetime_format=True).value // 10 ** 9
|
||||
d = pd.to_datetime(row.split(',')[1],
|
||||
infer_datetime_format=True).value // 10 ** 9
|
||||
return tId, d
|
||||
|
||||
def retrieve_trade_history(self, currencyPair, start=DT_START,
|
||||
|
||||
def retrieve_trade_history(self, currencyPair, start=DT_START,
|
||||
end=DT_END, temp=None):
|
||||
'''
|
||||
Retrieves TradeHistory from exchange for a given currencyPair
|
||||
between start and end dates. If no start date is provided, uses
|
||||
Retrieves TradeHistory from exchange for a given currencyPair
|
||||
between start and end dates. If no start date is provided, uses
|
||||
a system-wide one (beginning of time for cryptotrading).
|
||||
If no end date is provided, 'now' is used.
|
||||
|
||||
Stores results in CSV file on disk.
|
||||
|
||||
This function is called recursively to work around the
|
||||
|
||||
This function is called recursively to work around the
|
||||
limitations imposed by the provider API.
|
||||
'''
|
||||
csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
|
||||
|
||||
'''
|
||||
Check what data we already have on disk, reading first and last
|
||||
Check what data we already have on disk, reading first and last
|
||||
lines from file. Data is stored on file from NEWEST to OLDEST.
|
||||
'''
|
||||
try:
|
||||
with open(csv_fn, 'ab+') as f:
|
||||
with open(csv_fn, 'ab+') as f:
|
||||
f.seek(0, os.SEEK_END)
|
||||
if(f.tell() > 2): # Check file size is not 0
|
||||
f.seek(0) # Go to start to read
|
||||
last_tradeID, end_file = self._retrieve_tradeID_date(
|
||||
f.readline())
|
||||
f.seek(0) # Go to start to read
|
||||
last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
|
||||
f.seek(-2, os.SEEK_END) # Jump to the 2nd last byte
|
||||
while f.read(1) != b"\n": # Until EOL is found...
|
||||
# ...jump back the read byte plus one more.
|
||||
f.seek(-2, os.SEEK_CUR)
|
||||
first_tradeID, start_file = self._retrieve_tradeID_date(
|
||||
f.readline())
|
||||
f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
|
||||
first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
|
||||
|
||||
if(end_file + 3600 * 6 > DT_END
|
||||
and (first_tradeID == 1
|
||||
or (currencyPair == 'BTC_HUC'
|
||||
and first_tradeID == 2)
|
||||
or (currencyPair == 'BTC_RIC'
|
||||
and first_tradeID == 2)
|
||||
or (currencyPair == 'BTC_XCP'
|
||||
and first_tradeID == 2)
|
||||
or (currencyPair == 'BTC_NAV'
|
||||
and first_tradeID == 4569)
|
||||
or (currencyPair == 'BTC_POT'
|
||||
and first_tradeID == 23511))):
|
||||
if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
|
||||
or (currencyPair == 'BTC_HUC' and first_tradeID == 2)
|
||||
or (currencyPair == 'BTC_RIC' and first_tradeID == 2)
|
||||
or (currencyPair == 'BTC_XCP' and first_tradeID == 2)
|
||||
or (currencyPair == 'BTC_NAV' and first_tradeID == 4569)
|
||||
or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
@@ -122,11 +109,11 @@ class PoloniexCurator(object):
|
||||
log.exception(e)
|
||||
|
||||
'''
|
||||
Poloniex API limits querying TradeHistory to intervals smaller
|
||||
Poloniex API limits querying TradeHistory to intervals smaller
|
||||
than 1 month, so we make sure that start date is never more than
|
||||
1 month apart from end date
|
||||
'''
|
||||
if(end - start > 2419200): # 60s/min * 60min/hr * 24hr/day * 28days
|
||||
if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days
|
||||
newstart = end - 2419200
|
||||
else:
|
||||
newstart = start
|
||||
@@ -137,11 +124,12 @@ class PoloniexCurator(object):
|
||||
|
||||
url = '{path}command=returnTradeHistory¤cyPair={pair}' \
|
||||
'&start={start}&end={end}'.format(
|
||||
path=self._api_path,
|
||||
pair=currencyPair,
|
||||
start=str(newstart),
|
||||
end=str(end)
|
||||
path = self._api_path,
|
||||
pair = currencyPair,
|
||||
start = str(newstart),
|
||||
end = str(end)
|
||||
)
|
||||
print url
|
||||
|
||||
attempts = 0
|
||||
success = 0
|
||||
@@ -149,14 +137,14 @@ class PoloniexCurator(object):
|
||||
try:
|
||||
response = requests.get(url)
|
||||
except Exception as e:
|
||||
log.error('Failed to retrieve trade history data'
|
||||
'for {}'.format(currencyPair))
|
||||
log.error('Failed to retrieve trade history data for {}'.format(
|
||||
currencyPair
|
||||
))
|
||||
log.exception(e)
|
||||
attempts += 1
|
||||
else:
|
||||
try:
|
||||
if(isinstance(response.json(), dict)
|
||||
and response.json()['error']):
|
||||
if isinstance(response.json(), dict) and response.json()['error']:
|
||||
log.error('Failed to to retrieve trade history data '
|
||||
'for {}: {}'.format(
|
||||
currencyPair,
|
||||
@@ -173,23 +161,24 @@ class PoloniexCurator(object):
|
||||
if not success:
|
||||
return None
|
||||
|
||||
|
||||
'''
|
||||
If we get to transactionId == 1, and we already have that on
|
||||
If we get to transactionId == 1, and we already have that on
|
||||
disk, we got to the end of TradeHistory for this coin.
|
||||
'''
|
||||
if('first_tradeID' in locals()
|
||||
and response.json()[-1]['tradeID'] == first_tradeID):
|
||||
if('first_tradeID' in locals()
|
||||
and response.json()[-1]['tradeID'] == first_tradeID):
|
||||
return
|
||||
|
||||
'''
|
||||
There are primarily two scenarios:
|
||||
a) There is newer data available that we need to add at
|
||||
the beginning of the file. We'll retrieve all what we
|
||||
need until we get to what we already have, writing it
|
||||
to a temporary file; and we will write that at the
|
||||
a) There is newer data available that we need to add at
|
||||
the beginning of the file. We'll retrieve all what we
|
||||
need until we get to what we already have, writing it
|
||||
to a temporary file; and we will write that at the
|
||||
beginning of our existing file.
|
||||
b) We are going back in time, appending at the end of
|
||||
our existing TradeHistory until the first transaction
|
||||
b) We are going back in time, appending at the end of
|
||||
our existing TradeHistory until the first transaction
|
||||
for this currencyPair
|
||||
'''
|
||||
try:
|
||||
@@ -199,7 +188,7 @@ class PoloniexCurator(object):
|
||||
temp = os.tmpfile()
|
||||
tempcsv = csv.writer(temp)
|
||||
for item in response.json():
|
||||
if(item['tradeID'] <= last_tradeID):
|
||||
if( item['tradeID'] <= last_tradeID ):
|
||||
continue
|
||||
tempcsv.writerow([
|
||||
item['tradeID'],
|
||||
@@ -208,28 +197,27 @@ class PoloniexCurator(object):
|
||||
item['rate'],
|
||||
item['amount'],
|
||||
item['total'],
|
||||
item['globalTradeID'],
|
||||
item['globalTradeID']
|
||||
])
|
||||
if(response.json()[-1]['tradeID'] > last_tradeID):
|
||||
end = pd.to_datetime(response.json()[-1]['date'],
|
||||
infer_datetime_format=True
|
||||
).value // 10**9
|
||||
self.retrieve_trade_history(currencyPair, start,
|
||||
end, temp=temp)
|
||||
if( response.json()[-1]['tradeID'] > last_tradeID ):
|
||||
end = pd.to_datetime( response.json()[-1]['date'],
|
||||
infer_datetime_format=True).value // 10 ** 9
|
||||
self.retrieve_trade_history(currencyPair, start,
|
||||
end, temp=temp)
|
||||
else:
|
||||
with open(csv_fn, 'rb+') as f:
|
||||
shutil.copyfileobj(f, temp)
|
||||
with open(csv_fn,'rb+') as f:
|
||||
shutil.copyfileobj(f,temp)
|
||||
f.seek(0)
|
||||
temp.seek(0)
|
||||
shutil.copyfileobj(temp, f)
|
||||
shutil.copyfileobj(temp,f)
|
||||
temp.close()
|
||||
end = start_file
|
||||
else:
|
||||
with open(csv_fn, 'ab') as csvfile:
|
||||
csvwriter = csv.writer(csvfile)
|
||||
for item in response.json():
|
||||
if('first_tradeID' in locals()
|
||||
and item['tradeID'] >= first_tradeID):
|
||||
if( 'first_tradeID' in locals()
|
||||
and item['tradeID'] >= first_tradeID ):
|
||||
continue
|
||||
csvwriter.writerow([
|
||||
item['tradeID'],
|
||||
@@ -240,66 +228,70 @@ class PoloniexCurator(object):
|
||||
item['total'],
|
||||
item['globalTradeID']
|
||||
])
|
||||
end = pd.to_datetime(response.json()[-1]['date'],
|
||||
infer_datetime_format=True).value//10**9
|
||||
end = pd.to_datetime(response.json()[-1]['date'],
|
||||
infer_datetime_format=True).value // 10 ** 9
|
||||
|
||||
except Exception as e:
|
||||
log.error('Error opening {}'.format(csv_fn))
|
||||
log.exception(e)
|
||||
|
||||
'''
|
||||
If we got here, we aren't done yet. Call recursively with
|
||||
If we got here, we aren't done yet. Call recursively with
|
||||
'end' times that go sequentially back in time.
|
||||
'''
|
||||
self.retrieve_trade_history(currencyPair, start, end)
|
||||
|
||||
|
||||
|
||||
def generate_ohlcv(self, df):
|
||||
'''
|
||||
Generates OHLCV dataframe from a dataframe containing all TradeHistory
|
||||
by resampling with 1-minute period
|
||||
'''
|
||||
df.set_index('date', inplace=True) # Index by date
|
||||
vol = df['total'].to_frame('volume') # set Vol aside
|
||||
df.drop('total', axis=1, inplace=True) # Drop volume data
|
||||
ohlc = df.resample('T').ohlc() # Resample OHLC 1min
|
||||
ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Rename cols
|
||||
closes = ohlc['close'].fillna(method='pad') # Pad fwd missing close
|
||||
ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill NA w/ last close
|
||||
vol = vol.resample('T').sum().fillna(0) # Add volumes by bin
|
||||
ohlcv = pd.concat([ohlc, vol], axis=1) # Concat OHLC + Vol
|
||||
df.set_index('date', inplace=True) # Index by date
|
||||
vol = df['total'].to_frame('volume') # set Vol aside
|
||||
df.drop('total', axis=1, inplace=True) # Drop volume data
|
||||
ohlc = df.resample('T').ohlc() # Resample OHLC 1min
|
||||
ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Raname columns by dropping 'rate'
|
||||
closes = ohlc['close'].fillna(method='pad') # Pad fwd missing 'close'
|
||||
ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill N/A with last close
|
||||
vol = vol.resample('T').sum().fillna(0) # Add volumes by bin
|
||||
ohlcv = pd.concat([ohlc,vol], axis=1) # Concatenate OHLC + Vol
|
||||
return ohlcv
|
||||
|
||||
def write_ohlcv_file(self, currencyPair):
|
||||
|
||||
|
||||
def write_ohlcv_file(self, currencyPair):
|
||||
'''
|
||||
Generates OHLCV data file with 1minute bars from TradeHistory on disk
|
||||
'''
|
||||
'''
|
||||
csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
|
||||
csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
|
||||
if(os.path.getmtime(csv_1min) > time.time() - 7200):
|
||||
csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
|
||||
if( os.path.getmtime(csv_1min) > time.time() - 7200 ):
|
||||
log.debug(currencyPair+': 1min data file already up to date. '
|
||||
'Delete the file if you want to rebuild it.')
|
||||
else:
|
||||
df = pd.read_csv(csv_trades,
|
||||
names=['tradeID',
|
||||
'date',
|
||||
'type',
|
||||
'rate',
|
||||
'amount',
|
||||
'total',
|
||||
'globalTradeID'],
|
||||
dtype={'tradeID': int,
|
||||
'date': str,
|
||||
'type': str,
|
||||
'rate': float,
|
||||
'amount': float,
|
||||
'total': float,
|
||||
'globalTradeID': int}
|
||||
)
|
||||
df.drop(['tradeID', 'type', 'amount', 'globalTradeID'],
|
||||
df = pd.read_csv(csv_trades,
|
||||
names=['tradeID',
|
||||
'date',
|
||||
'type',
|
||||
'rate',
|
||||
'amount',
|
||||
'total',
|
||||
'globalTradeID'],
|
||||
dtype = {'tradeID': int,
|
||||
'date': str,
|
||||
'type': str,
|
||||
'rate': float,
|
||||
'amount': float,
|
||||
'total': float,
|
||||
'globalTradeID': int }
|
||||
)
|
||||
df.drop(['tradeID','type','amount','globalTradeID'],
|
||||
axis=1, inplace=True)
|
||||
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
|
||||
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
|
||||
ohlcv = self.generate_ohlcv(df)
|
||||
try:
|
||||
try:
|
||||
with open(csv_1min, 'w') as csvfile:
|
||||
csvwriter = csv.writer(csvfile)
|
||||
for item in ohlcv.itertuples():
|
||||
@@ -314,28 +306,32 @@ class PoloniexCurator(object):
|
||||
item.volume,
|
||||
])
|
||||
except Exception as e:
|
||||
log.error('Error opening {}'.format(csv_1min))
|
||||
log.error('Error opening {}'.format(csv_fn))
|
||||
log.exception(e)
|
||||
log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
|
||||
|
||||
|
||||
|
||||
def onemin_to_dataframe(self, currencyPair, start, end):
|
||||
'''
|
||||
Returns a data frame for a given currencyPair from data on disk
|
||||
'''
|
||||
csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
|
||||
df = pd.read_csv(csv_fn, names=['date',
|
||||
'open',
|
||||
'high',
|
||||
'low',
|
||||
'close',
|
||||
'volume'])
|
||||
df['date'] = pd.to_datetime(df['date'], unit='s')
|
||||
csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
|
||||
df = pd.read_csv(csv_fn, names=['date',
|
||||
'open',
|
||||
'high',
|
||||
'low',
|
||||
'close',
|
||||
'volume']
|
||||
)
|
||||
df['date'] = pd.to_datetime(df['date'],unit='s')
|
||||
df.set_index('date', inplace=True)
|
||||
return df[start:end]
|
||||
return df[start : end]
|
||||
|
||||
|
||||
def generate_symbols_json(self, filename=None):
|
||||
'''
|
||||
Generates a symbols.json file with corresponding start_date
|
||||
Generates a symbols.json file with corresponding start_date
|
||||
for each currencyPair
|
||||
'''
|
||||
symbol_map = {}
|
||||
@@ -346,37 +342,36 @@ class PoloniexCurator(object):
|
||||
with open(filename, 'w') as symbols:
|
||||
for currencyPair in self.currency_pairs:
|
||||
start = None
|
||||
csv_fn = '{}crypto_trades-{}.csv'.format(
|
||||
CSV_OUT_FOLDER,
|
||||
currencyPair)
|
||||
with open(csv_fn, 'r') as f:
|
||||
csv_fn = '{}crypto_trades-{}.csv'.format(
|
||||
CSV_OUT_FOLDER, currencyPair)
|
||||
with open(csv_fn, 'r') as f:
|
||||
f.seek(0, os.SEEK_END)
|
||||
if(f.tell() > 2): # Check file size is not 0
|
||||
f.seek(-2, os.SEEK_END) # Jump to 2nd last byte
|
||||
while f.read(1) != b"\n": # Until EOL is found...
|
||||
# ...jump back the read byte plus one more.
|
||||
f.seek(-2, os.SEEK_CUR)
|
||||
start = pd.to_datetime(f.readline().split(',')[1],
|
||||
infer_datetime_format=True)
|
||||
f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
|
||||
start = pd.to_datetime( f.readline().split(',')[1],
|
||||
infer_datetime_format=True)
|
||||
|
||||
if(start is None):
|
||||
start = time.gmtime()
|
||||
base, market = currencyPair.lower().split('_')
|
||||
symbol = '{market}_{base}'.format(market=market, base=base)
|
||||
symbol = '{market}_{base}'.format( market=market, base=base )
|
||||
symbol_map[currencyPair] = dict(
|
||||
symbol=symbol,
|
||||
start_date=start.strftime("%Y-%m-%d")
|
||||
symbol = symbol,
|
||||
start_date = start.strftime("%Y-%m-%d")
|
||||
)
|
||||
json.dump(symbol_map, symbols, sort_keys=True, indent=2,
|
||||
separators=(',', ':'))
|
||||
json.dump(symbol_map, symbols, sort_keys=True, indent=2,
|
||||
separators=(',',':'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pc = PoloniexCurator()
|
||||
pc.get_currency_pairs()
|
||||
# pc.generate_symbols_json()
|
||||
|
||||
#pc.generate_symbols_json()
|
||||
|
||||
for currencyPair in pc.currency_pairs:
|
||||
pc.retrieve_trade_history(currencyPair)
|
||||
log.debug('{} up to date.'.format(currencyPair))
|
||||
pc.write_ohlcv_file(currencyPair)
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# These imports are necessary to force module-scope register calls to happen.
|
||||
from . import quandl # noqa
|
||||
from . import poloniex
|
||||
from .core import (
|
||||
UnknownBundle,
|
||||
bundles,
|
||||
|
||||
@@ -13,9 +13,10 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
from itertools import count
|
||||
import tarfile
|
||||
from time import sleep
|
||||
from time import time, sleep
|
||||
|
||||
from abc import abstractmethod, abstractproperty
|
||||
import logbook
|
||||
@@ -36,7 +37,6 @@ log = logbook.Logger(__name__, level=LOG_LEVEL)
|
||||
|
||||
DEFAULT_RETRIES = 5
|
||||
|
||||
|
||||
class BaseBundle(object):
|
||||
def __init__(self, asset_filter=[]):
|
||||
self._asset_filter = asset_filter
|
||||
@@ -104,11 +104,11 @@ class BaseBundle(object):
|
||||
|
||||
def post_process_symbol_metadata(self, metadata, data):
|
||||
return metadata
|
||||
|
||||
|
||||
@abstractmethod
|
||||
def fetch_raw_symbol_frame(self, api_key, symbol, start_date, end_date):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
def ingest(self,
|
||||
environ,
|
||||
asset_db_writer,
|
||||
@@ -128,7 +128,7 @@ class BaseBundle(object):
|
||||
retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5)
|
||||
|
||||
if is_compile:
|
||||
# User has instructed local compilation & ingestion of bundle.
|
||||
# User has instructed local compilation and ingestion of bundle.
|
||||
# Fetch raw metadata for all symbols.
|
||||
raw_metadata = self._fetch_metadata_frame(
|
||||
api_key,
|
||||
@@ -157,9 +157,9 @@ class BaseBundle(object):
|
||||
show_progress=show_progress,
|
||||
)
|
||||
|
||||
# Post-process metadata using cached symbol frames, and write
|
||||
# to disk. This metadata must be written before any attempt
|
||||
# to write minute data.
|
||||
# Post-process metadata using cached symbol frames, and write to
|
||||
# disk. This metadata must be written before any attempt to write
|
||||
# minute data.
|
||||
metadata = self._post_process_metadata(
|
||||
raw_metadata,
|
||||
cache,
|
||||
@@ -184,11 +184,10 @@ class BaseBundle(object):
|
||||
show_progress=show_progress,
|
||||
)
|
||||
|
||||
# For legacy purposes, this call is required to ensure the
|
||||
# database contains an appropriately initialized file
|
||||
# structure. We don't forsee a usecase for adjustments at
|
||||
# this time, but may later choose to expose this functionality
|
||||
# in the future.
|
||||
# For legacy purposes, this call is required to ensure the database
|
||||
# contains an appropriately initialized file structure. We don't
|
||||
# forsee a usecase for adjustments at this time, but may later
|
||||
# choose to expose this functionality in the future.
|
||||
adjustment_writer.write(
|
||||
splits=(
|
||||
pd.concat(self.splits, ignore_index=True)
|
||||
@@ -233,12 +232,12 @@ class BaseBundle(object):
|
||||
tar.extractall(output_dir)
|
||||
|
||||
def _fetch_metadata_frame(self,
|
||||
api_key,
|
||||
cache,
|
||||
retries=DEFAULT_RETRIES,
|
||||
environ=None,
|
||||
show_progress=False):
|
||||
|
||||
api_key,
|
||||
cache,
|
||||
retries=DEFAULT_RETRIES,
|
||||
environ=None,
|
||||
show_progress=False):
|
||||
|
||||
# Setup raw metadata iterator to fetch pages if necessary.
|
||||
raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ)
|
||||
|
||||
@@ -252,7 +251,7 @@ class BaseBundle(object):
|
||||
show_percent=False,
|
||||
) as blocks:
|
||||
metadata = pd.concat(blocks, ignore_index=True)
|
||||
|
||||
|
||||
return metadata
|
||||
|
||||
def _fetch_metadata_iter(self, api_key, cache, retries, environ):
|
||||
@@ -270,20 +269,21 @@ class BaseBundle(object):
|
||||
page_number,
|
||||
)
|
||||
break
|
||||
except ValueError:
|
||||
except ValueError as e:
|
||||
raw = pd.DataFrame([])
|
||||
break
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
log.exception(
|
||||
'Failed to load metadata from {}. '
|
||||
'Retrying.'.format(self.name)
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
'Failed to download metadata page {} after {} '
|
||||
'attempts.'.format(page_number, retries)
|
||||
)
|
||||
|
||||
|
||||
if raw.empty:
|
||||
# Empty DataFrame signals completion.
|
||||
break
|
||||
@@ -305,7 +305,7 @@ class BaseBundle(object):
|
||||
columns=self.md_column_names,
|
||||
index=metadata.index,
|
||||
)
|
||||
|
||||
|
||||
# Iterate over the available symbols, loading the asset's raw symbol
|
||||
# data from the cache. The final metadata is computed and recorded in
|
||||
# the appropriate row depending on the asset's id.
|
||||
@@ -318,22 +318,22 @@ class BaseBundle(object):
|
||||
show_percent=False,
|
||||
) as symbols_map:
|
||||
for asset_id, symbol in symbols_map:
|
||||
# Attempt to load data from disk, the cache should have an
|
||||
# entry for each symbol at this point of the execution. If one
|
||||
# does not exist, we should fail.
|
||||
# Attempt to load data from disk, the cache should have an entry
|
||||
# for each symbol at this point of the execution. If one does
|
||||
# not exist, we should fail.
|
||||
key = '{sym}.daily.frame'.format(sym=symbol)
|
||||
try:
|
||||
raw_data = cache[key]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
'Unable to find cached data for symbol:'
|
||||
' {0}'.format(symbol))
|
||||
'Unable to find cached data for symbol: {0}'.format(symbol)
|
||||
)
|
||||
|
||||
# Perform and require post-processing of metadata.
|
||||
final_symbol_metadata = self.post_process_symbol_metadata(
|
||||
asset_id,
|
||||
metadata.iloc[asset_id],
|
||||
raw_data,
|
||||
raw_data,
|
||||
)
|
||||
|
||||
# Record symbol's final metadata.
|
||||
@@ -363,8 +363,8 @@ class BaseBundle(object):
|
||||
# returns the cached data unaltered. The `should_sleep` flag
|
||||
# indicates that an API call was attempted, and that we should be
|
||||
# ensure aren't exceeding our rate limit before proceeding to the
|
||||
# next symbol. If the raw_data is updated, it is cached before
|
||||
# being returned.
|
||||
# next symbol. If the raw_data is updated, it is cached before being
|
||||
# returned.
|
||||
raw_data, should_sleep = self._maybe_update_symbol_frame(
|
||||
start_time,
|
||||
api_key,
|
||||
@@ -414,7 +414,7 @@ class BaseBundle(object):
|
||||
last = start_session
|
||||
if raw_data is not None and len(raw_data) > 0:
|
||||
last = raw_data.index[-1].tz_localize('UTC')
|
||||
|
||||
|
||||
should_sleep = False
|
||||
|
||||
# Determine time at which cached data will be considered stale.
|
||||
@@ -455,7 +455,7 @@ class BaseBundle(object):
|
||||
retries=DEFAULT_RETRIES):
|
||||
|
||||
# Data for symbol is old enough to attempt an update or is not
|
||||
# present in the cache. Fetch raw data for a single symbol
|
||||
# present in the cache. Fetch raw data for a single symbol
|
||||
# with requested intervals and frequency. Retry as necessary.
|
||||
for _ in range(retries):
|
||||
try:
|
||||
@@ -468,6 +468,7 @@ class BaseBundle(object):
|
||||
data_frequency,
|
||||
)
|
||||
raw_data.index = pd.to_datetime(raw_data.index, utc=True)
|
||||
#raw_data.index = raw_data.index.tz_localize('UTC')
|
||||
|
||||
# Filter incoming data to fit start and end sessions.
|
||||
raw_data = raw_data[
|
||||
@@ -481,7 +482,7 @@ class BaseBundle(object):
|
||||
|
||||
return raw_data
|
||||
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
log.exception(
|
||||
'Exception raised fetching {name} data. Retrying.'
|
||||
.format(name=self.name)
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
from catalyst.data.bundles.base import BaseBundle
|
||||
from catalyst.utils.memoize import lazyval
|
||||
|
||||
|
||||
class BasePricingBundle(BaseBundle):
|
||||
@lazyval
|
||||
def md_dtypes(self):
|
||||
@@ -39,7 +38,6 @@ class BasePricingBundle(BaseBundle):
|
||||
('volume', 'float64'),
|
||||
]
|
||||
|
||||
|
||||
class BaseCryptoPricingBundle(BasePricingBundle):
|
||||
@lazyval
|
||||
def calendar_name(self):
|
||||
@@ -57,7 +55,6 @@ class BaseCryptoPricingBundle(BasePricingBundle):
|
||||
def dividends(self):
|
||||
return []
|
||||
|
||||
|
||||
class BaseEquityPricingBundle(BasePricingBundle):
|
||||
@lazyval
|
||||
def calendar_name(self):
|
||||
|
||||
@@ -37,7 +37,6 @@ from catalyst.utils.cli import maybe_show_progress
|
||||
|
||||
ONE_MEGABYTE = 1024 * 1024
|
||||
|
||||
|
||||
def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
|
||||
return pth.data_path(
|
||||
asset_db_relative(bundle_name, timestr, environ, db_version),
|
||||
@@ -136,7 +135,6 @@ def ingestions_for_bundle(bundle, environ=None):
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def download_with_progress(url, chunk_size, **progress_kwargs):
|
||||
"""
|
||||
Download streaming data from a URL, printing progress information to the
|
||||
@@ -707,5 +705,4 @@ def _make_bundle_core():
|
||||
)
|
||||
|
||||
|
||||
bundles, register_bundle, register, unregister, ingest, load, clean = \
|
||||
_make_bundle_core()
|
||||
bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core()
|
||||
|
||||
@@ -14,17 +14,19 @@
|
||||
# limitations under the License.
|
||||
|
||||
import sys
|
||||
from six.moves.urllib.parse import urlencode
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from six.moves.urllib.parse import urlencode
|
||||
|
||||
from catalyst.data.bundles.core import register_bundle
|
||||
from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
|
||||
from catalyst.utils.memoize import lazyval
|
||||
|
||||
from catalyst.curate.poloniex import PoloniexCurator
|
||||
|
||||
|
||||
class PoloniexBundle(BaseCryptoPricingBundle):
|
||||
@lazyval
|
||||
def name(self):
|
||||
@@ -44,8 +46,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
|
||||
@lazyval
|
||||
def tar_url(self):
|
||||
return (
|
||||
'https://s3.amazonaws.com/enigmaco/catalyst-bundles/'
|
||||
'poloniex/poloniex-bundle.tar.gz'
|
||||
'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
|
||||
)
|
||||
|
||||
@lazyval
|
||||
@@ -66,11 +67,12 @@ class PoloniexBundle(BaseCryptoPricingBundle):
|
||||
|
||||
raw = raw.sort_index().reset_index()
|
||||
raw.rename(
|
||||
columns={'index': 'symbol'},
|
||||
columns={'index':'symbol'},
|
||||
inplace=True,
|
||||
)
|
||||
|
||||
raw = raw[raw['isFrozen'] == 0]
|
||||
|
||||
return raw
|
||||
|
||||
def post_process_symbol_metadata(self, asset_id, sym_md, sym_data):
|
||||
@@ -96,8 +98,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
|
||||
frequency):
|
||||
|
||||
# TODO: replace this with direct exchange call
|
||||
# The end date and frequency should be used to
|
||||
# calculate the number of bars
|
||||
# The end date and frequency should be used to calculate the number of bars
|
||||
if(frequency == 'minute'):
|
||||
pc = PoloniexCurator()
|
||||
raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
|
||||
@@ -115,9 +116,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
|
||||
)
|
||||
raw.set_index('date', inplace=True)
|
||||
|
||||
# BcolzDailyBarReader introduces a 1/1000 factor in the way
|
||||
# pricing is stored on disk, which we compensate here to get
|
||||
# the right pricing amounts
|
||||
# BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
|
||||
# on disk, which we compensate here to get the right pricing amounts
|
||||
# ref: data/us_equity_pricing.py
|
||||
scale = 1
|
||||
raw.loc[:, 'open'] /= scale
|
||||
@@ -139,6 +139,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
|
||||
|
||||
return self._format_polo_query(query_params)
|
||||
|
||||
|
||||
def _format_data_url(self,
|
||||
api_key,
|
||||
symbol,
|
||||
@@ -161,26 +162,27 @@ class PoloniexBundle(BaseCryptoPricingBundle):
|
||||
('end', end_date.value / 10**9),
|
||||
('period', period),
|
||||
]
|
||||
|
||||
|
||||
return self._format_polo_query(query_params)
|
||||
|
||||
|
||||
def _format_polo_query(self, query_params):
|
||||
# TODO: got against the exchange object
|
||||
return 'https://poloniex.com/public?{query}'.format(
|
||||
query=urlencode(query_params),
|
||||
)
|
||||
|
||||
|
||||
'''
|
||||
As a second parameter, you can pass an array of currency pairs
|
||||
that will be processed as an asset_filter to only process that
|
||||
'''
|
||||
As a second parameter, you can pass an array of currency pairs
|
||||
that will be processed as an asset_filter to only process that
|
||||
subset of assets in the bundle, such as:
|
||||
register_bundle(PoloniexBundle, ['USDT_BTC',])
|
||||
|
||||
For a production environment make sure to use (to bundle all pairs):
|
||||
register_bundle(PoloniexBundle)
|
||||
'''
|
||||
|
||||
if 'ingest' in sys.argv and '-c' in sys.argv:
|
||||
register_bundle(PoloniexBundle)
|
||||
else:
|
||||
register_bundle(PoloniexBundle, create_writers=False)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from six.moves.urllib.parse import urlencode
|
||||
|
||||
from catalyst.data.bundles.core import register_bundle
|
||||
@@ -25,16 +26,25 @@ from catalyst.utils.memoize import lazyval
|
||||
"""
|
||||
Module for building a complete daily dataset from Quandl's WIKI dataset.
|
||||
"""
|
||||
from itertools import count
|
||||
import tarfile
|
||||
from time import time, sleep
|
||||
from datetime import datetime
|
||||
|
||||
from logbook import Logger
|
||||
import pandas as pd
|
||||
from six.moves.urllib.parse import urlencode
|
||||
|
||||
from catalyst.utils.calendars import register_calendar_alias
|
||||
from catalyst.utils.cli import maybe_show_progress
|
||||
|
||||
from . import core as bundles
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.utils.calendars import register_calendar_alias
|
||||
|
||||
|
||||
log = Logger(__name__, level=LOG_LEVEL)
|
||||
seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
|
||||
|
||||
|
||||
class QuandlBundle(BaseEquityPricingBundle):
|
||||
@lazyval
|
||||
def name(self):
|
||||
@@ -99,8 +109,8 @@ class QuandlBundle(BaseEquityPricingBundle):
|
||||
# Filter out invalid symbols
|
||||
raw = raw[~raw.symbol.isin(self._excluded_symbols)]
|
||||
|
||||
# cut out all the other stuff in the name column. We need to
|
||||
# escape the paren because it is actually splitting on a regex
|
||||
# cut out all the other stuff in the name column
|
||||
# we need to escape the paren because it is actually splitting on a regex
|
||||
raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0)
|
||||
|
||||
return raw
|
||||
@@ -165,6 +175,7 @@ class QuandlBundle(BaseEquityPricingBundle):
|
||||
df['sid'] = asset_id
|
||||
self.splits.append(df)
|
||||
|
||||
|
||||
def _update_dividends(self, asset_id, raw_data):
|
||||
divs = raw_data.ex_dividend
|
||||
df = pd.DataFrame({'amount': divs[divs != 0]})
|
||||
@@ -175,6 +186,7 @@ class QuandlBundle(BaseEquityPricingBundle):
|
||||
df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
|
||||
self.dividends.append(df)
|
||||
|
||||
|
||||
def _format_metadata_url(self, api_key, page_number):
|
||||
"""Build the query RL for the quandl WIKI metadata.
|
||||
"""
|
||||
@@ -188,10 +200,10 @@ class QuandlBundle(BaseEquityPricingBundle):
|
||||
query_params = [('api_key', api_key)] + query_params
|
||||
|
||||
return (
|
||||
'https://www.quandl.com/api/v3/datasets.csv?'
|
||||
+ urlencode(query_params)
|
||||
'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
|
||||
)
|
||||
|
||||
|
||||
def _format_wiki_url(self,
|
||||
api_key,
|
||||
symbol,
|
||||
@@ -217,6 +229,5 @@ class QuandlBundle(BaseEquityPricingBundle):
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
register_calendar_alias('QUANDL', 'NYSE')
|
||||
register_bundle(QuandlBundle)
|
||||
|
||||
@@ -656,11 +656,11 @@ class DataPortal(object):
|
||||
return spot_value
|
||||
|
||||
def _get_minutely_spot_value(self,
|
||||
asset,
|
||||
column,
|
||||
dt,
|
||||
data_frequency,
|
||||
ffill=False):
|
||||
asset,
|
||||
column,
|
||||
dt,
|
||||
data_frequency,
|
||||
ffill=False):
|
||||
|
||||
reader = self._get_pricing_reader(data_frequency)
|
||||
|
||||
@@ -706,7 +706,7 @@ class DataPortal(object):
|
||||
asset,
|
||||
column,
|
||||
dt,
|
||||
ffill,
|
||||
ffill,
|
||||
'minute',
|
||||
)
|
||||
|
||||
|
||||
@@ -88,11 +88,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
|
||||
if self._last_available_dt is not None:
|
||||
return self._last_available_dt
|
||||
else:
|
||||
return min(r.last_available_dt for r in list(self._readers.values()))
|
||||
return min(r.last_available_dt for r in self._readers.values())
|
||||
|
||||
@lazyval
|
||||
def first_trading_day(self):
|
||||
return max(r.first_trading_day for r in list(self._readers.values()))
|
||||
return max(r.first_trading_day for r in self._readers.values())
|
||||
|
||||
def get_value(self, sid, dt, field):
|
||||
asset = self._asset_finder.retrieve_asset(sid)
|
||||
@@ -133,13 +133,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
|
||||
|
||||
def _dt_window_size(self, start_dt, end_dt):
|
||||
return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
|
||||
|
||||
|
||||
class AssetDispatchSessionBarReader(AssetDispatchBarReader):
|
||||
|
||||
def _dt_window_size(self, start_dt, end_dt):
|
||||
|
||||
+83
-26
@@ -12,6 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import datetime
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
|
||||
@@ -22,7 +23,6 @@ from pandas_datareader.data import DataReader
|
||||
from six import iteritems
|
||||
from six.moves.urllib_error import HTTPError
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.utils.calendars import get_calendar
|
||||
from . import treasuries, treasuries_can
|
||||
from .benchmarks import get_benchmark_returns
|
||||
@@ -32,6 +32,8 @@ from ..utils.paths import (
|
||||
data_root,
|
||||
)
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
|
||||
logger = logbook.Logger('Loader', level=LOG_LEVEL)
|
||||
|
||||
# Mapping from index symbol to appropriate bond data
|
||||
@@ -101,7 +103,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
|
||||
trading_day = get_calendar('OPEN').trading_day
|
||||
|
||||
# TODO: consider making configurable
|
||||
bm_symbol = 'btc_usd'
|
||||
bm_symbol = 'btc_usdt'
|
||||
# if trading_days is None:
|
||||
# trading_days = get_calendar('OPEN').schedule
|
||||
|
||||
@@ -127,13 +129,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
|
||||
# before this date.
|
||||
'''
|
||||
if(bundle_data):
|
||||
# If we are using the bundle to retrieve the cryptobenchmark, find
|
||||
# the last date for which there is trading data in the bundle
|
||||
asset = bundle_data.asset_finder.lookup_symbol(
|
||||
symbol=bm_symbol,as_of_date=None)
|
||||
# If we are using the bundle to retrieve the cryptobenchmark, find the last
|
||||
# date for which there is trading data in the bundle
|
||||
asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
|
||||
ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
|
||||
last_date = pd.to_datetime(
|
||||
bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
|
||||
last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
|
||||
else:
|
||||
last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
|
||||
'''
|
||||
@@ -142,11 +142,8 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
|
||||
if exchange is None:
|
||||
# This is exceptional, since placing the import at the module scope
|
||||
# breaks things and it's only needed here
|
||||
from catalyst.exchange.utils.factory import get_exchange
|
||||
exchange = get_exchange(
|
||||
exchange_name='bitfinex', base_currency='usd'
|
||||
)
|
||||
exchange.init()
|
||||
from catalyst.exchange.poloniex.poloniex import Poloniex
|
||||
exchange = Poloniex('', '', '')
|
||||
|
||||
benchmark_asset = exchange.get_asset(bm_symbol)
|
||||
|
||||
@@ -165,8 +162,8 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
|
||||
br.loc[start_dt] = 0
|
||||
br = br.sort_index()
|
||||
|
||||
# Override first_date for treasury data since we have it for many more
|
||||
# years and is independent of crypto data
|
||||
# Override first_date for treasury data since we have it for many more years
|
||||
# and is independent of crypto data
|
||||
first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC')
|
||||
tc = ensure_treasury_data(
|
||||
bm_symbol,
|
||||
@@ -302,14 +299,14 @@ def ensure_crypto_benchmark_data(symbol,
|
||||
|
||||
if (bundle == 'poloniex'):
|
||||
'''
|
||||
If we're using the Poloniex bundle, we'll get the benchmark from the
|
||||
bundle instead of downloading it from Poloniex every time we need it.
|
||||
Poloniex has a captcha for API queries originating from outside the US
|
||||
that prevents users abroad from getting Catalyst to work
|
||||
If we're using the Poloniex bundle, we'll get the benchmark from the bundle
|
||||
instead of downloading it from Poloniex every time we need it.
|
||||
Poloniex has a captcha for API queries originating from outside the US that
|
||||
prevents users abroad from getting Catalyst to work
|
||||
'''
|
||||
logger.info(
|
||||
('Retrieving benchmark data from bundle for {symbol!r}'
|
||||
' from {first_date} to {last_date}'),
|
||||
(
|
||||
'Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
|
||||
symbol=symbol, first_date=first_date, last_date=last_date)
|
||||
|
||||
asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,
|
||||
@@ -331,12 +328,11 @@ def ensure_crypto_benchmark_data(symbol,
|
||||
last_date)]
|
||||
|
||||
else:
|
||||
# This is how it used to be: downloading the benchmark everytime.
|
||||
# Leaving this code here to be repurposed in the future for
|
||||
# other bundles.
|
||||
# This is how it used to be: downloading the benchmark everytime.
|
||||
# Leaving this code here to be repurposed in the future for other bundles.
|
||||
logger.info(
|
||||
('Downloading benchmark data for {symbol!r}'
|
||||
' from {first_date} to {last_date}'),
|
||||
(
|
||||
'Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
|
||||
symbol=symbol, first_date=first_date, last_date=last_date)
|
||||
|
||||
raise DeprecationWarning('poloniex bundle deprecated')
|
||||
@@ -433,6 +429,67 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
|
||||
return data
|
||||
|
||||
|
||||
def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
|
||||
environ=None):
|
||||
"""
|
||||
Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
|
||||
|
||||
Parameters
|
||||
----------
|
||||
symbol : str
|
||||
The symbol for the benchmark to load.
|
||||
first_date : pd.Timestamp
|
||||
First required date for the cache.
|
||||
last_date : pd.Timestamp
|
||||
Last required date for the cache.
|
||||
now : pd.Timestamp
|
||||
The current time. This is used to prevent repeated attempts to
|
||||
re-download data that isn't available due to scheduling quirks or other
|
||||
failures.
|
||||
trading_day : pd.CustomBusinessDay
|
||||
A trading day delta. Used to find the day before first_date so we can
|
||||
get the close of the day prior to first_date.
|
||||
|
||||
We attempt to download data unless we already have data stored at the data
|
||||
cache for `symbol` whose first entry is before or on `first_date` and whose
|
||||
last entry is on or after `last_date`.
|
||||
|
||||
If we perform a download and the cache criteria are not satisfied, we wait
|
||||
at least one hour before attempting a redownload. This is determined by
|
||||
comparing the current time to the result of os.path.getmtime on the cache
|
||||
path.
|
||||
"""
|
||||
filename = get_benchmark_filename(symbol)
|
||||
data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
|
||||
environ)
|
||||
if data is not None:
|
||||
return data
|
||||
|
||||
# If no cached data was found or it was missing any dates then download the
|
||||
# necessary data.
|
||||
logger.info(
|
||||
('Downloading benchmark data for {symbol!r} '
|
||||
'from {first_date} to {last_date}'),
|
||||
symbol=symbol,
|
||||
first_date=first_date - trading_day,
|
||||
last_date=last_date
|
||||
)
|
||||
|
||||
try:
|
||||
data = get_benchmark_returns(
|
||||
symbol,
|
||||
first_date - trading_day,
|
||||
last_date,
|
||||
)
|
||||
data.to_csv(get_data_filepath(filename, environ))
|
||||
except (OSError, IOError, HTTPError):
|
||||
logger.exception('Failed to cache the new benchmark returns')
|
||||
raise
|
||||
if not has_data_for_dates(data, first_date, last_date):
|
||||
logger.warn("Still don't have expected data after redownload!")
|
||||
return data
|
||||
|
||||
|
||||
def ensure_treasury_data(symbol, first_date, last_date, now, environ=None):
|
||||
"""
|
||||
Ensure we have treasury data from treasury module associated with
|
||||
|
||||
@@ -341,10 +341,12 @@ class BcolzMinuteBarMetadata(object):
|
||||
'end_session': str(self.end_session.date()),
|
||||
# Write these values for backwards compatibility
|
||||
'first_trading_day': str(self.start_session.date()),
|
||||
'market_opens': (market_opens.values.astype('datetime64[m]').
|
||||
astype(np.int64).tolist()),
|
||||
'market_closes': (market_closes.values.astype('datetime64[m]').
|
||||
astype(np.int64).tolist()),
|
||||
'market_opens': (
|
||||
market_opens.values.astype('datetime64[m]').
|
||||
astype(np.int64).tolist()),
|
||||
'market_closes': (
|
||||
market_closes.values.astype('datetime64[m]').
|
||||
astype(np.int64).tolist()),
|
||||
}
|
||||
with open(self.metadata_path(rootdir), 'w+') as fp:
|
||||
json.dump(metadata, fp)
|
||||
@@ -1254,8 +1256,8 @@ class BcolzMinuteBarReader(MinuteBarReader):
|
||||
values = carray[start_idx:end_idx + 1]
|
||||
if indices_to_exclude is not None:
|
||||
for excl_start, excl_stop in indices_to_exclude[::-1]:
|
||||
excl_slice = np.s_[excl_start - start_idx:excl_stop
|
||||
- start_idx + 1]
|
||||
excl_slice = np.s_[
|
||||
excl_start - start_idx:excl_stop - start_idx + 1]
|
||||
values = np.delete(values, excl_slice)
|
||||
|
||||
where = values != 0
|
||||
@@ -1318,8 +1320,9 @@ class H5MinuteBarUpdateWriter(object):
|
||||
|
||||
def __init__(self, path, complevel=None, complib=None):
|
||||
self._complevel = complevel if complevel \
|
||||
is not None else self._COMPLEVEL
|
||||
self._complib = complib if complib is not None else self._COMPLIB
|
||||
is not None else self._COMPLEVEL
|
||||
self._complib = complib if complib \
|
||||
is not None else self._COMPLIB
|
||||
self._path = path
|
||||
|
||||
def write(self, frames):
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import division # Python2 req for division of ints yield float
|
||||
from __future__ import division # Python2 req to have division of ints yield float
|
||||
|
||||
from errno import ENOENT
|
||||
from functools import partial
|
||||
@@ -120,8 +120,7 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
|
||||
UINT32_MAX = iinfo(uint32).max
|
||||
UINT64_MAX = iinfo(uint64).max
|
||||
|
||||
# Provides 9 decimals resolution. Also affects _equities.pyx L220
|
||||
PRICE_ADJUSTMENT_FACTOR = 1000000000
|
||||
PRICE_ADJUSTMENT_FACTOR = 1000000000 # Provides 9 decimals resolution. Also affects _equities.pyx L220
|
||||
|
||||
|
||||
def check_uint32_safe(value, colname):
|
||||
@@ -131,7 +130,6 @@ def check_uint32_safe(value, colname):
|
||||
"for uint32" % (value, colname)
|
||||
)
|
||||
|
||||
|
||||
def check_uint64_safe(value, colname):
|
||||
if value >= UINT64_MAX:
|
||||
raise ValueError(
|
||||
@@ -324,8 +322,8 @@ class BcolzDailyBarWriter(object):
|
||||
# Maps column name -> output carray.
|
||||
columns = {
|
||||
k: carray(array([], dtype=uint64))
|
||||
if k in OHLCV
|
||||
else carray(array([], dtype=uint32))
|
||||
if k in OHLCV
|
||||
else carray(array([], dtype=uint32))
|
||||
for k in US_EQUITY_PRICING_BCOLZ_COLUMNS
|
||||
}
|
||||
|
||||
@@ -441,13 +439,11 @@ class BcolzDailyBarWriter(object):
|
||||
return raw_data
|
||||
|
||||
winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
|
||||
processed = (raw_data[list(OHLC)]
|
||||
* PRICE_ADJUSTMENT_FACTOR).astype('uint64')
|
||||
processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
|
||||
dates = raw_data.index.values.astype('datetime64[s]')
|
||||
check_uint32_safe(dates.max().view(np.int64), 'day')
|
||||
processed['day'] = dates.astype('uint32')
|
||||
processed['volume'] = (raw_data.volume
|
||||
* PRICE_ADJUSTMENT_FACTOR).astype('uint64')
|
||||
processed['volume'] = (raw_data.volume * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
|
||||
return ctable.fromdataframe(processed)
|
||||
|
||||
|
||||
@@ -500,7 +496,7 @@ class BcolzDailyBarReader(SessionBarReader):
|
||||
|
||||
The data in these columns is interpreted as follows:
|
||||
|
||||
- Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
|
||||
- Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
|
||||
as 10^9 * as-traded dollar value.
|
||||
- Day is interpreted as seconds since midnight UTC, Jan 1, 1970.
|
||||
- Id is the asset id of the row.
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
An overview of most of the trading strategies in this folder can be found in the
|
||||
`Examples Algorithms <https://enigmampc.github.io/catalyst/example-algos.html>`_
|
||||
section of our documentation website.
|
||||
@@ -6,7 +6,7 @@ from catalyst.api import (
|
||||
symbol,
|
||||
get_open_orders
|
||||
)
|
||||
from catalyst.exchange.utils.stats_utils import get_pretty_stats
|
||||
from catalyst.exchange.stats_utils import get_pretty_stats
|
||||
from catalyst.utils.run_algo import run_algorithm
|
||||
|
||||
algo_namespace = 'arbitrage_eth_btc'
|
||||
@@ -83,15 +83,15 @@ def place_orders(context, amount, buying_price, selling_price, action):
|
||||
else:
|
||||
raise ValueError('invalid order action')
|
||||
|
||||
quote_currency = enter_exchange.quote_currency
|
||||
quote_currency_amount = enter_exchange.portfolio.cash
|
||||
base_currency = enter_exchange.base_currency
|
||||
base_currency_amount = enter_exchange.portfolio.cash
|
||||
|
||||
exit_balances = exit_exchange.get_balances()
|
||||
exit_currency = context.trading_pairs[
|
||||
context.selling_exchange].quote_currency
|
||||
context.selling_exchange].market_currency
|
||||
|
||||
if exit_currency in exit_balances:
|
||||
quote_currency_amount = exit_balances[exit_currency]
|
||||
market_currency_amount = exit_balances[exit_currency]
|
||||
else:
|
||||
log.warn(
|
||||
'the selling exchange {exchange_name} does not hold '
|
||||
@@ -102,25 +102,25 @@ def place_orders(context, amount, buying_price, selling_price, action):
|
||||
)
|
||||
return
|
||||
|
||||
if quote_currency_amount < (amount * entry_price):
|
||||
adj_amount = quote_currency_amount / entry_price
|
||||
if base_currency_amount < (amount * entry_price):
|
||||
adj_amount = base_currency_amount / entry_price
|
||||
log.warn(
|
||||
'not enough {quote_currency} ({quote_currency_amount}) to buy '
|
||||
'not enough {base_currency} ({base_currency_amount}) to buy '
|
||||
'{amount}, adjusting the amount to {adj_amount}'.format(
|
||||
quote_currency=quote_currency,
|
||||
quote_currency_amount=quote_currency_amount,
|
||||
base_currency=base_currency,
|
||||
base_currency_amount=base_currency_amount,
|
||||
amount=amount,
|
||||
adj_amount=adj_amount
|
||||
)
|
||||
)
|
||||
amount = adj_amount
|
||||
|
||||
elif quote_currency_amount < amount:
|
||||
elif market_currency_amount < amount:
|
||||
log.warn(
|
||||
'not enough {currency} ({currency_amount}) to sell '
|
||||
'{amount}, aborting'.format(
|
||||
currency=exit_currency,
|
||||
currency_amount=quote_currency_amount,
|
||||
currency_amount=market_currency_amount,
|
||||
amount=amount
|
||||
)
|
||||
)
|
||||
@@ -263,20 +263,13 @@ def analyze(context, stats):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# The execution mode: backtest or live
|
||||
MODE = 'live'
|
||||
if MODE == 'live':
|
||||
run_algorithm(
|
||||
capital_base=0.1,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex,bitfinex',
|
||||
live=True,
|
||||
algo_namespace=algo_namespace,
|
||||
base_currency='btc',
|
||||
live_graph=False,
|
||||
simulate_orders=True,
|
||||
stats_output=None,
|
||||
)
|
||||
run_algorithm(
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex,bitfinex',
|
||||
live=True,
|
||||
algo_namespace=algo_namespace,
|
||||
base_currency='btc',
|
||||
live_graph=False
|
||||
)
|
||||
|
||||
@@ -15,15 +15,19 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import (order_target_value, symbol, record,
|
||||
cancel_order, get_open_orders, )
|
||||
from catalyst.api import (
|
||||
order_target_value,
|
||||
symbol,
|
||||
record,
|
||||
cancel_order,
|
||||
get_open_orders,
|
||||
)
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.ASSET_NAME = 'btc_usdt'
|
||||
context.ASSET_NAME = 'btc_usd'
|
||||
context.TARGET_HODL_RATIO = 0.8
|
||||
context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
|
||||
|
||||
@@ -61,6 +65,7 @@ def handle_data(context, data):
|
||||
context.asset,
|
||||
target_hodl_value,
|
||||
limit_price=price * 1.1,
|
||||
stop_price=price * 0.9,
|
||||
)
|
||||
|
||||
record(
|
||||
@@ -73,14 +78,15 @@ def handle_data(context, data):
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Plot the portfolio and asset data.
|
||||
ax1 = plt.subplot(611)
|
||||
results[['portfolio_value']].plot(ax=ax1)
|
||||
ax1.set_ylabel('Portfolio\nValue\n(USD)')
|
||||
ax1.set_ylabel('Portfolio Value (USD)')
|
||||
|
||||
ax2 = plt.subplot(612, sharex=ax1)
|
||||
ax2.set_ylabel('{asset}\n(USD)'.format(asset=context.ASSET_NAME))
|
||||
ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
|
||||
results[['price']].plot(ax=ax2)
|
||||
|
||||
trans = results.ix[[t != [] for t in results.transactions]]
|
||||
@@ -120,11 +126,11 @@ def analyze(context=None, results=None):
|
||||
'algorithm',
|
||||
'benchmark',
|
||||
]].plot(ax=ax5)
|
||||
ax5.set_ylabel('Percent\nChange')
|
||||
ax5.set_ylabel('Percent Change')
|
||||
|
||||
ax6 = plt.subplot(616, sharex=ax1)
|
||||
results[['volume']].plot(ax=ax6)
|
||||
ax6.set_ylabel('Volume')
|
||||
ax6.set_ylabel('Volume (mCoins/5min)')
|
||||
|
||||
plt.legend(loc=3)
|
||||
|
||||
@@ -136,13 +142,13 @@ def analyze(context=None, results=None):
|
||||
if __name__ == '__main__':
|
||||
run_algorithm(
|
||||
capital_base=10000,
|
||||
data_frequency='daily',
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
exchange_name='bitfinex',
|
||||
algo_namespace='buy_and_hodl',
|
||||
base_currency='usdt',
|
||||
start=pd.to_datetime('2015-03-01', utc=True),
|
||||
end=pd.to_datetime('2017-10-31', utc=True),
|
||||
base_currency='usd',
|
||||
start=pd.to_datetime('2017-11-01', utc=True),
|
||||
end=pd.to_datetime('2017-11-10', utc=True),
|
||||
)
|
||||
|
||||
@@ -1,49 +1,29 @@
|
||||
'''
|
||||
This is a very simple example referenced in the beginner's tutorial:
|
||||
https://enigmampc.github.io/catalyst/beginner-tutorial.html
|
||||
This is a very simple example referenced in the beginner's tutorial:
|
||||
https://enigmampc.github.io/catalyst/beginner-tutorial.html
|
||||
|
||||
Run this example, by executing the following from your terminal:
|
||||
catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
|
||||
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \
|
||||
--end 2017-9-30 -o buy_btc_simple_out.pickle
|
||||
Run this example, by executing the following from your terminal:
|
||||
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
|
||||
|
||||
If you want to run this code using another exchange, make sure that
|
||||
the asset is available on that exchange. For example, if you were to run
|
||||
it for exchange Poloniex, you would need to edit the following line:
|
||||
If you want to run this code using another exchange, make sure that
|
||||
the asset is available on that exchange. For example, if you were to run
|
||||
it for exchange Poloniex, you would need to edit the following line:
|
||||
|
||||
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
|
||||
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
|
||||
|
||||
and specify exchange poloniex as follows:
|
||||
catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
|
||||
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \
|
||||
--end 2017-9-30 -o buy_btc_simple_out.pickle
|
||||
and specify exchange poloniex as follows:
|
||||
|
||||
To see which assets are available on each exchange, visit:
|
||||
https://www.enigma.co/catalyst/status
|
||||
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
|
||||
|
||||
To see which assets are available on each exchange, visit:
|
||||
https://www.enigma.co/catalyst/status
|
||||
'''
|
||||
from catalyst import run_algorithm
|
||||
|
||||
from catalyst.api import order, record, symbol
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.asset = symbol('btc_usdt')
|
||||
|
||||
context.asset = symbol('btc_usd')
|
||||
|
||||
def handle_data(context, data):
|
||||
order(context.asset, 1)
|
||||
record(btc=data.current(context.asset, 'price'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_algorithm(
|
||||
capital_base=10000,
|
||||
data_frequency='daily',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='buy_and_hodl',
|
||||
base_currency='usdt',
|
||||
start=pd.to_datetime('2015-03-01', utc=True),
|
||||
end=pd.to_datetime('2017-10-31', utc=True),
|
||||
)
|
||||
record(btc = data.current(context.asset, 'price'))
|
||||
@@ -1,5 +1,15 @@
|
||||
'''
|
||||
This algorithm requires an additional library (ta-lib) beyond those required by catalyst.
|
||||
Install it first by running:
|
||||
$ pip install TA-Lib
|
||||
|
||||
If you get build errors like "fatal error: ta-lib/ta_libc.h: No such file or directory"
|
||||
it typically means that it can't find the underlying TA-Lib library and needs to be installed.
|
||||
See https://mrjbq7.github.io/ta-lib/install.html for instructions on how to install
|
||||
the required dependencies.
|
||||
'''
|
||||
|
||||
import talib
|
||||
import pandas as pd
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.api import (
|
||||
@@ -9,52 +19,58 @@ from catalyst.api import (
|
||||
record,
|
||||
get_open_orders,
|
||||
)
|
||||
from catalyst.exchange.utils.stats_utils import get_pretty_stats
|
||||
from catalyst.utils.run_algo import run_algorithm
|
||||
from catalyst.exchange.stats_utils import get_pretty_stats
|
||||
|
||||
algo_namespace = 'buy_the_dip_live'
|
||||
log = Logger('buy low sell high')
|
||||
algo_namespace = 'buy_low_sell_high_xrp'
|
||||
log = Logger(algo_namespace)
|
||||
|
||||
|
||||
def initialize(context):
|
||||
log.info('initializing algo')
|
||||
context.ASSET_NAME = 'btc_usdt'
|
||||
context.ASSET_NAME = 'XRP_USDT'
|
||||
context.asset = symbol(context.ASSET_NAME)
|
||||
|
||||
context.TARGET_POSITIONS = 30
|
||||
context.TARGET_POSITIONS = 5000
|
||||
context.PROFIT_TARGET = 0.1
|
||||
context.SLIPPAGE_ALLOWED = 0.02
|
||||
context.SLIPPAGE_ALLOWED = 0.05
|
||||
|
||||
context.retry_check_open_orders = 10
|
||||
context.retry_update_portfolio = 10
|
||||
context.retry_order = 5
|
||||
|
||||
context.swallow_errors = True
|
||||
|
||||
context.errors = []
|
||||
pass
|
||||
|
||||
|
||||
def _handle_data(context, data):
|
||||
price = data.current(context.asset, 'price')
|
||||
log.info('got price {price}'.format(price=price))
|
||||
|
||||
prices = data.history(
|
||||
context.asset,
|
||||
fields='price',
|
||||
bar_count=20,
|
||||
frequency='1D'
|
||||
frequency='15m'
|
||||
)
|
||||
|
||||
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
|
||||
log.info('got rsi: {}'.format(rsi))
|
||||
|
||||
# Buying more when RSI is low, this should lower our cost basis
|
||||
if rsi <= 30:
|
||||
buy_increment = 1
|
||||
buy_increment = 50
|
||||
elif rsi <= 40:
|
||||
buy_increment = 0.5
|
||||
buy_increment = 20
|
||||
elif rsi <= 70:
|
||||
buy_increment = 0.2
|
||||
buy_increment = 5
|
||||
else:
|
||||
buy_increment = 0.1
|
||||
buy_increment = None
|
||||
|
||||
cash = context.portfolio.cash
|
||||
log.info('base currency available: {cash}'.format(cash=cash))
|
||||
|
||||
price = data.current(context.asset, 'price')
|
||||
log.info('got price {price}'.format(price=price))
|
||||
|
||||
record(
|
||||
price=price,
|
||||
rsi=rsi,
|
||||
@@ -84,8 +100,8 @@ def _handle_data(context, data):
|
||||
|
||||
if price < cost_basis:
|
||||
is_buy = True
|
||||
elif (position.amount > 0
|
||||
and price > cost_basis * (1 + context.PROFIT_TARGET)):
|
||||
elif position.amount > 0 and \
|
||||
price > cost_basis * (1 + context.PROFIT_TARGET):
|
||||
profit = (price * position.amount) - (cost_basis * position.amount)
|
||||
log.info('closing position, taking profit: {}'.format(profit))
|
||||
order_target_percent(
|
||||
@@ -122,11 +138,11 @@ def _handle_data(context, data):
|
||||
|
||||
def handle_data(context, data):
|
||||
log.info('handling bar {}'.format(data.current_dt))
|
||||
# try:
|
||||
_handle_data(context, data)
|
||||
# except Exception as e:
|
||||
# log.warn('aborting the bar on error {}'.format(e))
|
||||
# context.errors.append(e)
|
||||
try:
|
||||
_handle_data(context, data)
|
||||
except Exception as e:
|
||||
log.warn('aborting the bar on error {}'.format(e))
|
||||
context.errors.append(e)
|
||||
|
||||
log.info('completed bar {}, total execution errors {}'.format(
|
||||
data.current_dt,
|
||||
@@ -140,32 +156,3 @@ def handle_data(context, data):
|
||||
def analyze(context, stats):
|
||||
log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
live = True
|
||||
if live:
|
||||
run_algorithm(
|
||||
capital_base=0.001,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='binance',
|
||||
live=True,
|
||||
algo_namespace=algo_namespace,
|
||||
base_currency='btc',
|
||||
simulate_orders=True,
|
||||
)
|
||||
else:
|
||||
run_algorithm(
|
||||
capital_base=10000,
|
||||
data_frequency='daily',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='buy_and_hodl',
|
||||
base_currency='usdt',
|
||||
start=pd.to_datetime('2015-03-01', utc=True),
|
||||
end=pd.to_datetime('2017-10-31', utc=True),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import talib
|
||||
from logbook import Logger
|
||||
|
||||
import pandas as pd
|
||||
from catalyst.api import (
|
||||
order,
|
||||
order_target_percent,
|
||||
symbol,
|
||||
record,
|
||||
get_open_orders,
|
||||
)
|
||||
from catalyst.exchange.stats_utils import get_pretty_stats
|
||||
from catalyst.utils.run_algo import run_algorithm
|
||||
|
||||
algo_namespace = 'buy_the_dip_live'
|
||||
log = Logger('buy low sell high')
|
||||
|
||||
|
||||
def initialize(context):
|
||||
log.info('initializing algo')
|
||||
context.ASSET_NAME = 'btc_usdt'
|
||||
context.asset = symbol(context.ASSET_NAME)
|
||||
|
||||
context.TARGET_POSITIONS = 30
|
||||
context.PROFIT_TARGET = 0.1
|
||||
context.SLIPPAGE_ALLOWED = 0.02
|
||||
|
||||
context.retry_check_open_orders = 10
|
||||
context.retry_update_portfolio = 10
|
||||
context.retry_order = 5
|
||||
|
||||
context.errors = []
|
||||
pass
|
||||
|
||||
|
||||
def _handle_data(context, data):
|
||||
price = data.current(context.asset, 'price')
|
||||
log.info('got price {price}'.format(price=price))
|
||||
|
||||
prices = data.history(
|
||||
context.asset,
|
||||
fields='price',
|
||||
bar_count=20,
|
||||
frequency='1d'
|
||||
)
|
||||
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
|
||||
log.info('got rsi: {}'.format(rsi))
|
||||
|
||||
# Buying more when RSI is low, this should lower our cost basis
|
||||
if rsi <= 30:
|
||||
buy_increment = 1
|
||||
elif rsi <= 40:
|
||||
buy_increment = 0.5
|
||||
elif rsi <= 70:
|
||||
buy_increment = 0.2
|
||||
else:
|
||||
buy_increment = 0.1
|
||||
|
||||
cash = context.portfolio.cash
|
||||
log.info('base currency available: {cash}'.format(cash=cash))
|
||||
|
||||
record(
|
||||
price=price,
|
||||
rsi=rsi,
|
||||
)
|
||||
|
||||
orders = get_open_orders(context.asset)
|
||||
if orders:
|
||||
log.info('skipping bar until all open orders execute')
|
||||
return
|
||||
|
||||
is_buy = False
|
||||
cost_basis = None
|
||||
if context.asset in context.portfolio.positions:
|
||||
position = context.portfolio.positions[context.asset]
|
||||
|
||||
cost_basis = position.cost_basis
|
||||
log.info(
|
||||
'found {amount} positions with cost basis {cost_basis}'.format(
|
||||
amount=position.amount,
|
||||
cost_basis=cost_basis
|
||||
)
|
||||
)
|
||||
|
||||
if position.amount >= context.TARGET_POSITIONS:
|
||||
log.info('reached positions target: {}'.format(position.amount))
|
||||
return
|
||||
|
||||
if price < cost_basis:
|
||||
is_buy = True
|
||||
elif position.amount > 0 and \
|
||||
price > cost_basis * (1 + context.PROFIT_TARGET):
|
||||
profit = (price * position.amount) - (cost_basis * position.amount)
|
||||
log.info('closing position, taking profit: {}'.format(profit))
|
||||
order_target_percent(
|
||||
asset=context.asset,
|
||||
target=0,
|
||||
limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
|
||||
)
|
||||
else:
|
||||
log.info('no buy or sell opportunity found')
|
||||
else:
|
||||
is_buy = True
|
||||
|
||||
if is_buy:
|
||||
if buy_increment is None:
|
||||
log.info('the rsi is too high to consider buying {}'.format(rsi))
|
||||
return
|
||||
|
||||
if price * buy_increment > cash:
|
||||
log.info('not enough base currency to consider buying')
|
||||
return
|
||||
|
||||
log.info(
|
||||
'buying position cheaper than cost basis {} < {}'.format(
|
||||
price,
|
||||
cost_basis
|
||||
)
|
||||
)
|
||||
order(
|
||||
asset=context.asset,
|
||||
amount=buy_increment,
|
||||
limit_price=price * (1 + context.SLIPPAGE_ALLOWED)
|
||||
)
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
log.info('handling bar {}'.format(data.current_dt))
|
||||
# try:
|
||||
_handle_data(context, data)
|
||||
# except Exception as e:
|
||||
# log.warn('aborting the bar on error {}'.format(e))
|
||||
# context.errors.append(e)
|
||||
|
||||
log.info('completed bar {}, total execution errors {}'.format(
|
||||
data.current_dt,
|
||||
len(context.errors)
|
||||
))
|
||||
|
||||
if len(context.errors) > 0:
|
||||
log.info('the errors:\n{}'.format(context.errors))
|
||||
|
||||
|
||||
def analyze(context, stats):
|
||||
log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
|
||||
pass
|
||||
|
||||
|
||||
run_algorithm(
|
||||
capital_base=100000,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
start=pd.to_datetime('2017-5-01', utc=True),
|
||||
end=pd.to_datetime('2017-10-16', utc=True),
|
||||
base_currency='usdt',
|
||||
data_frequency='daily'
|
||||
)
|
||||
# run_algorithm(
|
||||
# initialize=initialize,
|
||||
# handle_data=handle_data,
|
||||
# analyze=analyze,
|
||||
# exchange_name='poloniex',
|
||||
# live=True,
|
||||
# algo_namespace=algo_namespace,
|
||||
# base_currency='btc'
|
||||
# )
|
||||
@@ -1,17 +1,16 @@
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from logbook import Logger
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import (record, symbol, order_target_percent,
|
||||
get_open_orders)
|
||||
from catalyst.exchange.utils.stats_utils import extract_transactions
|
||||
from catalyst.api import (order, record, symbol, order_target_percent,
|
||||
get_open_orders)
|
||||
from catalyst.exchange.stats_utils import extract_transactions
|
||||
|
||||
NAMESPACE = 'dual_moving_average'
|
||||
log = Logger(NAMESPACE)
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.i = 0
|
||||
context.asset = symbol('ltc_usd')
|
||||
@@ -26,22 +25,16 @@ def handle_data(context, data):
|
||||
# Skip as many bars as long_window to properly compute the average
|
||||
context.i += 1
|
||||
if context.i < long_window:
|
||||
return
|
||||
return
|
||||
|
||||
# Compute moving averages calling data.history() for each
|
||||
# moving average with the appropriate parameters. We choose to use
|
||||
# minute bars for this simulation -> freq="1m"
|
||||
# Returns a pandas dataframe.
|
||||
short_mavg = data.history(context.asset,
|
||||
'price',
|
||||
bar_count=short_window,
|
||||
frequency="1m",
|
||||
).mean()
|
||||
long_mavg = data.history(context.asset,
|
||||
'price',
|
||||
bar_count=long_window,
|
||||
frequency="1m",
|
||||
).mean()
|
||||
short_mavg = data.history(context.asset, 'price',
|
||||
bar_count=short_window, frequency="1m").mean()
|
||||
long_mavg = data.history(context.asset, 'price',
|
||||
bar_count=long_window, frequency="1m").mean()
|
||||
|
||||
# Let's keep the price of our asset in a more handy variable
|
||||
price = data.current(context.asset, 'price')
|
||||
@@ -74,18 +67,17 @@ def handle_data(context, data):
|
||||
|
||||
# Trading logic
|
||||
if short_mavg > long_mavg and pos_amount == 0:
|
||||
# we buy 100% of our portfolio for this asset
|
||||
order_target_percent(context.asset, 1)
|
||||
# we buy 100% of our portfolio for this asset
|
||||
order_target_percent(context.asset, 1)
|
||||
elif short_mavg < long_mavg and pos_amount > 0:
|
||||
# we sell all our positions for this asset
|
||||
order_target_percent(context.asset, 0)
|
||||
# we sell all our positions for this asset
|
||||
order_target_percent(context.asset, 0)
|
||||
|
||||
|
||||
def analyze(context, perf):
|
||||
|
||||
# Get the base_currency that was passed as a parameter to the simulation
|
||||
exchange = list(context.exchanges.values())[0]
|
||||
base_currency = exchange.base_currency.upper()
|
||||
base_currency = context.exchanges.values()[0].base_currency.upper()
|
||||
|
||||
# First chart: Plot portfolio value using base_currency
|
||||
ax1 = plt.subplot(411)
|
||||
@@ -97,13 +89,11 @@ def analyze(context, perf):
|
||||
|
||||
# Second chart: Plot asset price, moving averages and buys/sells
|
||||
ax2 = plt.subplot(412, sharex=ax1)
|
||||
perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot(
|
||||
ax=ax2,
|
||||
label='Price')
|
||||
perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
|
||||
ax2.legend_.remove()
|
||||
ax2.set_ylabel('{asset}\n({base})'.format(
|
||||
asset=context.asset.symbol,
|
||||
base=base_currency
|
||||
asset = context.asset.symbol,
|
||||
base = base_currency
|
||||
))
|
||||
start, end = ax2.get_ylim()
|
||||
ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
|
||||
@@ -160,4 +150,4 @@ if __name__ == '__main__':
|
||||
base_currency='usd',
|
||||
start=pd.to_datetime('2017-9-22', utc=True),
|
||||
end=pd.to_datetime('2017-9-23', utc=True),
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright 2017 Enigma MPC, Inc.
|
||||
# Copyright 2014 Quantopian, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from catalyst.api import (
|
||||
order_target_percent,
|
||||
record,
|
||||
symbol,
|
||||
get_open_orders,
|
||||
set_max_leverage,
|
||||
schedule_function,
|
||||
date_rules,
|
||||
attach_pipeline,
|
||||
pipeline_output,
|
||||
)
|
||||
|
||||
from catalyst.pipeline import Pipeline
|
||||
from catalyst.pipeline.data import CryptoPricing
|
||||
from catalyst.pipeline.factors.crypto import VWAP
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.ASSET_NAME = 'USDT_BTC'
|
||||
context.TARGET_INVESTMENT_RATIO = 0.8
|
||||
context.SHORT_WINDOW = 30
|
||||
context.LONG_WINDOW = 100
|
||||
|
||||
# For all trading pairs in the poloniex bundle, the default denomination
|
||||
# currently supported by Catalyst is 1/1000th of a full coin. Use this
|
||||
# constant to scale the price of up to that of a full coin if desired.
|
||||
context.TICK_SIZE = 1000.0
|
||||
|
||||
context.i = 0
|
||||
context.asset = symbol(context.ASSET_NAME)
|
||||
|
||||
set_max_leverage(1.0)
|
||||
|
||||
attach_pipeline(make_pipeline(context), 'vwap_pipeline')
|
||||
|
||||
schedule_function(
|
||||
rebalance,
|
||||
time_rules=times_rules.every_minute(),
|
||||
)
|
||||
|
||||
|
||||
def before_trading_start(context, data):
|
||||
context.pipeline_data = pipeline_output('vwap_pipeline')
|
||||
|
||||
def make_pipeline(context):
|
||||
return Pipeline(
|
||||
columns={
|
||||
'price': CryptoPricing.open.latest,
|
||||
'volume': CryptoPricing.volume.latest,
|
||||
'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
|
||||
'long_mavg': VWAP(window_length=context.LONG_WINDOW),
|
||||
}
|
||||
)
|
||||
|
||||
def rebalance(context, data):
|
||||
context.i += 1
|
||||
|
||||
# skip first LONG_WINDOW bars to fill windows
|
||||
if context.i < context.LONG_WINDOW:
|
||||
return
|
||||
|
||||
# get pipeline data for asset of interest
|
||||
pipeline_data = context.pipeline_data
|
||||
pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]
|
||||
|
||||
# retrieve long and short moving averages from pipeline
|
||||
short_mavg = pipeline_data.short_mavg
|
||||
long_mavg = pipeline_data.long_mavg
|
||||
price = pipeline_data.price
|
||||
volume = pipeline_data.volume
|
||||
|
||||
# check that order has not already been placed
|
||||
open_orders = get_open_orders()
|
||||
if context.asset not in open_orders:
|
||||
# check that the asset of interest can currently be traded
|
||||
if data.can_trade(context.asset):
|
||||
# adjust portfolio based on comparison of long and short vwap
|
||||
if short_mavg > long_mavg:
|
||||
order_target_percent(
|
||||
context.asset,
|
||||
context.TARGET_INVESTMENT_RATIO,
|
||||
)
|
||||
elif short_mavg < long_mavg:
|
||||
order_target_percent(
|
||||
context.asset,
|
||||
0.0,
|
||||
)
|
||||
|
||||
record(
|
||||
price=price,
|
||||
cash=context.portfolio.cash,
|
||||
leverage=context.account.leverage,
|
||||
short_mavg=short_mavg,
|
||||
long_mavg=long_mavg,
|
||||
volume=volume,
|
||||
)
|
||||
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Plot the portfolio and asset data.
|
||||
ax1 = plt.subplot(611)
|
||||
results[['portfolio_value']].plot(ax=ax1)
|
||||
ax1.set_ylabel('Portfolio value (USD)')
|
||||
|
||||
ax2 = plt.subplot(612, sharex=ax1)
|
||||
ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
|
||||
(context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
|
||||
|
||||
trans = results.ix[[t != [] for t in results.transactions]]
|
||||
amounts = [t[0]['amount'] for t in trans.transactions]
|
||||
|
||||
buys = trans.ix[
|
||||
[t[0]['amount'] > 0 for t in trans.transactions]
|
||||
]
|
||||
sells = trans.ix[
|
||||
[t[0]['amount'] < 0 for t in trans.transactions]
|
||||
]
|
||||
|
||||
ax2.plot(
|
||||
buys.index,
|
||||
context.TICK_SIZE * results.price[buys.index],
|
||||
'^',
|
||||
markersize=10,
|
||||
color='g',
|
||||
)
|
||||
ax2.plot(
|
||||
sells.index,
|
||||
context.TICK_SIZE * results.price[sells.index],
|
||||
'v',
|
||||
markersize=10,
|
||||
color='r',
|
||||
)
|
||||
|
||||
ax3 = plt.subplot(613, sharex=ax1)
|
||||
results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
|
||||
ax3.set_ylabel('Leverage (USD)')
|
||||
|
||||
ax4 = plt.subplot(614, sharex=ax1)
|
||||
results[['cash']].plot(ax=ax4)
|
||||
ax4.set_ylabel('Cash (USD)')
|
||||
|
||||
results[[
|
||||
'treasury',
|
||||
'algorithm',
|
||||
'benchmark',
|
||||
]] = results[[
|
||||
'treasury_period_return',
|
||||
'algorithm_period_return',
|
||||
'benchmark_period_return',
|
||||
]]
|
||||
|
||||
ax5 = plt.subplot(615, sharex=ax1)
|
||||
results[[
|
||||
'treasury',
|
||||
'algorithm',
|
||||
'benchmark',
|
||||
]].plot(ax=ax5)
|
||||
ax5.set_ylabel('Percent Change')
|
||||
|
||||
ax6 = plt.subplot(616, sharex=ax1)
|
||||
results[['volume']].plot(ax=ax6)
|
||||
ax6.set_ylabel('Volume (mBTC/day)')
|
||||
|
||||
plt.legend(loc=3)
|
||||
|
||||
# Show the plot.
|
||||
plt.gcf().set_size_inches(18, 8)
|
||||
plt.show()
|
||||
@@ -1,18 +1,17 @@
|
||||
# For this example, we're going to write a simple momentum script. When the
|
||||
# For this example, we're going to write a simple momentum script. When the
|
||||
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
|
||||
# going to sell. Hopefully we'll ride the waves.
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import talib
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import symbol, record, order_target_percent, get_open_orders
|
||||
from catalyst.exchange.utils.stats_utils import extract_transactions
|
||||
from catalyst.exchange.stats_utils import extract_transactions
|
||||
# We give a name to the algorithm which Catalyst will use to persist its state.
|
||||
# In this example, Catalyst will create the `.catalyst/data/live_algos`
|
||||
# directory. If we stop and start the algorithm, Catalyst will resume its
|
||||
@@ -32,20 +31,17 @@ def initialize(context):
|
||||
# trading pairs) you want to backtest. You'll also want to define any
|
||||
# parameters or values you're going to use.
|
||||
|
||||
# In our example, we're looking at Neo in Ether.
|
||||
context.market = symbol('eth_btc')
|
||||
# In our example, we're looking at Ether in USD Tether.
|
||||
context.neo_eth = symbol('neo_eth')
|
||||
context.base_price = None
|
||||
context.current_day = None
|
||||
|
||||
context.RSI_OVERSOLD = 55
|
||||
context.RSI_OVERBOUGHT = 60
|
||||
context.CANDLE_SIZE = '15T'
|
||||
context.RSI_OVERSOLD = 50
|
||||
context.RSI_OVERBOUGHT = 80
|
||||
context.CANDLE_SIZE = '5T'
|
||||
|
||||
context.start_time = time.time()
|
||||
|
||||
context.set_commission(maker=0.001, taker=0.002)
|
||||
context.set_slippage(spread=0.001)
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
# This handle_data function is where the real work is done. Our data is
|
||||
@@ -62,14 +58,14 @@ def handle_data(context, data):
|
||||
context.current_day = today
|
||||
|
||||
# We're computing the volume-weighted-average-price of the security
|
||||
# defined above, in the context.market variable. For this example, we're
|
||||
# defined above, in the context.neo_eth variable. For this example, we're
|
||||
# using three bars on the 15 min bars.
|
||||
|
||||
# The frequency attribute determine the bar size. We use this convention
|
||||
# for the frequency alias:
|
||||
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
|
||||
prices = data.history(
|
||||
context.market,
|
||||
context.neo_eth,
|
||||
fields='close',
|
||||
bar_count=50,
|
||||
frequency=context.CANDLE_SIZE
|
||||
@@ -84,7 +80,7 @@ def handle_data(context, data):
|
||||
# We need a variable for the current price of the security to compare to
|
||||
# the average. Since we are requesting two fields, data.current()
|
||||
# returns a DataFrame with
|
||||
current = data.current(context.market, fields=['close', 'volume'])
|
||||
current = data.current(context.neo_eth, fields=['close', 'volume'])
|
||||
price = current['close']
|
||||
|
||||
# If base_price is not set, we use the current value. This is the
|
||||
@@ -98,36 +94,34 @@ def handle_data(context, data):
|
||||
# Now that we've collected all current data for this frame, we use
|
||||
# the record() method to save it. This data will be available as
|
||||
# a parameter of the analyze() function for further analysis.
|
||||
|
||||
record(
|
||||
volume=current['volume'],
|
||||
price=price,
|
||||
volume=current['volume'],
|
||||
price_change=price_change,
|
||||
rsi=rsi[-1],
|
||||
cash=cash
|
||||
)
|
||||
|
||||
# We are trying to avoid over-trading by limiting our trades to
|
||||
# one per day.
|
||||
if context.traded_today:
|
||||
return
|
||||
|
||||
# TODO: retest with open orders
|
||||
# Since we are using limit orders, some orders may not execute immediately
|
||||
# we wait until all orders are executed before considering more trades.
|
||||
orders = context.blotter.open_orders
|
||||
orders = get_open_orders(context.neo_eth)
|
||||
if len(orders) > 0:
|
||||
log.info('exiting because orders are open: {}'.format(orders))
|
||||
return
|
||||
|
||||
# Exit if we cannot trade
|
||||
if not data.can_trade(context.market):
|
||||
if not data.can_trade(context.neo_eth):
|
||||
return
|
||||
|
||||
# Another powerful built-in feature of the Catalyst backtester is the
|
||||
# portfolio object. The portfolio object tracks your positions, cash,
|
||||
# cost basis of specific holdings, and more. In this line, we calculate
|
||||
# how long or short our position is at this minute.
|
||||
pos_amount = context.portfolio.positions[context.market].amount
|
||||
# how long or short our position is at this minute.
|
||||
pos_amount = context.portfolio.positions[context.neo_eth].amount
|
||||
|
||||
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
|
||||
log.info(
|
||||
@@ -138,7 +132,7 @@ def handle_data(context, data):
|
||||
# Set a style for limit orders,
|
||||
limit_price = price * 1.005
|
||||
order_target_percent(
|
||||
context.market, 1, limit_price=limit_price
|
||||
context.neo_eth, 1, limit_price=limit_price
|
||||
)
|
||||
context.traded_today = True
|
||||
|
||||
@@ -150,7 +144,7 @@ def handle_data(context, data):
|
||||
)
|
||||
limit_price = price * 0.995
|
||||
order_target_percent(
|
||||
context.market, 0, limit_price=limit_price
|
||||
context.neo_eth, 0, limit_price=limit_price
|
||||
)
|
||||
context.traded_today = True
|
||||
|
||||
@@ -161,19 +155,19 @@ def analyze(context=None, perf=None):
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
# The base currency of the algo exchange
|
||||
base_currency = list(context.exchanges.values())[0].base_currency.upper()
|
||||
base_currency = context.exchanges.values()[0].base_currency.upper()
|
||||
|
||||
# Plot the portfolio value over time.
|
||||
ax1 = plt.subplot(611)
|
||||
perf.loc[:, 'portfolio_value'].plot(ax=ax1)
|
||||
ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
|
||||
ax1.set_ylabel('Portfolio Value ({})'.format(base_currency))
|
||||
|
||||
# Plot the price increase or decrease over time.
|
||||
ax2 = plt.subplot(612, sharex=ax1)
|
||||
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
|
||||
|
||||
ax2.set_ylabel('{asset}\n({base})'.format(
|
||||
asset=context.market.symbol, base=base_currency
|
||||
ax2.set_ylabel('{asset} ({base})'.format(
|
||||
asset=context.neo_eth.symbol, base=base_currency
|
||||
))
|
||||
|
||||
transaction_df = extract_transactions(perf)
|
||||
@@ -201,19 +195,18 @@ def analyze(context=None, perf=None):
|
||||
perf.loc[:, 'cash'].plot(
|
||||
ax=ax4, label='Base Currency ({})'.format(base_currency)
|
||||
)
|
||||
ax4.set_ylabel('Cash\n({})'.format(base_currency))
|
||||
ax4.set_ylabel('Cash ({})'.format(base_currency))
|
||||
|
||||
perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
|
||||
|
||||
ax5 = plt.subplot(614, sharex=ax1)
|
||||
perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
|
||||
ax5.set_ylabel('Percent\nChange')
|
||||
ax5.set_ylabel('Percent Change')
|
||||
|
||||
ax6 = plt.subplot(615, sharex=ax1)
|
||||
perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
|
||||
ax6.set_ylabel('RSI')
|
||||
ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
|
||||
ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
|
||||
ax6.axhline(70, color='darkgoldenrod')
|
||||
ax6.axhline(30, color='darkgoldenrod')
|
||||
|
||||
if not transaction_df.empty:
|
||||
ax6.scatter(
|
||||
@@ -233,8 +226,6 @@ def analyze(context=None, perf=None):
|
||||
label=''
|
||||
)
|
||||
plt.legend(loc=3)
|
||||
start, end = ax6.get_ylim()
|
||||
ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
|
||||
|
||||
# Show the plot.
|
||||
plt.gcf().set_size_inches(18, 8)
|
||||
@@ -244,25 +235,9 @@ def analyze(context=None, perf=None):
|
||||
|
||||
if __name__ == '__main__':
|
||||
# The execution mode: backtest or live
|
||||
live = True
|
||||
MODE = 'backtest'
|
||||
|
||||
if live:
|
||||
run_algorithm(
|
||||
capital_base=0.01,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
live=True,
|
||||
algo_namespace=NAMESPACE,
|
||||
base_currency='btc',
|
||||
live_graph=False,
|
||||
simulate_orders=False,
|
||||
stats_output=None,
|
||||
# auth_aliases=dict(poloniex='auth2')
|
||||
)
|
||||
|
||||
else:
|
||||
if MODE == 'backtest':
|
||||
folder = os.path.join(
|
||||
tempfile.gettempdir(), 'catalyst', NAMESPACE
|
||||
)
|
||||
@@ -270,20 +245,31 @@ if __name__ == '__main__':
|
||||
|
||||
timestr = time.strftime('%Y%m%d-%H%M%S')
|
||||
out = os.path.join(folder, '{}.p'.format(timestr))
|
||||
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
|
||||
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
|
||||
# --data-frequency minute --capital-base 10000
|
||||
# catalyst run -f catalyst/examples/mean_reversion_simple.py -x poloniex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
|
||||
run_algorithm(
|
||||
capital_base=0.1,
|
||||
capital_base=10000,
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='bitfinex',
|
||||
algo_namespace=NAMESPACE,
|
||||
base_currency='btc',
|
||||
base_currency='usd',
|
||||
start=pd.to_datetime('2017-10-01', utc=True),
|
||||
end=pd.to_datetime('2017-11-10', utc=True),
|
||||
output=out
|
||||
)
|
||||
log.info('saved perf stats: {}'.format(out))
|
||||
|
||||
elif MODE == 'live':
|
||||
run_algorithm(
|
||||
capital_base=0.5,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='bittrex',
|
||||
live=True,
|
||||
algo_namespace=NAMESPACE,
|
||||
base_currency='eth',
|
||||
live_graph=False
|
||||
)
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
# For this example, we're going to write a simple momentum script. When the
|
||||
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
|
||||
# going to sell. Hopefully we'll ride the waves.
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import talib
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import symbol, record, order_target_percent, get_open_orders
|
||||
from catalyst.exchange.utils.stats_utils import extract_transactions
|
||||
# We give a name to the algorithm which Catalyst will use to persist its state.
|
||||
# In this example, Catalyst will create the `.catalyst/data/live_algos`
|
||||
# directory. If we stop and start the algorithm, Catalyst will resume its
|
||||
# state using the files included in the folder.
|
||||
from catalyst.utils.paths import ensure_directory
|
||||
|
||||
NAMESPACE = 'mean_reversion_simple'
|
||||
log = Logger(NAMESPACE)
|
||||
|
||||
|
||||
# To run an algorithm in Catalyst, you need two functions: initialize and
|
||||
# handle_data.
|
||||
|
||||
def initialize(context):
|
||||
# This initialize function sets any data or variables that you'll use in
|
||||
# your algorithm. For instance, you'll want to define the trading pair (or
|
||||
# trading pairs) you want to backtest. You'll also want to define any
|
||||
# parameters or values you're going to use.
|
||||
|
||||
# In our example, we're looking at Neo in Ether.
|
||||
context.market = symbol('eth_btc')
|
||||
context.base_price = None
|
||||
context.current_day = None
|
||||
|
||||
context.RSI_OVERSOLD = 50
|
||||
context.RSI_OVERBOUGHT = 60
|
||||
context.CANDLE_SIZE = '5T'
|
||||
|
||||
context.start_time = time.time()
|
||||
|
||||
context.set_commission(maker=0.001, taker=0.002)
|
||||
# context.set_slippage(spread=0.001)
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
# This handle_data function is where the real work is done. Our data is
|
||||
# minute-level tick data, and each minute is called a frame. This function
|
||||
# runs on each frame of the data.
|
||||
|
||||
# We flag the first period of each day.
|
||||
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
|
||||
# would only execute once. This method works with minute and daily
|
||||
# frequencies.
|
||||
today = data.current_dt.floor('1D')
|
||||
if today != context.current_day:
|
||||
context.traded_today = False
|
||||
context.current_day = today
|
||||
|
||||
# We're computing the volume-weighted-average-price of the security
|
||||
# defined above, in the context.market variable. For this example, we're
|
||||
# using three bars on the 15 min bars.
|
||||
|
||||
# The frequency attribute determine the bar size. We use this convention
|
||||
# for the frequency alias:
|
||||
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
|
||||
prices = data.history(
|
||||
context.market,
|
||||
fields='close',
|
||||
bar_count=50,
|
||||
frequency=context.CANDLE_SIZE
|
||||
)
|
||||
|
||||
# Ta-lib calculates various technical indicator based on price and
|
||||
# volume arrays.
|
||||
|
||||
# In this example, we are comp
|
||||
rsi = talib.RSI(prices.values, timeperiod=14)
|
||||
|
||||
# We need a variable for the current price of the security to compare to
|
||||
# the average. Since we are requesting two fields, data.current()
|
||||
# returns a DataFrame with
|
||||
current = data.current(context.market, fields=['close', 'volume'])
|
||||
price = current['close']
|
||||
|
||||
# If base_price is not set, we use the current value. This is the
|
||||
# price at the first bar which we reference to calculate price_change.
|
||||
if context.base_price is None:
|
||||
context.base_price = price
|
||||
|
||||
price_change = (price - context.base_price) / context.base_price
|
||||
cash = context.portfolio.cash
|
||||
|
||||
# Now that we've collected all current data for this frame, we use
|
||||
# the record() method to save it. This data will be available as
|
||||
# a parameter of the analyze() function for further analysis.
|
||||
|
||||
record(
|
||||
volume=current['volume'],
|
||||
price=price,
|
||||
price_change=price_change,
|
||||
rsi=rsi[-1],
|
||||
cash=cash
|
||||
)
|
||||
# We are trying to avoid over-trading by limiting our trades to
|
||||
# one per day.
|
||||
if context.traded_today:
|
||||
return
|
||||
|
||||
# TODO: retest with open orders
|
||||
# Since we are using limit orders, some orders may not execute immediately
|
||||
# we wait until all orders are executed before considering more trades.
|
||||
orders = get_open_orders(context.market)
|
||||
if len(orders) > 0:
|
||||
log.info('exiting because orders are open: {}'.format(orders))
|
||||
return
|
||||
|
||||
# Exit if we cannot trade
|
||||
if not data.can_trade(context.market):
|
||||
return
|
||||
|
||||
# Another powerful built-in feature of the Catalyst backtester is the
|
||||
# portfolio object. The portfolio object tracks your positions, cash,
|
||||
# cost basis of specific holdings, and more. In this line, we calculate
|
||||
# how long or short our position is at this minute.
|
||||
pos_amount = context.portfolio.positions[context.market].amount
|
||||
|
||||
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
|
||||
log.info(
|
||||
'{}: buying - price: {}, rsi: {}'.format(
|
||||
data.current_dt, price, rsi[-1]
|
||||
)
|
||||
)
|
||||
# Set a style for limit orders,
|
||||
limit_price = price * 1.005
|
||||
order_target_percent(
|
||||
context.market, 1, limit_price=limit_price
|
||||
)
|
||||
context.traded_today = True
|
||||
|
||||
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
|
||||
log.info(
|
||||
'{}: selling - price: {}, rsi: {}'.format(
|
||||
data.current_dt, price, rsi[-1]
|
||||
)
|
||||
)
|
||||
limit_price = price * 0.995
|
||||
order_target_percent(
|
||||
context.market, 0, limit_price=limit_price
|
||||
)
|
||||
context.traded_today = True
|
||||
|
||||
|
||||
def analyze(context=None, perf=None):
|
||||
end = time.time()
|
||||
log.info('elapsed time: {}'.format(end - context.start_time))
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
# The base currency of the algo exchange
|
||||
base_currency = list(context.exchanges.values())[0].base_currency.upper()
|
||||
|
||||
# Plot the portfolio value over time.
|
||||
ax1 = plt.subplot(611)
|
||||
perf.loc[:, 'portfolio_value'].plot(ax=ax1)
|
||||
ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
|
||||
|
||||
# Plot the price increase or decrease over time.
|
||||
ax2 = plt.subplot(612, sharex=ax1)
|
||||
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
|
||||
|
||||
ax2.set_ylabel('{asset}\n({base})'.format(
|
||||
asset=context.market.symbol, base=base_currency
|
||||
))
|
||||
|
||||
transaction_df = extract_transactions(perf)
|
||||
if not transaction_df.empty:
|
||||
buy_df = transaction_df[transaction_df['amount'] > 0]
|
||||
sell_df = transaction_df[transaction_df['amount'] < 0]
|
||||
ax2.scatter(
|
||||
buy_df.index.to_pydatetime(),
|
||||
perf.loc[buy_df.index.floor('1 min'), 'price'],
|
||||
marker='^',
|
||||
s=100,
|
||||
c='green',
|
||||
label=''
|
||||
)
|
||||
ax2.scatter(
|
||||
sell_df.index.to_pydatetime(),
|
||||
perf.loc[sell_df.index.floor('1 min'), 'price'],
|
||||
marker='v',
|
||||
s=100,
|
||||
c='red',
|
||||
label=''
|
||||
)
|
||||
|
||||
ax4 = plt.subplot(613, sharex=ax1)
|
||||
perf.loc[:, 'cash'].plot(
|
||||
ax=ax4, label='Base Currency ({})'.format(base_currency)
|
||||
)
|
||||
ax4.set_ylabel('Cash\n({})'.format(base_currency))
|
||||
|
||||
perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
|
||||
|
||||
ax5 = plt.subplot(614, sharex=ax1)
|
||||
perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
|
||||
ax5.set_ylabel('Percent\nChange')
|
||||
|
||||
ax6 = plt.subplot(615, sharex=ax1)
|
||||
perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
|
||||
ax6.set_ylabel('RSI')
|
||||
ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
|
||||
ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
|
||||
|
||||
if not transaction_df.empty:
|
||||
ax6.scatter(
|
||||
buy_df.index.to_pydatetime(),
|
||||
perf.loc[buy_df.index.floor('1 min'), 'rsi'],
|
||||
marker='^',
|
||||
s=100,
|
||||
c='green',
|
||||
label=''
|
||||
)
|
||||
ax6.scatter(
|
||||
sell_df.index.to_pydatetime(),
|
||||
perf.loc[sell_df.index.floor('1 min'), 'rsi'],
|
||||
marker='v',
|
||||
s=100,
|
||||
c='red',
|
||||
label=''
|
||||
)
|
||||
plt.legend(loc=3)
|
||||
start, end = ax6.get_ylim()
|
||||
ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
|
||||
|
||||
# Show the plot.
|
||||
plt.gcf().set_size_inches(18, 8)
|
||||
plt.show()
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# The execution mode: backtest or live
|
||||
live = False
|
||||
|
||||
if live:
|
||||
run_algorithm(
|
||||
capital_base=0.025,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
live=True,
|
||||
algo_namespace=NAMESPACE,
|
||||
base_currency='btc',
|
||||
live_graph=False,
|
||||
simulate_orders=False,
|
||||
stats_output=None,
|
||||
)
|
||||
|
||||
else:
|
||||
folder = os.path.join(
|
||||
tempfile.gettempdir(), 'catalyst', NAMESPACE
|
||||
)
|
||||
ensure_directory(folder)
|
||||
|
||||
timestr = time.strftime('%Y%m%d-%H%M%S')
|
||||
out = os.path.join(folder, '{}.p'.format(timestr))
|
||||
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
|
||||
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
|
||||
# --data-frequency minute --capital-base 10000
|
||||
run_algorithm(
|
||||
capital_base=0.1,
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='bitfinex',
|
||||
algo_namespace=NAMESPACE,
|
||||
base_currency='eth',
|
||||
start=pd.to_datetime('2017-10-01', utc=True),
|
||||
end=pd.to_datetime('2017-11-10', utc=True),
|
||||
output=out
|
||||
)
|
||||
log.info('saved perf stats: {}'.format(out))
|
||||
@@ -11,6 +11,7 @@ from catalyst.api import (
|
||||
record,
|
||||
get_open_orders,
|
||||
)
|
||||
from catalyst.exchange.stats_utils import crossover, crossunder
|
||||
from catalyst.utils.run_algo import run_algorithm
|
||||
|
||||
algo_namespace = 'rsi'
|
||||
@@ -54,7 +55,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
|
||||
stop=None
|
||||
)
|
||||
|
||||
# action = None
|
||||
action = None
|
||||
if context.position is not None:
|
||||
cost_basis = context.position['cost_basis']
|
||||
amount = context.position['amount']
|
||||
@@ -79,7 +80,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
|
||||
amount=-amount,
|
||||
limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
|
||||
)
|
||||
# action = 0
|
||||
action = 0
|
||||
context.position = None
|
||||
|
||||
else:
|
||||
@@ -96,7 +97,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
|
||||
amount=buy_amount,
|
||||
stop=None
|
||||
)
|
||||
# action = 0
|
||||
action = 0
|
||||
|
||||
|
||||
def _handle_data_rsi_only(context, data):
|
||||
@@ -114,7 +115,7 @@ def _handle_data_rsi_only(context, data):
|
||||
prices = data.history(
|
||||
context.asset,
|
||||
fields='price',
|
||||
bar_count=20,
|
||||
bar_count=17,
|
||||
frequency='30T'
|
||||
)
|
||||
except Exception as e:
|
||||
@@ -156,7 +157,7 @@ def handle_data(context, data):
|
||||
dt = data.current_dt
|
||||
|
||||
if context.last_bar is None or (
|
||||
context.last_bar + timedelta(minutes=15)) <= dt:
|
||||
context.last_bar + timedelta(minutes=15)) <= dt:
|
||||
context.last_bar = dt
|
||||
else:
|
||||
return
|
||||
@@ -175,7 +176,7 @@ def handle_data(context, data):
|
||||
def analyze(context=None, results=None):
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
base_currency = list(context.exchanges.values())[0].base_currency.upper()
|
||||
base_currency = context.exchanges.values()[0].base_currency.upper()
|
||||
# Plot the portfolio and asset data.
|
||||
ax1 = plt.subplot(611)
|
||||
results.loc[:, 'portfolio_value'].plot(ax=ax1)
|
||||
@@ -249,17 +250,27 @@ def analyze(context=None, results=None):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Backtest
|
||||
run_algorithm(
|
||||
capital_base=0.5,
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace=algo_namespace,
|
||||
base_currency='btc',
|
||||
start=pd.to_datetime('2017-9-1', utc=True),
|
||||
end=pd.to_datetime('2017-10-1', utc=True),
|
||||
)
|
||||
# run_algorithm(
|
||||
# initialize=initialize,
|
||||
# handle_data=handle_data,
|
||||
# analyze=analyze,
|
||||
# exchange_name='bittrex',
|
||||
# live=True,
|
||||
# algo_namespace=algo_namespace,
|
||||
# base_currency='btc',
|
||||
# live_graph=False
|
||||
# )
|
||||
|
||||
# Backtest
|
||||
run_algorithm(
|
||||
capital_base=0.5,
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace=algo_namespace,
|
||||
base_currency='btc',
|
||||
start=pd.to_datetime('2017-9-1', utc=True),
|
||||
end=pd.to_datetime('2017-10-1', utc=True),
|
||||
)
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,38 +1,35 @@
|
||||
import pandas as pd
|
||||
import talib
|
||||
from logbook import Logger, INFO
|
||||
import pandas as pd
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import symbol, record
|
||||
from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
|
||||
from catalyst.exchange.stats_utils import get_pretty_stats, \
|
||||
extract_transactions
|
||||
|
||||
log = Logger('simple_loop', level=INFO)
|
||||
|
||||
|
||||
def initialize(context):
|
||||
log.info('initializing')
|
||||
context.asset = symbol('eth_btc')
|
||||
print('initializing')
|
||||
context.asset = symbol('neo_usd')
|
||||
context.base_price = None
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
log.info('handling bar: {}'.format(data.current_dt))
|
||||
print('handling bar: {}'.format(data.current_dt))
|
||||
|
||||
price = data.current(context.asset, 'close')
|
||||
log.info('got price {price}'.format(price=price))
|
||||
print('got price {price}'.format(price=price))
|
||||
|
||||
prices = data.history(
|
||||
context.asset,
|
||||
fields='price',
|
||||
bar_count=20,
|
||||
frequency='30T'
|
||||
)
|
||||
last_traded = prices.index[-1]
|
||||
log.info('last candle date: {}'.format(last_traded))
|
||||
|
||||
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
|
||||
log.info('got rsi: {}'.format(rsi))
|
||||
try:
|
||||
prices = data.history(
|
||||
context.asset,
|
||||
fields='price',
|
||||
bar_count=14,
|
||||
frequency='15T'
|
||||
)
|
||||
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
|
||||
print('got rsi: {}'.format(rsi))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
# If base_price is not set, we use the current value. This is the
|
||||
# price at the first bar which we reference to calculate price_change.
|
||||
@@ -54,10 +51,10 @@ def handle_data(context, data):
|
||||
|
||||
def analyze(context, perf):
|
||||
import matplotlib.pyplot as plt
|
||||
log.info('the stats: {}'.format(get_pretty_stats(perf)))
|
||||
print('the stats: {}'.format(get_pretty_stats(perf)))
|
||||
|
||||
# The base currency of the algo exchange
|
||||
base_currency = list(context.exchanges.values())[0].base_currency.upper()
|
||||
base_currency = context.exchanges.values()[0].base_currency.upper()
|
||||
|
||||
# Plot the portfolio value over time.
|
||||
ax1 = plt.subplot(611)
|
||||
@@ -113,32 +110,24 @@ def analyze(context, perf):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
mode = 'backtest'
|
||||
|
||||
if mode == 'backtest':
|
||||
run_algorithm(
|
||||
capital_base=1,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=None,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='simple_loop',
|
||||
base_currency='eth',
|
||||
data_frequency='minute',
|
||||
start=pd.to_datetime('2017-9-1', utc=True),
|
||||
end=pd.to_datetime('2017-12-1', utc=True),
|
||||
)
|
||||
else:
|
||||
run_algorithm(
|
||||
capital_base=1,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=None,
|
||||
exchange_name='binance',
|
||||
live=True,
|
||||
algo_namespace='simple_loop',
|
||||
base_currency='eth',
|
||||
live_graph=False,
|
||||
simulate_orders=True
|
||||
)
|
||||
run_algorithm(
|
||||
capital_base=250,
|
||||
start=pd.to_datetime('2017-11-1 0:00', utc=True),
|
||||
end=pd.to_datetime('2017-11-10 23:59', utc=True),
|
||||
data_frequency='daily',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='bitfinex',
|
||||
algo_namespace='simple_loop',
|
||||
base_currency='usd'
|
||||
)
|
||||
# run_algorithm(
|
||||
# initialize=initialize,
|
||||
# handle_data=handle_data,
|
||||
# analyze=None,
|
||||
# exchange_name='poloniex',
|
||||
# live=True,
|
||||
# algo_namespace='simple_loop',
|
||||
# base_currency='eth',
|
||||
# live_graph=False
|
||||
|
||||
@@ -2,117 +2,73 @@
|
||||
Requires Catalyst version 0.3.0 or above
|
||||
Tested on Catalyst version 0.3.3
|
||||
|
||||
This example aims to provide an easy way for users to learn how to
|
||||
collect data from any given exchange and select a subset of the available
|
||||
currency pairs for trading. You simply need to specify the exchange and
|
||||
the market (base_currency) that you want to focus on. You will then see
|
||||
how to create a universe of assets, and filter it based the market you
|
||||
desire.
|
||||
These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
|
||||
You simply need to specify the exchange and the market that you want to focus on.
|
||||
You will all see how to create a universe and filter it base on the exchange and the market you desire.
|
||||
|
||||
The example prints out the closing price of all the pairs for a given
|
||||
market in a given exchange every 30 minutes. The example also contains
|
||||
the OHLCV data with minute-resolution for the past seven days which
|
||||
could be used to create indicators. Use this code as the backbone to
|
||||
create your own trading strategy.
|
||||
|
||||
The lookback_date variable is used to ensure data for a coin existed on
|
||||
the lookback period specified.
|
||||
|
||||
To run, execute the following two commands in a terminal (inside catalyst
|
||||
environment). The first one retrieves all the pricing data needed for this
|
||||
script to run (only needs to be run once), and the second one executes this
|
||||
script with the parameters specified in the run_algorithm() call at the end
|
||||
of the file:
|
||||
|
||||
catalyst ingest-exchange -x bitfinex -f minute
|
||||
|
||||
python simple_universe.py
|
||||
The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
|
||||
The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
|
||||
Use this as the backbone to create your own trading strategies.
|
||||
|
||||
Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from datetime import timedelta
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import (symbols, )
|
||||
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols
|
||||
|
||||
from catalyst.api import (
|
||||
symbols,
|
||||
)
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.i = -1 # minute counter
|
||||
context.exchange = list(context.exchanges.values())[0].name.lower()
|
||||
context.base_currency = list(context.exchanges.values())[0].base_currency.lower()
|
||||
context.i = -1 # counts the minutes
|
||||
context.exchange = context.exchanges.values()[0].name.lower() # exchange name
|
||||
context.base_currency = context.exchanges.values()[0].base_currency.lower() # market base currency
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
context.i += 1
|
||||
lookback_days = 7 # 7 days
|
||||
|
||||
# current date & time in each iteration formatted into a string
|
||||
now = data.current_dt
|
||||
date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ')
|
||||
lookback_date = now - timedelta(days=lookback_days)
|
||||
# keep only the date as a string, discard the time
|
||||
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
|
||||
# current date formatted into a string
|
||||
today = data.current_dt
|
||||
date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
|
||||
lookback_date = today - timedelta(days=lookback_days) # subtract the amount of days specified in lookback
|
||||
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] # get only the date as a string
|
||||
|
||||
one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute'
|
||||
# update universe everyday at midnight
|
||||
if not context.i % one_day_in_minutes:
|
||||
# update universe everyday
|
||||
new_day = 60 * 24 # assuming data_frequency='minute'
|
||||
if not context.i % new_day:
|
||||
context.universe = universe(context, lookback_date, date)
|
||||
|
||||
# get data every 30 minutes
|
||||
minutes = 30
|
||||
|
||||
# get lookback_days of history data: that is 'lookback' number of bins
|
||||
lookback = int(one_day_in_minutes / minutes * lookback_days)
|
||||
one_day_in_minutes = 1440 # 1440 assumes data_frequency='minute'
|
||||
lookback = one_day_in_minutes / minutes * lookback_days # get N lookback_days of history data
|
||||
if not context.i % minutes and context.universe:
|
||||
# we iterate for every pair in the current universe
|
||||
for coin in context.coins:
|
||||
pair = str(coin.symbol)
|
||||
|
||||
# Get 30 minute interval OHLCV data. This is the standard data
|
||||
# required for candlestick or indicators/signals. Return Pandas
|
||||
# DataFrames. 30T means 30-minute re-sampling of one minute data.
|
||||
# Adjust it to your desired time interval as needed.
|
||||
opened = fill(data.history(coin,
|
||||
'open',
|
||||
bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
high = fill(data.history(coin,
|
||||
'high',
|
||||
bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
low = fill(data.history(coin,
|
||||
'low',
|
||||
bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
close = fill(data.history(coin,
|
||||
'price',
|
||||
bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
volume = fill(data.history(coin,
|
||||
'volume',
|
||||
bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
# 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
|
||||
# 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
|
||||
opened = fill(data.history(coin, 'open', bar_count=lookback, frequency='30T')).values
|
||||
high = fill(data.history(coin, 'high', bar_count=lookback, frequency='30T')).values
|
||||
low = fill(data.history(coin, 'low', bar_count=lookback, frequency='30T')).values
|
||||
close = fill(data.history(coin, 'price', bar_count=lookback, frequency='30T')).values
|
||||
volume = fill(data.history(coin, 'volume', bar_count=lookback, frequency='30T')).values
|
||||
|
||||
# close[-1] is the last value in the set, which is the equivalent
|
||||
# to current price (as in the most recent value)
|
||||
# close[-1] is the equivalent to current price
|
||||
# displays the minute price for each pair every 30 minutes
|
||||
print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},'
|
||||
'\tV:{v}'.format(
|
||||
now=now,
|
||||
pair=pair,
|
||||
o=opened[-1],
|
||||
h=high[-1],
|
||||
l=low[-1],
|
||||
c=close[-1],
|
||||
v=volume[-1],
|
||||
))
|
||||
print(today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
|
||||
|
||||
# -------------------------------------------------------------
|
||||
# --------------- Insert Your Strategy Here -------------------
|
||||
# -------------------------------------------------------------
|
||||
# ----------------------------------------------------------------------------------------------------------
|
||||
# -------------------------------------- Insert Your Strategy Here -----------------------------------------
|
||||
# ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
@@ -122,24 +78,23 @@ def analyze(context=None, results=None):
|
||||
# Get the universe for a given exchange and a given base_currency market
|
||||
# Example: Poloniex BTC Market
|
||||
def universe(context, lookback_date, current_date):
|
||||
# get all the pairs for the given exchange
|
||||
json_symbols = get_exchange_symbols(context.exchange)
|
||||
# convert into a DataFrame for easier processing
|
||||
df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)
|
||||
df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],
|
||||
axis=1)
|
||||
df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],
|
||||
axis=1)
|
||||
json_symbols = get_exchange_symbols(context.exchange) # get all the pairs for the exchange
|
||||
universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) # convert into a dataframe
|
||||
universe_df['base_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[1],
|
||||
axis=1)
|
||||
universe_df['market_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[0],
|
||||
axis=1)
|
||||
|
||||
# Filter all the pairs to get only the ones for a given base_currency
|
||||
df = df[df['base_currency'] == context.base_currency]
|
||||
# Filter all the exchange pairs to only the ones for a give base currency
|
||||
universe_df = universe_df[universe_df['base_currency'] == context.base_currency]
|
||||
|
||||
# Filter all pairs to ensure that pair existed in the current date range
|
||||
df = df[df.start_date < lookback_date]
|
||||
df = df[df.end_daily >= current_date]
|
||||
context.coins = symbols(*df.symbol) # convert all the pairs to symbols
|
||||
# Filter all the pairs to ensure that pair existed in the current date range
|
||||
universe_df = universe_df[universe_df.start_date < lookback_date]
|
||||
universe_df = universe_df[universe_df.end_daily >= current_date]
|
||||
context.coins = symbols(*universe_df.symbol) # convert all the pairs to symbols
|
||||
|
||||
return df.symbol.tolist()
|
||||
# print(universe_df.symbol.tolist())
|
||||
return universe_df.symbol.tolist()
|
||||
|
||||
|
||||
# Replace all NA, NAN or infinite values with its nearest value
|
||||
@@ -147,9 +102,7 @@ def fill(series):
|
||||
if isinstance(series, pd.Series):
|
||||
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
|
||||
elif isinstance(series, np.ndarray):
|
||||
return pd.Series(series).replace(
|
||||
[np.inf, -np.inf], np.nan
|
||||
).ffill().bfill().values
|
||||
return pd.Series(series).replace([np.inf, -np.inf], np.nan).ffill().bfill().values
|
||||
else:
|
||||
return series
|
||||
|
||||
@@ -159,13 +112,18 @@ if __name__ == '__main__':
|
||||
end_date = pd.to_datetime('2017-11-13', utc=True)
|
||||
|
||||
performance = run_algorithm(start=start_date, end=end_date,
|
||||
capital_base=100.0, # amount of base_currency
|
||||
capital_base=100.0, # amount of base_currency, not always in dollars unless usd
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
exchange_name='bitfinex',
|
||||
data_frequency='minute',
|
||||
base_currency='btc',
|
||||
live=False,
|
||||
live_graph=False,
|
||||
algo_namespace='simple_universe')
|
||||
|
||||
"""
|
||||
Run in Terminal (inside catalyst environment):
|
||||
python simple_universe.py
|
||||
"""
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
# Run Command
|
||||
# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \
|
||||
# -f talib_simple.py -x poloniex
|
||||
#
|
||||
# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle -f talib_simple.py -x poloniex
|
||||
#
|
||||
# Description
|
||||
# Simple TALib Example showing how to use various indicators
|
||||
# in you strategy. Based loosly on
|
||||
# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
|
||||
# Simple TALib Example showing how to use various indicators in you strategy
|
||||
# Based loosly on https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
|
||||
|
||||
import os
|
||||
|
||||
@@ -23,7 +21,7 @@ from catalyst.api import (
|
||||
order_target_percent,
|
||||
symbol,
|
||||
)
|
||||
from catalyst.exchange.utils.stats_utils import get_pretty_stats
|
||||
from catalyst.exchange.stats_utils import get_pretty_stats
|
||||
|
||||
algo_namespace = 'talib_sample'
|
||||
log = Logger(algo_namespace)
|
||||
@@ -90,7 +88,7 @@ def _handle_data(context, data):
|
||||
prices.close.as_matrix(), fastperiod=context.MACD_FAST,
|
||||
slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL)
|
||||
|
||||
# Stochastics %K %D
|
||||
# Stochastics %K %D
|
||||
# %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100
|
||||
# %D = 3-day SMA of %K
|
||||
analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH(
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
|
||||
log = Logger('AssetFinderExchange', level=LOG_LEVEL)
|
||||
|
||||
|
||||
class AssetFinderExchange(object):
|
||||
def __init__(self):
|
||||
self._asset_cache = {}
|
||||
|
||||
@property
|
||||
def sids(self):
|
||||
"""
|
||||
This seems to be used to pre-fetch assets.
|
||||
I don't think that we need this for live-trading.
|
||||
Leaving the list empty.
|
||||
"""
|
||||
return list()
|
||||
|
||||
def retrieve_all(self, sids, default_none=False):
|
||||
"""
|
||||
Retrieve all assets in `sids`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sids : iterable of int
|
||||
Assets to retrieve.
|
||||
default_none : bool
|
||||
If True, return None for failed lookups.
|
||||
If False, raise `SidsNotFound`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
assets : list[Asset or None]
|
||||
A list of the same length as `sids` containing Assets (or Nones)
|
||||
corresponding to the requested sids.
|
||||
|
||||
Raises
|
||||
------
|
||||
SidsNotFound
|
||||
When a requested sid is not found and default_none=False.
|
||||
"""
|
||||
# for sid in sids:
|
||||
# if sid in self._asset_cache:
|
||||
# log.debug('got asset from cache: {}'.format(sid))
|
||||
# else:
|
||||
# log.debug('fetching asset: {}'.format(sid))
|
||||
return list()
|
||||
|
||||
def lookup_symbol(self, symbol, exchange, data_frequency=None,
|
||||
as_of_date=None, fuzzy=False):
|
||||
"""Lookup an asset by symbol.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
symbol : str
|
||||
The ticker symbol to resolve.
|
||||
as_of_date : datetime or None
|
||||
Look up the last owner of this symbol as of this datetime.
|
||||
If ``as_of_date`` is None, then this can only resolve the equity
|
||||
if exactly one equity has ever owned the ticker.
|
||||
fuzzy : bool, optional
|
||||
Should fuzzy symbol matching be used? Fuzzy symbol matching
|
||||
attempts to resolve differences in representations for
|
||||
shareclasses. For example, some people may represent the ``A``
|
||||
shareclass of ``BRK`` as ``BRK.A``, where others could write
|
||||
``BRK_A``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
equity : Asset
|
||||
The equity that held ``symbol`` on the given ``as_of_date``, or the
|
||||
only equity to hold ``symbol`` if ``as_of_date`` is None.
|
||||
|
||||
Raises
|
||||
------
|
||||
SymbolNotFound
|
||||
Raised when no equity has ever held the given symbol.
|
||||
MultipleSymbolsFound
|
||||
Raised when no ``as_of_date`` is given and more than one equity
|
||||
has held ``symbol``. This is also raised when ``fuzzy=True`` and
|
||||
there are multiple candidates for the given ``symbol`` on the
|
||||
``as_of_date``.
|
||||
"""
|
||||
log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
|
||||
|
||||
if data_frequency is not None:
|
||||
key = ','.join([exchange.name, symbol, data_frequency])
|
||||
|
||||
else:
|
||||
key = ','.join([exchange.name, symbol])
|
||||
|
||||
if key in self._asset_cache:
|
||||
return self._asset_cache[key]
|
||||
else:
|
||||
asset = exchange.get_asset(symbol, data_frequency)
|
||||
self._asset_cache[key] = asset
|
||||
return asset
|
||||
@@ -0,0 +1,705 @@
|
||||
import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytz
|
||||
import requests
|
||||
import six
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.exchange.exchange import Exchange
|
||||
from catalyst.exchange.exchange_bundle import ExchangeBundle
|
||||
from catalyst.exchange.exchange_errors import (
|
||||
ExchangeRequestError,
|
||||
InvalidHistoryFrequencyError,
|
||||
InvalidOrderStyle, OrderCancelError)
|
||||
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
|
||||
ExchangeStopLimitOrder, ExchangeStopOrder
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
|
||||
download_exchange_symbols, get_symbols_string
|
||||
from catalyst.finance.order import Order, ORDER_STATUS
|
||||
from catalyst.protocol import Account
|
||||
|
||||
# Trying to account for REST api instability
|
||||
# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
|
||||
requests.adapters.DEFAULT_RETRIES = 20
|
||||
|
||||
BITFINEX_URL = 'https://api.bitfinex.com'
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
|
||||
log = Logger('Bitfinex', level=LOG_LEVEL)
|
||||
warning_logger = Logger('AlgoWarning')
|
||||
|
||||
|
||||
class Bitfinex(Exchange):
|
||||
def __init__(self, key, secret, base_currency, portfolio=None):
|
||||
self.url = BITFINEX_URL
|
||||
self.key = key
|
||||
self.secret = secret.encode('UTF-8')
|
||||
self.name = 'bitfinex'
|
||||
self.color = 'green'
|
||||
|
||||
self.assets = dict()
|
||||
self.load_assets()
|
||||
|
||||
self.local_assets = dict()
|
||||
self.load_assets(is_local=True)
|
||||
|
||||
self.base_currency = base_currency
|
||||
self._portfolio = portfolio
|
||||
self.minute_writer = None
|
||||
self.minute_reader = None
|
||||
|
||||
# The candle limit for each request
|
||||
self.num_candles_limit = 1000
|
||||
|
||||
# Max is 90 but playing it safe
|
||||
# https://www.bitfinex.com/posts/188
|
||||
self.max_requests_per_minute = 9
|
||||
self.request_cpt = dict()
|
||||
|
||||
self.bundle = ExchangeBundle(self.name)
|
||||
|
||||
def _request(self, operation, data, version='v1'):
|
||||
payload_object = {
|
||||
'request': '/{}/{}'.format(version, operation),
|
||||
'nonce': '{0:f}'.format(time.time() * 1000000),
|
||||
# convert to string
|
||||
'options': {}
|
||||
}
|
||||
|
||||
if data is None:
|
||||
payload_dict = payload_object
|
||||
else:
|
||||
payload_dict = payload_object.copy()
|
||||
payload_dict.update(data)
|
||||
|
||||
payload_json = json.dumps(payload_dict)
|
||||
if six.PY3:
|
||||
payload = base64.b64encode(bytes(payload_json, 'utf-8'))
|
||||
else:
|
||||
payload = base64.b64encode(payload_json)
|
||||
|
||||
m = hmac.new(self.secret, payload, hashlib.sha384)
|
||||
m = m.hexdigest()
|
||||
|
||||
# headers
|
||||
headers = {
|
||||
'X-BFX-APIKEY': self.key,
|
||||
'X-BFX-PAYLOAD': payload,
|
||||
'X-BFX-SIGNATURE': m
|
||||
}
|
||||
|
||||
if data is None:
|
||||
request = requests.get(
|
||||
'{url}/{version}/{operation}'.format(
|
||||
url=self.url,
|
||||
version=version,
|
||||
operation=operation
|
||||
), data={},
|
||||
headers=headers)
|
||||
else:
|
||||
request = requests.post(
|
||||
'{url}/{version}/{operation}'.format(
|
||||
url=self.url,
|
||||
version=version,
|
||||
operation=operation
|
||||
),
|
||||
headers=headers)
|
||||
|
||||
return request
|
||||
|
||||
def _get_v2_symbol(self, asset):
|
||||
pair = asset.symbol.split('_')
|
||||
symbol = 't' + pair[0].upper() + pair[1].upper()
|
||||
return symbol
|
||||
|
||||
def _get_v2_symbols(self, assets):
|
||||
"""
|
||||
Workaround to support Bitfinex v2
|
||||
TODO: Might require a separate asset dictionary
|
||||
|
||||
:param assets:
|
||||
:return:
|
||||
"""
|
||||
|
||||
v2_symbols = []
|
||||
for asset in assets:
|
||||
v2_symbols.append(self._get_v2_symbol(asset))
|
||||
|
||||
return v2_symbols
|
||||
|
||||
def _create_order(self, order_status):
|
||||
"""
|
||||
Create a Catalyst order object from a Bitfinex order dictionary
|
||||
:param order_status:
|
||||
:return: Order
|
||||
"""
|
||||
if order_status['is_cancelled']:
|
||||
status = ORDER_STATUS.CANCELLED
|
||||
elif not order_status['is_live']:
|
||||
log.info('found executed order {}'.format(order_status))
|
||||
status = ORDER_STATUS.FILLED
|
||||
else:
|
||||
status = ORDER_STATUS.OPEN
|
||||
|
||||
amount = float(order_status['original_amount'])
|
||||
filled = float(order_status['executed_amount'])
|
||||
|
||||
if order_status['side'] == 'sell':
|
||||
amount = -amount
|
||||
filled = -filled
|
||||
|
||||
price = float(order_status['price'])
|
||||
order_type = order_status['type']
|
||||
|
||||
stop_price = None
|
||||
limit_price = None
|
||||
|
||||
# TODO: is this comprehensive enough?
|
||||
if order_type.endswith('limit'):
|
||||
limit_price = price
|
||||
elif order_type.endswith('stop'):
|
||||
stop_price = price
|
||||
|
||||
executed_price = float(order_status['avg_execution_price'])
|
||||
|
||||
# TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
|
||||
commission = None
|
||||
|
||||
date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
|
||||
date = pytz.utc.localize(date)
|
||||
order = Order(
|
||||
dt=date,
|
||||
asset=self.assets[order_status['symbol']],
|
||||
amount=amount,
|
||||
stop=stop_price,
|
||||
limit=limit_price,
|
||||
filled=filled,
|
||||
id=str(order_status['id']),
|
||||
commission=commission
|
||||
)
|
||||
order.status = status
|
||||
|
||||
return order, executed_price
|
||||
|
||||
def get_balances(self):
|
||||
log.debug('retrieving wallets balances')
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('balances', None)
|
||||
balances = response.json()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'message' in balances:
|
||||
raise ExchangeRequestError(
|
||||
error='unable to fetch balance {}'.format(balances['message'])
|
||||
)
|
||||
|
||||
std_balances = dict()
|
||||
for balance in balances:
|
||||
currency = balance['currency'].lower()
|
||||
std_balances[currency] = float(balance['available'])
|
||||
|
||||
return std_balances
|
||||
|
||||
@property
|
||||
def account(self):
|
||||
account = Account()
|
||||
|
||||
account.settled_cash = None
|
||||
account.accrued_interest = None
|
||||
account.buying_power = None
|
||||
account.equity_with_loan = None
|
||||
account.total_positions_value = None
|
||||
account.total_positions_exposure = None
|
||||
account.regt_equity = None
|
||||
account.regt_margin = None
|
||||
account.initial_margin_requirement = None
|
||||
account.maintenance_margin_requirement = None
|
||||
account.available_funds = None
|
||||
account.excess_liquidity = None
|
||||
account.cushion = None
|
||||
account.day_trades_remaining = None
|
||||
account.leverage = None
|
||||
account.net_leverage = None
|
||||
account.net_liquidation = None
|
||||
|
||||
return account
|
||||
|
||||
@property
|
||||
def time_skew(self):
|
||||
# TODO: research the time skew conditions
|
||||
return pd.Timedelta('0s')
|
||||
|
||||
def get_account(self):
|
||||
# TODO: fetch account data and keep in cache
|
||||
return None
|
||||
|
||||
def get_candles(self, freq, assets, bar_count=None,
|
||||
start_dt=None, end_dt=None):
|
||||
"""
|
||||
Retrieve OHLVC candles from Bitfinex
|
||||
|
||||
:param data_frequency:
|
||||
:param assets:
|
||||
:param bar_count:
|
||||
:return:
|
||||
|
||||
Available Frequencies
|
||||
---------------------
|
||||
'1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
|
||||
'1M'
|
||||
"""
|
||||
log.debug(
|
||||
'retrieving {bars} {freq} candles on {exchange} from '
|
||||
'{end_dt} for markets {symbols}, '.format(
|
||||
bars=bar_count,
|
||||
freq=freq,
|
||||
exchange=self.name,
|
||||
end_dt=end_dt,
|
||||
symbols=get_symbols_string(assets)
|
||||
)
|
||||
)
|
||||
|
||||
allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
|
||||
'360T', '720T', '1D', '7D', '14D', '30D']
|
||||
if freq not in allowed_frequencies:
|
||||
raise InvalidHistoryFrequencyError(frequency=freq)
|
||||
|
||||
freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
|
||||
if freq_match:
|
||||
number = int(freq_match.group(1))
|
||||
unit = freq_match.group(2)
|
||||
|
||||
if unit == 'T':
|
||||
if number in [60, 180, 360, 720]:
|
||||
number = number / 60
|
||||
converted_unit = 'h'
|
||||
else:
|
||||
converted_unit = 'm'
|
||||
else:
|
||||
converted_unit = unit
|
||||
|
||||
frequency = '{}{}'.format(number, converted_unit)
|
||||
|
||||
else:
|
||||
raise InvalidHistoryFrequencyError(frequency=freq)
|
||||
|
||||
# Making sure that assets are iterable
|
||||
asset_list = [assets] if isinstance(assets, TradingPair) else assets
|
||||
ohlc_map = dict()
|
||||
for asset in asset_list:
|
||||
symbol = self._get_v2_symbol(asset)
|
||||
url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
|
||||
url=self.url,
|
||||
frequency=frequency,
|
||||
symbol=symbol
|
||||
)
|
||||
|
||||
if bar_count:
|
||||
is_list = True
|
||||
url += '/hist?limit={}'.format(int(bar_count))
|
||||
|
||||
def get_ms(date):
|
||||
epoch = datetime.datetime.utcfromtimestamp(0)
|
||||
epoch = epoch.replace(tzinfo=pytz.UTC)
|
||||
|
||||
return (date - epoch).total_seconds() * 1000.0
|
||||
|
||||
if start_dt is not None:
|
||||
start_ms = get_ms(start_dt)
|
||||
url += '&start={0:f}'.format(start_ms)
|
||||
|
||||
if end_dt is not None:
|
||||
end_ms = get_ms(end_dt)
|
||||
url += '&end={0:f}'.format(end_ms)
|
||||
|
||||
else:
|
||||
is_list = False
|
||||
url += '/last'
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
response = requests.get(url)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'error' in response.content:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to retrieve candles: {}'.format(
|
||||
response.content)
|
||||
)
|
||||
|
||||
candles = response.json()
|
||||
|
||||
def ohlc_from_candle(candle):
|
||||
last_traded = pd.Timestamp.utcfromtimestamp(
|
||||
candle[0] / 1000.0)
|
||||
last_traded = last_traded.replace(tzinfo=pytz.UTC)
|
||||
ohlc = dict(
|
||||
open=np.float64(candle[1]),
|
||||
high=np.float64(candle[3]),
|
||||
low=np.float64(candle[4]),
|
||||
close=np.float64(candle[2]),
|
||||
volume=np.float64(candle[5]),
|
||||
price=np.float64(candle[2]),
|
||||
last_traded=last_traded
|
||||
)
|
||||
return ohlc
|
||||
|
||||
if is_list:
|
||||
ohlc_bars = []
|
||||
# We can to list candles from old to new
|
||||
for candle in reversed(candles):
|
||||
ohlc = ohlc_from_candle(candle)
|
||||
ohlc_bars.append(ohlc)
|
||||
|
||||
ohlc_map[asset] = ohlc_bars
|
||||
|
||||
else:
|
||||
ohlc = ohlc_from_candle(candles)
|
||||
ohlc_map[asset] = ohlc
|
||||
|
||||
return ohlc_map[assets] \
|
||||
if isinstance(assets, TradingPair) else ohlc_map
|
||||
|
||||
def create_order(self, asset, amount, is_buy, style):
|
||||
"""
|
||||
Creating order on the exchange.
|
||||
|
||||
:param asset:
|
||||
:param amount:
|
||||
:param is_buy:
|
||||
:param style:
|
||||
:return:
|
||||
"""
|
||||
exchange_symbol = self.get_symbol(asset)
|
||||
if isinstance(style, ExchangeLimitOrder) \
|
||||
or isinstance(style, ExchangeStopLimitOrder):
|
||||
price = style.get_limit_price(is_buy)
|
||||
order_type = 'limit'
|
||||
|
||||
elif isinstance(style, ExchangeStopOrder):
|
||||
price = style.get_stop_price(is_buy)
|
||||
order_type = 'stop'
|
||||
|
||||
else:
|
||||
raise InvalidOrderStyle(exchange=self.name,
|
||||
style=style.__class__.__name__)
|
||||
|
||||
req = dict(
|
||||
symbol=exchange_symbol,
|
||||
amount=str(float(abs(amount))),
|
||||
price="{:.20f}".format(float(price)),
|
||||
side='buy' if is_buy else 'sell',
|
||||
type='exchange ' + order_type, # TODO: support margin trades
|
||||
exchange=self.name,
|
||||
is_hidden=False,
|
||||
is_postonly=False,
|
||||
use_all_available=0,
|
||||
ocoorder=False,
|
||||
buy_price_oco=0,
|
||||
sell_price_oco=0
|
||||
)
|
||||
|
||||
date = pd.Timestamp.utcnow()
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('order/new', req)
|
||||
order_status = response.json()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'message' in order_status:
|
||||
raise ExchangeRequestError(
|
||||
error='unable to create Bitfinex order {}'.format(
|
||||
order_status['message'])
|
||||
)
|
||||
|
||||
order_id = str(order_status['id'])
|
||||
order = Order(
|
||||
dt=date,
|
||||
asset=asset,
|
||||
amount=amount,
|
||||
stop=style.get_stop_price(is_buy),
|
||||
limit=style.get_limit_price(is_buy),
|
||||
id=order_id
|
||||
)
|
||||
|
||||
return order
|
||||
|
||||
def get_open_orders(self, asset=None):
|
||||
"""Retrieve all of the current open orders.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
asset : Asset
|
||||
If passed and not None, return only the open orders for the given
|
||||
asset instead of all open orders.
|
||||
|
||||
Returns
|
||||
-------
|
||||
open_orders : dict[list[Order]] or list[Order]
|
||||
If no asset is passed this will return a dict mapping Assets
|
||||
to a list containing all the open orders for the asset.
|
||||
If an asset is passed then this will return a list of the open
|
||||
orders for this asset.
|
||||
"""
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('orders', None)
|
||||
order_statuses = response.json()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'message' in order_statuses:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to retrieve open orders: {}'.format(
|
||||
order_statuses['message'])
|
||||
)
|
||||
|
||||
orders = []
|
||||
for order_status in order_statuses:
|
||||
order, executed_price = self._create_order(order_status)
|
||||
if asset is None or asset == order.sid:
|
||||
orders.append(order)
|
||||
|
||||
return orders
|
||||
|
||||
def get_order(self, order_id):
|
||||
"""Lookup an order based on the order id returned from one of the
|
||||
order functions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
order_id : str
|
||||
The unique identifier for the order.
|
||||
|
||||
Returns
|
||||
-------
|
||||
order : Order
|
||||
The order object.
|
||||
"""
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request(
|
||||
'order/status', {'order_id': int(order_id)})
|
||||
order_status = response.json()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'message' in order_status:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to retrieve order status: {}'.format(
|
||||
order_status['message'])
|
||||
)
|
||||
return self._create_order(order_status)
|
||||
|
||||
def cancel_order(self, order_param):
|
||||
"""Cancel an open order.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
order_param : str or Order
|
||||
The order_id or order object to cancel.
|
||||
"""
|
||||
order_id = order_param.id \
|
||||
if isinstance(order_param, Order) else order_param
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('order/cancel', {'order_id': order_id})
|
||||
status = response.json()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'message' in status:
|
||||
raise OrderCancelError(
|
||||
order_id=order_id,
|
||||
exchange=self.name,
|
||||
error=status['message']
|
||||
)
|
||||
|
||||
def tickers(self, assets):
|
||||
"""
|
||||
Fetch ticket data for assets
|
||||
https://docs.bitfinex.com/v2/reference#rest-public-tickers
|
||||
|
||||
:param assets:
|
||||
:return:
|
||||
"""
|
||||
symbols = self._get_v2_symbols(assets)
|
||||
log.debug('fetching tickers {}'.format(symbols))
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
response = requests.get(
|
||||
'{url}/v2/tickers?symbols={symbols}'.format(
|
||||
url=self.url,
|
||||
symbols=','.join(symbols),
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'error' in response.content:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to retrieve tickers: {}'.format(
|
||||
response.content)
|
||||
)
|
||||
|
||||
try:
|
||||
tickers = response.json()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
ticks = dict()
|
||||
for index, ticker in enumerate(tickers):
|
||||
if not len(ticker) == 11:
|
||||
raise ExchangeRequestError(
|
||||
error='Invalid ticker in response: {}'.format(ticker)
|
||||
)
|
||||
|
||||
ticks[assets[index]] = dict(
|
||||
timestamp=pd.Timestamp.utcnow(),
|
||||
bid=ticker[1],
|
||||
ask=ticker[3],
|
||||
last_price=ticker[7],
|
||||
low=ticker[10],
|
||||
high=ticker[9],
|
||||
volume=ticker[8],
|
||||
)
|
||||
|
||||
log.debug('got tickers {}'.format(ticks))
|
||||
return ticks
|
||||
|
||||
def generate_symbols_json(self, filename=None, source_dates=False):
|
||||
symbol_map = {}
|
||||
|
||||
fn, r = download_exchange_symbols(self.name)
|
||||
with open(fn) as data_file:
|
||||
cached_symbols = json.load(data_file)
|
||||
|
||||
response = self._request('symbols', None)
|
||||
|
||||
for symbol in response.json():
|
||||
if (source_dates):
|
||||
start_date = self.get_symbol_start_date(symbol)
|
||||
else:
|
||||
try:
|
||||
start_date = cached_symbols[symbol]['start_date']
|
||||
except KeyError as e:
|
||||
start_date = time.strftime('%Y-%m-%d')
|
||||
|
||||
try:
|
||||
end_daily = cached_symbols[symbol]['end_daily']
|
||||
except KeyError as e:
|
||||
end_daily = 'N/A'
|
||||
|
||||
try:
|
||||
end_minute = cached_symbols[symbol]['end_minute']
|
||||
except KeyError as e:
|
||||
end_minute = 'N/A'
|
||||
|
||||
symbol_map[symbol] = dict(
|
||||
symbol=symbol[:-3] + '_' + symbol[-3:],
|
||||
start_date=start_date,
|
||||
end_daily=end_daily,
|
||||
end_minute=end_minute,
|
||||
)
|
||||
|
||||
if (filename is None):
|
||||
filename = get_exchange_symbols_filename(self.name)
|
||||
|
||||
with open(filename, 'w') as f:
|
||||
json.dump(symbol_map, f, sort_keys=True, indent=2,
|
||||
separators=(',', ':'))
|
||||
|
||||
def get_symbol_start_date(self, symbol):
|
||||
|
||||
print(symbol)
|
||||
symbol_v2 = 't' + symbol.upper()
|
||||
|
||||
"""
|
||||
For each symbol we retrieve candles with Monhtly resolution
|
||||
We get the first month, and query again with daily resolution
|
||||
around that date, and we get the first date
|
||||
"""
|
||||
url = '{url}/v2/candles/trade:1M:{symbol}/hist'.format(
|
||||
url=self.url,
|
||||
symbol=symbol_v2
|
||||
)
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
time.sleep(60 / self.max_requests_per_minute)
|
||||
response = requests.get(url)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
"""
|
||||
If we don't get any data back for our monthly-resolution query
|
||||
it means that symbol started trading less than a month ago, so
|
||||
arbitrarily set the ref. date to 15 days ago to be safe with
|
||||
+/- 31 days
|
||||
"""
|
||||
if (len(response.json())):
|
||||
startmonth = int(response.json()[-1][0])
|
||||
else:
|
||||
startmonth = int((time.time() - 15 * 24 * 3600) * 1000)
|
||||
|
||||
"""
|
||||
Query again with daily resolution setting the start and end around
|
||||
the startmonth we got above. Avoid end dates greater than now: time.time()
|
||||
"""
|
||||
url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format(
|
||||
url=self.url,
|
||||
symbol=symbol_v2,
|
||||
start=startmonth - 3600 * 24 * 31 * 1000,
|
||||
end=min(startmonth + 3600 * 24 * 31 * 1000,
|
||||
int(time.time() * 1000))
|
||||
)
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
response = requests.get(url)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
return time.strftime('%Y-%m-%d',
|
||||
time.gmtime(int(response.json()[-1][0] / 1000)))
|
||||
|
||||
def get_orderbook(self, asset, order_type='all', limit=100):
|
||||
exchange_symbol = asset.exchange_symbol
|
||||
try:
|
||||
self.ask_request()
|
||||
# TODO: implement limit
|
||||
response = self._request(
|
||||
'book/{}'.format(exchange_symbol), None)
|
||||
data = response.json()
|
||||
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
# TODO: filter by type
|
||||
result = dict()
|
||||
for order_type in data:
|
||||
result[order_type] = []
|
||||
|
||||
for entry in data[order_type]:
|
||||
result[order_type].append(dict(
|
||||
rate=float(entry['price']),
|
||||
quantity=float(entry['amount'])
|
||||
))
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,127 @@
|
||||
{
|
||||
"neobtc": {
|
||||
"symbol": "neo_btc",
|
||||
"start_date": "2017-09-07",
|
||||
"precision": 5
|
||||
},
|
||||
"neousd": {
|
||||
"symbol": "neo_usd",
|
||||
"start_date": "2017-09-07"
|
||||
},
|
||||
"neoeth": {
|
||||
"symbol": "neo_eth",
|
||||
"start_date": "2017-09-07"
|
||||
},
|
||||
"btcusd": {
|
||||
"symbol": "btc_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"bchusd": {
|
||||
"symbol": "bch_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"ltcusd": {
|
||||
"symbol": "ltc_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"ltcbtc": {
|
||||
"symbol": "ltc_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"ethusd": {
|
||||
"symbol": "eth_usd",
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"ethbtc": {
|
||||
"symbol": "eth_btc",
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"etcbtc": {
|
||||
"symbol": "etc_btc",
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"etcusd": {
|
||||
"symbol": "etc_usd",
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"rrtusd": {
|
||||
"symbol": "rrt_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"rrtbtc": {
|
||||
"symbol": "rrt_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"zecusd": {
|
||||
"symbol": "zec_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"zecbtc": {
|
||||
"symbol": "zec_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"xmrusd": {
|
||||
"symbol": "xmr_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"xmrbtc": {
|
||||
"symbol": "xmr_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"dshusd": {
|
||||
"symbol": "dsh_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"dshbtc": {
|
||||
"symbol": "dsh_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"bccbtc": {
|
||||
"symbol": "bcc_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"bcubtc": {
|
||||
"symbol": "bcu_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"bccusd": {
|
||||
"symbol": "bcc_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"bcuusd": {
|
||||
"symbol": "bcu_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"xrpusd": {
|
||||
"symbol": "xrp_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"xrpbtc": {
|
||||
"symbol": "xrp_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"iotusd": {
|
||||
"symbol": "iot_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"iotbtc": {
|
||||
"symbol": "iot_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"ioteth": {
|
||||
"symbol": "iot_eth",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"eosusd": {
|
||||
"symbol": "eos_usd",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"eosbtc": {
|
||||
"symbol": "eos_btc",
|
||||
"start_date": "2010-01-01"
|
||||
},
|
||||
"eoseth": {
|
||||
"symbol": "eos_eth",
|
||||
"start_date": "2010-01-01"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
import json
|
||||
import time
|
||||
|
||||
import pandas as pd
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
from six.moves import urllib
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
|
||||
from catalyst.exchange.exchange import Exchange
|
||||
from catalyst.exchange.exchange_bundle import ExchangeBundle
|
||||
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
|
||||
ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
|
||||
CreateOrderError
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
|
||||
download_exchange_symbols, get_symbols_string
|
||||
from catalyst.finance.execution import LimitOrder, StopLimitOrder
|
||||
from catalyst.finance.order import Order, ORDER_STATUS
|
||||
|
||||
# TODO: consider using this: https://github.com/mondeja/bittrex_v2
|
||||
|
||||
log = Logger('Bittrex', level=LOG_LEVEL)
|
||||
|
||||
URL2 = 'https://bittrex.com/Api/v2.0'
|
||||
|
||||
|
||||
class Bittrex(Exchange):
|
||||
def __init__(self, key, secret, base_currency, portfolio=None):
|
||||
self.api = Bittrex_api(key=key, secret=secret)
|
||||
self.name = 'bittrex'
|
||||
self.color = 'blue'
|
||||
self.base_currency = base_currency
|
||||
self._portfolio = portfolio
|
||||
|
||||
self.num_candles_limit = 2000
|
||||
|
||||
# Not sure what the rate limit is but trying to play it safe
|
||||
# https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit
|
||||
self.max_requests_per_minute = 60
|
||||
self.request_cpt = dict()
|
||||
|
||||
self.minute_writer = None
|
||||
self.minute_reader = None
|
||||
|
||||
self.assets = dict()
|
||||
self.load_assets()
|
||||
|
||||
self.local_assets = dict()
|
||||
self.load_assets(is_local=True)
|
||||
|
||||
self.bundle = ExchangeBundle(self.name)
|
||||
|
||||
@property
|
||||
def account(self):
|
||||
pass
|
||||
|
||||
@property
|
||||
def time_skew(self):
|
||||
# TODO: research the time skew conditions
|
||||
return pd.Timedelta('0s')
|
||||
|
||||
def sanitize_curency_symbol(self, exchange_symbol):
|
||||
"""
|
||||
Helper method used to build the universal pair.
|
||||
Include any symbol mapping here if appropriate.
|
||||
|
||||
:param exchange_symbol:
|
||||
:return universal_symbol:
|
||||
"""
|
||||
return exchange_symbol.lower()
|
||||
|
||||
def get_balances(self):
|
||||
balances = self.api.getbalances()
|
||||
try:
|
||||
log.debug('retrieving wallet balances')
|
||||
self.ask_request()
|
||||
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
std_balances = dict()
|
||||
try:
|
||||
for balance in balances:
|
||||
currency = balance['Currency'].lower()
|
||||
std_balances[currency] = balance['Available']
|
||||
|
||||
except TypeError:
|
||||
raise ExchangeRequestError(error=balances)
|
||||
|
||||
return std_balances
|
||||
|
||||
def create_order(self, asset, amount, is_buy, style):
|
||||
log.info('creating {} order'.format('buy' if is_buy else 'sell'))
|
||||
exchange_symbol = self.get_symbol(asset)
|
||||
|
||||
if isinstance(style, LimitOrder) or isinstance(style, StopLimitOrder):
|
||||
if isinstance(style, StopLimitOrder):
|
||||
log.warn('{} will ignore the stop price'.format(self.name))
|
||||
|
||||
price = style.get_limit_price(is_buy)
|
||||
try:
|
||||
self.ask_request()
|
||||
if is_buy:
|
||||
order_status = self.api.buylimit(exchange_symbol, amount,
|
||||
price)
|
||||
else:
|
||||
order_status = self.api.selllimit(exchange_symbol,
|
||||
abs(amount), price)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'uuid' in order_status:
|
||||
order_id = order_status['uuid']
|
||||
order = Order(
|
||||
dt=pd.Timestamp.utcnow(),
|
||||
asset=asset,
|
||||
amount=amount,
|
||||
stop=style.get_stop_price(is_buy),
|
||||
limit=style.get_limit_price(is_buy),
|
||||
id=order_id
|
||||
)
|
||||
return order
|
||||
else:
|
||||
if order_status == 'INSUFFICIENT_FUNDS':
|
||||
log.warn('not enough funds to create order')
|
||||
return None
|
||||
elif order_status == 'DUST_TRADE_DISALLOWED_MIN_VALUE_50K_SAT':
|
||||
log.warn('Your order is too small, order at least 50K'
|
||||
' Satoshi')
|
||||
return None
|
||||
else:
|
||||
raise CreateOrderError(
|
||||
exchange=self.name,
|
||||
error=order_status
|
||||
)
|
||||
else:
|
||||
raise InvalidOrderStyle(exchange=self.name,
|
||||
style=style.__class__.__name__)
|
||||
|
||||
def get_open_orders(self, asset):
|
||||
symbol = self.get_symbol(asset)
|
||||
try:
|
||||
self.ask_request()
|
||||
open_orders = self.api.getopenorders(symbol)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
orders = list()
|
||||
for order_status in open_orders:
|
||||
order = self._create_order(order_status)
|
||||
orders.append(order)
|
||||
|
||||
return orders
|
||||
|
||||
def _create_order(self, order_status):
|
||||
log.info(
|
||||
'creating catalyst order from Bittrex {}'.format(order_status))
|
||||
if order_status['CancelInitiated']:
|
||||
status = ORDER_STATUS.CANCELLED
|
||||
elif order_status['Closed'] is not None:
|
||||
status = ORDER_STATUS.FILLED
|
||||
else:
|
||||
status = ORDER_STATUS.OPEN
|
||||
|
||||
date = pd.to_datetime(order_status['Opened'], utc=True)
|
||||
amount = order_status['Quantity']
|
||||
filled = amount - order_status['QuantityRemaining']
|
||||
order = Order(
|
||||
dt=date,
|
||||
asset=self.assets[order_status['Exchange']],
|
||||
amount=amount,
|
||||
stop=None, # Not yet supported by Bittrex
|
||||
limit=order_status['Limit'],
|
||||
filled=filled,
|
||||
id=order_status['OrderUuid'],
|
||||
commission=order_status['CommissionPaid']
|
||||
)
|
||||
order.status = status
|
||||
|
||||
executed_price = order_status['PricePerUnit']
|
||||
|
||||
return order, executed_price
|
||||
|
||||
def get_order(self, order_id):
|
||||
log.info('retrieving order {}'.format(order_id))
|
||||
try:
|
||||
self.ask_request()
|
||||
order_status = self.api.getorder(order_id)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if order_status is None:
|
||||
raise OrderNotFound(order_id=order_id, exchange=self.name)
|
||||
|
||||
return self._create_order(order_status)
|
||||
|
||||
def cancel_order(self, order_param):
|
||||
order_id = order_param.id \
|
||||
if isinstance(order_param, Order) else order_param
|
||||
log.info('cancelling order {}'.format(order_id))
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
status = self.api.cancel(order_id)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'message' in status:
|
||||
raise OrderCancelError(
|
||||
order_id=order_id,
|
||||
exchange=self.name,
|
||||
error=status['message']
|
||||
)
|
||||
|
||||
def get_candles(self, freq, assets, bar_count=None,
|
||||
start_dt=None, end_dt=None):
|
||||
"""
|
||||
Supported Intervals
|
||||
-------------------
|
||||
day, oneMin, fiveMin, thirtyMin, hour
|
||||
|
||||
:param freq:
|
||||
:param assets:
|
||||
:param bar_count:
|
||||
:param start_dt
|
||||
:param end_dt
|
||||
:return:
|
||||
"""
|
||||
|
||||
# TODO: this has no effect at the moment
|
||||
if end_dt is None:
|
||||
end_dt = pd.Timestamp.utcnow()
|
||||
|
||||
log.debug(
|
||||
'retrieving {bars} {freq} candles on {exchange} from '
|
||||
'{end_dt} for markets {symbols}, '.format(
|
||||
bars=bar_count,
|
||||
freq=freq,
|
||||
exchange=self.name,
|
||||
end_dt=end_dt,
|
||||
symbols=get_symbols_string(assets)
|
||||
)
|
||||
)
|
||||
|
||||
if freq == '1T':
|
||||
frequency = 'oneMin'
|
||||
elif freq == '5T':
|
||||
frequency = 'fiveMin'
|
||||
elif freq == '30T':
|
||||
frequency = 'thirtyMin'
|
||||
elif freq == '60T':
|
||||
frequency = 'hour'
|
||||
elif freq == '1D':
|
||||
frequency = 'day'
|
||||
else:
|
||||
raise InvalidHistoryFrequencyError(frequency=freq)
|
||||
|
||||
# Making sure that assets are iterable
|
||||
asset_list = [assets] if isinstance(assets, TradingPair) else assets
|
||||
for asset in asset_list:
|
||||
end = int(time.mktime(end_dt.timetuple()))
|
||||
url = '{url}/pub/market/GetTicks?marketName={symbol}' \
|
||||
'&tickInterval={frequency}&_={end}'.format(
|
||||
url=URL2,
|
||||
symbol=self.get_symbol(asset),
|
||||
frequency=frequency,
|
||||
end=end
|
||||
)
|
||||
|
||||
try:
|
||||
data = json.loads(urllib.request.urlopen(url).read().decode())
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if data['message']:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to fetch candles {}'.format(data['message'])
|
||||
)
|
||||
|
||||
candles = data['result']
|
||||
|
||||
def ohlc_from_candle(candle):
|
||||
ohlc = dict(
|
||||
open=candle['O'],
|
||||
high=candle['H'],
|
||||
low=candle['L'],
|
||||
close=candle['C'],
|
||||
volume=candle['V'],
|
||||
price=candle['C'],
|
||||
last_traded=pd.to_datetime(candle['T'], utc=True)
|
||||
)
|
||||
return ohlc
|
||||
|
||||
ordered_candles = list(reversed(candles))
|
||||
ohlc_map = dict()
|
||||
if bar_count is None:
|
||||
ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
|
||||
else:
|
||||
# TODO: optimize
|
||||
ohlc_bars = []
|
||||
for candle in ordered_candles[:bar_count]:
|
||||
ohlc = ohlc_from_candle(candle)
|
||||
ohlc_bars.append(ohlc)
|
||||
|
||||
ohlc_map[asset] = ohlc_bars
|
||||
|
||||
return ohlc_map[assets] \
|
||||
if isinstance(assets, TradingPair) else ohlc_map
|
||||
|
||||
def tickers(self, assets):
|
||||
"""
|
||||
As of v1.1, Bittrex only allows one ticker at the time.
|
||||
So we have to make multiple calls to fetch multiple assets.
|
||||
|
||||
:param assets:
|
||||
:return:
|
||||
"""
|
||||
log.info('retrieving tickers')
|
||||
|
||||
ticks = dict()
|
||||
for asset in assets:
|
||||
symbol = self.get_symbol(asset)
|
||||
try:
|
||||
self.ask_request()
|
||||
ticker = self.api.getticker(symbol)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
# TODO: catch invalid ticker
|
||||
ticks[asset] = dict(
|
||||
timestamp=pd.Timestamp.utcnow(),
|
||||
bid=ticker['Bid'],
|
||||
ask=ticker['Ask'],
|
||||
last_price=ticker['Last']
|
||||
)
|
||||
|
||||
log.debug('got tickers {}'.format(ticks))
|
||||
return ticks
|
||||
|
||||
def get_account(self):
|
||||
log.info('retrieving account data')
|
||||
pass
|
||||
|
||||
def generate_symbols_json(self, filename=None):
|
||||
symbol_map = {}
|
||||
|
||||
fn, r = download_exchange_symbols(self.name)
|
||||
with open(fn) as data_file:
|
||||
cached_symbols = json.load(data_file)
|
||||
|
||||
markets = self.api.getmarkets()
|
||||
for market in markets:
|
||||
exchange_symbol = market['MarketName']
|
||||
symbol = '{market}_{base}'.format(
|
||||
market=self.sanitize_curency_symbol(market['MarketCurrency']),
|
||||
base=self.sanitize_curency_symbol(market['BaseCurrency'])
|
||||
)
|
||||
|
||||
try:
|
||||
end_daily = cached_symbols[exchange_symbol]['end_daily']
|
||||
except KeyError as e:
|
||||
end_daily = 'N/A'
|
||||
|
||||
try:
|
||||
end_minute = cached_symbols[exchange_symbol]['end_minute']
|
||||
except KeyError as e:
|
||||
end_minute = 'N/A'
|
||||
|
||||
symbol_map[exchange_symbol] = dict(
|
||||
symbol=symbol,
|
||||
start_date=pd.to_datetime(market['Created'],
|
||||
utc=True).strftime("%Y-%m-%d"),
|
||||
end_daily=end_daily,
|
||||
end_minute=end_minute,
|
||||
)
|
||||
|
||||
if (filename is None):
|
||||
filename = get_exchange_symbols_filename(self.name)
|
||||
|
||||
with open(filename, 'w') as f:
|
||||
json.dump(symbol_map, f, sort_keys=True, indent=2,
|
||||
separators=(',', ':'))
|
||||
|
||||
def get_orderbook(self, asset, order_type='all', limit=100):
|
||||
if order_type == 'all':
|
||||
order_type = 'both'
|
||||
elif order_type == 'bid':
|
||||
order_type = 'buy'
|
||||
elif order_type == 'ask':
|
||||
order_type = 'sell'
|
||||
else:
|
||||
raise ValueError('invalid type')
|
||||
|
||||
exchange_symbol = asset.exchange_symbol
|
||||
data = self.api.getorderbook(
|
||||
market=exchange_symbol,
|
||||
type=order_type,
|
||||
depth=100
|
||||
)
|
||||
|
||||
result = dict()
|
||||
for exchange_type in data:
|
||||
if exchange_type == 'buy':
|
||||
order_type = 'bids'
|
||||
elif exchange_type == 'sell':
|
||||
order_type = 'asks'
|
||||
|
||||
result[order_type] = []
|
||||
for entry in data[exchange_type]:
|
||||
result[order_type].append(dict(
|
||||
rate=entry['Rate'],
|
||||
quantity=entry['Quantity']
|
||||
))
|
||||
|
||||
return result
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python
|
||||
import json
|
||||
import time
|
||||
import hmac
|
||||
import hashlib
|
||||
import ssl
|
||||
|
||||
# Workaround for backwards compatibility
|
||||
# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
|
||||
from six.moves import urllib
|
||||
|
||||
urlopen = urllib.request.urlopen
|
||||
|
||||
|
||||
class Bittrex_api(object):
|
||||
def __init__(self, key, secret):
|
||||
self.key = key
|
||||
self.secret = secret
|
||||
self.public = ['getmarkets', 'getcurrencies', 'getticker',
|
||||
'getmarketsummaries', 'getmarketsummary',
|
||||
'getorderbook', 'getmarkethistory']
|
||||
self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket',
|
||||
'cancel', 'getopenorders']
|
||||
self.account = ['getbalances', 'getbalance', 'getdepositaddress',
|
||||
'withdraw', 'getorder', 'getorderhistory',
|
||||
'getwithdrawalhistory', 'getdeposithistory']
|
||||
|
||||
def query(self, method, values={}):
|
||||
if method in self.public:
|
||||
url = 'https://bittrex.com/api/v1.1/public/'
|
||||
elif method in self.market:
|
||||
url = 'https://bittrex.com/api/v1.1/market/'
|
||||
elif method in self.account:
|
||||
url = 'https://bittrex.com/api/v1.1/account/'
|
||||
else:
|
||||
return 'Something went wrong, sorry.'
|
||||
|
||||
url += method + '?' + urllib.parse.urlencode(values)
|
||||
|
||||
if method not in self.public:
|
||||
url += '&apikey=' + self.key
|
||||
url += '&nonce=' + str(int(time.time()))
|
||||
|
||||
signature = hmac.new(self.secret.encode('utf-8'),
|
||||
url.encode('utf-8'),
|
||||
hashlib.sha512).hexdigest()
|
||||
headers = {'apisign': signature}
|
||||
else:
|
||||
headers = {}
|
||||
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
response = json.loads(urlopen(
|
||||
req, context=ssl._create_unverified_context()).read())
|
||||
|
||||
if response["result"]:
|
||||
return response["result"]
|
||||
else:
|
||||
return response["message"]
|
||||
|
||||
def getmarkets(self):
|
||||
return self.query('getmarkets')
|
||||
|
||||
def getcurrencies(self):
|
||||
return self.query('getcurrencies')
|
||||
|
||||
def getticker(self, market):
|
||||
return self.query('getticker', {'market': market})
|
||||
|
||||
def getmarketsummaries(self):
|
||||
return self.query('getmarketsummaries')
|
||||
|
||||
def getmarketsummary(self, market):
|
||||
return self.query('getmarketsummary', {'market': market})
|
||||
|
||||
def getorderbook(self, market, type, depth=20):
|
||||
return self.query('getorderbook',
|
||||
{'market': market, 'type': type, 'depth': depth})
|
||||
|
||||
def getmarkethistory(self, market, count=20):
|
||||
return self.query('getmarkethistory',
|
||||
{'market': market, 'count': count})
|
||||
|
||||
def buylimit(self, market, quantity, rate):
|
||||
return self.query('buylimit', {'market': market, 'quantity': quantity,
|
||||
'rate': rate})
|
||||
|
||||
def buymarket(self, market, quantity):
|
||||
return self.query('buymarket',
|
||||
{'market': market, 'quantity': quantity})
|
||||
|
||||
def selllimit(self, market, quantity, rate):
|
||||
return self.query('selllimit', {'market': market, 'quantity': quantity,
|
||||
'rate': rate})
|
||||
|
||||
def sellmarket(self, market, quantity):
|
||||
return self.query('sellmarket',
|
||||
{'market': market, 'quantity': quantity})
|
||||
|
||||
def cancel(self, uuid):
|
||||
return self.query('cancel', {'uuid': uuid})
|
||||
|
||||
def getopenorders(self, market):
|
||||
return self.query('getopenorders', {'market': market})
|
||||
|
||||
def getbalances(self):
|
||||
return self.query('getbalances')
|
||||
|
||||
def getbalance(self, currency):
|
||||
return self.query('getbalance', {'currency': currency})
|
||||
|
||||
def getdepositaddress(self, currency):
|
||||
return self.query('getdepositaddress', {'currency': currency})
|
||||
|
||||
def withdraw(self, currency, quantity, address):
|
||||
return self.query('withdraw',
|
||||
{'currency': currency, 'quantity': quantity,
|
||||
'address': address})
|
||||
|
||||
def getorder(self, uuid):
|
||||
return self.query('getorder', {'uuid': uuid})
|
||||
|
||||
def getorderhistory(self, market, count):
|
||||
return self.query('getorderhistory',
|
||||
{'market': market, 'count': count})
|
||||
|
||||
def getwithdrawalhistory(self, currency, count):
|
||||
return self.query('getwithdrawalhistory',
|
||||
{'currency': currency, 'count': count})
|
||||
|
||||
def getdeposithistory(self, currency, count):
|
||||
return self.query('getdeposithistory',
|
||||
{'currency': currency, 'count': count})
|
||||
@@ -0,0 +1,7 @@
|
||||
from catalyst.data.bundles import register
|
||||
from catalyst.exchange.exchange_bundle import exchange_bundle
|
||||
|
||||
symbols = (
|
||||
'neo_btc',
|
||||
)
|
||||
register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))
|
||||
@@ -1,12 +1,19 @@
|
||||
import calendar
|
||||
import re
|
||||
from datetime import datetime, timedelta, date
|
||||
import os
|
||||
import tarfile
|
||||
from datetime import timedelta, datetime, date
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytz
|
||||
from catalyst.assets._assets import TradingPair
|
||||
|
||||
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
|
||||
InvalidHistoryFrequencyAlias
|
||||
from catalyst.data.bundles.core import download_without_progress
|
||||
from catalyst.exchange.exchange_utils import get_exchange_bundles_folder, \
|
||||
get_exchange_symbols
|
||||
|
||||
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
|
||||
API_URL = 'http://data.enigma.co/api/v1'
|
||||
|
||||
|
||||
def get_date_from_ms(ms):
|
||||
@@ -44,6 +51,46 @@ def get_seconds_from_date(date):
|
||||
return int((date - epoch).total_seconds())
|
||||
|
||||
|
||||
def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
|
||||
"""
|
||||
Download and extract a bcolz bundle.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exchange_name: str
|
||||
symbol: str
|
||||
data_frequency: str
|
||||
period: str
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
|
||||
|
||||
"""
|
||||
root = get_exchange_bundles_folder(exchange_name)
|
||||
name = '{exchange}-{frequency}-{symbol}-{period}'.format(
|
||||
exchange=exchange_name,
|
||||
frequency=data_frequency,
|
||||
symbol=symbol,
|
||||
period=period
|
||||
)
|
||||
path = os.path.join(root, name)
|
||||
|
||||
if not os.path.isdir(path):
|
||||
url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
|
||||
'exchange-{exchange}/{name}.tar.gz'.format(
|
||||
exchange=exchange_name,
|
||||
name=name
|
||||
)
|
||||
|
||||
bytes = download_without_progress(url)
|
||||
with tarfile.open('r', fileobj=bytes) as tar:
|
||||
tar.extractall(path)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def get_delta(periods, data_frequency):
|
||||
"""
|
||||
Get a time delta based on the specified data frequency.
|
||||
@@ -62,7 +109,7 @@ def get_delta(periods, data_frequency):
|
||||
if data_frequency == 'minute' else timedelta(days=periods)
|
||||
|
||||
|
||||
def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
|
||||
def get_periods_range(start_dt, end_dt, freq):
|
||||
"""
|
||||
Get a date range for the specified parameters.
|
||||
|
||||
@@ -83,38 +130,7 @@ def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
|
||||
elif freq == 'daily':
|
||||
freq = 'D'
|
||||
|
||||
if start_dt is not None and end_dt is not None and periods is None:
|
||||
|
||||
return pd.date_range(start_dt, end_dt, freq=freq)
|
||||
|
||||
elif periods is not None and (start_dt is not None or end_dt is not None):
|
||||
_, unit_periods, unit, _ = get_frequency(freq)
|
||||
adj_periods = periods * unit_periods
|
||||
|
||||
# TODO: standardize time aliases to avoid any mapping
|
||||
unit = 'd' if unit == 'D' else 'm'
|
||||
delta = pd.Timedelta(adj_periods, unit)
|
||||
|
||||
if start_dt is not None:
|
||||
return pd.date_range(
|
||||
start=start_dt,
|
||||
end=start_dt + delta,
|
||||
freq=freq,
|
||||
closed='left',
|
||||
)
|
||||
|
||||
else:
|
||||
return pd.date_range(
|
||||
start=end_dt - delta,
|
||||
end=end_dt,
|
||||
freq=freq,
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
'Choose only two parameters between start_dt, end_dt '
|
||||
'and periods.'
|
||||
)
|
||||
return pd.date_range(start_dt, end_dt, freq=freq)
|
||||
|
||||
|
||||
def get_periods(start_dt, end_dt, freq):
|
||||
@@ -132,7 +148,7 @@ def get_periods(start_dt, end_dt, freq):
|
||||
int
|
||||
|
||||
"""
|
||||
return len(get_periods_range(start_dt=start_dt, end_dt=end_dt, freq=freq))
|
||||
return len(get_periods_range(start_dt, end_dt, freq))
|
||||
|
||||
|
||||
def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
|
||||
@@ -144,7 +160,6 @@ def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
|
||||
end_dt: datetime
|
||||
bar_count: int
|
||||
data_frequency: str
|
||||
include_first
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -178,10 +193,8 @@ def get_period_label(dt, data_frequency):
|
||||
str
|
||||
|
||||
"""
|
||||
if data_frequency == 'minute':
|
||||
return '{}-{:02d}'.format(dt.year, dt.month)
|
||||
else:
|
||||
return '{}'.format(dt.year)
|
||||
return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \
|
||||
else '{}'.format(dt.year)
|
||||
|
||||
|
||||
def get_month_start_end(dt, first_day=None, last_day=None):
|
||||
@@ -248,80 +261,99 @@ def get_year_start_end(dt, first_day=None, last_day=None):
|
||||
return year_start, year_end
|
||||
|
||||
|
||||
def get_frequency(freq, data_frequency=None):
|
||||
def get_df_from_arrays(arrays, periods):
|
||||
"""
|
||||
Get the frequency parameters.
|
||||
|
||||
Notes
|
||||
-----
|
||||
We're trying to use Pandas convention for frequency aliases.
|
||||
A DataFrame from the specified OHCLV arrays.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
freq: str
|
||||
data_frequency: str
|
||||
arrays: Object
|
||||
periods: DateTimeIndex
|
||||
|
||||
Returns
|
||||
-------
|
||||
str, int, str, str
|
||||
DataFrame
|
||||
|
||||
"""
|
||||
if data_frequency is None:
|
||||
data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
|
||||
ohlcv = dict()
|
||||
for index, field in enumerate(
|
||||
['open', 'high', 'low', 'close', 'volume']):
|
||||
ohlcv[field] = arrays[index].flatten()
|
||||
|
||||
if freq == 'minute':
|
||||
unit = 'T'
|
||||
candle_size = 1
|
||||
df = pd.DataFrame(
|
||||
data=ohlcv,
|
||||
index=periods
|
||||
)
|
||||
return df
|
||||
|
||||
elif freq == 'daily':
|
||||
unit = 'D'
|
||||
candle_size = 1
|
||||
|
||||
def range_in_bundle(asset, start_dt, end_dt, reader):
|
||||
"""
|
||||
Evaluate whether price data of an asset is included has been ingested in
|
||||
the exchange bundle for the given date range.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
asset: TradingPair
|
||||
start_dt: datetime
|
||||
end_dt: datetime
|
||||
reader: BcolzBarMinuteReader
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
|
||||
"""
|
||||
has_data = True
|
||||
dates = [start_dt, end_dt]
|
||||
|
||||
while dates and has_data:
|
||||
try:
|
||||
dt = dates.pop(0)
|
||||
close = reader.get_value(asset.sid, dt, 'close')
|
||||
|
||||
if np.isnan(close):
|
||||
has_data = False
|
||||
|
||||
except Exception as e:
|
||||
has_data = False
|
||||
|
||||
return has_data
|
||||
|
||||
|
||||
def get_assets(exchange, include_symbols, exclude_symbols):
|
||||
"""
|
||||
Get assets from an exchange, including or excluding the specified
|
||||
symbols.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exchange: Exchange
|
||||
include_symbols: str
|
||||
exclude_symbols: str
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[TradingPair]
|
||||
|
||||
"""
|
||||
if include_symbols is not None:
|
||||
include_symbols_list = include_symbols.split(',')
|
||||
|
||||
return exchange.get_assets(include_symbols_list)
|
||||
|
||||
else:
|
||||
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
|
||||
if freq_match:
|
||||
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
|
||||
else 1
|
||||
unit = freq_match.group(2)
|
||||
all_assets = exchange.get_assets()
|
||||
|
||||
if exclude_symbols is not None:
|
||||
exclude_symbols_list = exclude_symbols.split(',')
|
||||
|
||||
assets = []
|
||||
for asset in all_assets:
|
||||
if asset.symbol not in exclude_symbols_list:
|
||||
assets.append(asset)
|
||||
|
||||
return assets
|
||||
|
||||
else:
|
||||
raise InvalidHistoryFrequencyError(frequency=freq)
|
||||
|
||||
# TODO: some exchanges support H and W frequencies but not bundles
|
||||
# Find a way to pass-through these parameters to exchanges
|
||||
# but resample from minute or daily in backtest mode
|
||||
# see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
|
||||
# Pandas offet aliases (used by Catalyst) and the CCXT timeframes
|
||||
if unit.lower() == 'd':
|
||||
unit = 'D'
|
||||
alias = '{}D'.format(candle_size)
|
||||
|
||||
if data_frequency == 'minute':
|
||||
data_frequency = 'daily'
|
||||
|
||||
elif unit.lower() == 'm' or unit == 'T':
|
||||
unit = 'T'
|
||||
alias = '{}T'.format(candle_size)
|
||||
|
||||
if data_frequency == 'daily':
|
||||
data_frequency = 'minute'
|
||||
|
||||
# elif unit.lower() == 'h':
|
||||
# candle_size = candle_size * 60
|
||||
#
|
||||
# alias = '{}T'.format(candle_size)
|
||||
# if data_frequency == 'daily':
|
||||
# data_frequency = 'minute'
|
||||
|
||||
else:
|
||||
raise InvalidHistoryFrequencyAlias(freq=freq)
|
||||
|
||||
return alias, candle_size, unit, data_frequency
|
||||
|
||||
|
||||
def from_ms_timestamp(ms):
|
||||
return pd.to_datetime(ms, unit='ms', utc=True)
|
||||
|
||||
|
||||
def get_epoch():
|
||||
return pd.to_datetime('1970-1-1', utc=True)
|
||||
return all_assets
|
||||
File diff suppressed because it is too large
Load Diff
+267
-310
@@ -5,20 +5,25 @@ from time import sleep
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.data.data_portal import BASE_FIELDS
|
||||
from catalyst.exchange.bundle_utils import get_start_dt, \
|
||||
get_delta, get_periods, get_periods_range
|
||||
from catalyst.exchange.exchange_bundle import ExchangeBundle
|
||||
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
|
||||
SymbolNotFoundOnExchange, \
|
||||
InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
|
||||
PricingDataNotLoadedError, \
|
||||
NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
|
||||
TickerNotFoundError, NotEnoughCashError
|
||||
from catalyst.exchange.utils.datetime_utils import get_delta, \
|
||||
get_periods_range, \
|
||||
get_periods, get_start_dt, get_frequency
|
||||
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
|
||||
resample_history_df, has_bundle
|
||||
from logbook import Logger
|
||||
NoDataAvailableOnExchange, ExchangeSymbolsNotFound
|
||||
from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
|
||||
ExchangeLimitOrder, ExchangeStopOrder
|
||||
from catalyst.exchange.exchange_portfolio import ExchangePortfolio
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols, \
|
||||
get_frequency, resample_history_df
|
||||
from catalyst.finance.order import ORDER_STATUS
|
||||
from catalyst.finance.transaction import Transaction
|
||||
|
||||
log = Logger('Exchange', level=LOG_LEVEL)
|
||||
|
||||
@@ -28,8 +33,9 @@ class Exchange:
|
||||
|
||||
def __init__(self):
|
||||
self.name = None
|
||||
self.assets = []
|
||||
self._symbol_maps = [None, None]
|
||||
self.assets = dict()
|
||||
self.local_assets = dict()
|
||||
self._portfolio = None
|
||||
self.minute_writer = None
|
||||
self.minute_reader = None
|
||||
self.base_currency = None
|
||||
@@ -39,7 +45,26 @@ class Exchange:
|
||||
self.request_cpt = None
|
||||
self.bundle = ExchangeBundle(self.name)
|
||||
|
||||
self.low_balance_threshold = None
|
||||
@property
|
||||
def positions(self):
|
||||
return self.portfolio.positions
|
||||
|
||||
@property
|
||||
def portfolio(self):
|
||||
"""
|
||||
The exchange portfolio
|
||||
|
||||
Returns
|
||||
-------
|
||||
ExchangePortfolio
|
||||
"""
|
||||
if self._portfolio is None:
|
||||
self._portfolio = ExchangePortfolio(
|
||||
start_date=pd.Timestamp.utcnow()
|
||||
)
|
||||
self.synchronize_portfolio()
|
||||
|
||||
return self._portfolio
|
||||
|
||||
@abstractproperty
|
||||
def account(self):
|
||||
@@ -49,9 +74,6 @@ class Exchange:
|
||||
def time_skew(self):
|
||||
pass
|
||||
|
||||
def has_bundle(self, data_frequency):
|
||||
return has_bundle(self.name, data_frequency)
|
||||
|
||||
def is_open(self, dt):
|
||||
"""
|
||||
Is the exchange open
|
||||
@@ -123,9 +145,9 @@ class Exchange:
|
||||
"""
|
||||
symbol = None
|
||||
|
||||
for a in self.assets:
|
||||
if not symbol and a.symbol == asset.symbol:
|
||||
symbol = a.symbol
|
||||
for key in self.assets:
|
||||
if not symbol and self.assets[key].symbol == asset.symbol:
|
||||
symbol = key
|
||||
|
||||
if not symbol:
|
||||
raise ValueError('Currency %s not supported by exchange %s' %
|
||||
@@ -152,164 +174,85 @@ class Exchange:
|
||||
|
||||
return symbols
|
||||
|
||||
def get_assets(self, symbols=None, data_frequency=None,
|
||||
is_exchange_symbol=False,
|
||||
is_local=None, quote_currency=None):
|
||||
def get_assets(self, symbols=None, data_frequency=None):
|
||||
"""
|
||||
The list of markets for the specified symbols.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
symbols: list[str]
|
||||
data_frequency: str
|
||||
is_exchange_symbol: bool
|
||||
is_local: bool
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[TradingPair]
|
||||
A list of asset objects.
|
||||
|
||||
Notes
|
||||
-----
|
||||
See get_asset for details of each parameter.
|
||||
|
||||
"""
|
||||
if symbols is None:
|
||||
# Make a distinct list of all symbols
|
||||
symbols = list(set([asset.symbol for asset in self.assets]))
|
||||
|
||||
if quote_currency is not None:
|
||||
for symbol in symbols[:]:
|
||||
suffix = '_{}'.format(quote_currency.lower())
|
||||
|
||||
if not symbol.endswith(suffix):
|
||||
symbols.remove(symbol)
|
||||
|
||||
is_exchange_symbol = False
|
||||
|
||||
assets = []
|
||||
for symbol in symbols:
|
||||
try:
|
||||
asset = self.get_asset(
|
||||
symbol, data_frequency, is_exchange_symbol, is_local
|
||||
)
|
||||
assets.append(asset)
|
||||
|
||||
except SymbolNotFoundOnExchange:
|
||||
log.debug(
|
||||
'skipping non-existent market {} {}'.format(
|
||||
self.name, symbol
|
||||
)
|
||||
)
|
||||
if symbols is not None:
|
||||
for symbol in symbols:
|
||||
asset = self.get_asset(symbol, data_frequency)
|
||||
assets.append(asset)
|
||||
else:
|
||||
for key in self.assets:
|
||||
assets.append(self.assets[key])
|
||||
|
||||
return assets
|
||||
|
||||
def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
|
||||
is_local=None):
|
||||
def _find_asset(self, asset, symbol, data_frequency, is_local=False):
|
||||
assets = self.assets if not is_local else self.local_assets
|
||||
|
||||
for key in assets:
|
||||
has_data = (data_frequency == 'minute'
|
||||
and assets[key].end_minute is not None) \
|
||||
or (data_frequency == 'daily'
|
||||
and assets[key].end_daily is not None)
|
||||
if not asset and assets[key].symbol.lower() == symbol.lower() \
|
||||
and (not data_frequency or has_data):
|
||||
asset = assets[key]
|
||||
|
||||
return asset
|
||||
|
||||
def get_asset(self, symbol, data_frequency=None):
|
||||
"""
|
||||
The market for the specified symbol.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
symbol: str
|
||||
The Catalyst or exchange symbol.
|
||||
|
||||
data_frequency: str
|
||||
Check for asset corresponding to the specified data_frequency.
|
||||
The same asset might exist in the Catalyst repository or
|
||||
locally (following a CSV ingestion). Filtering by
|
||||
data_frequency picks the right asset.
|
||||
|
||||
is_exchange_symbol: bool
|
||||
Whether the symbol uses the Catalyst or exchange convention.
|
||||
|
||||
is_local: bool
|
||||
For the local or Catalyst asset.
|
||||
|
||||
Returns
|
||||
-------
|
||||
TradingPair
|
||||
The asset object.
|
||||
|
||||
"""
|
||||
asset = None
|
||||
|
||||
# TODO: temp mapping, fix to use a single symbol convention
|
||||
og_symbol = symbol
|
||||
symbol = self.get_symbol(symbol) if not is_exchange_symbol else symbol
|
||||
log.debug(
|
||||
'searching assets for: {} {}'.format(
|
||||
self.name, symbol
|
||||
)
|
||||
)
|
||||
# TODO: simplify and loose the loop
|
||||
for a in self.assets:
|
||||
if asset is not None:
|
||||
break
|
||||
log.debug('searching asset {} on the server'.format(symbol))
|
||||
asset = self._find_asset(asset, symbol, data_frequency, False)
|
||||
|
||||
if is_local is not None:
|
||||
data_source = 'local' if is_local else 'catalyst'
|
||||
applies = (a.data_source == data_source)
|
||||
log.debug('asset {} not found on the server, searching local '
|
||||
'assets'.format(symbol))
|
||||
asset = self._find_asset(asset, symbol, data_frequency, True)
|
||||
|
||||
elif data_frequency is not None:
|
||||
applies = (
|
||||
(
|
||||
data_frequency == 'minute' and a.end_minute is not None)
|
||||
or (
|
||||
data_frequency == 'daily' and a.end_daily is not None)
|
||||
)
|
||||
|
||||
else:
|
||||
applies = True
|
||||
|
||||
# The symbol provided may use the Catalyst or the exchange
|
||||
# convention
|
||||
key = a.exchange_symbol if \
|
||||
is_exchange_symbol else self.get_symbol(a)
|
||||
if not asset and key.lower() == symbol.lower():
|
||||
if applies:
|
||||
asset = a
|
||||
|
||||
else:
|
||||
raise NoDataAvailableOnExchange(
|
||||
symbol=key,
|
||||
exchange=self.name,
|
||||
data_frequency=data_frequency,
|
||||
)
|
||||
|
||||
if asset is None:
|
||||
supported_symbols = sorted([a.symbol for a in self.assets])
|
||||
if not asset:
|
||||
all_values = list(self.assets.values()) + \
|
||||
list(self.local_assets.values())
|
||||
supported_symbols = sorted([
|
||||
asset.symbol for asset in all_values
|
||||
])
|
||||
|
||||
raise SymbolNotFoundOnExchange(
|
||||
symbol=og_symbol,
|
||||
symbol=symbol,
|
||||
exchange=self.name.title(),
|
||||
supported_symbols=supported_symbols
|
||||
)
|
||||
|
||||
log.debug('found asset: {}'.format(asset))
|
||||
return asset
|
||||
|
||||
def fetch_symbol_map(self, is_local=False):
|
||||
index = 1 if is_local else 0
|
||||
if self._symbol_maps[index] is not None:
|
||||
return self._symbol_maps[index]
|
||||
return get_exchange_symbols(self.name, is_local)
|
||||
|
||||
else:
|
||||
symbol_map = get_exchange_symbols(self.name, is_local)
|
||||
self._symbol_maps[index] = symbol_map
|
||||
return symbol_map
|
||||
|
||||
@abstractmethod
|
||||
def init(self):
|
||||
"""
|
||||
Load the asset list from the network.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def load_assets(self, is_local=False):
|
||||
"""
|
||||
Populate the 'assets' attribute with a dictionary of Assets.
|
||||
@@ -327,7 +270,112 @@ class Exchange:
|
||||
via its api.
|
||||
|
||||
"""
|
||||
pass
|
||||
try:
|
||||
symbol_map = self.fetch_symbol_map(is_local)
|
||||
except ExchangeSymbolsNotFound:
|
||||
return None
|
||||
|
||||
for exchange_symbol in symbol_map:
|
||||
asset = symbol_map[exchange_symbol]
|
||||
|
||||
if 'start_date' in asset:
|
||||
start_date = pd.to_datetime(asset['start_date'], utc=True)
|
||||
else:
|
||||
start_date = None
|
||||
|
||||
if 'end_date' in asset:
|
||||
end_date = pd.to_datetime(asset['end_date'], utc=True)
|
||||
else:
|
||||
end_date = None
|
||||
|
||||
if 'leverage' in asset:
|
||||
leverage = asset['leverage']
|
||||
else:
|
||||
leverage = 1.0
|
||||
|
||||
if 'asset_name' in asset:
|
||||
asset_name = asset['asset_name']
|
||||
else:
|
||||
asset_name = None
|
||||
|
||||
if 'min_trade_size' in asset:
|
||||
min_trade_size = asset['min_trade_size']
|
||||
else:
|
||||
min_trade_size = 0.0000001
|
||||
|
||||
if 'end_daily' in asset and asset['end_daily'] != 'N/A':
|
||||
end_daily = pd.to_datetime(asset['end_daily'], utc=True)
|
||||
else:
|
||||
end_daily = None
|
||||
|
||||
if 'end_minute' in asset and asset['end_minute'] != 'N/A':
|
||||
end_minute = pd.to_datetime(asset['end_minute'], utc=True)
|
||||
else:
|
||||
end_minute = None
|
||||
|
||||
trading_pair = TradingPair(
|
||||
symbol=asset['symbol'],
|
||||
exchange=self.name,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
leverage=leverage,
|
||||
asset_name=asset_name,
|
||||
min_trade_size=min_trade_size,
|
||||
end_daily=end_daily,
|
||||
end_minute=end_minute,
|
||||
exchange_symbol=exchange_symbol
|
||||
)
|
||||
|
||||
if is_local:
|
||||
self.local_assets[exchange_symbol] = trading_pair
|
||||
else:
|
||||
self.assets[exchange_symbol] = trading_pair
|
||||
|
||||
def check_open_orders(self):
|
||||
"""
|
||||
Loop through the list of open orders in the Portfolio object.
|
||||
For each executed order found, create a transaction and apply to the
|
||||
Portfolio.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Transaction]
|
||||
|
||||
"""
|
||||
transactions = list()
|
||||
if self.portfolio.open_orders:
|
||||
for order_id in list(self.portfolio.open_orders):
|
||||
log.debug('found open order: {}'.format(order_id))
|
||||
|
||||
order, executed_price = self.get_order(order_id)
|
||||
log.debug('got updated order {} {}'.format(
|
||||
order, executed_price))
|
||||
|
||||
if order.status == ORDER_STATUS.FILLED:
|
||||
transaction = Transaction(
|
||||
asset=order.asset,
|
||||
amount=order.amount,
|
||||
dt=pd.Timestamp.utcnow(),
|
||||
price=executed_price,
|
||||
order_id=order.id,
|
||||
commission=order.commission
|
||||
)
|
||||
transactions.append(transaction)
|
||||
|
||||
self.portfolio.execute_order(order, transaction)
|
||||
|
||||
elif order.status == ORDER_STATUS.CANCELLED:
|
||||
self.portfolio.remove_order(order)
|
||||
|
||||
else:
|
||||
delta = pd.Timestamp.utcnow() - order.dt
|
||||
log.info(
|
||||
'order {order_id} still open after {delta}'.format(
|
||||
order_id=order_id,
|
||||
delta=delta
|
||||
)
|
||||
)
|
||||
return transactions
|
||||
|
||||
def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
|
||||
"""
|
||||
@@ -364,15 +412,12 @@ class Exchange:
|
||||
if field not in BASE_FIELDS:
|
||||
raise KeyError('Invalid column: {}'.format(field))
|
||||
|
||||
tickers = self.tickers(assets)
|
||||
if field == 'close' or field == 'price':
|
||||
return [tickers[asset]['last'] for asset in tickers]
|
||||
values = []
|
||||
for asset in assets:
|
||||
value = self.get_single_spot_value(asset, field, data_frequency)
|
||||
values.append(value)
|
||||
|
||||
elif field == 'volume':
|
||||
return [tickers[asset]['volume'] for asset in tickers]
|
||||
|
||||
else:
|
||||
raise NoValueForField(field=field)
|
||||
return values
|
||||
|
||||
def get_single_spot_value(self, asset, field, data_frequency):
|
||||
"""
|
||||
@@ -414,7 +459,6 @@ class Exchange:
|
||||
|
||||
return value
|
||||
|
||||
# TODO: replace with catalyst.exchange.exchange_utils.get_candles_df
|
||||
def get_series_from_candles(self, candles, start_dt, end_dt,
|
||||
data_frequency, field, previous_value=None):
|
||||
"""
|
||||
@@ -439,7 +483,7 @@ class Exchange:
|
||||
series = pd.Series(values, index=dates)
|
||||
|
||||
periods = get_periods_range(
|
||||
start_dt=start_dt, end_dt=end_dt, freq=data_frequency
|
||||
start_dt, end_dt, data_frequency
|
||||
)
|
||||
# TODO: ensure that this working as expected, if not use fillna
|
||||
series = series.reindex(
|
||||
@@ -447,7 +491,7 @@ class Exchange:
|
||||
method='ffill',
|
||||
fill_value=previous_value,
|
||||
)
|
||||
series.sort_index(inplace=True)
|
||||
|
||||
return series
|
||||
|
||||
def get_history_window(self,
|
||||
@@ -457,7 +501,7 @@ class Exchange:
|
||||
frequency,
|
||||
field,
|
||||
data_frequency=None,
|
||||
is_current=False):
|
||||
ffill=True):
|
||||
|
||||
"""
|
||||
Public API method that returns a dataframe containing the requested
|
||||
@@ -484,15 +528,10 @@ class Exchange:
|
||||
The frequency of the data to query; i.e. whether the data is
|
||||
'daily' or 'minute' bars.
|
||||
|
||||
is_current: bool
|
||||
Skip date filters when current data is requested (last few bars
|
||||
until now).
|
||||
|
||||
Notes
|
||||
-----
|
||||
Catalysts requires an end data with bar count both CCXT wants a
|
||||
start data with bar count. Since we have to make calculations here,
|
||||
we ensure that the last candle match the end_dt parameter.
|
||||
# TODO: fill how?
|
||||
ffill: boolean
|
||||
Forward-fill missing values. Only has effect if field
|
||||
is 'price'.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -503,37 +542,27 @@ class Exchange:
|
||||
freq, candle_size, unit, data_frequency = get_frequency(
|
||||
frequency, data_frequency
|
||||
)
|
||||
adj_bar_count = candle_size * bar_count
|
||||
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
|
||||
|
||||
# The get_history method supports multiple asset
|
||||
candles = self.get_candles(
|
||||
freq=freq,
|
||||
assets=assets,
|
||||
bar_count=bar_count,
|
||||
end_dt=end_dt if not is_current else None,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt
|
||||
)
|
||||
|
||||
series = dict()
|
||||
for asset in candles:
|
||||
first_candle = candles[asset][0]
|
||||
asset_series = self.get_series_from_candles(
|
||||
candles=candles[asset],
|
||||
start_dt=first_candle['last_traded'],
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
data_frequency=frequency,
|
||||
field=field,
|
||||
)
|
||||
|
||||
# Checking to make sure that the dates match
|
||||
delta = get_delta(candle_size, data_frequency)
|
||||
adj_end_dt = end_dt - delta
|
||||
last_traded = asset_series.index[-1]
|
||||
|
||||
if last_traded < adj_end_dt:
|
||||
raise LastCandleTooEarlyError(
|
||||
last_traded=last_traded,
|
||||
end_dt=adj_end_dt,
|
||||
exchange=self.name,
|
||||
)
|
||||
|
||||
series[asset] = asset_series
|
||||
|
||||
df = pd.DataFrame(series)
|
||||
@@ -587,7 +616,6 @@ class Exchange:
|
||||
A dataframe containing the requested data.
|
||||
|
||||
"""
|
||||
# TODO: this function needs some work, we're currently using it just for benchmark data
|
||||
freq, candle_size, unit, data_frequency = get_frequency(
|
||||
frequency, data_frequency
|
||||
)
|
||||
@@ -601,7 +629,6 @@ class Exchange:
|
||||
data_frequency=data_frequency,
|
||||
force_auto_ingest=force_auto_ingest
|
||||
)
|
||||
|
||||
except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
|
||||
series = dict()
|
||||
|
||||
@@ -618,14 +645,15 @@ class Exchange:
|
||||
# The get_history method supports multiple asset
|
||||
# Use the original frequency to let each api optimize
|
||||
# the size of result sets
|
||||
trailing_bars = get_periods(
|
||||
trailing_bar_count = get_periods(
|
||||
trailing_dt, end_dt, freq
|
||||
)
|
||||
candles = self.get_candles(
|
||||
freq=freq,
|
||||
assets=asset,
|
||||
end_dt=end_dt,
|
||||
bar_count=trailing_bars if trailing_bars < 500 else 500,
|
||||
bar_count=trailing_bar_count,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt
|
||||
)
|
||||
|
||||
last_value = series[asset].iloc(0) if asset in series \
|
||||
@@ -654,100 +682,50 @@ class Exchange:
|
||||
|
||||
return df
|
||||
|
||||
def _check_low_balance(self, currency, balances, amount):
|
||||
free = balances[currency]['free'] if currency in balances else 0.0
|
||||
|
||||
if free < amount:
|
||||
return free, True
|
||||
|
||||
else:
|
||||
return free, False
|
||||
|
||||
def sync_positions(self, positions, cash=None, check_balances=False):
|
||||
def synchronize_portfolio(self):
|
||||
"""
|
||||
Update the portfolio cash and position balances based on the
|
||||
latest ticker prices.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
positions:
|
||||
The positions to synchronize.
|
||||
|
||||
check_balances:
|
||||
Check balances amounts against the exchange.
|
||||
|
||||
"""
|
||||
free_cash = 0.0
|
||||
if check_balances:
|
||||
log.debug('fetching {} balances'.format(self.name))
|
||||
balances = self.get_balances()
|
||||
log.debug(
|
||||
'got free balances for {} currencies'.format(
|
||||
len(balances)
|
||||
)
|
||||
)
|
||||
if cash is not None:
|
||||
free_cash, is_lower = self._check_low_balance(
|
||||
currency=self.base_currency,
|
||||
balances=balances,
|
||||
amount=cash,
|
||||
)
|
||||
if is_lower:
|
||||
raise NotEnoughCashError(
|
||||
currency=self.base_currency,
|
||||
exchange=self.name,
|
||||
free=free_cash,
|
||||
cash=cash,
|
||||
)
|
||||
log.debug('synchronizing portfolio with exchange {}'.format(self.name))
|
||||
balances = self.get_balances()
|
||||
|
||||
positions_value = 0.0
|
||||
if positions is not None:
|
||||
assets = set([position.asset for position in positions])
|
||||
base_position_available = balances[self.base_currency] \
|
||||
if self.base_currency in balances else None
|
||||
|
||||
if base_position_available is None:
|
||||
raise BaseCurrencyNotFoundError(
|
||||
base_currency=self.base_currency,
|
||||
exchange=self.name.title()
|
||||
)
|
||||
|
||||
portfolio = self._portfolio
|
||||
portfolio.cash = base_position_available
|
||||
log.debug('found base currency balance: {}'.format(portfolio.cash))
|
||||
|
||||
if portfolio.starting_cash is None:
|
||||
portfolio.starting_cash = portfolio.cash
|
||||
|
||||
if portfolio.positions:
|
||||
assets = list(portfolio.positions.keys())
|
||||
tickers = self.tickers(assets)
|
||||
|
||||
for position in positions:
|
||||
asset = position.asset
|
||||
if asset not in tickers:
|
||||
raise TickerNotFoundError(
|
||||
symbol=asset.symbol,
|
||||
exchange=self.name,
|
||||
)
|
||||
|
||||
portfolio.positions_value = 0.0
|
||||
for asset in tickers:
|
||||
# TODO: convert if the position is not in the base currency
|
||||
ticker = tickers[asset]
|
||||
log.debug(
|
||||
'updating {symbol} position, last traded on {dt} for '
|
||||
'{price}{currency}'.format(
|
||||
symbol=asset.symbol,
|
||||
dt=ticker['last_traded'],
|
||||
price=ticker['last_price'],
|
||||
currency=asset.quote_currency,
|
||||
)
|
||||
)
|
||||
position = portfolio.positions[asset]
|
||||
position.last_sale_price = ticker['last_price']
|
||||
position.last_sale_date = ticker['last_traded']
|
||||
position.last_sale_date = ticker['timestamp']
|
||||
|
||||
positions_value += \
|
||||
portfolio.positions_value += \
|
||||
position.amount * position.last_sale_price
|
||||
portfolio.portfolio_value = \
|
||||
portfolio.positions_value + portfolio.cash
|
||||
|
||||
if check_balances:
|
||||
free, is_lower = self._check_low_balance(
|
||||
currency=asset.base_currency,
|
||||
balances=balances,
|
||||
amount=position.amount,
|
||||
)
|
||||
|
||||
if is_lower:
|
||||
log.warn(
|
||||
'detected lower balance for {} on {}: {} < {}, '
|
||||
'updating position amount'.format(
|
||||
asset.symbol, self.name, free, position.amount
|
||||
)
|
||||
)
|
||||
position.amount = free
|
||||
|
||||
return free_cash, positions_value
|
||||
|
||||
def order(self, asset, amount, style):
|
||||
def order(self, asset, amount, limit_price=None, stop_price=None,
|
||||
style=None):
|
||||
"""Place an order.
|
||||
|
||||
Parameters
|
||||
@@ -796,30 +774,45 @@ class Exchange:
|
||||
log.warn('skipping order amount of 0')
|
||||
return None
|
||||
|
||||
if self.base_currency is None:
|
||||
raise ValueError('no base_currency defined for this exchange')
|
||||
|
||||
if asset.quote_currency != self.base_currency.lower():
|
||||
if asset.base_currency != self.base_currency.lower():
|
||||
raise MismatchingBaseCurrencies(
|
||||
base_currency=asset.quote_currency,
|
||||
base_currency=asset.base_currency,
|
||||
algo_currency=self.base_currency
|
||||
)
|
||||
|
||||
is_buy = (amount > 0)
|
||||
display_price = style.get_limit_price(is_buy)
|
||||
|
||||
if limit_price is not None and stop_price is not None:
|
||||
style = ExchangeStopLimitOrder(limit_price, stop_price,
|
||||
exchange=self.name)
|
||||
elif limit_price is not None:
|
||||
style = ExchangeLimitOrder(limit_price, exchange=self.name)
|
||||
|
||||
elif stop_price is not None:
|
||||
style = ExchangeStopOrder(stop_price, exchange=self.name)
|
||||
|
||||
elif style is not None:
|
||||
raise InvalidOrderStyle(exchange=self.name.title(),
|
||||
style=style.__class__.__name__)
|
||||
else:
|
||||
raise ValueError('Incomplete order data.')
|
||||
|
||||
display_price = limit_price if limit_price is not None else stop_price
|
||||
log.debug(
|
||||
'issuing {side} order of {amount} {symbol} for {type}:'
|
||||
' {price}'.format(
|
||||
'issuing {side} order of {amount} {symbol} for {type}: {price}'.format(
|
||||
side='buy' if is_buy else 'sell',
|
||||
amount=amount,
|
||||
symbol=asset.symbol,
|
||||
type=style.__class__.__name__,
|
||||
price='{}{}'.format(display_price, asset.quote_currency)
|
||||
price='{}{}'.format(display_price, asset.base_currency)
|
||||
)
|
||||
)
|
||||
|
||||
return self.create_order(asset, amount, is_buy, style)
|
||||
order = self.create_order(asset, amount, is_buy, style)
|
||||
if order:
|
||||
self._portfolio.create_order(order)
|
||||
return order.id
|
||||
else:
|
||||
return None
|
||||
|
||||
# The methods below must be implemented for each exchange.
|
||||
@abstractmethod
|
||||
@@ -882,7 +875,7 @@ class Exchange:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_order(self, order_id, symbol_or_asset=None):
|
||||
def get_order(self, order_id):
|
||||
"""Lookup an order based on the order id returned from one of the
|
||||
order functions.
|
||||
|
||||
@@ -890,8 +883,6 @@ class Exchange:
|
||||
----------
|
||||
order_id : str
|
||||
The unique identifier for the order.
|
||||
symbol_or_asset: str|TradingPair
|
||||
The catalyst symbol, some exchanges need this
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -903,36 +894,19 @@ class Exchange:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def process_order(self, order):
|
||||
"""
|
||||
Similar to get_order but looks only for executed orders.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
order: Order
|
||||
|
||||
Returns
|
||||
-------
|
||||
float
|
||||
Avg execution price
|
||||
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def cancel_order(self, order_param, symbol_or_asset=None):
|
||||
def cancel_order(self, order_param):
|
||||
"""Cancel an open order.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
order_param : str or Order
|
||||
The order_id or order object to cancel.
|
||||
symbol_or_asset: str|TradingPair
|
||||
The catalyst symbol, some exchanges need this
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_candles(self, freq, assets, bar_count, start_dt=None, end_dt=None):
|
||||
def get_candles(self, freq, assets, bar_count=None,
|
||||
start_dt=None, end_dt=None):
|
||||
"""
|
||||
Retrieve OHLCV candles for the given assets
|
||||
|
||||
@@ -997,7 +971,7 @@ class Exchange:
|
||||
@abc.abstractmethod
|
||||
def get_orderbook(self, asset, order_type, limit):
|
||||
"""
|
||||
Retrieve the orderbook for the given trading pair.
|
||||
Retrieve the the orderbook for the given trading pair.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -1011,20 +985,3 @@ class Exchange:
|
||||
list[dict[str, float]
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_trades(self, asset, my_trades, start_dt, limit):
|
||||
"""
|
||||
Retrieve a list of trades.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
my_trades: bool
|
||||
List only my trades.
|
||||
start_dt
|
||||
limit
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,179 +0,0 @@
|
||||
import pandas as pd
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.utils.factory import find_exchanges
|
||||
from logbook import Logger
|
||||
|
||||
log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
|
||||
|
||||
|
||||
class ExchangeAssetFinder(object):
|
||||
def __init__(self, exchanges):
|
||||
self.exchanges = exchanges
|
||||
|
||||
@property
|
||||
def sids(self):
|
||||
"""
|
||||
This seems to be used to pre-fetch assets.
|
||||
I don't think that we need this for live-trading.
|
||||
Leaving the list empty.
|
||||
"""
|
||||
all_sids = []
|
||||
for exchange_name in self.exchanges:
|
||||
# This is what initializes each exchanges at the beginning
|
||||
# of an algo
|
||||
exchange = self.exchanges[exchange_name]
|
||||
exchange.init()
|
||||
|
||||
all_sids += [asset.sid for asset in exchange.assets]
|
||||
|
||||
sids = list(set(all_sids))
|
||||
return sids
|
||||
|
||||
def retrieve_asset(self, sid, default_none=False):
|
||||
"""
|
||||
Retrieve the first Asset found for a given sid.
|
||||
"""
|
||||
asset = None
|
||||
for exchange_name in self.exchanges:
|
||||
if asset is not None:
|
||||
break
|
||||
|
||||
exchange = self.exchanges[exchange_name]
|
||||
assets = [asset for asset in exchange.assets if asset.sid == sid]
|
||||
if assets:
|
||||
asset = assets[0]
|
||||
|
||||
return asset
|
||||
|
||||
def retrieve_all(self, sids, default_none=False):
|
||||
"""
|
||||
Retrieve all assets in `sids`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
sids : iterable of int
|
||||
Assets to retrieve.
|
||||
default_none : bool
|
||||
If True, return None for failed lookups.
|
||||
If False, raise `SidsNotFound`.
|
||||
|
||||
Returns
|
||||
-------
|
||||
assets : list[Asset or None]
|
||||
A list of the same length as `sids` containing Assets (or Nones)
|
||||
corresponding to the requested sids.
|
||||
|
||||
Raises
|
||||
------
|
||||
SidsNotFound
|
||||
When a requested sid is not found and default_none=False.
|
||||
"""
|
||||
assets = []
|
||||
for exchange_name in self.exchanges:
|
||||
exchange = self.exchanges[exchange_name]
|
||||
xas = [asset for asset in exchange.assets if asset.sid in sids]
|
||||
assets += xas
|
||||
|
||||
return assets
|
||||
|
||||
def lookup_symbol(self, symbol, exchange, data_frequency=None,
|
||||
as_of_date=None, fuzzy=False):
|
||||
"""Lookup an asset by symbol.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
symbol : str
|
||||
The ticker symbol to resolve.
|
||||
as_of_date : datetime or None
|
||||
Look up the last owner of this symbol as of this datetime.
|
||||
If ``as_of_date`` is None, then this can only resolve the equity
|
||||
if exactly one equity has ever owned the ticker.
|
||||
fuzzy : bool, optional
|
||||
Should fuzzy symbol matching be used? Fuzzy symbol matching
|
||||
attempts to resolve differences in representations for
|
||||
shareclasses. For example, some people may represent the ``A``
|
||||
shareclass of ``BRK`` as ``BRK.A``, where others could write
|
||||
``BRK_A``.
|
||||
|
||||
Returns
|
||||
-------
|
||||
equity : Asset
|
||||
The equity that held ``symbol`` on the given ``as_of_date``, or the
|
||||
only equity to hold ``symbol`` if ``as_of_date`` is None.
|
||||
|
||||
Raises
|
||||
------
|
||||
SymbolNotFound
|
||||
Raised when no equity has ever held the given symbol.
|
||||
MultipleSymbolsFound
|
||||
Raised when no ``as_of_date`` is given and more than one equity
|
||||
has held ``symbol``. This is also raised when ``fuzzy=True`` and
|
||||
there are multiple candidates for the given ``symbol`` on the
|
||||
``as_of_date``.
|
||||
"""
|
||||
log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
|
||||
|
||||
return exchange.get_asset(symbol, data_frequency)
|
||||
|
||||
def lifetimes(self, dates, include_start_date):
|
||||
"""
|
||||
Compute a DataFrame representing asset lifetimes for the specified date
|
||||
range.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
dates : pd.DatetimeIndex
|
||||
The dates for which to compute lifetimes.
|
||||
include_start_date : bool
|
||||
Whether or not to count the asset as alive on its start_date.
|
||||
|
||||
This is useful in a backtesting context where `lifetimes` is being
|
||||
used to signify "do I have data for this asset as of the morning of
|
||||
this date?" For many financial metrics, (e.g. daily close), data
|
||||
isn't available for an asset until the end of the asset's first
|
||||
day.
|
||||
|
||||
Returns
|
||||
-------
|
||||
lifetimes : pd.DataFrame
|
||||
A frame of dtype bool with `dates` as index and an Int64Index of
|
||||
assets as columns. The value at `lifetimes.loc[date, asset]` will
|
||||
be True iff `asset` existed on `date`. If `include_start_date` is
|
||||
False, then lifetimes.loc[date, asset] will be false when date ==
|
||||
asset.start_date.
|
||||
|
||||
See Also
|
||||
--------
|
||||
numpy.putmask
|
||||
catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask
|
||||
"""
|
||||
exchanges = find_exchanges(features=['minuteBundle'])
|
||||
if not exchanges:
|
||||
raise ValueError('exchange with minute bundles not found')
|
||||
|
||||
# TODO: find a way to support multiple exchanges
|
||||
exchange = exchanges[0]
|
||||
# Using a single exchange for now because are not unique for the
|
||||
# same asset in different exchanges. I'd like to avoid binding
|
||||
# pipeline to a single exchange.
|
||||
exchange.init()
|
||||
|
||||
data = []
|
||||
for dt in dates:
|
||||
exists = []
|
||||
|
||||
for asset in exchange.assets:
|
||||
if include_start_date:
|
||||
condition = (asset.start_date <= dt < asset.end_minute)
|
||||
|
||||
else:
|
||||
condition = (asset.start_date < dt < asset.end_minute)
|
||||
|
||||
exists.append(condition)
|
||||
|
||||
data.append(exists)
|
||||
|
||||
sids = [asset.sid for asset in exchange.assets]
|
||||
df = pd.DataFrame(data, index=dates, columns=exchange.assets)
|
||||
|
||||
return df
|
||||
@@ -1,20 +1,21 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from logbook import Logger
|
||||
from redo import retry
|
||||
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.exchange_errors import ExchangeRequestError
|
||||
from catalyst.finance.blotter import Blotter
|
||||
from catalyst.finance.commission import CommissionModel
|
||||
from catalyst.finance.order import ORDER_STATUS
|
||||
from catalyst.finance.slippage import SlippageModel
|
||||
from catalyst.finance.transaction import create_transaction, Transaction
|
||||
from catalyst.utils.input_validation import expect_types
|
||||
from catalyst.finance.transaction import create_transaction
|
||||
|
||||
log = Logger('exchange_blotter', level=LOG_LEVEL)
|
||||
|
||||
# It seems like we need to accept greater slippage risk in cryptos
|
||||
# Orders won't often close at Equity levels.
|
||||
# TODO: should work with set_commission and set_slippage
|
||||
DEFAULT_SLIPPAGE_SPREAD = 0.0001
|
||||
DEFAULT_MAKER_FEE = 0.0015
|
||||
DEFAULT_TAKER_FEE = 0.0025
|
||||
|
||||
|
||||
class TradingPairFeeSchedule(CommissionModel):
|
||||
"""
|
||||
@@ -22,55 +23,40 @@ class TradingPairFeeSchedule(CommissionModel):
|
||||
|
||||
Parameters
|
||||
----------
|
||||
maker : float, optional
|
||||
The percentage maker fee.
|
||||
|
||||
taker: float, optional
|
||||
The percentage taker fee.
|
||||
fee : float, optional
|
||||
The percentage fee.
|
||||
"""
|
||||
|
||||
def __init__(self, maker=None, taker=None):
|
||||
self.maker = maker
|
||||
self.taker = taker
|
||||
def __init__(self,
|
||||
maker_fee=DEFAULT_MAKER_FEE,
|
||||
taker_fee=DEFAULT_TAKER_FEE):
|
||||
self.maker_fee = maker_fee
|
||||
self.taker_fee = taker_fee
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
'{class_name}(maker={maker}, '
|
||||
'taker={taker})'.format(
|
||||
'{class_name}(maker_fee={maker_fee}, '
|
||||
'taker_fee={taker_fee})'.format(
|
||||
class_name=self.__class__.__name__,
|
||||
maker=self.maker,
|
||||
taker=self.taker,
|
||||
maker_fee=self.maker_fee,
|
||||
taker_fee=self.taker_fee,
|
||||
)
|
||||
)
|
||||
|
||||
def get_maker_taker(self, asset):
|
||||
maker = self.maker if self.maker is not None else asset.maker
|
||||
taker = self.taker if self.taker is not None else asset.taker
|
||||
return maker, taker
|
||||
|
||||
def calculate(self, order, transaction):
|
||||
"""
|
||||
Calculate the final fee based on the order parameters.
|
||||
|
||||
:param order: Order
|
||||
:param transaction: Transaction
|
||||
:param order:
|
||||
:param transaction:
|
||||
|
||||
:return float:
|
||||
The total commission.
|
||||
"""
|
||||
cost = abs(transaction.amount) * transaction.price
|
||||
|
||||
asset = order.asset
|
||||
maker, taker = self.get_maker_taker(asset)
|
||||
|
||||
multiplier = taker
|
||||
if order.limit is not None:
|
||||
multiplier = maker \
|
||||
if ((order.amount > 0 and order.limit < transaction.price)
|
||||
or (order.amount < 0 and order.limit > transaction.price)) \
|
||||
and order.limit_reached else taker
|
||||
|
||||
fee = cost * multiplier
|
||||
# Assuming just the taker fee for now
|
||||
fee = cost * self.taker_fee
|
||||
return fee
|
||||
|
||||
|
||||
@@ -84,7 +70,7 @@ class TradingPairFixedSlippage(SlippageModel):
|
||||
spread / 2 will be added to buys and subtracted from sells.
|
||||
"""
|
||||
|
||||
def __init__(self, spread=0.0001):
|
||||
def __init__(self, spread=DEFAULT_SLIPPAGE_SPREAD):
|
||||
super(TradingPairFixedSlippage, self).__init__()
|
||||
self.spread = spread
|
||||
|
||||
@@ -95,6 +81,7 @@ class TradingPairFixedSlippage(SlippageModel):
|
||||
|
||||
def simulate(self, data, asset, orders_for_asset):
|
||||
self._volume_for_bar = 0
|
||||
|
||||
price = data.current(asset, 'close')
|
||||
|
||||
dt = data.current_dt
|
||||
@@ -104,20 +91,18 @@ class TradingPairFixedSlippage(SlippageModel):
|
||||
|
||||
order.check_triggers(price, dt)
|
||||
if not order.triggered:
|
||||
log.info(
|
||||
'order has not reached the trigger at current '
|
||||
'price {}'.format(price)
|
||||
)
|
||||
log.debug('order has not reached the trigger at current '
|
||||
'price {}'.format(price))
|
||||
continue
|
||||
|
||||
execution_price, execution_volume = self.process_order(data, order)
|
||||
if execution_price is not None:
|
||||
transaction = create_transaction(
|
||||
order, dt, execution_price, execution_volume
|
||||
)
|
||||
|
||||
self._volume_for_bar += abs(transaction.amount)
|
||||
yield order, transaction
|
||||
transaction = create_transaction(
|
||||
order, dt, execution_price, execution_volume
|
||||
)
|
||||
|
||||
self._volume_for_bar += abs(transaction.amount)
|
||||
yield order, transaction
|
||||
|
||||
def process_order(self, data, order):
|
||||
price = data.current(order.asset, 'close')
|
||||
@@ -136,15 +121,6 @@ class TradingPairFixedSlippage(SlippageModel):
|
||||
|
||||
class ExchangeBlotter(Blotter):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.simulate_orders = kwargs.pop('simulate_orders', False)
|
||||
self.attempts = kwargs.pop('attempts', False)
|
||||
|
||||
self.exchanges = kwargs.pop('exchanges', None)
|
||||
if not self.exchanges:
|
||||
raise ValueError(
|
||||
'ExchangeBlotter must have an `exchanges` attribute.'
|
||||
)
|
||||
|
||||
super(ExchangeBlotter, self).__init__(*args, **kwargs)
|
||||
|
||||
# Using the equity models for now
|
||||
@@ -156,119 +132,3 @@ class ExchangeBlotter(Blotter):
|
||||
self.commission_models = {
|
||||
TradingPair: TradingPairFeeSchedule()
|
||||
}
|
||||
|
||||
def exchange_order(self, asset, amount, style=None):
|
||||
exchange = self.exchanges[asset.exchange]
|
||||
return exchange.order(
|
||||
asset, amount, style
|
||||
)
|
||||
|
||||
@expect_types(asset=TradingPair)
|
||||
def order(self, asset, amount, style, order_id=None):
|
||||
log.debug('ordering {} {}'.format(amount, asset.symbol))
|
||||
if amount == 0:
|
||||
log.warn('skipping 0 amount orders')
|
||||
return None
|
||||
|
||||
if self.simulate_orders:
|
||||
return super(ExchangeBlotter, self).order(
|
||||
asset, amount, style, order_id
|
||||
)
|
||||
|
||||
else:
|
||||
order = retry(
|
||||
action=self.exchange_order,
|
||||
attempts=self.attempts['order_attempts'],
|
||||
sleeptime=self.attempts['retry_sleeptime'],
|
||||
retry_exceptions=(ExchangeRequestError,),
|
||||
cleanup=lambda: log.warn('Ordering again.'),
|
||||
args=(asset, amount, style),
|
||||
)
|
||||
|
||||
self.open_orders[order.asset].append(order)
|
||||
self.orders[order.id] = order
|
||||
self.new_orders.append(order)
|
||||
|
||||
return order.id
|
||||
|
||||
def check_open_orders(self):
|
||||
"""
|
||||
Loop through the list of open orders in the Portfolio object.
|
||||
For each executed order found, create a transaction and apply to the
|
||||
Portfolio.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Transaction]
|
||||
|
||||
"""
|
||||
for asset in self.open_orders:
|
||||
exchange = self.exchanges[asset.exchange]
|
||||
|
||||
for order in self.open_orders[asset]:
|
||||
log.debug('found open order: {}'.format(order.id))
|
||||
|
||||
transactions = exchange.process_order(order)
|
||||
# This is a temporary measure, we should really update all
|
||||
# trades, not just when the order gets filled. I just think
|
||||
# that this is safer until we have a robust way to track
|
||||
# the trades already processed by the algo. We can't loose
|
||||
# them if the algo shuts down.
|
||||
if transactions and order.open_amount == 0:
|
||||
avg_price = np.average(
|
||||
a=[t.price for t in transactions],
|
||||
weights=[t.amount for t in transactions],
|
||||
)
|
||||
ostatus = 'filled' if order.open_amount == 0 else 'partial'
|
||||
log.info(
|
||||
'{} order {} / {}: {}, avg price: {}'.format(
|
||||
ostatus,
|
||||
order.id,
|
||||
asset.symbol,
|
||||
order.filled,
|
||||
avg_price,
|
||||
)
|
||||
)
|
||||
for transaction in transactions:
|
||||
yield order, transaction
|
||||
|
||||
elif order.status == ORDER_STATUS.CANCELLED:
|
||||
yield order, None
|
||||
|
||||
else:
|
||||
delta = pd.Timestamp.utcnow() - order.dt
|
||||
log.info(
|
||||
'order {order_id} still open after {delta}'.format(
|
||||
order_id=order.id,
|
||||
delta=delta
|
||||
)
|
||||
)
|
||||
|
||||
def get_exchange_transactions(self):
|
||||
closed_orders = []
|
||||
transactions = []
|
||||
commissions = []
|
||||
|
||||
for order, txn in self.check_open_orders():
|
||||
order.dt = txn.dt
|
||||
transactions.append(txn)
|
||||
|
||||
if not order.open:
|
||||
closed_orders.append(order)
|
||||
|
||||
return transactions, commissions, closed_orders
|
||||
|
||||
def get_transactions(self, bar_data):
|
||||
if self.simulate_orders:
|
||||
return super(ExchangeBlotter, self).get_transactions(bar_data)
|
||||
|
||||
else:
|
||||
return retry(
|
||||
action=self.get_exchange_transactions,
|
||||
attempts=self.attempts['get_transactions_attempts'],
|
||||
sleeptime=self.attempts['retry_sleeptime'],
|
||||
retry_exceptions=(ExchangeRequestError,),
|
||||
cleanup=lambda: log.warn(
|
||||
'Fetching exchange transactions again.'
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import os
|
||||
import shutil
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from operator import is_not
|
||||
@@ -8,29 +9,31 @@ from operator import is_not
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytz
|
||||
from catalyst import get_calendar
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
from pytz import UTC
|
||||
from six import itervalues
|
||||
|
||||
from catalyst import get_calendar
|
||||
from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
|
||||
BcolzMinuteBarMetadata
|
||||
from catalyst.exchange.bundle_utils import range_in_bundle, \
|
||||
get_bcolz_chunk, get_month_start_end, \
|
||||
get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
|
||||
get_delta, get_assets
|
||||
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
|
||||
BcolzExchangeBarWriter
|
||||
from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
|
||||
TempBundleNotFoundError, \
|
||||
NoDataAvailableOnExchange, \
|
||||
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
|
||||
from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
|
||||
get_bcolz_chunk, get_df_from_arrays, get_assets
|
||||
from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
|
||||
get_period_label, get_month_start_end, get_year_start_end
|
||||
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
|
||||
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
|
||||
PricingDataNotLoadedError, DataCorruptionError, ExchangeSymbolsNotFound, \
|
||||
PricingDataValueError
|
||||
from catalyst.exchange.exchange_utils import get_exchange_folder, \
|
||||
get_exchange_symbols, save_exchange_symbols
|
||||
from catalyst.utils.cli import maybe_show_progress
|
||||
from catalyst.utils.paths import ensure_directory
|
||||
from logbook import Logger
|
||||
from pytz import UTC
|
||||
from six import itervalues
|
||||
|
||||
log = Logger('exchange_bundle', level=LOG_LEVEL)
|
||||
|
||||
@@ -233,12 +236,10 @@ class ExchangeBundle:
|
||||
problem = '{name} ({start_dt} to {end_dt}) has empty ' \
|
||||
'periods: {dates}'.format(
|
||||
name=asset.symbol,
|
||||
start_dt=asset.start_date.strftime(
|
||||
DATE_TIME_FORMAT),
|
||||
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
|
||||
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
|
||||
dates=[date.strftime(
|
||||
DATE_TIME_FORMAT) for date in dates])
|
||||
|
||||
dates=[date.strftime(DATE_TIME_FORMAT) for date in dates]
|
||||
)
|
||||
if empty_rows_behavior == 'warn':
|
||||
log.warn(problem)
|
||||
|
||||
@@ -246,7 +247,8 @@ class ExchangeBundle:
|
||||
raise EmptyValuesInBundleError(
|
||||
name=asset.symbol,
|
||||
end_minute=end_dt,
|
||||
dates=dates, )
|
||||
dates=dates
|
||||
)
|
||||
|
||||
else:
|
||||
ohlcv_df.dropna(inplace=True)
|
||||
@@ -291,7 +293,8 @@ class ExchangeBundle:
|
||||
end_dt=end_dt.strftime(DATE_TIME_FORMAT),
|
||||
threshold=threshold,
|
||||
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
|
||||
for date in dates])
|
||||
for date in dates]
|
||||
)
|
||||
|
||||
problems.append(problem)
|
||||
|
||||
@@ -461,7 +464,7 @@ class ExchangeBundle:
|
||||
(earliest_trade is not None and earliest_trade > start):
|
||||
start = earliest_trade
|
||||
|
||||
if last_entry is not None and (end is None or end > last_entry):
|
||||
if end is None or (last_entry is not None and end > last_entry):
|
||||
end = last_entry.replace(minute=59, hour=23) \
|
||||
if data_frequency == 'minute' else last_entry
|
||||
|
||||
@@ -664,11 +667,12 @@ class ExchangeBundle:
|
||||
|
||||
"""
|
||||
log.info('ingesting csv file: {}'.format(path))
|
||||
|
||||
if self.exchange is None:
|
||||
# Avoid circular dependencies
|
||||
from catalyst.exchange.utils.factory import get_exchange
|
||||
self.exchange = get_exchange(self.exchange_name)
|
||||
try:
|
||||
symbols_def = get_exchange_symbols(
|
||||
self.exchange_name, is_local=True
|
||||
)
|
||||
except ExchangeSymbolsNotFound:
|
||||
symbols_def = dict()
|
||||
|
||||
problems = []
|
||||
df = pd.read_csv(
|
||||
@@ -680,7 +684,6 @@ class ExchangeBundle:
|
||||
last_traded=np.object_,
|
||||
open=np.float64,
|
||||
high=np.float64,
|
||||
low=np.float64,
|
||||
close=np.float64,
|
||||
volume=np.float64
|
||||
),
|
||||
@@ -702,40 +705,24 @@ class ExchangeBundle:
|
||||
end_dt = df.index.get_level_values(1).max()
|
||||
end_dt_key = 'end_{}'.format(data_frequency)
|
||||
|
||||
market = self.exchange.get_market(symbol)
|
||||
if market is None:
|
||||
raise ValueError('symbol not available in the exchange.')
|
||||
if symbol is symbols_def:
|
||||
symbol_def = symbols_def[symbol]
|
||||
|
||||
params = dict(
|
||||
exchange=self.exchange.name,
|
||||
data_source='local',
|
||||
exchange_symbol=market['id'],
|
||||
)
|
||||
mixin_market_params(self.exchange_name, params, market)
|
||||
start_dt = symbol_def['start_date'] \
|
||||
if symbol_def['start_date'] < start_dt else start_dt
|
||||
|
||||
asset_def = self.exchange.get_asset_def(market, True)
|
||||
if asset_def is not None:
|
||||
params['symbol'] = asset_def['symbol']
|
||||
end_dt = symbol_def[end_dt_key] \
|
||||
if symbol_def[end_dt_key] > end_dt else end_dt
|
||||
|
||||
params['start_date'] = asset_def['start_date'] \
|
||||
if asset_def['start_date'] < start_dt else start_dt
|
||||
end_daily = end_dt \
|
||||
if data_frequency == 'daily' else symbol_def['end_daily']
|
||||
|
||||
params['end_date'] = asset_def[end_dt_key] \
|
||||
if asset_def[end_dt_key] > end_dt else end_dt
|
||||
|
||||
params['end_daily'] = end_dt \
|
||||
if data_frequency == 'daily' else asset_def['end_daily']
|
||||
|
||||
params['end_minute'] = end_dt \
|
||||
if data_frequency == 'minute' else asset_def['end_minute']
|
||||
end_minute = end_dt \
|
||||
if data_frequency == 'minute' else symbol_def['end_minute']
|
||||
|
||||
else:
|
||||
params['symbol'] = get_catalyst_symbol(market)
|
||||
|
||||
params['end_daily'] = end_dt \
|
||||
if data_frequency == 'daily' else 'N/A'
|
||||
params['end_minute'] = end_dt \
|
||||
if data_frequency == 'minute' else 'N/A'
|
||||
end_daily = end_dt if data_frequency == 'daily' else 'N/A'
|
||||
end_minute = end_dt if data_frequency == 'minute' else 'N/A'
|
||||
|
||||
if min_start_dt is None or start_dt < min_start_dt:
|
||||
min_start_dt = start_dt
|
||||
@@ -743,8 +730,19 @@ class ExchangeBundle:
|
||||
if max_end_dt is None or end_dt > max_end_dt:
|
||||
max_end_dt = end_dt
|
||||
|
||||
asset = TradingPair(**params)
|
||||
assets[market['id']] = asset
|
||||
asset = TradingPair(
|
||||
symbol=symbol,
|
||||
exchange=self.exchange_name,
|
||||
start_date=start_dt,
|
||||
end_date=end_dt,
|
||||
leverage=0, # TODO: add as an optional column
|
||||
asset_name=symbol,
|
||||
min_trade_size=0, # TODO: add as an optional column
|
||||
end_daily=end_daily,
|
||||
end_minute=end_minute,
|
||||
exchange_symbol=symbol
|
||||
)
|
||||
assets[symbol] = asset
|
||||
|
||||
save_exchange_symbols(self.exchange_name, assets, True)
|
||||
|
||||
@@ -755,10 +753,9 @@ class ExchangeBundle:
|
||||
)
|
||||
|
||||
for symbol in assets:
|
||||
# here the symbol is the market['id']
|
||||
asset = assets[symbol]
|
||||
ohlcv_df = df.loc[
|
||||
(df.index.get_level_values(0) == asset.symbol)
|
||||
(df.index.get_level_values(0) == symbol)
|
||||
] # type: pd.DataFrame
|
||||
ohlcv_df.index = ohlcv_df.index.droplevel(0)
|
||||
|
||||
@@ -806,7 +803,7 @@ class ExchangeBundle:
|
||||
else:
|
||||
if self.exchange is None:
|
||||
# Avoid circular dependencies
|
||||
from catalyst.exchange.utils.factory import get_exchange
|
||||
from catalyst.exchange.factory import get_exchange
|
||||
self.exchange = get_exchange(self.exchange_name)
|
||||
|
||||
assets = get_assets(
|
||||
@@ -964,15 +961,15 @@ class ExchangeBundle:
|
||||
data_frequency,
|
||||
trailing_bar_count=None,
|
||||
reset_reader=False):
|
||||
if trailing_bar_count:
|
||||
delta = get_delta(trailing_bar_count, data_frequency)
|
||||
end_dt += delta
|
||||
|
||||
start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
|
||||
start_dt, _ = self.get_adj_dates(
|
||||
start_dt, end_dt, assets, data_frequency
|
||||
)
|
||||
|
||||
if trailing_bar_count:
|
||||
delta = get_delta(trailing_bar_count, data_frequency)
|
||||
end_dt += delta
|
||||
|
||||
# This is an attempt to resolve some caching with the reader
|
||||
# when auto-ingesting data.
|
||||
# TODO: needs more work
|
||||
|
||||
@@ -1,29 +1,30 @@
|
||||
import abc
|
||||
from time import sleep
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.constants import LOG_LEVEL, AUTO_INGEST
|
||||
from catalyst.data.data_portal import DataPortal
|
||||
from catalyst.exchange.exchange_bundle import ExchangeBundle
|
||||
from catalyst.exchange.exchange_errors import (
|
||||
ExchangeRequestError,
|
||||
ExchangeBarDataError,
|
||||
PricingDataNotLoadedError)
|
||||
from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange
|
||||
from catalyst.exchange.utils.datetime_utils import get_frequency
|
||||
from logbook import Logger
|
||||
from redo import retry
|
||||
from catalyst.exchange.exchange_utils import get_frequency, resample_history_df
|
||||
|
||||
log = Logger('DataPortalExchange', level=LOG_LEVEL)
|
||||
|
||||
|
||||
class DataPortalExchangeBase(DataPortal):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.attempts = dict(
|
||||
get_spot_value_attempts=5,
|
||||
get_history_window_attempts=5,
|
||||
retry_sleeptime=5,
|
||||
)
|
||||
|
||||
# TODO: put somewhere accessible by each algo
|
||||
self.retry_get_history_window = 5
|
||||
self.retry_get_spot_value = 5
|
||||
self.retry_delay = 5
|
||||
|
||||
super(DataPortalExchangeBase, self).__init__(*args, **kwargs)
|
||||
|
||||
@@ -34,14 +35,39 @@ class DataPortalExchangeBase(DataPortal):
|
||||
frequency,
|
||||
field,
|
||||
data_frequency,
|
||||
ffill=True):
|
||||
exchange_assets = group_assets_by_exchange(assets)
|
||||
if len(exchange_assets) > 1:
|
||||
df_list = []
|
||||
for exchange_name in exchange_assets:
|
||||
assets = exchange_assets[exchange_name]
|
||||
ffill=True,
|
||||
attempt_index=0):
|
||||
try:
|
||||
exchange_assets = dict()
|
||||
for asset in assets:
|
||||
if asset.exchange not in exchange_assets:
|
||||
exchange_assets[asset.exchange] = list()
|
||||
|
||||
df_exchange = self.get_exchange_history_window(
|
||||
exchange_assets[asset.exchange].append(asset)
|
||||
|
||||
if len(exchange_assets) > 1:
|
||||
df_list = []
|
||||
for exchange_name in exchange_assets:
|
||||
assets = exchange_assets[exchange_name]
|
||||
|
||||
df_exchange = self.get_exchange_history_window(
|
||||
exchange_name,
|
||||
assets,
|
||||
end_dt,
|
||||
bar_count,
|
||||
frequency,
|
||||
field,
|
||||
data_frequency,
|
||||
ffill)
|
||||
|
||||
df_list.append(df_exchange)
|
||||
|
||||
# Merging the values values of each exchange
|
||||
return pd.concat(df_list)
|
||||
|
||||
else:
|
||||
exchange_name = list(exchange_assets.keys())[0]
|
||||
return self.get_exchange_history_window(
|
||||
exchange_name,
|
||||
assets,
|
||||
end_dt,
|
||||
@@ -51,22 +77,26 @@ class DataPortalExchangeBase(DataPortal):
|
||||
data_frequency,
|
||||
ffill)
|
||||
|
||||
df_list.append(df_exchange)
|
||||
|
||||
# Merging the values values of each exchange
|
||||
return pd.concat(df_list)
|
||||
|
||||
else:
|
||||
exchange_name = list(exchange_assets.keys())[0]
|
||||
return self.get_exchange_history_window(
|
||||
exchange_name,
|
||||
assets,
|
||||
end_dt,
|
||||
bar_count,
|
||||
frequency,
|
||||
field,
|
||||
data_frequency,
|
||||
ffill)
|
||||
except ExchangeRequestError as e:
|
||||
log.warn(
|
||||
'get history attempt {}: {}'.format(attempt_index, e)
|
||||
)
|
||||
if attempt_index < self.retry_get_history_window:
|
||||
sleep(self.retry_delay)
|
||||
return self._get_history_window(assets,
|
||||
end_dt,
|
||||
bar_count,
|
||||
frequency,
|
||||
field,
|
||||
data_frequency,
|
||||
ffill,
|
||||
attempt_index + 1)
|
||||
else:
|
||||
raise ExchangeBarDataError(
|
||||
data_type='history',
|
||||
attempts=attempt_index,
|
||||
error=e
|
||||
)
|
||||
|
||||
def get_history_window(self,
|
||||
assets,
|
||||
@@ -80,19 +110,13 @@ class DataPortalExchangeBase(DataPortal):
|
||||
if field == 'price':
|
||||
field = 'close'
|
||||
|
||||
return retry(
|
||||
action=self._get_history_window,
|
||||
attempts=self.attempts['get_history_window_attempts'],
|
||||
sleeptime=self.attempts['retry_sleeptime'],
|
||||
retry_exceptions=(ExchangeRequestError,),
|
||||
cleanup=lambda: log.warn('fetching history again.'),
|
||||
args=(assets,
|
||||
end_dt,
|
||||
bar_count,
|
||||
frequency,
|
||||
field,
|
||||
data_frequency,
|
||||
ffill))
|
||||
return self._get_history_window(assets,
|
||||
end_dt,
|
||||
bar_count,
|
||||
frequency,
|
||||
field,
|
||||
data_frequency,
|
||||
ffill)
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_exchange_history_window(self,
|
||||
@@ -106,58 +130,69 @@ class DataPortalExchangeBase(DataPortal):
|
||||
ffill=True):
|
||||
pass
|
||||
|
||||
def _get_spot_value(self, assets, field, dt, data_frequency):
|
||||
if isinstance(assets, TradingPair):
|
||||
spot_values = self.get_exchange_spot_value(
|
||||
assets.exchange, [assets], field, dt, data_frequency)
|
||||
def _get_spot_value(self, assets, field, dt, data_frequency,
|
||||
attempt_index=0):
|
||||
try:
|
||||
if isinstance(assets, TradingPair):
|
||||
spot_values = self.get_exchange_spot_value(
|
||||
assets.exchange, [assets], field, dt, data_frequency)
|
||||
|
||||
if not spot_values:
|
||||
return np.nan
|
||||
if not spot_values:
|
||||
return np.nan
|
||||
|
||||
return spot_values[0]
|
||||
|
||||
else:
|
||||
exchange_assets = dict()
|
||||
for asset in assets:
|
||||
if asset.exchange not in exchange_assets:
|
||||
exchange_assets[asset.exchange] = list()
|
||||
|
||||
exchange_assets[asset.exchange].append(asset)
|
||||
|
||||
if len(list(exchange_assets.keys())) == 1:
|
||||
exchange_name = list(exchange_assets.keys())[0]
|
||||
return self.get_exchange_spot_value(
|
||||
exchange_name, assets, field, dt, data_frequency)
|
||||
return spot_values[0]
|
||||
|
||||
else:
|
||||
spot_values = []
|
||||
for exchange_name in exchange_assets:
|
||||
assets = exchange_assets[exchange_name]
|
||||
exchange_spot_values = self.get_exchange_spot_value(
|
||||
exchange_name,
|
||||
assets,
|
||||
field,
|
||||
dt,
|
||||
data_frequency
|
||||
)
|
||||
if len(assets) == 1:
|
||||
spot_values.append(exchange_spot_values)
|
||||
else:
|
||||
spot_values += exchange_spot_values
|
||||
exchange_assets = dict()
|
||||
for asset in assets:
|
||||
if asset.exchange not in exchange_assets:
|
||||
exchange_assets[asset.exchange] = list()
|
||||
|
||||
return spot_values
|
||||
exchange_assets[asset.exchange].append(asset)
|
||||
|
||||
if len(list(exchange_assets.keys())) == 1:
|
||||
exchange_name = list(exchange_assets.keys())[0]
|
||||
return self.get_exchange_spot_value(
|
||||
exchange_name, assets, field, dt, data_frequency)
|
||||
|
||||
else:
|
||||
spot_values = []
|
||||
for exchange_name in exchange_assets:
|
||||
assets = exchange_assets[exchange_name]
|
||||
exchange_spot_values = self.get_exchange_spot_value(
|
||||
exchange_name,
|
||||
assets,
|
||||
field,
|
||||
dt,
|
||||
data_frequency
|
||||
)
|
||||
if len(assets) == 1:
|
||||
spot_values.append(exchange_spot_values)
|
||||
else:
|
||||
spot_values += exchange_spot_values
|
||||
|
||||
return spot_values
|
||||
|
||||
except ExchangeRequestError as e:
|
||||
log.warn(
|
||||
'get spot value attempt {}: {}'.format(attempt_index, e)
|
||||
)
|
||||
if attempt_index < self.retry_get_spot_value:
|
||||
sleep(self.retry_delay)
|
||||
return self._get_spot_value(assets, field, dt, data_frequency,
|
||||
attempt_index + 1)
|
||||
else:
|
||||
raise ExchangeBarDataError(
|
||||
data_type='spot',
|
||||
attempts=attempt_index,
|
||||
error=e
|
||||
)
|
||||
|
||||
def get_spot_value(self, assets, field, dt, data_frequency):
|
||||
if field == 'price':
|
||||
field = 'close'
|
||||
|
||||
return retry(
|
||||
action=self._get_spot_value,
|
||||
attempts=self.attempts['get_spot_value_attempts'],
|
||||
sleeptime=self.attempts['retry_sleeptime'],
|
||||
retry_exceptions=(ExchangeRequestError,),
|
||||
cleanup=lambda: log.warn('fetching spot value again.'),
|
||||
args=(assets, field, dt, data_frequency))
|
||||
return self._get_spot_value(assets, field, dt, data_frequency)
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_exchange_spot_value(self, exchange_name, assets, field, dt,
|
||||
@@ -207,7 +242,6 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
|
||||
|
||||
"""
|
||||
exchange = self.exchanges[exchange_name]
|
||||
|
||||
df = exchange.get_history_window(
|
||||
assets,
|
||||
end_dt,
|
||||
@@ -215,7 +249,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
|
||||
frequency,
|
||||
field,
|
||||
data_frequency,
|
||||
False)
|
||||
ffill)
|
||||
return df
|
||||
|
||||
def get_exchange_spot_value(self, exchange_name, assets, field, dt,
|
||||
@@ -291,7 +325,6 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
|
||||
DataFrame
|
||||
|
||||
"""
|
||||
# TODO: verify that the exchange supports the timeframe
|
||||
bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle
|
||||
|
||||
freq, candle_size, unit, adj_data_frequency = get_frequency(
|
||||
@@ -310,7 +343,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
|
||||
field=field,
|
||||
data_frequency=adj_data_frequency,
|
||||
algo_end_dt=self._last_available_session,
|
||||
trailing_bar_count=trailing_bar_count,
|
||||
trailing_bar_count=trailing_bar_count
|
||||
)
|
||||
|
||||
df = resample_history_df(pd.DataFrame(series), freq, field)
|
||||
|
||||
@@ -100,19 +100,6 @@ class InvalidHistoryFrequencyError(ZiplineError):
|
||||
).strip()
|
||||
|
||||
|
||||
class UnsupportedHistoryFrequencyError(ZiplineError):
|
||||
msg = (
|
||||
'{exchange} does not support candle frequency {freq}, please choose '
|
||||
'from: {freqs}.'
|
||||
).strip()
|
||||
|
||||
|
||||
class InvalidHistoryTimeframeError(ZiplineError):
|
||||
msg = (
|
||||
'CCXT timeframe {timeframe} not supported by the exchange.'
|
||||
).strip()
|
||||
|
||||
|
||||
class MismatchingFrequencyError(ZiplineError):
|
||||
msg = (
|
||||
'Bar aggregate frequency {frequency} not compatible with '
|
||||
@@ -156,8 +143,7 @@ class OrphanOrderError(ZiplineError):
|
||||
|
||||
class OrphanOrderReverseError(ZiplineError):
|
||||
msg = (
|
||||
'Order {order_id} tracked by algorithm, but not found in exchange '
|
||||
'{exchange}.'
|
||||
'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.'
|
||||
).strip()
|
||||
|
||||
|
||||
@@ -175,8 +161,8 @@ class SidHashError(ZiplineError):
|
||||
|
||||
class BaseCurrencyNotFoundError(ZiplineError):
|
||||
msg = (
|
||||
'Algorithm base currency {base_currency} not found in account '
|
||||
'balances on {exchange}: {balances}'
|
||||
'Algorithm base currency {base_currency} not found in exchange '
|
||||
'{exchange}.'
|
||||
).strip()
|
||||
|
||||
|
||||
@@ -220,9 +206,8 @@ class EmptyValuesInBundleError(ZiplineError):
|
||||
|
||||
class PricingDataBeforeTradingError(ZiplineError):
|
||||
msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} '
|
||||
'starts on {first_trading_day}, but you are either trying to trade '
|
||||
'or retrieve pricing data on {dt}. Adjust your dates accordingly.'
|
||||
).strip()
|
||||
'starts on {first_trading_day}, but you are either trying to trade or '
|
||||
'retrieve pricing data on {dt}. Adjust your dates accordingly.').strip()
|
||||
|
||||
|
||||
class PricingDataNotLoadedError(ZiplineError):
|
||||
@@ -232,93 +217,26 @@ class PricingDataNotLoadedError(ZiplineError):
|
||||
'{data_frequency} -i {symbol_list}`. See catalyst documentation '
|
||||
'for details.').strip()
|
||||
|
||||
|
||||
class PricingDataValueError(ZiplineError):
|
||||
msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
|
||||
'[{start_dt} - {end_dt}]: {error}').strip()
|
||||
|
||||
|
||||
class DataCorruptionError(ZiplineError):
|
||||
msg = (
|
||||
'Unable to validate data for {exchange} {symbols} in date range '
|
||||
'[{start_dt} - {end_dt}]. The data is either corrupted or '
|
||||
'unavailable. Please try deleting this bundle:'
|
||||
'\n`catalyst clean-exchange -x {exchange}\n'
|
||||
'Then, ingest the data again. Please contact the Catalyst team if '
|
||||
'the issue persists.'
|
||||
).strip()
|
||||
msg = ('Unable to validate data for {exchange} {symbols} in date range '
|
||||
'[{start_dt} - {end_dt}]. The data is either corrupted or '
|
||||
'unavailable. Please try deleting this bundle:'
|
||||
'\n`catalyst clean-exchange -x {exchange}\n'
|
||||
'Then, ingest the data again. Please contact the Catalyst team if '
|
||||
'the issue persists.').strip()
|
||||
|
||||
|
||||
class ApiCandlesError(ZiplineError):
|
||||
msg = (
|
||||
'Unable to fetch candles from the remote API: {error}.'
|
||||
).strip()
|
||||
msg = ('Unable to fetch candles from the remote API: {error}.').strip()
|
||||
|
||||
|
||||
class NoDataAvailableOnExchange(ZiplineError):
|
||||
msg = (
|
||||
'Requested data for trading pair {symbol} is not available on '
|
||||
'exchange {exchange} '
|
||||
'Requested data for trading pair {symbol} is not available on exchange {exchange} '
|
||||
'in `{data_frequency}` frequency at this time. '
|
||||
'Check `http://enigma.co/catalyst/status` for market coverage.'
|
||||
).strip()
|
||||
|
||||
|
||||
class NoValueForField(ZiplineError):
|
||||
msg = (
|
||||
'Value not found for field: {field}.'
|
||||
).strip()
|
||||
|
||||
|
||||
class OrderTypeNotSupported(ZiplineError):
|
||||
msg = (
|
||||
'Order type `{order_type}` not currency supported by Catalyst. '
|
||||
'Please use `limit` or `market` orders only.'
|
||||
).strip()
|
||||
|
||||
|
||||
class NotEnoughCapitalError(ZiplineError):
|
||||
msg = (
|
||||
'Not enough capital on exchange {exchange} for trading. Each '
|
||||
'exchange should contain at least as much {base_currency} '
|
||||
'as the specified `capital_base`. The current balance {balance} is '
|
||||
'lower than the `capital_base`: {capital_base}'
|
||||
).strip()
|
||||
|
||||
|
||||
class NotEnoughCashError(ZiplineError):
|
||||
msg = (
|
||||
'Total {currency} amount on {exchange} is lower than the cash '
|
||||
'reserved for this algo: {free} < {cash}. While trades can be made on '
|
||||
'the exchange accounts outside of the algo, exchange must have enough '
|
||||
'free {currency} to cover the algo cash.'
|
||||
).strip()
|
||||
|
||||
|
||||
class LastCandleTooEarlyError(ZiplineError):
|
||||
msg = (
|
||||
'The trade date of the last candle {last_traded} is before the '
|
||||
'specified end date minus one candle {end_dt}. Please verify how '
|
||||
'{exchange} calculates the start date of OHLCV candles.'
|
||||
).strip()
|
||||
|
||||
|
||||
class TickerNotFoundError(ZiplineError):
|
||||
msg = (
|
||||
'Unable to fetch ticker for {symbol} on {exchange}.'
|
||||
).strip()
|
||||
|
||||
|
||||
class BalanceNotFoundError(ZiplineError):
|
||||
msg = (
|
||||
'{currency} not found in account balance on {exchange}: {balances}.'
|
||||
).strip()
|
||||
|
||||
|
||||
class BalanceTooLowError(ZiplineError):
|
||||
msg = (
|
||||
'Balance for {currency} on {exchange} too low: {free} < {amount}. '
|
||||
'Positions have likely been sold outside of this algorithm. Please '
|
||||
'add positions to hold a free amount greater than {amount}, or clean '
|
||||
'the state of this algo and restart.'
|
||||
).strip()
|
||||
'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import numpy as np
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.protocol import Portfolio, Positions, Position
|
||||
from logbook import Logger
|
||||
from catalyst.utils.deprecate import deprecated
|
||||
|
||||
log = Logger('ExchangePortfolio', level=LOG_LEVEL)
|
||||
|
||||
@@ -9,8 +11,7 @@ log = Logger('ExchangePortfolio', level=LOG_LEVEL)
|
||||
class ExchangePortfolio(Portfolio):
|
||||
"""
|
||||
Since the goal is to support multiple exchanges, it makes sense to
|
||||
include additional stats in the portfolio object. This fills the role
|
||||
of Blotter and Portfolio in live mode.
|
||||
include additional stats in the portfolio object.
|
||||
|
||||
Instead of relying on the performance tracker, each exchange portfolio
|
||||
tracks its own holding. This offers a separation between tracking an
|
||||
@@ -39,13 +40,7 @@ class ExchangePortfolio(Portfolio):
|
||||
|
||||
"""
|
||||
log.debug('creating order {}'.format(order.id))
|
||||
|
||||
open_orders = self.open_orders[order.asset] \
|
||||
if order.asset is self.open_orders else []
|
||||
|
||||
open_orders.append(order)
|
||||
|
||||
self.open_orders[order.asset] = open_orders
|
||||
self.open_orders[order.id] = order
|
||||
|
||||
order_position = self.positions[order.asset] \
|
||||
if order.asset in self.positions else None
|
||||
@@ -57,17 +52,6 @@ class ExchangePortfolio(Portfolio):
|
||||
order_position.amount += order.amount
|
||||
log.debug('open order added to portfolio')
|
||||
|
||||
def _remove_open_order(self, order):
|
||||
try:
|
||||
open_orders = self.open_orders[order.asset]
|
||||
if order in open_orders:
|
||||
open_orders.remove(order)
|
||||
|
||||
except Exception:
|
||||
raise ValueError(
|
||||
'unable to clear order not found in open order list.'
|
||||
)
|
||||
|
||||
def execute_order(self, order, transaction):
|
||||
"""
|
||||
Update the open orders and positions to apply an executed order.
|
||||
@@ -82,15 +66,14 @@ class ExchangePortfolio(Portfolio):
|
||||
|
||||
"""
|
||||
log.debug('executing order {}'.format(order.id))
|
||||
self._remove_open_order(order)
|
||||
del self.open_orders[order.id]
|
||||
|
||||
order_position = self.positions[order.asset] \
|
||||
if order.asset in self.positions else None
|
||||
|
||||
if order_position is None:
|
||||
raise ValueError(
|
||||
'Trying to execute order for a position not held:'
|
||||
' {}'.format(order.id)
|
||||
'Trying to execute order for a position not held: %s' % order.id
|
||||
)
|
||||
|
||||
self.capital_used += order.amount * transaction.price
|
||||
@@ -106,6 +89,32 @@ class ExchangePortfolio(Portfolio):
|
||||
|
||||
log.debug('updated portfolio with executed order')
|
||||
|
||||
@deprecated
|
||||
def execute_transaction(self, transaction):
|
||||
# TODO: almost duplicate of execute_order. Not sure why Poloniex needs this.
|
||||
log.debug('executing transaction {}'.format(transaction.order_id))
|
||||
|
||||
order_position = self.positions[transaction.asset] \
|
||||
if transaction.asset in self.positions else None
|
||||
|
||||
if order_position is None:
|
||||
raise ValueError(
|
||||
'Trying to execute transaction for a position not held: %s' % transaction.order_id
|
||||
)
|
||||
|
||||
self.capital_used += transaction.amount * transaction.price
|
||||
|
||||
if transaction.amount > 0:
|
||||
if order_position.cost_basis > 0:
|
||||
order_position.cost_basis = np.average(
|
||||
[order_position.cost_basis, transaction.price],
|
||||
weights=[order_position.amount, transaction.amount]
|
||||
)
|
||||
else:
|
||||
order_position.cost_basis = transaction.price
|
||||
|
||||
log.debug('updated portfolio with executed order')
|
||||
|
||||
def remove_order(self, order):
|
||||
"""
|
||||
Removing an open order.
|
||||
@@ -116,7 +125,7 @@ class ExchangePortfolio(Portfolio):
|
||||
|
||||
"""
|
||||
log.info('removing cancelled order {}'.format(order.id))
|
||||
self._remove_open_order(order)
|
||||
del self.open_orders[order.id]
|
||||
|
||||
order_position = self.positions[order.asset] \
|
||||
if order.asset in self.positions else None
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
# Copyright 2015 Quantopian, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.data.us_equity_pricing import BcolzDailyBarReader
|
||||
from catalyst.errors import NoFurtherDataError
|
||||
from catalyst.exchange.utils.factory import get_exchange
|
||||
from catalyst.lib.adjusted_array import AdjustedArray
|
||||
from catalyst.pipeline.data import DataSet, Column
|
||||
from catalyst.pipeline.loaders.base import PipelineLoader
|
||||
from catalyst.utils.calendars import get_calendar
|
||||
from catalyst.utils.numpy_utils import float64_dtype
|
||||
from logbook import Logger
|
||||
from numpy import (
|
||||
iinfo,
|
||||
uint32,
|
||||
)
|
||||
|
||||
UINT32_MAX = iinfo(uint32).max
|
||||
|
||||
log = Logger('ExchangePriceLoader', level=LOG_LEVEL)
|
||||
|
||||
|
||||
class TradingPairPricing(DataSet):
|
||||
"""
|
||||
Dataset representing daily trading prices and volumes.
|
||||
"""
|
||||
open = Column(float64_dtype)
|
||||
high = Column(float64_dtype)
|
||||
low = Column(float64_dtype)
|
||||
close = Column(float64_dtype)
|
||||
volume = Column(float64_dtype)
|
||||
|
||||
|
||||
class ExchangePricingLoader(PipelineLoader):
|
||||
"""
|
||||
PipelineLoader for Crypto Pricing data
|
||||
|
||||
Delegates loading of baselines and adjustments.
|
||||
"""
|
||||
|
||||
def __init__(self, data_frequency):
|
||||
|
||||
cal = get_calendar('OPEN')
|
||||
|
||||
if data_frequency == 'daily':
|
||||
reader = None
|
||||
all_sessions = cal.all_sessions
|
||||
|
||||
elif data_frequency == 'minute':
|
||||
reader = None
|
||||
all_sessions = cal.all_minutes
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
'Invalid data frequency: {}'.format(data_frequency)
|
||||
)
|
||||
|
||||
self.data_frequency = data_frequency
|
||||
self.raw_price_loader = reader
|
||||
self._columns = TradingPairPricing.columns
|
||||
self._all_sessions = all_sessions
|
||||
|
||||
@classmethod
|
||||
def from_files(cls, pricing_path):
|
||||
"""
|
||||
Create a loader from a bcolz equity pricing dir and a SQLite
|
||||
adjustments path.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
pricing_path : str
|
||||
Path to a bcolz directory written by a BcolzDailyBarWriter.
|
||||
"""
|
||||
return cls(
|
||||
BcolzDailyBarReader(pricing_path),
|
||||
)
|
||||
|
||||
def load_adjusted_array(self, columns, dates, assets, mask):
|
||||
# load_adjusted_array is called with dates on which the user's algo
|
||||
# will be shown data, which means we need to return the data that would
|
||||
# be known at the start of each date. We assume that the latest data
|
||||
# known on day N is the data from day (N - 1), so we shift all query
|
||||
# dates back by a day.
|
||||
start_date, end_date = _shift_dates(
|
||||
self._all_sessions, dates[0], dates[-1], shift=1,
|
||||
)
|
||||
colnames = [c.name for c in columns]
|
||||
|
||||
if len(assets) == 0:
|
||||
raise ValueError(
|
||||
'Pipeline cannot load data with eligible assets.'
|
||||
)
|
||||
|
||||
exchange_names = []
|
||||
for asset in assets:
|
||||
if asset.exchange not in exchange_names:
|
||||
exchange_names.append(asset.exchange)
|
||||
|
||||
exchange = get_exchange(exchange_names[0])
|
||||
reader = exchange.bundle.get_reader(self.data_frequency)
|
||||
|
||||
raw_arrays = reader.load_raw_arrays(
|
||||
colnames,
|
||||
start_date,
|
||||
end_date,
|
||||
assets,
|
||||
)
|
||||
|
||||
out = {}
|
||||
for c, c_raw in zip(columns, raw_arrays):
|
||||
out[c] = AdjustedArray(
|
||||
c_raw.astype(c.dtype),
|
||||
mask,
|
||||
{},
|
||||
c.missing_value,
|
||||
)
|
||||
return out
|
||||
|
||||
@property
|
||||
def columns(self):
|
||||
return self._columns
|
||||
|
||||
|
||||
def _shift_dates(dates, start_date, end_date, shift):
|
||||
try:
|
||||
start = dates.get_loc(start_date)
|
||||
except KeyError:
|
||||
if start_date < dates[0]:
|
||||
raise NoFurtherDataError(
|
||||
msg=(
|
||||
"Pipeline Query requested data starting on {query_start}, "
|
||||
"but first known date is {calendar_start}"
|
||||
).format(
|
||||
query_start=str(start_date),
|
||||
calendar_start=str(dates[0]),
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise ValueError("Query start %s not in calendar" % start_date)
|
||||
|
||||
# Make sure that shifting doesn't push us out of the calendar.
|
||||
if start < shift:
|
||||
raise NoFurtherDataError(
|
||||
msg=(
|
||||
"Pipeline Query requested data from {shift}"
|
||||
" days before {query_start}, but first known date is only "
|
||||
"{start} days earlier."
|
||||
).format(shift=shift, query_start=start_date, start=start),
|
||||
)
|
||||
|
||||
try:
|
||||
end = dates.get_loc(end_date)
|
||||
except KeyError:
|
||||
if end_date > dates[-1]:
|
||||
raise NoFurtherDataError(
|
||||
msg=(
|
||||
"Pipeline Query requesting data up to {query_end}, "
|
||||
"but last known date is {calendar_end}"
|
||||
).format(
|
||||
query_end=end_date,
|
||||
calendar_end=dates[-1],
|
||||
)
|
||||
)
|
||||
else:
|
||||
raise ValueError("Query end %s not in calendar" % end_date)
|
||||
return dates[start - shift], dates[end - shift]
|
||||
@@ -2,18 +2,17 @@ import hashlib
|
||||
import json
|
||||
import os
|
||||
import pickle
|
||||
import re
|
||||
import shutil
|
||||
from datetime import date, datetime
|
||||
|
||||
import pandas as pd
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from six import string_types
|
||||
from six.moves.urllib import request
|
||||
|
||||
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
|
||||
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
|
||||
from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \
|
||||
ExchangeJSONDecoder
|
||||
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
|
||||
InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
|
||||
from catalyst.utils.paths import data_root, ensure_directory, \
|
||||
last_modified_time
|
||||
|
||||
@@ -62,13 +61,6 @@ def get_exchange_folder(exchange_name, environ=None):
|
||||
return exchange_folder
|
||||
|
||||
|
||||
def is_blacklist(exchange_name, environ=None):
|
||||
exchange_folder = get_exchange_folder(exchange_name, environ)
|
||||
filename = os.path.join(exchange_folder, 'blacklist.txt')
|
||||
|
||||
return os.path.exists(filename)
|
||||
|
||||
|
||||
def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
|
||||
"""
|
||||
The absolute path of the exchange's symbol.json file.
|
||||
@@ -128,15 +120,12 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
|
||||
if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
|
||||
pd.Timestamp('now', tz='UTC') - last_modified_time(
|
||||
filename)).days > 1):
|
||||
try:
|
||||
download_exchange_symbols(exchange_name, environ)
|
||||
except Exception as e:
|
||||
pass
|
||||
download_exchange_symbols(exchange_name, environ)
|
||||
|
||||
if os.path.isfile(filename):
|
||||
with open(filename) as data_file:
|
||||
try:
|
||||
data = json.load(data_file, cls=ExchangeJSONDecoder)
|
||||
data = json.load(data_file)
|
||||
return data
|
||||
|
||||
except ValueError:
|
||||
@@ -191,7 +180,7 @@ def get_symbols_string(assets):
|
||||
return ', '.join([asset.symbol for asset in array])
|
||||
|
||||
|
||||
def get_exchange_auth(exchange_name, alias=None, environ=None):
|
||||
def get_exchange_auth(exchange_name, environ=None):
|
||||
"""
|
||||
The de-serialized contend of the exchange's auth.json file.
|
||||
|
||||
@@ -206,8 +195,7 @@ def get_exchange_auth(exchange_name, alias=None, environ=None):
|
||||
|
||||
"""
|
||||
exchange_folder = get_exchange_folder(exchange_name, environ)
|
||||
name = 'auth' if alias is None else alias
|
||||
filename = os.path.join(exchange_folder, '{}.json'.format(name))
|
||||
filename = os.path.join(exchange_folder, 'auth.json')
|
||||
|
||||
if os.path.isfile(filename):
|
||||
with open(filename) as data_file:
|
||||
@@ -263,7 +251,7 @@ def get_algo_folder(algo_name, environ=None):
|
||||
return algo_folder
|
||||
|
||||
|
||||
def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
|
||||
def get_algo_object(algo_name, key, environ=None, rel_path=None):
|
||||
"""
|
||||
The de-serialized object of the algo name and key.
|
||||
|
||||
@@ -287,25 +275,19 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
|
||||
if rel_path is not None:
|
||||
folder = os.path.join(folder, rel_path)
|
||||
|
||||
name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key)
|
||||
filename = os.path.join(folder, name)
|
||||
filename = os.path.join(folder, key + '.p')
|
||||
|
||||
if os.path.isfile(filename):
|
||||
if how == 'pickle':
|
||||
try:
|
||||
with open(filename, 'rb') as handle:
|
||||
return pickle.load(handle)
|
||||
|
||||
else:
|
||||
with open(filename) as data_file:
|
||||
data = json.load(data_file, cls=ExchangeJSONDecoder)
|
||||
return data
|
||||
|
||||
except Exception as e:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
|
||||
how='pickle'):
|
||||
def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
|
||||
"""
|
||||
Serialize and save an object by algo name and key.
|
||||
|
||||
@@ -324,15 +306,10 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
|
||||
folder = os.path.join(folder, rel_path)
|
||||
ensure_directory(folder)
|
||||
|
||||
if how == 'json':
|
||||
filename = os.path.join(folder, '{}.json'.format(key))
|
||||
with open(filename, 'wt') as handle:
|
||||
json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)
|
||||
filename = os.path.join(folder, key + '.p')
|
||||
|
||||
else:
|
||||
filename = os.path.join(folder, '{}.p'.format(key))
|
||||
with open(filename, 'wb') as handle:
|
||||
pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
with open(filename, 'wb') as handle:
|
||||
pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
|
||||
def get_algo_df(algo_name, key, environ=None, rel_path=None):
|
||||
@@ -436,15 +413,6 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
|
||||
return temp_bundles
|
||||
|
||||
|
||||
def has_bundle(exchange_name, data_frequency, environ=None):
|
||||
exchange_folder = get_exchange_folder(exchange_name, environ)
|
||||
|
||||
folder_name = '{}_bundle'.format(data_frequency.lower())
|
||||
folder = os.path.join(exchange_folder, folder_name)
|
||||
|
||||
return os.path.isdir(folder)
|
||||
|
||||
|
||||
def symbols_serial(obj):
|
||||
"""
|
||||
JSON serializer for objects not serializable by default json code
|
||||
@@ -512,6 +480,67 @@ def get_common_assets(exchanges):
|
||||
return assets
|
||||
|
||||
|
||||
def get_frequency(freq, data_frequency):
|
||||
"""
|
||||
Get the frequency parameters.
|
||||
|
||||
Notes
|
||||
-----
|
||||
We're trying to use Pandas convention for frequency aliases.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
freq: str
|
||||
data_frequency: str
|
||||
|
||||
Returns
|
||||
-------
|
||||
str, int, str, str
|
||||
|
||||
"""
|
||||
if freq == 'minute':
|
||||
unit = 'T'
|
||||
candle_size = 1
|
||||
|
||||
elif freq == 'daily':
|
||||
unit = 'D'
|
||||
candle_size = 1
|
||||
|
||||
else:
|
||||
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
|
||||
if freq_match:
|
||||
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
|
||||
else 1
|
||||
unit = freq_match.group(2)
|
||||
|
||||
else:
|
||||
raise InvalidHistoryFrequencyError(frequency=freq)
|
||||
|
||||
if unit.lower() == 'd':
|
||||
alias = '{}D'.format(candle_size)
|
||||
|
||||
if data_frequency == 'minute':
|
||||
data_frequency = 'daily'
|
||||
|
||||
elif unit.lower() == 'm' or unit == 'T':
|
||||
alias = '{}T'.format(candle_size)
|
||||
|
||||
if data_frequency == 'daily':
|
||||
data_frequency = 'minute'
|
||||
|
||||
# elif unit.lower() == 'h':
|
||||
# candle_size = candle_size * 60
|
||||
#
|
||||
# alias = '{}T'.format(candle_size)
|
||||
# if data_frequency == 'daily':
|
||||
# data_frequency = 'minute'
|
||||
|
||||
else:
|
||||
raise InvalidHistoryFrequencyAlias(freq=freq)
|
||||
|
||||
return alias, candle_size, unit, data_frequency
|
||||
|
||||
|
||||
def resample_history_df(df, freq, field):
|
||||
"""
|
||||
Resample the OHCLV DataFrame using the specified frequency.
|
||||
@@ -542,122 +571,3 @@ def resample_history_df(df, freq, field):
|
||||
|
||||
resampled_df = df.resample(freq).agg(agg)
|
||||
return resampled_df
|
||||
|
||||
|
||||
def mixin_market_params(exchange_name, params, market):
|
||||
"""
|
||||
Applies a CCXT market dict to parameters of TradingPair init.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
params: dict[Object]
|
||||
market: dict[Object]
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
# TODO: make this more externalized / configurable
|
||||
if 'lot' in market:
|
||||
params['min_trade_size'] = market['lot']
|
||||
params['lot'] = market['lot']
|
||||
|
||||
if exchange_name == 'bitfinex':
|
||||
params['maker'] = 0.001
|
||||
params['taker'] = 0.002
|
||||
|
||||
elif 'maker' in market and 'taker' in market \
|
||||
and market['maker'] is not None and market['taker'] is not None:
|
||||
params['maker'] = market['maker']
|
||||
params['taker'] = market['taker']
|
||||
|
||||
else:
|
||||
# TODO: default commission, make configurable
|
||||
params['maker'] = 0.0015
|
||||
params['taker'] = 0.0025
|
||||
|
||||
info = market['info'] if 'info' in market else None
|
||||
if info:
|
||||
if 'minimum_order_size' in info:
|
||||
params['min_trade_size'] = float(info['minimum_order_size'])
|
||||
|
||||
if 'lot' not in params:
|
||||
params['lot'] = params['min_trade_size']
|
||||
|
||||
|
||||
def group_assets_by_exchange(assets):
|
||||
exchange_assets = dict()
|
||||
for asset in assets:
|
||||
if asset.exchange not in exchange_assets:
|
||||
exchange_assets[asset.exchange] = list()
|
||||
|
||||
exchange_assets[asset.exchange].append(asset)
|
||||
|
||||
return exchange_assets
|
||||
|
||||
|
||||
def get_catalyst_symbol(market_or_symbol):
|
||||
"""
|
||||
The Catalyst symbol.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
market_or_symbol
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
if isinstance(market_or_symbol, string_types):
|
||||
parts = market_or_symbol.split('/')
|
||||
return '{}_{}'.format(parts[0].lower(), parts[1].lower())
|
||||
|
||||
else:
|
||||
return '{}_{}'.format(
|
||||
market_or_symbol['base'].lower(),
|
||||
market_or_symbol['quote'].lower(),
|
||||
)
|
||||
|
||||
|
||||
def save_asset_data(folder, df, decimals=8):
|
||||
symbols = df.index.get_level_values('symbol')
|
||||
for symbol in symbols:
|
||||
symbol_df = df.loc[(symbols == symbol)] # Type: pd.DataFrame
|
||||
|
||||
filename = os.path.join(folder, '{}.csv'.format(symbol))
|
||||
if os.path.exists(filename):
|
||||
print_headers = False
|
||||
|
||||
else:
|
||||
print_headers = True
|
||||
|
||||
with open(filename, 'a') as f:
|
||||
symbol_df.to_csv(
|
||||
path_or_buf=f,
|
||||
header=print_headers,
|
||||
float_format='%.{}f'.format(decimals),
|
||||
)
|
||||
|
||||
|
||||
def get_candles_df(candles, field, freq, bar_count, end_dt,
|
||||
previous_value=None):
|
||||
all_series = dict()
|
||||
for asset in candles:
|
||||
periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
|
||||
|
||||
dates = [candle['last_traded'] for candle in candles[asset]]
|
||||
values = [candle[field] for candle in candles[asset]]
|
||||
series = pd.Series(values, index=dates)
|
||||
|
||||
series = series.reindex(
|
||||
periods,
|
||||
method='ffill',
|
||||
fill_value=previous_value,
|
||||
)
|
||||
series.sort_index(inplace=True)
|
||||
all_series[asset] = series
|
||||
|
||||
df = pd.DataFrame(all_series)
|
||||
df.dropna(inplace=True)
|
||||
|
||||
return df
|
||||
@@ -0,0 +1,43 @@
|
||||
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
|
||||
from catalyst.exchange.bittrex.bittrex import Bittrex
|
||||
from catalyst.exchange.exchange_errors import ExchangeNotFoundError
|
||||
from catalyst.exchange.exchange_utils import get_exchange_auth
|
||||
from catalyst.exchange.poloniex.poloniex import Poloniex
|
||||
|
||||
|
||||
def get_exchange(exchange_name, base_currency=None):
|
||||
exchange_auth = get_exchange_auth(exchange_name)
|
||||
if exchange_name == 'bitfinex':
|
||||
return Bitfinex(
|
||||
key=exchange_auth['key'],
|
||||
secret=exchange_auth['secret'],
|
||||
base_currency=base_currency,
|
||||
portfolio=None
|
||||
)
|
||||
|
||||
elif exchange_name == 'bittrex':
|
||||
return Bittrex(
|
||||
key=exchange_auth['key'],
|
||||
secret=exchange_auth['secret'],
|
||||
base_currency=base_currency,
|
||||
portfolio=None
|
||||
)
|
||||
|
||||
elif exchange_name == 'poloniex':
|
||||
return Poloniex(
|
||||
key=exchange_auth['key'],
|
||||
secret=exchange_auth['secret'],
|
||||
base_currency=base_currency,
|
||||
portfolio=None
|
||||
)
|
||||
|
||||
else:
|
||||
raise ExchangeNotFoundError(exchange_name=exchange_name)
|
||||
|
||||
|
||||
def get_exchanges(exchange_names):
|
||||
exchanges = dict()
|
||||
for exchange_name in exchange_names:
|
||||
exchanges[exchange_name] = get_exchange(exchange_name)
|
||||
|
||||
return exchanges
|
||||
@@ -1,12 +1,14 @@
|
||||
import pandas as pd
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.utils.stats_utils import prepare_stats
|
||||
from catalyst.gens.sim_engine import (
|
||||
BAR,
|
||||
SESSION_START
|
||||
)
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.exchange_errors import \
|
||||
MismatchingBaseCurrenciesExchanges
|
||||
|
||||
log = Logger('LiveGraphClock', level=LOG_LEVEL)
|
||||
|
||||
|
||||
@@ -36,23 +38,177 @@ class LiveGraphClock(object):
|
||||
the exchange and the live trading machine's clock. It's not used currently.
|
||||
"""
|
||||
|
||||
def __init__(self, sessions, context, callback=None,
|
||||
time_skew=pd.Timedelta('0s')):
|
||||
def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
|
||||
|
||||
global mdates, plt # TODO: Could be cleaner
|
||||
import matplotlib.dates as mdates
|
||||
from matplotlib import pyplot as plt
|
||||
from matplotlib import style
|
||||
|
||||
self.sessions = sessions
|
||||
self.time_skew = time_skew
|
||||
self._last_emit = None
|
||||
self._before_trading_start_bar_yielded = True
|
||||
self.context = context
|
||||
self.callback = callback
|
||||
self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
|
||||
|
||||
style.use('dark_background')
|
||||
|
||||
fig = plt.figure()
|
||||
fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
|
||||
self.context.algo_namespace))
|
||||
|
||||
self.ax_pnl = fig.add_subplot(311)
|
||||
|
||||
self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl)
|
||||
|
||||
self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl)
|
||||
|
||||
if len(context.minute_stats) > 0:
|
||||
self.draw_pnl()
|
||||
self.draw_custom_signals()
|
||||
self.draw_exposure()
|
||||
|
||||
# rotates and right aligns the x labels, and moves the bottom of the
|
||||
# axes up to make room for them
|
||||
fig.autofmt_xdate()
|
||||
fig.subplots_adjust(hspace=0.5)
|
||||
|
||||
plt.tight_layout()
|
||||
plt.ion()
|
||||
plt.show()
|
||||
|
||||
def format_ax(self, ax):
|
||||
"""
|
||||
Trying to assign reasonable parameters to the time axis.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ax:
|
||||
|
||||
"""
|
||||
# TODO: room for improvement
|
||||
ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
|
||||
ax.xaxis.set_major_formatter(self.fmt)
|
||||
|
||||
locator = mdates.HourLocator(interval=4)
|
||||
locator.MAXTICKS = 5000
|
||||
ax.xaxis.set_minor_locator(locator)
|
||||
|
||||
datemin = pd.Timestamp.utcnow()
|
||||
ax.set_xlim(datemin)
|
||||
|
||||
ax.grid(True)
|
||||
|
||||
def set_legend(self, ax):
|
||||
"""
|
||||
Set legend on the chart.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ax
|
||||
|
||||
"""
|
||||
ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
|
||||
|
||||
def draw_pnl(self):
|
||||
"""
|
||||
Draw p&l line on the chart.
|
||||
|
||||
"""
|
||||
ax = self.ax_pnl
|
||||
df = self.context.pnl_stats
|
||||
|
||||
ax.clear()
|
||||
ax.set_title('Performance')
|
||||
ax.plot(df.index, df['performance'], '-',
|
||||
color='green',
|
||||
linewidth=1.0,
|
||||
label='Performance'
|
||||
)
|
||||
|
||||
def perc(val):
|
||||
return '{:2f}'.format(val)
|
||||
|
||||
ax.format_ydata = perc
|
||||
|
||||
self.set_legend(ax)
|
||||
self.format_ax(ax)
|
||||
|
||||
def draw_custom_signals(self):
|
||||
"""
|
||||
Draw custom signals on the chart.
|
||||
|
||||
"""
|
||||
ax = self.ax_custom_signals
|
||||
df = self.context.custom_signals_stats
|
||||
|
||||
colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
|
||||
|
||||
ax.clear()
|
||||
ax.set_title('Custom Signals')
|
||||
for index, column in enumerate(df.columns.values.tolist()):
|
||||
ax.plot(df.index, df[column], '-',
|
||||
color=colors[index],
|
||||
linewidth=1.0,
|
||||
label=column
|
||||
)
|
||||
|
||||
self.set_legend(ax)
|
||||
self.format_ax(ax)
|
||||
|
||||
def draw_exposure(self):
|
||||
"""
|
||||
Draw exposure line on the chart.
|
||||
|
||||
"""
|
||||
ax = self.ax_exposure
|
||||
context = self.context
|
||||
df = context.exposure_stats
|
||||
|
||||
# TODO: list exchanges in graph
|
||||
base_currency = None
|
||||
positions = []
|
||||
for exchange_name in context.exchanges:
|
||||
exchange = context.exchanges[exchange_name]
|
||||
|
||||
if not base_currency:
|
||||
base_currency = exchange.base_currency
|
||||
elif base_currency != exchange.base_currency:
|
||||
raise MismatchingBaseCurrenciesExchanges(
|
||||
base_currency=base_currency,
|
||||
exchange_name=exchange.name,
|
||||
exchange_currency=exchange.base_currency
|
||||
)
|
||||
|
||||
positions += exchange.portfolio.positions
|
||||
|
||||
ax.clear()
|
||||
ax.set_title('Exposure')
|
||||
ax.plot(df.index, df['base_currency'], '-',
|
||||
color='green',
|
||||
linewidth=1.0,
|
||||
label='Base Currency: {}'.format(base_currency.upper())
|
||||
)
|
||||
|
||||
symbols = []
|
||||
for position in positions:
|
||||
symbols.append(position.symbol)
|
||||
|
||||
ax.plot(df.index, df['long_exposure'], '-',
|
||||
color='blue',
|
||||
linewidth=1.0,
|
||||
label='Long Exposure: {}'.format(', '.join(symbols).upper()))
|
||||
|
||||
self.set_legend(ax)
|
||||
self.format_ax(ax)
|
||||
|
||||
def __iter__(self):
|
||||
from matplotlib import pyplot as plt
|
||||
yield pd.Timestamp.utcnow(), SESSION_START
|
||||
|
||||
while True:
|
||||
current_time = pd.Timestamp.utcnow()
|
||||
current_minute = current_time.floor('1T')
|
||||
current_minute = current_time.floor('1 min')
|
||||
|
||||
if self._last_emit is None or current_minute > self._last_emit:
|
||||
log.debug('emitting minutely bar: {}'.format(current_minute))
|
||||
@@ -60,11 +216,14 @@ class LiveGraphClock(object):
|
||||
self._last_emit = current_minute
|
||||
yield current_minute, BAR
|
||||
|
||||
recorded_cols = list(self.context.recorded_vars.keys())
|
||||
df, _ = prepare_stats(
|
||||
self.context.frame_stats, recorded_cols=recorded_cols
|
||||
)
|
||||
self.callback(self.context, df)
|
||||
try:
|
||||
self.draw_pnl()
|
||||
self.draw_custom_signals()
|
||||
self.draw_exposure()
|
||||
|
||||
plt.draw()
|
||||
except Exception as e:
|
||||
log.warn('Unable to update the graph: {}'.format(e))
|
||||
|
||||
else:
|
||||
# I can't use the "animate" reactive approach here because
|
||||
|
||||
@@ -0,0 +1,655 @@
|
||||
import json
|
||||
import json
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytz
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
# import six
|
||||
from six import iteritems
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
# from websocket import create_connection
|
||||
from catalyst.exchange.exchange import Exchange
|
||||
from catalyst.exchange.exchange_bundle import ExchangeBundle
|
||||
from catalyst.exchange.exchange_errors import (
|
||||
ExchangeRequestError,
|
||||
InvalidHistoryFrequencyError,
|
||||
InvalidOrderStyle, OrphanOrderReverseError)
|
||||
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
|
||||
ExchangeStopLimitOrder
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
|
||||
download_exchange_symbols, get_symbols_string
|
||||
from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
|
||||
from catalyst.finance.order import Order, ORDER_STATUS
|
||||
from catalyst.finance.transaction import Transaction
|
||||
from catalyst.protocol import Account
|
||||
|
||||
log = Logger('Poloniex', level=LOG_LEVEL)
|
||||
|
||||
|
||||
class Poloniex(Exchange):
|
||||
def __init__(self, key, secret, base_currency, portfolio=None):
|
||||
self.api = Poloniex_api(key=key, secret=secret)
|
||||
self.name = 'poloniex'
|
||||
|
||||
self.assets = dict()
|
||||
self.load_assets()
|
||||
|
||||
self.local_assets = dict()
|
||||
self.load_assets(is_local=True)
|
||||
|
||||
self.base_currency = base_currency
|
||||
self._portfolio = portfolio
|
||||
self.minute_writer = None
|
||||
self.minute_reader = None
|
||||
self.transactions = defaultdict(list)
|
||||
|
||||
self.num_candles_limit = 2000
|
||||
self.max_requests_per_minute = 60
|
||||
self.request_cpt = dict()
|
||||
|
||||
self.bundle = ExchangeBundle(self.name)
|
||||
|
||||
def sanitize_curency_symbol(self, exchange_symbol):
|
||||
"""
|
||||
Helper method used to build the universal pair.
|
||||
Include any symbol mapping here if appropriate.
|
||||
|
||||
:param exchange_symbol:
|
||||
:return universal_symbol:
|
||||
"""
|
||||
return exchange_symbol.lower()
|
||||
|
||||
def _create_order(self, order_status):
|
||||
"""
|
||||
Create a Catalyst order object from the Exchange order dictionary
|
||||
:param order_status:
|
||||
:return: Order
|
||||
"""
|
||||
# if order_status['is_cancelled']:
|
||||
# status = ORDER_STATUS.CANCELLED
|
||||
# elif not order_status['is_live']:
|
||||
# log.info('found executed order {}'.format(order_status))
|
||||
# status = ORDER_STATUS.FILLED
|
||||
# else:
|
||||
status = ORDER_STATUS.OPEN
|
||||
|
||||
amount = float(order_status['amount'])
|
||||
# filled = float(order_status['executed_amount'])
|
||||
filled = None
|
||||
|
||||
if order_status['type'] == 'sell':
|
||||
amount = -amount
|
||||
# filled = -filled
|
||||
|
||||
price = float(order_status['rate'])
|
||||
order_type = order_status['type']
|
||||
|
||||
stop_price = None
|
||||
limit_price = None
|
||||
|
||||
# TODO: is this comprehensive enough?
|
||||
# if order_type.endswith('limit'):
|
||||
# limit_price = price
|
||||
# elif order_type.endswith('stop'):
|
||||
# stop_price = price
|
||||
|
||||
# executed_price = float(order_status['avg_execution_price'])
|
||||
executed_price = price
|
||||
|
||||
# TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
|
||||
commission = None
|
||||
|
||||
# date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
|
||||
# date = pytz.utc.localize(date)
|
||||
date = None
|
||||
|
||||
order = Order(
|
||||
dt=date,
|
||||
asset=self.assets[order_status['symbol']],
|
||||
# No such field in Poloniex
|
||||
amount=amount,
|
||||
stop=stop_price,
|
||||
limit=limit_price,
|
||||
filled=filled,
|
||||
id=str(order_status['orderNumber']),
|
||||
commission=commission
|
||||
)
|
||||
order.status = status
|
||||
|
||||
return order, executed_price
|
||||
|
||||
def get_balances(self):
|
||||
balances = self.api.returnbalances()
|
||||
try:
|
||||
log.debug('retrieving wallets balances')
|
||||
except Exception as e:
|
||||
log.debug(e)
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'error' in balances:
|
||||
raise ExchangeRequestError(
|
||||
error='unable to fetch balance {}'.format(balances['error'])
|
||||
)
|
||||
|
||||
std_balances = dict()
|
||||
for (key, value) in iteritems(balances):
|
||||
currency = key.lower()
|
||||
std_balances[currency] = float(value)
|
||||
|
||||
return std_balances
|
||||
|
||||
@property
|
||||
def account(self):
|
||||
account = Account()
|
||||
|
||||
account.settled_cash = None
|
||||
account.accrued_interest = None
|
||||
account.buying_power = None
|
||||
account.equity_with_loan = None
|
||||
account.total_positions_value = None
|
||||
account.total_positions_exposure = None
|
||||
account.regt_equity = None
|
||||
account.regt_margin = None
|
||||
account.initial_margin_requirement = None
|
||||
account.maintenance_margin_requirement = None
|
||||
account.available_funds = None
|
||||
account.excess_liquidity = None
|
||||
account.cushion = None
|
||||
account.day_trades_remaining = None
|
||||
account.leverage = None
|
||||
account.net_leverage = None
|
||||
account.net_liquidation = None
|
||||
|
||||
return account
|
||||
|
||||
@property
|
||||
def time_skew(self):
|
||||
# TODO: research the time skew conditions
|
||||
return pd.Timedelta('0s')
|
||||
|
||||
def get_account(self):
|
||||
# TODO: fetch account data and keep in cache
|
||||
return None
|
||||
|
||||
def get_candles(self, freq, assets, bar_count=None,
|
||||
start_dt=None, end_dt=None):
|
||||
"""
|
||||
Retrieve OHLVC candles from Poloniex
|
||||
|
||||
:param freq:
|
||||
:param assets:
|
||||
:param bar_count:
|
||||
:return:
|
||||
|
||||
Available Frequencies
|
||||
---------------------
|
||||
'5m', '15m', '30m', '2h', '4h', '1D'
|
||||
"""
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = pd.Timestamp.utcnow()
|
||||
|
||||
log.debug(
|
||||
'retrieving {bars} {freq} candles on {exchange} from '
|
||||
'{end_dt} for markets {symbols}, '.format(
|
||||
bars=bar_count,
|
||||
freq=freq,
|
||||
exchange=self.name,
|
||||
end_dt=end_dt,
|
||||
symbols=get_symbols_string(assets)
|
||||
)
|
||||
)
|
||||
|
||||
if freq == '1T' and (bar_count == 1 or bar_count is None):
|
||||
# TODO: use the order book instead
|
||||
# We use the 5m to fetch the last bar
|
||||
frequency = 300
|
||||
elif freq == '5T':
|
||||
frequency = 300
|
||||
elif freq == '15T':
|
||||
frequency = 900
|
||||
elif freq == '30T':
|
||||
frequency = 1800
|
||||
elif freq == '120T':
|
||||
frequency = 7200
|
||||
elif freq == '240T':
|
||||
frequency = 14400
|
||||
elif freq == '1D':
|
||||
frequency = 86400
|
||||
else:
|
||||
# Poloniex does not offer 1m data candles
|
||||
# It is likely to error out there frequently
|
||||
raise InvalidHistoryFrequencyError(frequency=freq)
|
||||
|
||||
# Making sure that assets are iterable
|
||||
asset_list = [assets] if isinstance(assets, TradingPair) else assets
|
||||
ohlc_map = dict()
|
||||
|
||||
for asset in asset_list:
|
||||
delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
|
||||
end = int(delta.total_seconds())
|
||||
|
||||
if bar_count is None:
|
||||
start = end - 2 * frequency
|
||||
else:
|
||||
start = end - bar_count * frequency
|
||||
|
||||
try:
|
||||
response = self.api.returnchartdata(
|
||||
self.get_symbol(asset), frequency, start, end
|
||||
)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'error' in response:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to retrieve candles: {}'.format(
|
||||
response.content)
|
||||
)
|
||||
|
||||
def ohlc_from_candle(candle):
|
||||
last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
|
||||
last_traded = last_traded.replace(tzinfo=pytz.UTC)
|
||||
|
||||
ohlc = dict(
|
||||
open=np.float64(candle['open']),
|
||||
high=np.float64(candle['high']),
|
||||
low=np.float64(candle['low']),
|
||||
close=np.float64(candle['close']),
|
||||
volume=np.float64(candle['volume']),
|
||||
price=np.float64(candle['close']),
|
||||
last_traded=last_traded
|
||||
)
|
||||
|
||||
return ohlc
|
||||
|
||||
if bar_count is None:
|
||||
ohlc_map[asset] = ohlc_from_candle(response[0])
|
||||
else:
|
||||
ohlc_bars = []
|
||||
for candle in response:
|
||||
ohlc = ohlc_from_candle(candle)
|
||||
ohlc_bars.append(ohlc)
|
||||
ohlc_map[asset] = ohlc_bars
|
||||
|
||||
return ohlc_map[assets] \
|
||||
if isinstance(assets, TradingPair) else ohlc_map
|
||||
|
||||
def create_order(self, asset, amount, is_buy, style):
|
||||
"""
|
||||
Creating order on the exchange.
|
||||
|
||||
:param asset:
|
||||
:param amount:
|
||||
:param is_buy:
|
||||
:param style:
|
||||
:return:
|
||||
"""
|
||||
exchange_symbol = self.get_symbol(asset)
|
||||
|
||||
if isinstance(style, ExchangeLimitOrder) or isinstance(style,
|
||||
ExchangeStopLimitOrder):
|
||||
if isinstance(style, ExchangeStopLimitOrder):
|
||||
log.warn('{} will ignore the stop price'.format(self.name))
|
||||
|
||||
price = style.get_limit_price(is_buy)
|
||||
|
||||
try:
|
||||
if (is_buy):
|
||||
response = self.api.buy(exchange_symbol, amount, price)
|
||||
else:
|
||||
response = self.api.sell(exchange_symbol, -amount, price)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
date = pd.Timestamp.utcnow()
|
||||
|
||||
if ('orderNumber' in response):
|
||||
order_id = str(response['orderNumber'])
|
||||
order = Order(
|
||||
dt=date,
|
||||
asset=asset,
|
||||
amount=amount,
|
||||
stop=style.get_stop_price(is_buy),
|
||||
limit=style.get_limit_price(is_buy),
|
||||
id=order_id
|
||||
)
|
||||
return order
|
||||
else:
|
||||
log.warn(
|
||||
'{} order failed: {}'.format('buy' if is_buy else 'sell',
|
||||
response['error']))
|
||||
return None
|
||||
else:
|
||||
raise InvalidOrderStyle(exchange=self.name,
|
||||
style=style.__class__.__name__)
|
||||
|
||||
def get_open_orders(self, asset='all'):
|
||||
"""Retrieve all of the current open orders.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
asset : Asset
|
||||
If passed and not 'all', return only the open orders for the given
|
||||
asset instead of all open orders.
|
||||
|
||||
Returns
|
||||
-------
|
||||
open_orders : dict[list[Order]] or list[Order]
|
||||
If 'all' is passed this will return a dict mapping Assets
|
||||
to a list containing all the open orders for the asset.
|
||||
If an asset is passed then this will return a list of the open
|
||||
orders for this asset.
|
||||
"""
|
||||
|
||||
return self.portfolio.open_orders
|
||||
|
||||
"""
|
||||
TODO: Why going to the exchange if we already have this info locally?
|
||||
And why creating all these Orders if we later discard them?
|
||||
"""
|
||||
|
||||
try:
|
||||
if (asset == 'all'):
|
||||
response = self.api.returnopenorders('all')
|
||||
else:
|
||||
response = self.api.returnopenorders(self.get_symbol(asset))
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'error' in response:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to retrieve open orders: {}'.format(
|
||||
order_statuses['message'])
|
||||
)
|
||||
|
||||
print(self.portfolio.open_orders)
|
||||
|
||||
# TODO: Need to handle openOrders for 'all'
|
||||
orders = list()
|
||||
for order_status in response:
|
||||
order, executed_price = self._create_order(
|
||||
order_status) # will Throw error b/c Polo doesn't track order['symbol']
|
||||
if asset is None or asset == order.sid:
|
||||
orders.append(order)
|
||||
|
||||
return orders
|
||||
|
||||
def get_order(self, order_id):
|
||||
"""Lookup an order based on the order id returned from one of the
|
||||
order functions.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
order_id : str
|
||||
The unique identifier for the order.
|
||||
|
||||
Returns
|
||||
-------
|
||||
order : Order
|
||||
The order object.
|
||||
"""
|
||||
|
||||
try:
|
||||
order = self._portfolio.open_orders[order_id]
|
||||
except Exception as e:
|
||||
raise OrphanOrderError(order_id=order_id, exchange=self.name)
|
||||
|
||||
return order
|
||||
|
||||
# TODO: Need to decide whether we fetch orders locally or from exchnage
|
||||
# The code below is ignored
|
||||
|
||||
try:
|
||||
response = self.api.returnopenorders(self.get_symbol(order.sid))
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
for o in response:
|
||||
if (int(o['orderNumber']) == int(order_id)):
|
||||
return order
|
||||
|
||||
return None
|
||||
|
||||
def cancel_order(self, order_param):
|
||||
"""Cancel an open order.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
order_param : str or Order
|
||||
The order_id or order object to cancel.
|
||||
"""
|
||||
|
||||
if (isinstance(order_param, Order)):
|
||||
order = order_param
|
||||
else:
|
||||
order = self._portfolio.open_orders[order_param]
|
||||
|
||||
try:
|
||||
response = self.api.cancelorder(order.id)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'error' in response:
|
||||
log.info(
|
||||
'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format(
|
||||
order_id=order.id,
|
||||
exchange=self.name,
|
||||
error=response['error']
|
||||
))
|
||||
|
||||
# raise OrderCancelError(
|
||||
# order_id=order.id,
|
||||
# exchange=self.name,
|
||||
# error=response['error']
|
||||
# )
|
||||
|
||||
self.portfolio.remove_order(order)
|
||||
|
||||
def tickers(self, assets):
|
||||
"""
|
||||
Fetch ticket data for assets
|
||||
https://docs.bitfinex.com/v2/reference#rest-public-tickers
|
||||
|
||||
:param assets:
|
||||
:return:
|
||||
"""
|
||||
symbols = self.get_symbols(assets)
|
||||
|
||||
log.debug('fetching tickers {}'.format(symbols))
|
||||
|
||||
try:
|
||||
response = self.api.returnticker()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if 'error' in response:
|
||||
raise ExchangeRequestError(
|
||||
error='Unable to retrieve tickers: {}'.format(
|
||||
response['error'])
|
||||
)
|
||||
|
||||
ticks = dict()
|
||||
|
||||
for index, symbol in enumerate(symbols):
|
||||
ticks[assets[index]] = dict(
|
||||
timestamp=pd.Timestamp.utcnow(),
|
||||
bid=float(response[symbol]['highestBid']),
|
||||
ask=float(response[symbol]['lowestAsk']),
|
||||
last_price=float(response[symbol]['last']),
|
||||
low=float(response[symbol]['lowestAsk']),
|
||||
# TODO: Polo does not provide low
|
||||
high=float(response[symbol]['highestBid']),
|
||||
# TODO: Polo does not provide high
|
||||
volume=float(response[symbol]['baseVolume']),
|
||||
)
|
||||
|
||||
log.debug('got tickers {}'.format(ticks))
|
||||
return ticks
|
||||
|
||||
def generate_symbols_json(self, filename=None, source_dates=False):
|
||||
symbol_map = {}
|
||||
|
||||
if not source_dates:
|
||||
fn, r = download_exchange_symbols(self.name)
|
||||
with open(fn) as data_file:
|
||||
cached_symbols = json.load(data_file)
|
||||
|
||||
response = self.api.returnticker()
|
||||
|
||||
for exchange_symbol in response:
|
||||
base, market = self.sanitize_curency_symbol(exchange_symbol).split(
|
||||
'_')
|
||||
symbol = '{market}_{base}'.format(market=market, base=base)
|
||||
|
||||
if (source_dates):
|
||||
start_date = self.get_symbol_start_date(exchange_symbol)
|
||||
else:
|
||||
try:
|
||||
start_date = cached_symbols[exchange_symbol]['start_date']
|
||||
except KeyError as e:
|
||||
start_date = time.strftime('%Y-%m-%d')
|
||||
|
||||
try:
|
||||
end_daily = cached_symbols[exchange_symbol]['end_daily']
|
||||
except KeyError as e:
|
||||
end_daily = 'N/A'
|
||||
|
||||
try:
|
||||
end_minute = cached_symbols[exchange_symbol]['end_minute']
|
||||
except KeyError as e:
|
||||
end_minute = 'N/A'
|
||||
|
||||
symbol_map[exchange_symbol] = dict(
|
||||
symbol=symbol,
|
||||
start_date=start_date,
|
||||
end_daily=end_daily,
|
||||
end_minute=end_minute,
|
||||
)
|
||||
|
||||
if (filename is None):
|
||||
filename = get_exchange_symbols_filename(self.name)
|
||||
|
||||
with open(filename, 'w') as f:
|
||||
json.dump(symbol_map, f, sort_keys=True, indent=2,
|
||||
separators=(',', ':'))
|
||||
|
||||
def get_symbol_start_date(self, symbol):
|
||||
try:
|
||||
r = self.api.returnchartdata(symbol, 86400, pd.to_datetime(
|
||||
'2010-1-1').value // 10 ** 9)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date'])))
|
||||
|
||||
def check_open_orders(self):
|
||||
"""
|
||||
Need to override this function for Poloniex:
|
||||
|
||||
Loop through the list of open orders in the Portfolio object.
|
||||
Check if any transactions have been executed:
|
||||
If so, create a transaction and apply to the Portfolio.
|
||||
Check if the order is still open:
|
||||
If not, remove it from open orders
|
||||
|
||||
:return:
|
||||
transactions: Transaction[]
|
||||
"""
|
||||
transactions = list()
|
||||
if self.portfolio.open_orders:
|
||||
for order_id in list(self.portfolio.open_orders):
|
||||
|
||||
order = self._portfolio.open_orders[order_id]
|
||||
log.debug('found open order: {}'.format(order_id))
|
||||
|
||||
try:
|
||||
order_open = self.get_order(order_id)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if (order_open):
|
||||
delta = pd.Timestamp.utcnow() - order.dt
|
||||
log.info(
|
||||
'order {order_id} still open after {delta}'.format(
|
||||
order_id=order_id,
|
||||
delta=delta)
|
||||
)
|
||||
|
||||
try:
|
||||
response = self.api.returnordertrades(order_id)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
if ('error' in response):
|
||||
if (not order_open):
|
||||
raise OrphanOrderReverseError(order_id=order_id,
|
||||
exchange=self.name)
|
||||
else:
|
||||
for tx in response:
|
||||
"""
|
||||
We maintain a list of dictionaries of transactions that correspond to
|
||||
partially filled orders, indexed by order_id. Every time we query
|
||||
executed transactions from the exchange, we check if we had that
|
||||
transaction for that order already. If not, we process it.
|
||||
|
||||
When an order if fully filled, we flush the dict of transactions
|
||||
associated with that order.
|
||||
"""
|
||||
if (not filter(
|
||||
lambda item: item['order_id'] == tx['tradeID'],
|
||||
self.transactions[order_id])):
|
||||
log.debug(
|
||||
'Got new transaction for order {}: amount {}, price {}'.format(
|
||||
order_id, tx['amount'], tx['rate']))
|
||||
tx['amount'] = float(tx['amount'])
|
||||
if (tx['type'] == 'sell'):
|
||||
tx['amount'] = -tx['amount']
|
||||
transaction = Transaction(
|
||||
asset=order.asset,
|
||||
amount=tx['amount'],
|
||||
dt=pd.to_datetime(tx['date'], utc=True),
|
||||
price=float(tx['rate']),
|
||||
order_id=tx['tradeID'],
|
||||
# it's a misnomer, but keeping it for compatibility
|
||||
commission=float(tx['fee'])
|
||||
)
|
||||
self.transactions[order_id].append(transaction)
|
||||
self.portfolio.execute_transaction(transaction)
|
||||
transactions.append(transaction)
|
||||
|
||||
if (not order_open):
|
||||
"""
|
||||
Since transactions have been executed individually
|
||||
the only thing left to do is remove them from list of open_orders
|
||||
"""
|
||||
del self.portfolio.open_orders[order_id]
|
||||
del self.transactions[order_id]
|
||||
|
||||
return transactions
|
||||
|
||||
def get_orderbook(self, asset, order_type='all'):
|
||||
exchange_symbol = asset.exchange_symbol
|
||||
data = self.api.returnOrderBook(market=exchange_symbol)
|
||||
|
||||
result = dict()
|
||||
for order_type in data:
|
||||
# TODO: filter by type
|
||||
if order_type != 'asks' and order_type != 'bids':
|
||||
continue
|
||||
|
||||
result[order_type] = []
|
||||
for entry in data[order_type]:
|
||||
if len(entry) == 2:
|
||||
result[order_type].append(
|
||||
dict(
|
||||
rate=float(entry[0]),
|
||||
quantity=float(entry[1])
|
||||
)
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env python
|
||||
import json
|
||||
import time
|
||||
import hmac
|
||||
import hashlib
|
||||
import ssl
|
||||
|
||||
from six.moves import urllib
|
||||
|
||||
# Workaround for backwards compatibility
|
||||
# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
|
||||
urlopen = urllib.request.urlopen
|
||||
|
||||
|
||||
class Poloniex_api(object):
|
||||
def __init__(self, key, secret):
|
||||
self.key = key
|
||||
self.secret = secret
|
||||
|
||||
self.max_requests_per_second = 6
|
||||
self.request_cpt = dict()
|
||||
|
||||
self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
|
||||
'returnTradeHistory', 'returnChartData',
|
||||
'returnCurrencies', 'returnLoanOrders']
|
||||
self.trading = ['returnBalances', 'returnCompleteBalances',
|
||||
'returnDepositAddresses',
|
||||
'generateNewAddress', 'returnDepositsWithdrawals',
|
||||
'returnOpenOrders',
|
||||
'returnTradeHistory', 'returnOrderTrades',
|
||||
'buy', 'sell', 'cancelOrder', 'moveOrder',
|
||||
'withdraw', 'returnFeeInfo',
|
||||
'returnAvailableAccountBalances',
|
||||
'returnTradableBalances', 'transferBalance',
|
||||
'returnMarginAccountSummary', 'marginBuy',
|
||||
'marginSell',
|
||||
'getMarginPosition', 'closeMarginPosition',
|
||||
'createLoanOffer',
|
||||
'cancelLoanOffer', 'returnOpenLoanOffers',
|
||||
'returnActiveLoans',
|
||||
'returnLendingHistory', 'toggleAutoRenew']
|
||||
|
||||
def ask_request(self):
|
||||
"""
|
||||
Asks permission to issue a request to the exchange.
|
||||
The primary purpose is to avoid hitting rate limits.
|
||||
|
||||
The application will pause if the maximum requests per minute
|
||||
permitted by the exchange is exceeded.
|
||||
|
||||
:return boolean:
|
||||
|
||||
"""
|
||||
now = time.time()
|
||||
if not self.request_cpt:
|
||||
self.request_cpt = dict()
|
||||
self.request_cpt[now] = 0
|
||||
return True
|
||||
|
||||
cpt_date = list(self.request_cpt.keys())[0]
|
||||
cpt = self.request_cpt[cpt_date]
|
||||
|
||||
if now > cpt_date + 1:
|
||||
self.request_cpt = dict()
|
||||
self.request_cpt[now] = 0
|
||||
return True
|
||||
|
||||
if cpt >= self.max_requests_per_second:
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
now = time.time()
|
||||
self.request_cpt = dict()
|
||||
self.request_cpt[now] = 0
|
||||
return True
|
||||
else:
|
||||
self.request_cpt[cpt_date] += 1
|
||||
|
||||
def query(self, method, req={}):
|
||||
|
||||
if method in self.public:
|
||||
url = 'https://poloniex.com/public?command=' + method + '&' + \
|
||||
urllib.parse.urlencode(req)
|
||||
headers = {}
|
||||
post_data = None
|
||||
elif method in self.trading:
|
||||
url = 'https://poloniex.com/tradingApi'
|
||||
req['command'] = method
|
||||
req['nonce'] = int(time.time() * 1000)
|
||||
post_data = urllib.parse.urlencode(req)
|
||||
|
||||
signature = hmac.new(self.secret.encode('utf-8'),
|
||||
post_data.encode('utf-8'),
|
||||
hashlib.sha512).hexdigest()
|
||||
headers = {'Sign': signature, 'Key': self.key}
|
||||
|
||||
post_data = post_data.encode('utf-8')
|
||||
else:
|
||||
raise ValueError(
|
||||
'Method "' + method + '" not found in neither the Public API '
|
||||
'or Trading API endpoints'
|
||||
)
|
||||
|
||||
self.ask_request()
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=post_data,
|
||||
headers=headers,
|
||||
)
|
||||
return json.loads(
|
||||
urlopen(req, context=ssl._create_unverified_context()).read())
|
||||
|
||||
def returnticker(self):
|
||||
return self.query('returnTicker', {})
|
||||
|
||||
def return24volume(self):
|
||||
return self.query('return24Volume', {})
|
||||
|
||||
def returnOrderBook(self, market='all'):
|
||||
return self.query('returnOrderBook', {'currencyPair': market})
|
||||
|
||||
def returntradehistory(self, market, start=None, end=None):
|
||||
if (start is not None and end is not None):
|
||||
return self.query('returntradehistory',
|
||||
{'currencyPair': market, 'start': start,
|
||||
'end': end})
|
||||
else:
|
||||
return self.query('returntradehistory', {'currencyPair': market})
|
||||
|
||||
def returnchartdata(self, market, period, start, end=9999999999):
|
||||
return self.query('returnChartData',
|
||||
{'currencyPair': market, 'period': period,
|
||||
'start': start, 'end': end})
|
||||
|
||||
def returncurrencies(self):
|
||||
return self.query('returnCurrencies', {})
|
||||
|
||||
def returnloadorders(self, market):
|
||||
return self.query('returnLoanOrders', {'currency': market})
|
||||
|
||||
def returnbalances(self):
|
||||
return self.query('returnBalances')
|
||||
|
||||
def returncompletebalances(self, account):
|
||||
if (account):
|
||||
return self.query('returnCompleteBalances', {'account': account})
|
||||
else:
|
||||
return self.query('returnCompleteBalances')
|
||||
|
||||
def returndepositaddresses(self):
|
||||
return self.query('returnDepositAddresses')
|
||||
|
||||
def generatenewaddress(self, currency):
|
||||
return self.query('generateNewAddress', {'currency': currency})
|
||||
|
||||
def returnDepositsWithdrawals(self, start, end):
|
||||
return self.query('returnDepositsWithdrawals',
|
||||
{'start': start, 'end': end})
|
||||
|
||||
def returnopenorders(self, market):
|
||||
return self.query('returnOpenOrders', {'currencyPair': market})
|
||||
|
||||
def returntradehistory(self, market):
|
||||
# TODO: optional start and/or end and limit
|
||||
return self.query('returnTradeHistory', {'currencyPair': market})
|
||||
|
||||
def returnordertrades(self, ordernumber):
|
||||
return self.query('returnOrderTrades', {'orderNumber': ordernumber})
|
||||
|
||||
def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
|
||||
postonly=0):
|
||||
if (fillorkill):
|
||||
return self.query('buy', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount,
|
||||
'fillOrKill': fillorkill, })
|
||||
elif (immediateorcancel):
|
||||
return self.query('buy', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount,
|
||||
'immediateOrCancel': immediateorcancel, })
|
||||
elif (postonly):
|
||||
return self.query('buy', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount,
|
||||
'postOnly': postonly, })
|
||||
else:
|
||||
return self.query('buy', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount, })
|
||||
|
||||
def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
|
||||
postonly=0):
|
||||
if (fillorkill):
|
||||
return self.query('sell', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount,
|
||||
'fillOrKill': fillorkill, })
|
||||
elif (immediateorcancel):
|
||||
return self.query('sell', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount,
|
||||
'immediateOrCancel': immediateorcancel, })
|
||||
elif (postonly):
|
||||
return self.query('sell', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount,
|
||||
'postOnly': postonly, })
|
||||
else:
|
||||
return self.query('sell', {'currencyPair': market, 'rate': rate,
|
||||
'amount': amount, })
|
||||
|
||||
def cancelorder(self, ordernumber):
|
||||
return self.query('cancelOrder', {'orderNumber': ordernumber})
|
||||
|
||||
def withdraw(self, currency, quantity, address):
|
||||
return self.query('withdraw',
|
||||
{'currency': currency, 'amount': quantity,
|
||||
'address': address})
|
||||
|
||||
def returnfeeinfo(self):
|
||||
return self.query('returnFeeInfo')
|
||||
@@ -14,13 +14,14 @@
|
||||
from time import sleep
|
||||
|
||||
import pandas as pd
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.gens.sim_engine import (
|
||||
BAR,
|
||||
SESSION_START
|
||||
)
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
|
||||
log = Logger('ExchangeClock', level=LOG_LEVEL)
|
||||
|
||||
|
||||
@@ -30,8 +31,7 @@ class SimpleClock(object):
|
||||
This class is a drop-in replacement for
|
||||
:class:`zipline.gens.sim_engine.MinuteSimulationClock`.
|
||||
|
||||
This is a stripped down version because crypto exchanges run
|
||||
around the clock.
|
||||
This is a stripped down version because crypto exchanges run around the clock.
|
||||
|
||||
The :param:`time_skew` parameter represents the time difference between
|
||||
the Broker and the live trading machine's clock.
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
import numbers
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def trend_direction(series):
|
||||
if series[-1] is np.nan or series[-1] is np.nan:
|
||||
return None
|
||||
|
||||
if series[-1] > series[-2]:
|
||||
return 'up'
|
||||
else:
|
||||
return 'down'
|
||||
|
||||
|
||||
def crossover(source, target):
|
||||
"""
|
||||
The `x`-series is defined as having crossed over `y`-series if the value
|
||||
of `x` is greater than the value of `y` and the value of `x` was less than
|
||||
the value of `y` on the bar immediately preceding the current bar.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source: Series
|
||||
target: Series
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
|
||||
"""
|
||||
if isinstance(target, numbers.Number):
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] >= target > source[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
else:
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target[-1] is np.nan or target[-2] is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] > target[-1] and source[-2] < target[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def crossunder(source, target):
|
||||
"""
|
||||
The `x`-series is defined as having crossed under `y`-series if the value
|
||||
of `x` is less than the value of `y` and the value of `x` was greater than
|
||||
the value of `y` on the bar immediately preceding the current bar.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source: Series
|
||||
target: Series
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
|
||||
"""
|
||||
if isinstance(target, numbers.Number):
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] < target <= source[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target[-1] is np.nan or target[-2] is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] < target[-1] and source[-2] >= target[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def vwap(df):
|
||||
"""
|
||||
Volume-weighted average price (VWAP) is a ratio generally used by
|
||||
institutional investors and mutual funds to make buys and sells so as not
|
||||
to disturb the market prices with large orders. It is the average share
|
||||
price of a stock weighted against its trading volume within a particular
|
||||
time frame, generally one day.
|
||||
|
||||
Read more: Volume Weighted Average Price - VWAP
|
||||
https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df: pd.DataFrame
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
if 'close' not in df.columns or 'volume' not in df.columns:
|
||||
raise ValueError('price data must include `volume` and `close`')
|
||||
|
||||
vol_sum = np.nansum(df['volume'].values)
|
||||
|
||||
try:
|
||||
ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
|
||||
except ZeroDivisionError:
|
||||
ret = np.nan
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
|
||||
"""
|
||||
Format and print the last few rows of a statistics DataFrame.
|
||||
See the pyfolio project for the data structure.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
stats_df: DataFrame
|
||||
num_rows: int
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
|
||||
"""
|
||||
stats_df.set_index('period_close', drop=True, inplace=True)
|
||||
stats_df.dropna(axis=1, how='all', inplace=True)
|
||||
|
||||
pd.set_option('display.expand_frame_repr', False)
|
||||
pd.set_option('precision', 3)
|
||||
pd.set_option('display.width', 1000)
|
||||
pd.set_option('display.max_colwidth', 1000)
|
||||
|
||||
columns = ['starting_cash', 'ending_cash', 'portfolio_value',
|
||||
'pnl', 'long_exposure', 'short_exposure', 'orders',
|
||||
'transactions', 'positions']
|
||||
|
||||
if recorded_cols is not None:
|
||||
for column in recorded_cols:
|
||||
columns.append(column)
|
||||
|
||||
def format_positions(positions):
|
||||
parts = []
|
||||
for position in positions:
|
||||
msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format(
|
||||
amount=position['amount'],
|
||||
market=position['sid'].market_currency,
|
||||
cost_basis=position['cost_basis'],
|
||||
base=position['sid'].base_currency
|
||||
)
|
||||
parts.append(msg)
|
||||
return ', '.join(parts)
|
||||
|
||||
formatters = {
|
||||
'orders': lambda orders: len(orders),
|
||||
'transactions': lambda transactions: len(transactions),
|
||||
'returns': lambda returns: "{0:.4f}".format(returns),
|
||||
'positions': format_positions
|
||||
}
|
||||
|
||||
return stats_df.tail(num_rows).to_string(
|
||||
columns=columns,
|
||||
formatters=formatters
|
||||
)
|
||||
|
||||
|
||||
def df_to_string(df):
|
||||
"""
|
||||
Create a formatted str representation of the DataFrame.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df: DataFrame
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
|
||||
"""
|
||||
pd.set_option('display.expand_frame_repr', False)
|
||||
pd.set_option('precision', 8)
|
||||
pd.set_option('display.width', 1000)
|
||||
pd.set_option('display.max_colwidth', 1000)
|
||||
|
||||
return df.to_string()
|
||||
|
||||
|
||||
def extract_transactions(perf):
|
||||
"""
|
||||
Compute indexes for buy and sell transactions
|
||||
|
||||
Parameters
|
||||
----------
|
||||
perf: DataFrame
|
||||
The algo performance DataFrame.
|
||||
|
||||
Returns
|
||||
-------
|
||||
DataFrame
|
||||
A DataFrame of transactions.
|
||||
|
||||
"""
|
||||
trans_list = perf.transactions.values
|
||||
all_trans = [t for sublist in trans_list for t in sublist]
|
||||
all_trans.sort(key=lambda t: t['dt'])
|
||||
|
||||
transactions = pd.DataFrame(all_trans)
|
||||
if not transactions.empty:
|
||||
transactions.set_index('dt', inplace=True, drop=True)
|
||||
return transactions
|
||||
@@ -1,158 +0,0 @@
|
||||
import os
|
||||
import tarfile
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from catalyst.data.bundles.core import download_without_progress
|
||||
from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
|
||||
import os
|
||||
import tarfile
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from catalyst.data.bundles.core import download_without_progress
|
||||
from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
|
||||
|
||||
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
|
||||
API_URL = 'http://data.enigma.co/api/v1'
|
||||
|
||||
|
||||
def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
|
||||
"""
|
||||
Download and extract a bcolz bundle.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exchange_name: str
|
||||
symbol: str
|
||||
data_frequency: str
|
||||
period: str
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
|
||||
|
||||
"""
|
||||
root = get_exchange_bundles_folder(exchange_name)
|
||||
name = '{exchange}-{frequency}-{symbol}-{period}'.format(
|
||||
exchange=exchange_name,
|
||||
frequency=data_frequency,
|
||||
symbol=symbol,
|
||||
period=period
|
||||
)
|
||||
path = os.path.join(root, name)
|
||||
|
||||
if not os.path.isdir(path):
|
||||
url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
|
||||
'exchange-{exchange}/{name}.tar.gz'.format(
|
||||
exchange=exchange_name,
|
||||
name=name)
|
||||
|
||||
bytes = download_without_progress(url)
|
||||
with tarfile.open('r', fileobj=bytes) as tar:
|
||||
tar.extractall(path)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def get_df_from_arrays(arrays, periods):
|
||||
"""
|
||||
A DataFrame from the specified OHCLV arrays.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
arrays: Object
|
||||
periods: DateTimeIndex
|
||||
|
||||
Returns
|
||||
-------
|
||||
DataFrame
|
||||
|
||||
"""
|
||||
ohlcv = dict()
|
||||
for index, field in enumerate(
|
||||
['open', 'high', 'low', 'close', 'volume']):
|
||||
ohlcv[field] = arrays[index].flatten()
|
||||
|
||||
df = pd.DataFrame(
|
||||
data=ohlcv,
|
||||
index=periods
|
||||
)
|
||||
return df
|
||||
|
||||
|
||||
def range_in_bundle(asset, start_dt, end_dt, reader):
|
||||
"""
|
||||
Evaluate whether price data of an asset is included has been ingested in
|
||||
the exchange bundle for the given date range.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
asset: TradingPair
|
||||
start_dt: datetime
|
||||
end_dt: datetime
|
||||
reader: BcolzBarMinuteReader
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
|
||||
"""
|
||||
has_data = True
|
||||
dates = [start_dt, end_dt]
|
||||
|
||||
while dates and has_data:
|
||||
try:
|
||||
dt = dates.pop(0)
|
||||
close = reader.get_value(asset.sid, dt, 'close')
|
||||
|
||||
if np.isnan(close):
|
||||
has_data = False
|
||||
|
||||
except Exception:
|
||||
has_data = False
|
||||
|
||||
return has_data
|
||||
|
||||
|
||||
def get_assets(exchange, include_symbols, exclude_symbols):
|
||||
"""
|
||||
Get assets from an exchange, including or excluding the specified
|
||||
symbols.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exchange: Exchange
|
||||
include_symbols: str
|
||||
exclude_symbols: str
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[TradingPair]
|
||||
|
||||
"""
|
||||
if include_symbols is not None:
|
||||
include_symbols_list = include_symbols.split(',')
|
||||
|
||||
return exchange.get_assets(include_symbols_list)
|
||||
|
||||
else:
|
||||
all_assets = exchange.get_assets()
|
||||
|
||||
if exclude_symbols is not None:
|
||||
exclude_symbols_list = exclude_symbols.split(',')
|
||||
|
||||
assets = []
|
||||
for asset in all_assets:
|
||||
if asset.symbol not in exclude_symbols_list:
|
||||
assets.append(asset)
|
||||
|
||||
return assets
|
||||
|
||||
else:
|
||||
return all_assets
|
||||
@@ -1,97 +0,0 @@
|
||||
import os
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
|
||||
from catalyst.exchange.exchange import Exchange
|
||||
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
|
||||
from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
|
||||
get_exchange_folder, is_blacklist
|
||||
from logbook import Logger
|
||||
|
||||
log = Logger('factory', level=LOG_LEVEL)
|
||||
exchange_cache = dict()
|
||||
|
||||
|
||||
def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
|
||||
skip_init=False, auth_alias=None):
|
||||
key = (exchange_name, base_currency)
|
||||
if key in exchange_cache:
|
||||
return exchange_cache[key]
|
||||
|
||||
exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
|
||||
|
||||
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
|
||||
if must_authenticate and not has_auth:
|
||||
raise ExchangeAuthEmpty(
|
||||
exchange=exchange_name.title(),
|
||||
filename=os.path.join(
|
||||
get_exchange_folder(exchange_name), 'auth.json'
|
||||
)
|
||||
)
|
||||
|
||||
exchange = CCXT(
|
||||
exchange_name=exchange_name,
|
||||
key=exchange_auth['key'],
|
||||
secret=exchange_auth['secret'],
|
||||
base_currency=base_currency,
|
||||
)
|
||||
exchange_cache[key] = exchange
|
||||
|
||||
if not skip_init:
|
||||
exchange.init()
|
||||
|
||||
return exchange
|
||||
|
||||
|
||||
def get_exchanges(exchange_names):
|
||||
exchanges = dict()
|
||||
for exchange_name in exchange_names:
|
||||
exchanges[exchange_name] = get_exchange(exchange_name)
|
||||
|
||||
return exchanges
|
||||
|
||||
|
||||
def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
|
||||
base_currency=None):
|
||||
"""
|
||||
Find exchanges filtered by a list of feature.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
features: str
|
||||
The list of features.
|
||||
|
||||
skip_blacklist: bool
|
||||
is_authenticated: bool
|
||||
base_currency: bool
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[Exchange]
|
||||
|
||||
"""
|
||||
exchange_names = CCXT.find_exchanges(features, is_authenticated)
|
||||
|
||||
exchanges = []
|
||||
for exchange_name in exchange_names:
|
||||
if skip_blacklist and is_blacklist(exchange_name):
|
||||
continue
|
||||
|
||||
exchange = get_exchange(
|
||||
exchange_name=exchange_name,
|
||||
skip_init=True,
|
||||
base_currency=base_currency,
|
||||
)
|
||||
|
||||
if features is not None:
|
||||
if 'dailyBundle' in features \
|
||||
and not exchange.has_bundle('daily'):
|
||||
continue
|
||||
|
||||
elif 'minuteBundle' in features \
|
||||
and not exchange.has_bundle('minute'):
|
||||
continue
|
||||
|
||||
exchanges.append(exchange)
|
||||
|
||||
return exchanges
|
||||
@@ -1,131 +0,0 @@
|
||||
import matplotlib.dates as mdates
|
||||
import pandas as pd
|
||||
|
||||
from catalyst.exchange.exchange_errors import \
|
||||
MismatchingBaseCurrenciesExchanges
|
||||
|
||||
fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
|
||||
|
||||
|
||||
def format_ax(ax):
|
||||
"""
|
||||
Trying to assign reasonable parameters to the time axis.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ax:
|
||||
|
||||
"""
|
||||
# TODO: room for improvement
|
||||
ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
|
||||
ax.xaxis.set_major_formatter(fmt)
|
||||
|
||||
locator = mdates.HourLocator(interval=4)
|
||||
locator.MAXTICKS = 5000
|
||||
ax.xaxis.set_minor_locator(locator)
|
||||
|
||||
datemin = pd.Timestamp.utcnow()
|
||||
ax.set_xlim(datemin)
|
||||
|
||||
ax.grid(True)
|
||||
|
||||
|
||||
def set_legend(ax):
|
||||
"""
|
||||
Set legend on the chart.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
ax
|
||||
|
||||
"""
|
||||
ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
|
||||
|
||||
|
||||
def draw_pnl(ax, df):
|
||||
"""
|
||||
Draw p&l line on the chart.
|
||||
|
||||
"""
|
||||
ax.clear()
|
||||
ax.set_title('Performance')
|
||||
index = df.index.unique()
|
||||
dt = index.get_level_values(level=0)
|
||||
pnl = index.get_level_values(level=4)
|
||||
ax.plot(
|
||||
dt, pnl, '-',
|
||||
color='green',
|
||||
linewidth=1.0,
|
||||
label='Performance'
|
||||
)
|
||||
|
||||
def perc(val):
|
||||
return '{:2f}'.format(val)
|
||||
|
||||
ax.format_ydata = perc
|
||||
|
||||
set_legend(ax)
|
||||
format_ax(ax)
|
||||
|
||||
|
||||
def draw_custom_signals(ax, df):
|
||||
"""
|
||||
Draw custom signals on the chart.
|
||||
|
||||
"""
|
||||
colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
|
||||
|
||||
ax.clear()
|
||||
ax.set_title('Custom Signals')
|
||||
for index, column in enumerate(df.columns.values.tolist()):
|
||||
ax.plot(df.index, df[column], '-',
|
||||
color=colors[index],
|
||||
linewidth=1.0,
|
||||
label=column
|
||||
)
|
||||
|
||||
set_legend(ax)
|
||||
format_ax(ax)
|
||||
|
||||
|
||||
def draw_exposure(ax, df, context):
|
||||
"""
|
||||
Draw exposure line on the chart.
|
||||
|
||||
"""
|
||||
# TODO: list exchanges in graph
|
||||
base_currency = None
|
||||
positions = []
|
||||
for exchange_name in context.exchanges:
|
||||
exchange = context.exchanges[exchange_name]
|
||||
|
||||
if not base_currency:
|
||||
base_currency = exchange.base_currency
|
||||
elif base_currency != exchange.base_currency:
|
||||
raise MismatchingBaseCurrenciesExchanges(
|
||||
base_currency=base_currency,
|
||||
exchange_name=exchange.name,
|
||||
exchange_currency=exchange.base_currency
|
||||
)
|
||||
|
||||
positions += exchange.portfolio.positions
|
||||
|
||||
ax.clear()
|
||||
ax.set_title('Exposure')
|
||||
ax.plot(df.index, df['base_currency'], '-',
|
||||
color='green',
|
||||
linewidth=1.0,
|
||||
label='Base Currency: {}'.format(base_currency.upper())
|
||||
)
|
||||
|
||||
symbols = []
|
||||
for position in positions:
|
||||
symbols.append(position.symbol)
|
||||
|
||||
ax.plot(df.index, df['long_exposure'], '-',
|
||||
color='blue',
|
||||
linewidth=1.0,
|
||||
label='Long Exposure: {}'.format(', '.join(symbols).upper()))
|
||||
|
||||
set_legend(ax)
|
||||
format_ax(ax)
|
||||
@@ -1,69 +0,0 @@
|
||||
import json
|
||||
import re
|
||||
from json import JSONEncoder
|
||||
|
||||
import pandas as pd
|
||||
from catalyst.constants import DATE_TIME_FORMAT
|
||||
from six import string_types
|
||||
|
||||
|
||||
class ExchangeJSONEncoder(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
if isinstance(obj, pd.Timestamp):
|
||||
return obj.strftime(DATE_TIME_FORMAT)
|
||||
|
||||
# Let the base class default method raise the TypeError
|
||||
return JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
class ExchangeJSONDecoder(json.JSONDecoder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
json.JSONDecoder.__init__(
|
||||
self, object_hook=self.object_hook, *args, **kwargs
|
||||
)
|
||||
|
||||
def recursive_iter(self, obj):
|
||||
if isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
match = isinstance(value, string_types) and re.search(
|
||||
r'(\d{4}-\d{2}-\d{2}).*', value
|
||||
)
|
||||
if match:
|
||||
try:
|
||||
obj[key] = pd.to_datetime(value, utc=True)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
elif any(isinstance(obj, t) for t in (list, tuple)):
|
||||
for item in obj:
|
||||
self.recursive_iter(item)
|
||||
|
||||
def object_hook(self, obj):
|
||||
self.recursive_iter(obj)
|
||||
return obj
|
||||
|
||||
|
||||
def portfolio_to_dict(portfolio):
|
||||
positions = []
|
||||
for asset in portfolio.positions:
|
||||
p = portfolio.positions[asset] # Type: Position
|
||||
|
||||
position = dict(
|
||||
symbol=asset.symbol,
|
||||
exchange=asset.exchange,
|
||||
amount=p.amount,
|
||||
cost_basis=p.cost_basis,
|
||||
last_sale_price=p.last_sale_price,
|
||||
last_sale_date=p.last_sale_date,
|
||||
)
|
||||
positions.append(position)
|
||||
|
||||
portfolio_dict = vars(portfolio)
|
||||
portfolio_dict['positions'] = positions
|
||||
|
||||
return portfolio_dict
|
||||
|
||||
|
||||
def portfolio_from_dict(self, portfolio_data):
|
||||
from catalyst.protocol import Portfolio
|
||||
return Portfolio()
|
||||
@@ -1,486 +0,0 @@
|
||||
import copy
|
||||
import csv
|
||||
import json
|
||||
import numbers
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from catalyst.exchange.utils.exchange_utils import get_algo_folder
|
||||
from catalyst.utils.paths import data_root, ensure_directory
|
||||
from operator import itemgetter
|
||||
|
||||
s3_conn = []
|
||||
mailgun = []
|
||||
|
||||
|
||||
def trend_direction(series):
|
||||
if series[-1] is np.nan or series[-1] is np.nan:
|
||||
return None
|
||||
|
||||
if series[-1] > series[-2]:
|
||||
return 'up'
|
||||
else:
|
||||
return 'down'
|
||||
|
||||
|
||||
def crossover(source, target):
|
||||
"""
|
||||
The `x`-series is defined as having crossed over `y`-series if the value
|
||||
of `x` is greater than the value of `y` and the value of `x` was less than
|
||||
the value of `y` on the bar immediately preceding the current bar.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source: Series
|
||||
target: Series
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
|
||||
"""
|
||||
if isinstance(target, numbers.Number):
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] >= target > source[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
else:
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target[-1] is np.nan or target[-2] is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] > target[-1] and source[-2] < target[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def crossunder(source, target):
|
||||
"""
|
||||
The `x`-series is defined as having crossed under `y`-series if the value
|
||||
of `x` is less than the value of `y` and the value of `x` was greater than
|
||||
the value of `y` on the bar immediately preceding the current bar.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
source: Series
|
||||
target: Series
|
||||
|
||||
Returns
|
||||
-------
|
||||
bool
|
||||
|
||||
"""
|
||||
if isinstance(target, numbers.Number):
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] < target <= source[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
if source[-1] is np.nan or source[-2] is np.nan \
|
||||
or target[-1] is np.nan or target[-2] is np.nan:
|
||||
return False
|
||||
|
||||
if source[-1] < target[-1] and source[-2] >= target[-2]:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def vwap(df):
|
||||
"""
|
||||
Volume-weighted average price (VWAP) is a ratio generally used by
|
||||
institutional investors and mutual funds to make buys and sells so as not
|
||||
to disturb the market prices with large orders. It is the average share
|
||||
price of a stock weighted against its trading volume within a particular
|
||||
time frame, generally one day.
|
||||
|
||||
Read more: Volume Weighted Average Price - VWAP
|
||||
https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df: pd.DataFrame
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
if 'close' not in df.columns or 'volume' not in df.columns:
|
||||
raise ValueError('price data must include `volume` and `close`')
|
||||
|
||||
vol_sum = np.nansum(df['volume'].values)
|
||||
|
||||
try:
|
||||
ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
|
||||
except ZeroDivisionError:
|
||||
ret = np.nan
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
def set_position_row(row, asset, asset_values=list()):
|
||||
"""
|
||||
Apply the position data as individual columns.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
row: dict[str, Object]
|
||||
asset: TradingPair
|
||||
asset_values: list[str]
|
||||
If a recorded_col contains a tuple which first value is an asset
|
||||
matching a position, its value will be displayed with the
|
||||
position and not in the index.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
asset_cols = ['symbol']
|
||||
row['symbol'] = asset.symbol
|
||||
|
||||
position = next((p for p in row['positions'] if p['sid'] == asset), None)
|
||||
|
||||
columns = ['amount', 'cost_basis', 'last_sale_price']
|
||||
for column in columns:
|
||||
if position is not None:
|
||||
row[column] = position[column]
|
||||
|
||||
else:
|
||||
row[column] = 0
|
||||
|
||||
asset_cols.append(column)
|
||||
|
||||
values = asset_values[asset] if asset in asset_values else list()
|
||||
for column in values:
|
||||
row[column] = values[column]
|
||||
|
||||
asset_cols.append(column)
|
||||
|
||||
return asset_cols
|
||||
|
||||
|
||||
def prepare_stats(stats, recorded_cols=list()):
|
||||
"""
|
||||
Prepare the stats DataFrame for user-friendly output.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
stats: list[Object]
|
||||
recorded_cols: list[str]
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
asset_cols = list()
|
||||
|
||||
stats = copy.deepcopy(stats)
|
||||
# Using a copy since we are adding rows inside the loop.
|
||||
for row_index, row_data in enumerate(list(stats)):
|
||||
assets = [p['sid'] for p in row_data['positions']]
|
||||
|
||||
asset_values = dict()
|
||||
if recorded_cols is not None:
|
||||
for column in recorded_cols[:]:
|
||||
value = row_data[column]
|
||||
if isinstance(value, pd.Series):
|
||||
value = value.to_dict()
|
||||
|
||||
if type(value) is dict:
|
||||
for asset in value:
|
||||
if not isinstance(asset, TradingPair):
|
||||
break
|
||||
|
||||
if asset not in assets:
|
||||
assets.append(asset)
|
||||
|
||||
if asset not in asset_values:
|
||||
asset_values[asset] = dict()
|
||||
|
||||
asset_values[asset][column] = value[asset]
|
||||
|
||||
if len(assets) == 1:
|
||||
row = stats[row_index]
|
||||
asset_cols = set_position_row(row, assets[0], asset_values)
|
||||
|
||||
elif len(assets) > 1:
|
||||
for asset_index, asset in enumerate(assets):
|
||||
if asset_index > 0:
|
||||
row = copy.deepcopy(row_data)
|
||||
stats.append(row)
|
||||
|
||||
else:
|
||||
row = stats[row_index]
|
||||
|
||||
asset_cols = set_position_row(row, assets[asset_index],
|
||||
asset_values)
|
||||
|
||||
df = pd.DataFrame(stats)
|
||||
|
||||
index_cols = [
|
||||
'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
|
||||
'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
|
||||
]
|
||||
|
||||
# Removing the asset specific entries
|
||||
if recorded_cols is not None:
|
||||
recorded_cols = [x for x in recorded_cols if x not in asset_cols]
|
||||
for column in recorded_cols:
|
||||
index_cols.append(column)
|
||||
|
||||
df['orders'] = df['orders'].apply(lambda orders: len(orders))
|
||||
df['transactions'] = df['transactions'].apply(
|
||||
lambda transactions: len(transactions)
|
||||
)
|
||||
|
||||
if asset_cols:
|
||||
columns = asset_cols
|
||||
df.set_index(index_cols, drop=True, inplace=True)
|
||||
|
||||
else:
|
||||
columns = index_cols
|
||||
columns.remove('period_close')
|
||||
df.set_index('period_close', drop=False, inplace=True)
|
||||
|
||||
df.dropna(axis=1, how='all', inplace=True)
|
||||
df.sort_index(axis=0, level=0, inplace=True)
|
||||
|
||||
return df, columns
|
||||
|
||||
|
||||
def set_print_settings():
|
||||
pd.set_option('display.expand_frame_repr', False)
|
||||
pd.set_option('precision', 8)
|
||||
pd.set_option('display.width', 1000)
|
||||
pd.set_option('display.max_colwidth', 1000)
|
||||
|
||||
|
||||
def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
|
||||
"""
|
||||
Format and print the last few rows of a statistics DataFrame.
|
||||
See the pyfolio project for the data structure.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
stats: list[Object]
|
||||
An array of statistics for the period.
|
||||
|
||||
num_rows: int
|
||||
The number of rows to display on the screen.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
|
||||
"""
|
||||
if isinstance(stats, pd.DataFrame):
|
||||
stats = list(stats.T.to_dict().values())
|
||||
stats.sort(key=itemgetter('period_close'))
|
||||
|
||||
if len(stats) > num_rows:
|
||||
display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
|
||||
else:
|
||||
display_stats = stats
|
||||
|
||||
df, columns = prepare_stats(
|
||||
display_stats, recorded_cols=recorded_cols
|
||||
)
|
||||
set_print_settings()
|
||||
return df.to_string(columns=columns)
|
||||
|
||||
|
||||
def get_csv_stats(stats, recorded_cols=None):
|
||||
"""
|
||||
Create a CSV buffer from the stats DataFrame.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path: str
|
||||
stats: list[Object]
|
||||
recorded_cols: list[str]
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
|
||||
|
||||
return df.to_csv(
|
||||
None,
|
||||
columns=columns,
|
||||
# encoding='utf-8',
|
||||
quoting=csv.QUOTE_NONNUMERIC
|
||||
).encode()
|
||||
|
||||
|
||||
def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
|
||||
folder='catalyst/stats', bytes_to_write=None):
|
||||
"""
|
||||
Uploads the performance stats to a S3 bucket.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
uri: str
|
||||
stats: list[Object]
|
||||
algo_namespace: str
|
||||
recorded_cols: list[str]
|
||||
folder: str
|
||||
bytes_to_write: str
|
||||
Option to reuse bytes instead of re-computing the csv
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
if not s3_conn:
|
||||
import boto3
|
||||
s3_conn.append(boto3.resource('s3'))
|
||||
|
||||
s3 = s3_conn[0]
|
||||
|
||||
if bytes_to_write is None:
|
||||
bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
|
||||
|
||||
now = pd.Timestamp.utcnow()
|
||||
timestr = now.strftime('%Y%m%d')
|
||||
pid = os.getpid()
|
||||
|
||||
parts = uri.split('//')
|
||||
path = '{folder}/{algo}/{time}-{algo}-{pid}.csv'.format(
|
||||
folder=folder,
|
||||
algo=algo_namespace,
|
||||
time=timestr,
|
||||
pid=pid,
|
||||
)
|
||||
obj = s3.Object(parts[1], path)
|
||||
obj.put(Body=bytes_to_write)
|
||||
|
||||
|
||||
def email_error(algo_name, dt, e, environ=None):
|
||||
import requests
|
||||
import traceback
|
||||
|
||||
if not mailgun:
|
||||
root = data_root(environ)
|
||||
filename = os.path.join(root, 'mailgun.json')
|
||||
if not os.path.exists(filename):
|
||||
raise ValueError(
|
||||
'mailgun.json not found in the catalyst data folder'
|
||||
)
|
||||
|
||||
with open(filename) as data_file:
|
||||
mailgun.append(json.load(data_file))
|
||||
|
||||
mg = mailgun[0]
|
||||
|
||||
return requests.post(
|
||||
mg['url'],
|
||||
auth=("api", mg['api']),
|
||||
data={
|
||||
"from": mg['from'],
|
||||
"to": mg['to'],
|
||||
"subject": 'Error: {}'.format(algo_name),
|
||||
"text": '{}\n\n{}\n{}'.format(
|
||||
dt, e, traceback.format_exc()
|
||||
)})
|
||||
|
||||
|
||||
def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None):
|
||||
"""
|
||||
Saves the performance stats to the algo local folder.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
stats: list[Object]
|
||||
algo_namespace: str
|
||||
recorded_cols: list[str]
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
|
||||
"""
|
||||
bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
|
||||
|
||||
timestr = time.strftime('%Y%m%d')
|
||||
folder = get_algo_folder(algo_namespace)
|
||||
|
||||
stats_folder = os.path.join(folder, 'stats')
|
||||
ensure_directory(stats_folder)
|
||||
|
||||
filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
|
||||
|
||||
with open(filename, 'wb') as handle:
|
||||
handle.write(bytes_to_write)
|
||||
|
||||
return bytes_to_write
|
||||
|
||||
|
||||
def df_to_string(df):
|
||||
"""
|
||||
Create a formatted str representation of the DataFrame.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df: DataFrame
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
|
||||
"""
|
||||
pd.set_option('display.expand_frame_repr', False)
|
||||
pd.set_option('precision', 8)
|
||||
pd.set_option('display.width', 1000)
|
||||
pd.set_option('display.max_colwidth', 1000)
|
||||
|
||||
return df.to_string()
|
||||
|
||||
|
||||
def extract_orders(perf):
|
||||
order_list = perf.orders.values
|
||||
all_orders = [t for sublist in order_list for t in sublist]
|
||||
all_orders.sort(key=lambda o: o['dt'])
|
||||
|
||||
orders = pd.DataFrame(all_orders)
|
||||
if not orders.empty:
|
||||
orders.set_index('dt', inplace=True, drop=True)
|
||||
return orders
|
||||
|
||||
|
||||
def extract_transactions(perf):
|
||||
"""
|
||||
Compute indexes for buy and sell transactions
|
||||
|
||||
Parameters
|
||||
----------
|
||||
perf: DataFrame
|
||||
The algo performance DataFrame.
|
||||
|
||||
Returns
|
||||
-------
|
||||
DataFrame
|
||||
A DataFrame of transactions.
|
||||
|
||||
"""
|
||||
trans_list = perf.transactions.values
|
||||
all_trans = [t for sublist in trans_list for t in sublist]
|
||||
all_trans.sort(key=lambda t: t['dt'])
|
||||
|
||||
transactions = pd.DataFrame(all_trans)
|
||||
if not transactions.empty:
|
||||
transactions.set_index('dt', inplace=True, drop=True)
|
||||
return transactions
|
||||
@@ -1,82 +0,0 @@
|
||||
import os
|
||||
import random
|
||||
import tempfile
|
||||
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
|
||||
from catalyst.exchange.utils.factory import find_exchanges
|
||||
from catalyst.utils.paths import ensure_directory
|
||||
|
||||
|
||||
def handle_exchange_error(exchange, e):
|
||||
try:
|
||||
message = '{}: {}'.format(
|
||||
e.__class__, e.message.decode('ascii', 'ignore')
|
||||
)
|
||||
except Exception:
|
||||
message = 'unexpected error'
|
||||
|
||||
folder = get_exchange_folder(exchange.name)
|
||||
filename = os.path.join(folder, 'blacklist.txt')
|
||||
with open(filename, 'wt') as handle:
|
||||
handle.write(message)
|
||||
|
||||
|
||||
def select_random_exchanges(population=3, features=None,
|
||||
is_authenticated=False, base_currency=None):
|
||||
all_exchanges = find_exchanges(
|
||||
features=features,
|
||||
is_authenticated=is_authenticated,
|
||||
base_currency=base_currency,
|
||||
)
|
||||
|
||||
if population is not None:
|
||||
if len(all_exchanges) < population:
|
||||
population = len(all_exchanges)
|
||||
|
||||
exchanges = random.sample(all_exchanges, population)
|
||||
|
||||
else:
|
||||
exchanges = all_exchanges
|
||||
|
||||
return exchanges
|
||||
|
||||
|
||||
def select_random_assets(all_assets, population=3):
|
||||
assets = random.sample(all_assets, population)
|
||||
return assets
|
||||
|
||||
|
||||
def output_df(df, assets, name=None):
|
||||
"""
|
||||
Outputs a price DataFrame to a temp folder.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df: pd.DataFrame
|
||||
assets
|
||||
name
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
if isinstance(assets, TradingPair):
|
||||
asset_folder = '{}_{}'.format(assets.exchange, assets.symbol)
|
||||
else:
|
||||
asset_folder = ','.join(
|
||||
['{}_{}'.format(a.exchange, a.symbol) for a in assets]
|
||||
)
|
||||
|
||||
folder = os.path.join(
|
||||
tempfile.gettempdir(), 'catalyst', asset_folder
|
||||
)
|
||||
ensure_directory(folder)
|
||||
|
||||
if name is None:
|
||||
name = 'output'
|
||||
|
||||
path = os.path.join(folder, '{}.csv'.format(name))
|
||||
df.to_csv(path)
|
||||
|
||||
return path, folder
|
||||
@@ -0,0 +1,142 @@
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pandas as pd
|
||||
import six
|
||||
from catalyst.assets._assets import TradingPair, get_calendar
|
||||
from logbook import Logger
|
||||
from pandas.util.testing import assert_frame_equal
|
||||
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
|
||||
from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
|
||||
from catalyst.exchange.factory import get_exchanges
|
||||
from catalyst.utils.paths import ensure_directory
|
||||
|
||||
log = Logger('Validator', level=LOG_LEVEL)
|
||||
|
||||
|
||||
def output_df(df, assets, name=None):
|
||||
"""
|
||||
Outputs a price DataFrame to a temp folder.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
df: pd.DataFrame
|
||||
assets
|
||||
name
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
if isinstance(assets, TradingPair):
|
||||
exchange_folder = assets.exchange
|
||||
asset_folder = assets.symbol
|
||||
else:
|
||||
exchange_folder = ','.join([asset.exchange for asset in assets])
|
||||
asset_folder = ','.join([asset.symbol for asset in assets])
|
||||
|
||||
folder = os.path.join(
|
||||
tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
|
||||
)
|
||||
ensure_directory(folder)
|
||||
|
||||
if name is None:
|
||||
name = 'output'
|
||||
|
||||
path = os.path.join(folder, '{}.csv'.format(name))
|
||||
df.to_csv(path)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
class Validator(object):
|
||||
def __init__(self, data_portal):
|
||||
self.data_portal = data_portal
|
||||
|
||||
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
|
||||
sample_minutes):
|
||||
"""
|
||||
Creates DataFrames from the bundle and exchange for the specified
|
||||
data set.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
exchange: Exchange
|
||||
assets
|
||||
end_dt
|
||||
bar_count
|
||||
sample_minutes
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
"""
|
||||
freq = '{}T'.format(sample_minutes)
|
||||
|
||||
log.info('creating data sample from bundle')
|
||||
df1 = self.data_portal.get_history_window(
|
||||
assets=assets,
|
||||
end_dt=end_dt,
|
||||
bar_count=bar_count,
|
||||
frequency=freq,
|
||||
field='close',
|
||||
data_frequency='minute'
|
||||
)
|
||||
path = output_df(df1, assets, '{}_resampled'.format(freq))
|
||||
log.info('saved resampled bundle candles: {}\n{}'.format(
|
||||
path, df1.tail(10))
|
||||
)
|
||||
|
||||
log.info('creating data sample from exchange api')
|
||||
candles = exchange.get_candles(
|
||||
end_dt=end_dt,
|
||||
freq='{}T'.format(sample_minutes),
|
||||
assets=assets,
|
||||
bar_count=bar_count
|
||||
)
|
||||
|
||||
series = dict()
|
||||
for asset in assets:
|
||||
series[asset] = pd.Series(
|
||||
data=[candle['close'] for candle in candles[asset]],
|
||||
index=[candle['last_traded'] for candle in candles[asset]]
|
||||
)
|
||||
|
||||
df2 = pd.DataFrame(series)
|
||||
path = output_df(df2, assets, '{}_api'.format(freq))
|
||||
log.info('saved exchange api candles: {}\n{}'.format(
|
||||
path, df2.tail(10))
|
||||
)
|
||||
|
||||
try:
|
||||
assert_frame_equal(df1, df2)
|
||||
return True
|
||||
except:
|
||||
log.warn('differences found in dataframes')
|
||||
return False
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
exchanges = get_exchanges(['poloniex'])
|
||||
exchange = six.next(six.itervalues(exchanges))
|
||||
assets = exchange.get_assets(symbols=['eth_btc'])
|
||||
|
||||
open_calendar = get_calendar('OPEN')
|
||||
asset_finder = AssetFinderExchange()
|
||||
data_portal = DataPortalExchangeBacktest(
|
||||
exchanges=exchanges,
|
||||
asset_finder=asset_finder,
|
||||
trading_calendar=open_calendar,
|
||||
first_trading_day=None # will set dynamically based on assets
|
||||
)
|
||||
validator = Validator(data_portal=data_portal)
|
||||
|
||||
validator.compare_bundle_with_exchange(
|
||||
exchange=exchange,
|
||||
assets=assets,
|
||||
end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
|
||||
bar_count=200,
|
||||
sample_minutes=30
|
||||
)
|
||||
@@ -15,9 +15,14 @@
|
||||
|
||||
import abc
|
||||
|
||||
from numpy import isfinite
|
||||
from sys import float_info
|
||||
|
||||
from six import with_metaclass
|
||||
|
||||
import catalyst.utils.math_utils as zp_math
|
||||
|
||||
from numpy import isfinite
|
||||
|
||||
from catalyst.errors import BadOrderParameters
|
||||
|
||||
|
||||
|
||||
@@ -27,15 +27,15 @@ from .risk import (
|
||||
choose_treasury
|
||||
)
|
||||
|
||||
from catalyst.patches.stats import (
|
||||
from empyrical import (
|
||||
alpha_beta_aligned,
|
||||
annual_volatility,
|
||||
cum_returns,
|
||||
downside_risk,
|
||||
information_ratio,
|
||||
max_drawdown,
|
||||
sharpe_ratio,
|
||||
sortino_ratio,
|
||||
cum_returns,
|
||||
)
|
||||
import warnings
|
||||
from catalyst.constants import LOG_LEVEL
|
||||
@@ -161,13 +161,9 @@ class RiskMetricsCumulative(object):
|
||||
if len(self.algorithm_returns) == 1:
|
||||
self.algorithm_returns = np.append(0.0, self.algorithm_returns)
|
||||
|
||||
try:
|
||||
self.algorithm_cumulative_returns[dt_loc] = cum_returns(
|
||||
self.algorithm_returns
|
||||
)[-1]
|
||||
except Exception as e:
|
||||
log.debug('unable to calculate cum returns: {}'.format(e))
|
||||
self.algorithm_cumulative_returns[dt_loc] = np.nan
|
||||
self.algorithm_cumulative_returns[dt_loc] = cum_returns(
|
||||
self.algorithm_returns
|
||||
)[-1]
|
||||
|
||||
algo_cumulative_returns_to_date = \
|
||||
self.algorithm_cumulative_returns[:dt_loc + 1]
|
||||
@@ -200,11 +196,8 @@ class RiskMetricsCumulative(object):
|
||||
self.benchmark_cumulative_returns[dt_loc] = cum_returns(
|
||||
self.benchmark_returns
|
||||
)[-1]
|
||||
except Exception as e:
|
||||
log.debug(
|
||||
'unable to calculate benchmark cum returns: {}'.format(e)
|
||||
)
|
||||
self.benchmark_cumulative_returns[dt_loc] = np.nan
|
||||
except Exception:
|
||||
self.benchmark_cumulative_returns[dt_loc] = 0
|
||||
|
||||
benchmark_cumulative_returns_to_date = \
|
||||
self.benchmark_cumulative_returns[:dt_loc + 1]
|
||||
@@ -276,16 +269,9 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
|
||||
self.sharpe[dt_loc] = sharpe_ratio(
|
||||
self.algorithm_returns,
|
||||
)
|
||||
|
||||
try:
|
||||
self.downside_risk[dt_loc] = downside_risk(
|
||||
self.algorithm_returns
|
||||
)
|
||||
except Exception as e:
|
||||
log.debug(
|
||||
'unable to calculate downside risk returns: {}'.format(e)
|
||||
)
|
||||
self.downside_risk[dt_loc] = np.nan
|
||||
self.downside_risk[dt_loc] = downside_risk(
|
||||
self.algorithm_returns
|
||||
)
|
||||
|
||||
try:
|
||||
risk = self.downside_risk[dt_loc]
|
||||
@@ -293,26 +279,17 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
|
||||
self.algorithm_returns,
|
||||
_downside_risk=risk
|
||||
)
|
||||
except Exception as e:
|
||||
log.debug(
|
||||
'unable to calculate benchmark cum returns: {}'.format(e)
|
||||
)
|
||||
self.sortino[dt_loc] = np.nan
|
||||
except Exception:
|
||||
# TODO: what causes it to error out?
|
||||
self.sortino[dt_loc] = 0
|
||||
|
||||
self.information[dt_loc] = information_ratio(
|
||||
self.algorithm_returns,
|
||||
self.benchmark_returns,
|
||||
)
|
||||
try:
|
||||
self.max_drawdown = max_drawdown(
|
||||
self.algorithm_returns
|
||||
)
|
||||
except Exception as e:
|
||||
log.debug(
|
||||
'unable to calculate max drawdown: {}'.format(e)
|
||||
)
|
||||
self.max_drawdown = np.nan
|
||||
|
||||
self.max_drawdown = max_drawdown(
|
||||
self.algorithm_returns
|
||||
)
|
||||
self.max_drawdowns[dt_loc] = self.max_drawdown
|
||||
self.max_leverage = self.calculate_max_leverage()
|
||||
self.max_leverages[dt_loc] = self.max_leverage
|
||||
|
||||
@@ -154,8 +154,8 @@ class RiskMetricsPeriod(object):
|
||||
self.algorithm_returns.values,
|
||||
self.benchmark_returns.values,
|
||||
)
|
||||
self.excess_return = self.algorithm_period_returns \
|
||||
- self.treasury_period_return
|
||||
self.excess_return = self.algorithm_period_returns - \
|
||||
self.treasury_period_return
|
||||
self.max_drawdown = max_drawdown(self.algorithm_returns.values)
|
||||
self.max_leverage = self.calculate_max_leverage()
|
||||
|
||||
|
||||
@@ -160,8 +160,7 @@ def choose_treasury(select_treasury, treasury_curves, start_session,
|
||||
)
|
||||
break
|
||||
|
||||
# Supress warning for 'OPEN' calendar
|
||||
if search_day and trading_calendar.name != 'OPEN':
|
||||
if search_day and trading_calendar.name != 'OPEN': # Supress warning for 'OPEN' calendar
|
||||
if (search_dist is None or search_dist > 1) and \
|
||||
search_days[0] <= end_session <= search_days[-1]:
|
||||
message = "No rate within 1 trading day of end date = \
|
||||
|
||||
@@ -41,6 +41,7 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025
|
||||
DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05
|
||||
|
||||
|
||||
|
||||
class LiquidityExceeded(Exception):
|
||||
pass
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,7 +7,6 @@ from abc import (
|
||||
)
|
||||
from uuid import uuid4
|
||||
|
||||
import six
|
||||
from six import (
|
||||
iteritems,
|
||||
with_metaclass,
|
||||
@@ -34,6 +33,7 @@ from catalyst.utils.sharedoc import copydoc
|
||||
|
||||
|
||||
class PipelineEngine(with_metaclass(ABCMeta)):
|
||||
|
||||
@abstractmethod
|
||||
def run_pipeline(self, pipeline, start_date, end_date):
|
||||
"""
|
||||
@@ -118,7 +118,6 @@ class ExplodingPipelineEngine(PipelineEngine):
|
||||
"""
|
||||
A PipelineEngine that doesn't do anything.
|
||||
"""
|
||||
|
||||
def run_pipeline(self, pipeline, start_date, end_date):
|
||||
raise NoEngineRegistered(
|
||||
"Attempted to run a pipeline but no pipeline "
|
||||
@@ -485,10 +484,8 @@ class SimplePipelineEngine(PipelineEngine):
|
||||
)
|
||||
|
||||
if isinstance(term, LoadableTerm):
|
||||
term_key = loader_group_key(term)
|
||||
# TODO: temp workaround
|
||||
to_load = sorted(
|
||||
six.next(six.itervalues(loader_groups)),
|
||||
loader_groups[loader_group_key(term)],
|
||||
key=lambda t: t.dataset
|
||||
)
|
||||
loader = get_loader(term)
|
||||
@@ -568,10 +565,9 @@ class SimplePipelineEngine(PipelineEngine):
|
||||
index=MultiIndex.from_arrays([empty_dates, empty_assets]),
|
||||
)
|
||||
|
||||
# TODO: not sure what's wrong with the resolved_assets
|
||||
# resolved_assets = array(self._finder.retrieve_all(assets))
|
||||
resolved_assets = array(self._finder.retrieve_all(assets))
|
||||
dates_kept = repeat_last_axis(dates.values, len(assets))[mask]
|
||||
assets_kept = repeat_first_axis(assets, len(dates))[mask]
|
||||
assets_kept = repeat_first_axis(resolved_assets, len(dates))[mask]
|
||||
|
||||
final_columns = {}
|
||||
for name in data:
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from .statistical import (
|
||||
RollingPearson,
|
||||
RollingLinearRegression,
|
||||
RollingLinearRegressionOfReturns,
|
||||
RollingPearsonOfReturns,
|
||||
RollingSpearman,
|
||||
RollingSpearmanOfReturns,
|
||||
)
|
||||
from .technical import (
|
||||
|
||||
@@ -142,7 +142,7 @@ class TermGraph(object):
|
||||
at the end of execution.
|
||||
"""
|
||||
refcounts = self.graph.out_degree()
|
||||
for t in list(self.outputs.values()):
|
||||
for t in self.outputs.values():
|
||||
refcounts[t] += 1
|
||||
|
||||
for t in initial_terms:
|
||||
@@ -238,7 +238,7 @@ class ExecutionPlan(TermGraph):
|
||||
min_extra_rows=0):
|
||||
super(ExecutionPlan, self).__init__(terms)
|
||||
|
||||
for term in list(terms.values()):
|
||||
for term in terms.values():
|
||||
self.set_extra_rows(
|
||||
term,
|
||||
all_dates,
|
||||
|
||||
@@ -38,11 +38,9 @@ class USEquityPricingLoader(PipelineLoader):
|
||||
|
||||
def __init__(self, bundle, data_frequency, dataset):
|
||||
|
||||
# TODO: This is currently broken, No Pipeline support for Catalyst
|
||||
# if data_frequency == 'daily':
|
||||
# reader = bundle.daily_bar_reader
|
||||
# elif daily_bar_reader == 'minute':
|
||||
if data_frequency == 'minute':
|
||||
if data_frequency == 'daily':
|
||||
reader = bundle.daily_bar_reader
|
||||
elif daily_bar_reader == 'minute':
|
||||
reader = bundle.minute_bar_reader
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -53,9 +51,7 @@ class USEquityPricingLoader(PipelineLoader):
|
||||
|
||||
if data_frequency == 'daily':
|
||||
all_sessions = cal.all_sessions
|
||||
# TODO: this cannot be right, but no pipeline support at the moment
|
||||
# elif daily_bar_reader == 'minute':
|
||||
elif data_frequency == 'minute':
|
||||
elif daily_bar_reader == 'minute':
|
||||
reader = bundle.minute_bar_reader
|
||||
all_sessions = cal.all_minutes
|
||||
|
||||
|
||||
@@ -231,7 +231,7 @@ class EventsLoader(PipelineLoader):
|
||||
self.load_next_events(n, dates, sids, mask),
|
||||
self.load_previous_events(p, dates, sids, mask),
|
||||
)
|
||||
|
||||
|
||||
@property
|
||||
def columns(self):
|
||||
return self._columns
|
||||
|
||||
@@ -180,3 +180,4 @@ class DataFrameLoader(PipelineLoader):
|
||||
@property
|
||||
def columns(self):
|
||||
return self._columns
|
||||
|
||||
|
||||
@@ -163,7 +163,7 @@ class SeededRandomLoader(PrecomputedLoader):
|
||||
bool_dtype: self._bool_values,
|
||||
object_dtype: self._object_values,
|
||||
}[dtype](shape)
|
||||
|
||||
|
||||
@property
|
||||
def columns(self):
|
||||
return self._columns
|
||||
|
||||
@@ -144,7 +144,7 @@ class SpecificEquityTrades(object):
|
||||
for identifier in self.identifiers:
|
||||
assets_by_identifier[identifier] = env.asset_finder.\
|
||||
lookup_generic(identifier, datetime.now())[0]
|
||||
self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
|
||||
self.sids = [asset.sid for asset in assets_by_identifier.values()]
|
||||
for event in self.event_list:
|
||||
event.sid = assets_by_identifier[event.sid].sid
|
||||
|
||||
@@ -167,7 +167,7 @@ class SpecificEquityTrades(object):
|
||||
for identifier in self.identifiers:
|
||||
assets_by_identifier[identifier] = env.asset_finder.\
|
||||
lookup_generic(identifier, datetime.now())[0]
|
||||
self.sids = [asset.sid for asset in list(assets_by_identifier.values())]
|
||||
self.sids = [asset.sid for asset in assets_by_identifier.values()]
|
||||
|
||||
# Hash_value for downstream sorting.
|
||||
self.arg_string = hash_args(*args, **kwargs)
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
import pandas as pd
|
||||
from catalyst import run_algorithm
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.i = -1 # counts the minutes
|
||||
context.exchange = 'cryptopia'
|
||||
context.base_currency = 'btc'
|
||||
context.coins = context.exchanges[context.exchange].assets
|
||||
context.coins = [c for c in context.coins if
|
||||
c.quote_currency == context.base_currency]
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
# current date formatted into a string
|
||||
today = data.current_dt
|
||||
|
||||
# update universe everyday
|
||||
new_day = 60 * 24 # assuming data_frequency='minute'
|
||||
if not context.i % new_day:
|
||||
context.coins = context.exchanges[context.exchange].assets
|
||||
context.coins = [c for c in context.coins if
|
||||
c.quote_currency == context.base_currency]
|
||||
|
||||
# get data every 30 minutes
|
||||
minutes = 1
|
||||
if not context.i % minutes:
|
||||
# we iterate for every pair in the current universe
|
||||
for coin in context.coins:
|
||||
pair = str(coin.symbol)
|
||||
|
||||
price = data.current(coin, 'price')
|
||||
print(today, pair, price)
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_date = pd.to_datetime('2018-01-17', utc=True)
|
||||
end_date = pd.to_datetime('2018-01-18', utc=True)
|
||||
|
||||
performance = run_algorithm(
|
||||
capital_base=1.0,
|
||||
# amount of base_currency, not always in dollars unless usd
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='cryptopia',
|
||||
data_frequency='minute',
|
||||
base_currency='btc',
|
||||
live=True,
|
||||
live_graph=False,
|
||||
simulate_orders=True,
|
||||
algo_namespace='simple_universe'
|
||||
)
|
||||
@@ -1,8 +0,0 @@
|
||||
import ccxt
|
||||
|
||||
bitfinex = ccxt.bitfinex()
|
||||
bitfinex.verbose = True
|
||||
ohlcvs = bitfinex.fetch_ohlcv('ETH/BTC', '30m', 1504224000000)
|
||||
|
||||
dt = bitfinex.iso8601(ohlcvs[0][0])
|
||||
print(dt) # should print '2017-09-01T00:00:00.000Z'
|
||||
@@ -1,50 +0,0 @@
|
||||
import pandas as pd
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import symbol
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.asset1 = symbol('fct_btc')
|
||||
context.asset2 = symbol('btc_usdt')
|
||||
context.coins = [context.asset1, context.asset2]
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
df = data.history(context.coins,
|
||||
'close',
|
||||
bar_count=10,
|
||||
frequency='5T',
|
||||
)
|
||||
print(df)
|
||||
print(data.current(context.asset1, 'close'))
|
||||
print(data.current(context.asset2, 'close'))
|
||||
exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
LIVE = True
|
||||
if LIVE:
|
||||
run_algorithm(
|
||||
capital_base=1,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='test_multi_assets',
|
||||
base_currency='usdt',
|
||||
live=True,
|
||||
simulate_orders=True,
|
||||
)
|
||||
else:
|
||||
run_algorithm(
|
||||
capital_base=1,
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='test_multi_assets',
|
||||
base_currency='usdt',
|
||||
live=False,
|
||||
start=pd.to_datetime('2017-12-1', utc=True),
|
||||
end=pd.to_datetime('2017-12-1', utc=True),
|
||||
)
|
||||
@@ -1,44 +0,0 @@
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import order_target_percent
|
||||
|
||||
NAMESPACE = 'goose7'
|
||||
log = Logger(NAMESPACE)
|
||||
|
||||
from catalyst.api import record, symbol
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.asset = symbol('trx_btc')
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
price = data.current(context.asset, 'price')
|
||||
record(btc=price)
|
||||
|
||||
# Only ordering if it does not have any position to avoid trying some
|
||||
# tiny orders with the leftover btc
|
||||
pos_amount = context.portfolio.positions[context.asset].amount
|
||||
if pos_amount > 0:
|
||||
return
|
||||
|
||||
# Adding a limit price to workaround an issue with performance
|
||||
# calculations of market orders
|
||||
order_target_percent(
|
||||
context.asset, 1, limit_price=price * 1.01
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_algorithm(
|
||||
capital_base=0.003,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='binance',
|
||||
live=True,
|
||||
algo_namespace=NAMESPACE,
|
||||
base_currency='btc',
|
||||
live_graph=False,
|
||||
simulate_orders=False,
|
||||
)
|
||||
@@ -1,44 +0,0 @@
|
||||
import pandas as pd
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import symbol
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.asset = symbol('btc_usdt')
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
df = data.history(context.asset,
|
||||
'close',
|
||||
bar_count=10,
|
||||
frequency='5T',
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
LIVE = True
|
||||
if LIVE:
|
||||
run_algorithm(
|
||||
capital_base=1,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='test_algo',
|
||||
base_currency='usdt',
|
||||
live=True,
|
||||
simulate_orders=True,
|
||||
)
|
||||
else:
|
||||
run_algorithm(
|
||||
capital_base=1,
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='test_algo',
|
||||
base_currency='usdt',
|
||||
live=False,
|
||||
start=pd.to_datetime('2017-12-1', utc=True),
|
||||
end=pd.to_datetime('2017-12-1', utc=True),
|
||||
)
|
||||
@@ -1,52 +0,0 @@
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import order, record, symbol
|
||||
import pandas as pd
|
||||
|
||||
from catalyst.exchange.utils.stats_utils import get_pretty_stats
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.assets = [symbol('eth_btc'), symbol('eth_usdt')]
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
order(context.assets[0], 1)
|
||||
|
||||
prices = data.current(context.assets, 'price')
|
||||
record(price=prices)
|
||||
pass
|
||||
|
||||
|
||||
def analyze(context, perf):
|
||||
stats = get_pretty_stats(perf)
|
||||
print(stats)
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
live = True
|
||||
if live:
|
||||
run_algorithm(
|
||||
capital_base=0.01,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='buy_btc_polo_jh',
|
||||
base_currency='btc',
|
||||
analyze=analyze,
|
||||
live=True,
|
||||
simulate_orders=True,
|
||||
)
|
||||
else:
|
||||
run_algorithm(
|
||||
capital_base=1000,
|
||||
data_frequency='daily',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
exchange_name='poloniex',
|
||||
algo_namespace='buy_btc_polo_jh',
|
||||
base_currency='usd',
|
||||
analyze=analyze,
|
||||
start=pd.to_datetime('2017-01-01', utc=True),
|
||||
end=pd.to_datetime('2017-12-25', utc=True),
|
||||
)
|
||||
@@ -1,44 +0,0 @@
|
||||
import pandas as pd
|
||||
from catalyst.utils.run_algo import run_algorithm
|
||||
from catalyst.api import symbol
|
||||
from exchange.utils.stats_utils import set_print_settings
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.i = 0
|
||||
context.data = []
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
prices = data.history(
|
||||
symbol('xlm_eth'),
|
||||
fields=['open', 'high', 'low', 'close'],
|
||||
bar_count=50,
|
||||
frequency='1T'
|
||||
)
|
||||
set_print_settings()
|
||||
print(prices.tail(10))
|
||||
context.data.append(prices)
|
||||
|
||||
context.i = context.i + 1
|
||||
if context.i == 3:
|
||||
context.interrupt_algorithm()
|
||||
|
||||
|
||||
def analyze(context, prefs):
|
||||
for dataset in context.data:
|
||||
print(dataset[-2:])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_algorithm(
|
||||
capital_base=0.1,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='binance',
|
||||
algo_namespace='Test candles',
|
||||
base_currency='eth',
|
||||
data_frequency='minute',
|
||||
live=True,
|
||||
simulate_orders=True)
|
||||
@@ -1,28 +0,0 @@
|
||||
from catalyst.api import symbol
|
||||
from catalyst.utils.run_algo import run_algorithm
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.asset = symbol('bcc_usdt')
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
data.history(context.asset, ['close'], bar_count=100, frequency='5T')
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_algorithm(
|
||||
capital_base=100,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='bittrex',
|
||||
algo_namespace="bittrex_is_broken",
|
||||
base_currency='usdt',
|
||||
data_frequency='minute',
|
||||
simulate_orders=True,
|
||||
live=True)
|
||||
@@ -0,0 +1,109 @@
|
||||
import pandas as pd
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols
|
||||
|
||||
from catalyst.api import (
|
||||
symbols,
|
||||
)
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.i = -1
|
||||
context.base_currency = 'btc'
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
lookback = 60 * 24 * 7 # (minutes, hours, days)
|
||||
context.i += 1
|
||||
if context.i < lookback:
|
||||
return
|
||||
|
||||
today = context.blotter.current_dt.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
try:
|
||||
# update universe everyday
|
||||
new_day = 60 * 24
|
||||
if not context.i % new_day:
|
||||
context.universe = universe(context, today)
|
||||
|
||||
# get data every 30 minutes
|
||||
minutes = 30
|
||||
if not context.i % minutes and context.universe:
|
||||
for coin in context.coins:
|
||||
pair = str(coin.symbol)
|
||||
|
||||
# ohlcv data
|
||||
open = data.history(coin, 'open', lookback,
|
||||
'1m').ffill().bfill().resample(
|
||||
'30T').first()
|
||||
high = data.history(coin, 'high', lookback,
|
||||
'1m').ffill().bfill().resample('30T').max()
|
||||
low = data.history(coin, 'low', lookback,
|
||||
'1m').ffill().bfill().resample('30T').min()
|
||||
close = data.history(coin, 'price', lookback,
|
||||
'1m').ffill().bfill().resample(
|
||||
'30T').last()
|
||||
volume = data.history(coin, 'volume', lookback,
|
||||
'1m').ffill().bfill().resample(
|
||||
'30T').sum()
|
||||
|
||||
print(today, pair, close[-1])
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
pass
|
||||
|
||||
|
||||
def universe(context, today):
|
||||
json_symbols = get_exchange_symbols('poloniex')
|
||||
poloniex_universe_df = pd.DataFrame.from_dict(
|
||||
json_symbols).transpose().astype(str)
|
||||
poloniex_universe_df['base_currency'] = poloniex_universe_df.apply(
|
||||
lambda row: row.symbol.split('_')[1],
|
||||
axis=1)
|
||||
poloniex_universe_df['market_currency'] = poloniex_universe_df.apply(
|
||||
lambda row: row.symbol.split('_')[0],
|
||||
axis=1)
|
||||
poloniex_universe_df = poloniex_universe_df[
|
||||
poloniex_universe_df['base_currency'] == context.base_currency]
|
||||
poloniex_universe_df = poloniex_universe_df[
|
||||
poloniex_universe_df.symbol != 'gas_btc']
|
||||
|
||||
# Markets currently not working on Catalyst 0.3.1
|
||||
# 2017-01-01
|
||||
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'bcn_btc']
|
||||
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'burst_btc']
|
||||
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'dgb_btc']
|
||||
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'doge_btc']
|
||||
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'emc2_btc']
|
||||
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'pink_btc']
|
||||
# poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'sc_btc']
|
||||
print(poloniex_universe_df.head())
|
||||
|
||||
date = str(today).split(' ')[0]
|
||||
|
||||
poloniex_universe_df = poloniex_universe_df[
|
||||
poloniex_universe_df.start_date < date]
|
||||
context.coins = symbols(*poloniex_universe_df.symbol)
|
||||
print(len(poloniex_universe_df))
|
||||
return poloniex_universe_df.symbol.tolist()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_date = pd.to_datetime('2017-01-01', utc=True)
|
||||
end_date = pd.to_datetime('2017-10-15', utc=True)
|
||||
|
||||
performance = run_algorithm(start=start_date, end=end_date,
|
||||
capital_base=10000.0,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
data_frequency='minute',
|
||||
base_currency='btc',
|
||||
live=False,
|
||||
live_graph=False,
|
||||
algo_namespace='test')
|
||||
@@ -0,0 +1,139 @@
|
||||
"""
|
||||
Requires Catalyst version 0.3.0 or above
|
||||
Tested on Catalyst version 0.3.3
|
||||
|
||||
These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
|
||||
You simply need to specify the exchange and the market that you want to focus on.
|
||||
You will all see how to create a universe and filter it base on the exchange and the market you desire.
|
||||
|
||||
The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
|
||||
The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
|
||||
Use this as the backbone to create your own trading strategies.
|
||||
|
||||
Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import timedelta
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.exchange.exchange_utils import get_exchange_symbols
|
||||
|
||||
from catalyst.api import (
|
||||
symbols,
|
||||
)
|
||||
|
||||
|
||||
def initialize(context):
|
||||
context.i = -1 # counts the minutes
|
||||
context.exchange = 'poloniex' # must match the exchange specified in run_algorithm
|
||||
context.base_currency = 'btc' # must match the base currency specified in run_algorithm
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
lookback = 60 * 24 * 7 # (minutes, hours, days) of how far to lookback in the data history
|
||||
context.i += 1
|
||||
|
||||
# current date formatted into a string
|
||||
today = context.blotter.current_dt
|
||||
date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
|
||||
lookback_date = today - timedelta(days=(
|
||||
lookback / (60 * 24))) # subtract the amount of days specified in lookback
|
||||
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[
|
||||
0] # get only the date as a string
|
||||
|
||||
# update universe everyday
|
||||
new_day = 60 * 24
|
||||
if not context.i % new_day:
|
||||
context.universe = universe(context, lookback_date, date)
|
||||
|
||||
# get data every 30 minutes
|
||||
minutes = 30
|
||||
if not context.i % minutes and context.universe:
|
||||
# we iterate for every pair in the current universe
|
||||
for coin in context.coins:
|
||||
pair = str(coin.symbol)
|
||||
|
||||
# 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
|
||||
# 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
|
||||
opened = fill(data.history(coin, 'open', bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
high = fill(data.history(coin, 'high', bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
low = fill(data.history(coin, 'low', bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
close = fill(data.history(coin, 'price', bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
volume = fill(data.history(coin, 'volume', bar_count=lookback,
|
||||
frequency='30T')).values
|
||||
|
||||
# close[-1] is the equivalent to current price
|
||||
# displays the minute price for each pair every 30 minutes
|
||||
print(
|
||||
today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------
|
||||
# -------------------------------------- Insert Your Strategy Here -----------------------------------------
|
||||
# ----------------------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
pass
|
||||
|
||||
|
||||
# Get the universe for a given exchange and a given base_currency market
|
||||
# Example: Poloniex btc Market
|
||||
def universe(context, lookback_date, current_date):
|
||||
json_symbols = get_exchange_symbols(
|
||||
context.exchange) # get all the pairs for the exchange
|
||||
universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(
|
||||
str) # convert into a dataframe
|
||||
universe_df['base_currency'] = universe_df.apply(
|
||||
lambda row: row.symbol.split('_')[1],
|
||||
axis=1)
|
||||
universe_df['market_currency'] = universe_df.apply(
|
||||
lambda row: row.symbol.split('_')[0],
|
||||
axis=1)
|
||||
# Filter all the exchange pairs to only the ones for a give base currency
|
||||
universe_df = universe_df[
|
||||
universe_df['base_currency'] == context.base_currency]
|
||||
|
||||
# Filter all the pairs to ensure that pair existed in the current date range
|
||||
universe_df = universe_df[universe_df.start_date < lookback_date]
|
||||
universe_df = universe_df[universe_df.end_daily >= current_date]
|
||||
context.coins = symbols(
|
||||
*universe_df.symbol) # convert all the pairs to symbols
|
||||
return universe_df.symbol.tolist()
|
||||
|
||||
|
||||
# Replace all NA, NAN or infinite values with its nearest value
|
||||
def fill(series):
|
||||
if isinstance(series, pd.Series):
|
||||
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
|
||||
elif isinstance(series, np.ndarray):
|
||||
return pd.Series(series).replace([np.inf, -np.inf],
|
||||
np.nan).ffill().bfill().values
|
||||
else:
|
||||
return series
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_date = pd.to_datetime('2017-01-08', utc=True)
|
||||
end_date = pd.to_datetime('2017-11-13', utc=True)
|
||||
|
||||
performance = run_algorithm(start=start_date, end=end_date,
|
||||
capital_base=10000.0,
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
exchange_name='poloniex',
|
||||
data_frequency='minute',
|
||||
base_currency='btc',
|
||||
live=False,
|
||||
live_graph=False,
|
||||
algo_namespace='simple_universe')
|
||||
|
||||
"""
|
||||
Run in Terminal (inside catalyst environment):
|
||||
python simple_universe.py
|
||||
"""
|
||||
@@ -1,3 +1,4 @@
|
||||
import talib
|
||||
import pandas as pd
|
||||
|
||||
from catalyst import run_algorithm
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import talib
|
||||
import pandas as pd
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import symbol
|
||||
|
||||
|
||||
def initialize(context):
|
||||
print('initializing')
|
||||
context.asset = symbol('btc_usdt')
|
||||
|
||||
|
||||
def handle_data(context, data):
|
||||
print('handling bar: {}'.format(data.current_dt))
|
||||
|
||||
price = data.current(context.asset, 'close')
|
||||
print('got price {price}'.format(price=price))
|
||||
|
||||
try:
|
||||
prices = data.history(
|
||||
context.asset,
|
||||
fields='close',
|
||||
bar_count=60,
|
||||
frequency='1D'
|
||||
)
|
||||
print('got {} price entries\n'.format(len(prices), prices))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
|
||||
run_algorithm(
|
||||
capital_base=1,
|
||||
start=pd.to_datetime('2016-2-11', utc=True),
|
||||
end=pd.to_datetime('2017-8-31', utc=True),
|
||||
data_frequency='daily',
|
||||
initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=None,
|
||||
exchange_name='bittrex',
|
||||
algo_namespace='issue_57',
|
||||
base_currency='btc'
|
||||
<<<<<<< HEAD
|
||||
)
|
||||
=======
|
||||
)
|
||||
>>>>>>> develop
|
||||
@@ -1,24 +1,13 @@
|
||||
'''Use this code to execute a portfolio optimization model. This code
|
||||
will select the portfolio with the maximum Sharpe Ratio. The parameters
|
||||
are set to use 180 days of historical data and rebalance every 30 days.
|
||||
|
||||
This is the code used in the following article:
|
||||
https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556
|
||||
|
||||
You can run this code using the Python interpreter:
|
||||
|
||||
$ python portfolio_optimization.py
|
||||
'''
|
||||
|
||||
from __future__ import division
|
||||
import os
|
||||
import pytz
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from scipy.optimize import minimize
|
||||
import matplotlib.pyplot as plt
|
||||
from datetime import datetime
|
||||
|
||||
from catalyst.api import record, symbols, order_target_percent
|
||||
from catalyst.api import record, symbol, symbols, order_target_percent
|
||||
from catalyst.utils.run_algo import run_algorithm
|
||||
|
||||
np.set_printoptions(threshold='nan', suppress=True)
|
||||
@@ -43,7 +32,7 @@ def handle_data(context, data):
|
||||
if context.i == 0 or context.i % context.rebalance_period == 0:
|
||||
n = context.window
|
||||
prices = data.history(context.assets, fields='price',
|
||||
bar_count=n + 1, frequency='1d')
|
||||
bar_count=n + 1, frequency='daily')
|
||||
pr = np.asmatrix(prices)
|
||||
t_prices = prices.iloc[1:n + 1]
|
||||
t_val = t_prices.values
|
||||
@@ -71,8 +60,8 @@ def handle_data(context, data):
|
||||
weights /= np.sum(weights)
|
||||
w = np.asmatrix(weights)
|
||||
p_r = np.sum(np.dot(w, np.transpose(m))) * 365
|
||||
p_std = np.sqrt(np.dot(np.dot(w, cov_m),
|
||||
np.transpose(w))) * np.sqrt(365)
|
||||
p_std = np.sqrt(
|
||||
np.dot(np.dot(w, cov_m), np.transpose(w))) * np.sqrt(365)
|
||||
|
||||
# store results in results array
|
||||
results_array[0, p] = p_r
|
||||
@@ -87,12 +76,12 @@ def handle_data(context, data):
|
||||
|
||||
# convert results array to Pandas DataFrame
|
||||
results_frame = pd.DataFrame(np.transpose(results_array),
|
||||
columns=['r', 'stdev', 'sharpe']
|
||||
+ context.assets)
|
||||
columns=['r', 'stdev',
|
||||
'sharpe'] + context.assets)
|
||||
# locate position of portfolio with highest Sharpe Ratio
|
||||
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
|
||||
# locate positon of portfolio with minimum standard deviation
|
||||
# min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
|
||||
min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
|
||||
|
||||
# order optimal weights for each asset
|
||||
for asset in context.assets:
|
||||
@@ -100,28 +89,18 @@ def handle_data(context, data):
|
||||
order_target_percent(asset, max_sharpe_port[asset])
|
||||
|
||||
# create scatter plot coloured by Sharpe Ratio
|
||||
plt.scatter(results_frame.stdev,
|
||||
results_frame.r,
|
||||
c=results_frame.sharpe,
|
||||
cmap='RdYlGn')
|
||||
plt.scatter(results_frame.stdev, results_frame.r,
|
||||
c=results_frame.sharpe, cmap='RdYlGn')
|
||||
plt.xlabel('Volatility')
|
||||
plt.ylabel('Returns')
|
||||
plt.colorbar()
|
||||
# plot red star to highlight position of portfolio
|
||||
# with highest Sharpe Ratio
|
||||
plt.scatter(max_sharpe_port[1],
|
||||
max_sharpe_port[0],
|
||||
marker='o',
|
||||
color='b',
|
||||
s=200)
|
||||
# plot red star to highlight position of portfolio with highest Sharpe Ratio
|
||||
plt.scatter(max_sharpe_port[1], max_sharpe_port[0], marker='o',
|
||||
color='b', s=200)
|
||||
# plot green star to highlight position of minimum variance portfolio
|
||||
plt.show()
|
||||
print(max_sharpe_port)
|
||||
record(pr=pr,
|
||||
r=r,
|
||||
m=m,
|
||||
stds=stds,
|
||||
max_sharpe_port=max_sharpe_port,
|
||||
record(pr=pr, r=r, m=m, stds=stds, max_sharpe_port=max_sharpe_port,
|
||||
corr_m=corr_m)
|
||||
context.i += 1
|
||||
|
||||
@@ -136,14 +115,13 @@ def analyze(context=None, results=None):
|
||||
data.to_csv(filename + '.csv')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
|
||||
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
|
||||
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
|
||||
results = run_algorithm(initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
start=start,
|
||||
end=end,
|
||||
exchange_name='poloniex',
|
||||
capital_base=100000, )
|
||||
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
|
||||
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
|
||||
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
|
||||
results = run_algorithm(initialize=initialize,
|
||||
handle_data=handle_data,
|
||||
analyze=analyze,
|
||||
start=start,
|
||||
end=end,
|
||||
exchange_name='poloniex',
|
||||
capital_base=100000, )
|
||||
@@ -0,0 +1,153 @@
|
||||
import pandas as pd
|
||||
from logbook import Logger, DEBUG
|
||||
|
||||
from catalyst import run_algorithm
|
||||
from catalyst.api import (schedule_function, order_target_percent, symbol,
|
||||
date_rules, get_open_orders, cancel_order, record,
|
||||
set_commission, set_slippage)
|
||||
|
||||
log = Logger('rodrigo_1', level=DEBUG)
|
||||
"""
|
||||
The initialize function sets any data or variables that
|
||||
you'll use in your algorithm.
|
||||
It's only called once at the beginning of your algorithm.
|
||||
"""
|
||||
|
||||
|
||||
def initialize(context):
|
||||
# Select asset of interest
|
||||
context.asset = symbol('BTC_USD')
|
||||
|
||||
# set_commission(TradingPairFeeSchedule(maker_fee=0.5, taker_fee=0.5))
|
||||
# set_slippage(TradingPairFixedSlippage(spread=0.5))
|
||||
# Set up a rebalance method to run every day
|
||||
schedule_function(rebalance, date_rule=date_rules.every_day())
|
||||
|
||||
|
||||
"""
|
||||
Rebalance function scheduled to run once per day.
|
||||
"""
|
||||
|
||||
|
||||
def rebalance(context, data):
|
||||
# To make market decisions, we're calculating the token's
|
||||
# moving average for the last 5 days.
|
||||
|
||||
# We get the price history for the last 5 days.
|
||||
price_history = data.history(context.asset, fields='price', bar_count=5,
|
||||
frequency='1d')
|
||||
|
||||
# Then we take an average of those 5 days.
|
||||
average_price = price_history.mean()
|
||||
|
||||
# We also get the coin's current price.
|
||||
price = data.current(context.asset, 'price')
|
||||
|
||||
# Cancel any outstanding orders
|
||||
orders = get_open_orders(context.asset) or []
|
||||
for order in orders:
|
||||
cancel_order(order)
|
||||
|
||||
# If our coin is currently listed on a major exchange
|
||||
if data.can_trade(context.asset):
|
||||
# If the current price is 1% above the 5-day average price,
|
||||
# we open a long position. If the current price is below the
|
||||
# average price, then we want to close our position to 0 shares.
|
||||
if price > (1.01 * average_price):
|
||||
# Place the buy order (positive means buy, negative means sell)
|
||||
order_target_percent(context.asset, .99)
|
||||
log.info("Buying %s" % (context.asset.symbol))
|
||||
elif price < average_price:
|
||||
# Sell all of our shares by setting the target position to zero
|
||||
order_target_percent(context.asset, 0)
|
||||
log.info("Selling %s" % (context.asset.symbol))
|
||||
|
||||
# Use the record() method to track up to five custom signals.
|
||||
# Record Apple's current price and the average price over the last
|
||||
# five days.
|
||||
cash = context.portfolio.cash
|
||||
leverage = context.account.leverage
|
||||
|
||||
record(price=price, average_price=average_price, cash=cash,
|
||||
leverage=leverage)
|
||||
|
||||
|
||||
def analyze(context=None, results=None):
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
# Plot the portfolio and asset data.
|
||||
ax1 = plt.subplot(511)
|
||||
results[['portfolio_value']].plot(ax=ax1)
|
||||
ax1.set_ylabel('Portfolio Value (USD)')
|
||||
|
||||
ax2 = plt.subplot(512, sharex=ax1)
|
||||
ax2.set_ylabel('{asset} (USD)'.format(asset=context.asset))
|
||||
(results[[
|
||||
'price',
|
||||
]]).plot(ax=ax2)
|
||||
|
||||
trans = results.ix[[t != [] for t in results.transactions]]
|
||||
buys = trans.ix[
|
||||
[t[0]['amount'] > 0 for t in trans.transactions]
|
||||
]
|
||||
sells = trans.ix[
|
||||
[t[0]['amount'] < 0 for t in trans.transactions]
|
||||
]
|
||||
|
||||
ax2.plot(
|
||||
buys.index,
|
||||
results.price[buys.index],
|
||||
'^',
|
||||
markersize=10,
|
||||
color='g',
|
||||
)
|
||||
ax2.plot(
|
||||
sells.index,
|
||||
results.price[sells.index],
|
||||
'v',
|
||||
markersize=10,
|
||||
color='r',
|
||||
)
|
||||
|
||||
ax3 = plt.subplot(513, sharex=ax1)
|
||||
results[['leverage']].plot(ax=ax3)
|
||||
ax3.set_ylabel('Leverage ')
|
||||
|
||||
ax4 = plt.subplot(514, sharex=ax1)
|
||||
results[['cash']].plot(ax=ax4)
|
||||
ax4.set_ylabel('Cash (USD)')
|
||||
|
||||
results[[
|
||||
'algorithm',
|
||||
'benchmark',
|
||||
]] = results[[
|
||||
'algorithm_period_return',
|
||||
'benchmark_period_return',
|
||||
]]
|
||||
|
||||
ax5 = plt.subplot(515, sharex=ax1)
|
||||
results[[
|
||||
'algorithm',
|
||||
'benchmark',
|
||||
]].plot(ax=ax5)
|
||||
ax5.set_ylabel('Percent Change')
|
||||
|
||||
plt.legend(loc=3)
|
||||
|
||||
# Show the plot.
|
||||
plt.gcf().set_size_inches(18, 8)
|
||||
plt.show()
|
||||
|
||||
|
||||
run_algorithm(
|
||||
capital_base=100000,
|
||||
start=pd.to_datetime('2017-1-1', utc=True),
|
||||
end=pd.to_datetime('2017-10-22', utc=True),
|
||||
data_frequency='minute',
|
||||
initialize=initialize,
|
||||
handle_data=None,
|
||||
analyze=analyze,
|
||||
exchange_name='bitfinex',
|
||||
algo_namespace='rodrigo_1',
|
||||
base_currency='usd'
|
||||
)
|
||||
@@ -31,5 +31,4 @@ class OpenExchangeCalendar(TradingCalendar):
|
||||
return DateOffset(days=1)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(OpenExchangeCalendar, self).__init__(
|
||||
start=Timestamp('2015-3-1', tz='UTC'), **kwargs)
|
||||
super(OpenExchangeCalendar, self).__init__(start=Timestamp('2015-3-1', tz='UTC'), **kwargs)
|
||||
|
||||
@@ -9,7 +9,6 @@ DEFAULT_BAR_TEMPLATE = ' [%(bar)s] %(label)s: %(info)s'
|
||||
DEFAULT_EMPTY_CHAR = ' '
|
||||
DEFAULT_FILL_CHAR = '='
|
||||
|
||||
|
||||
def item_show_count(total=None):
|
||||
def maybe_show_total(index):
|
||||
if total is not None:
|
||||
@@ -18,13 +17,12 @@ def item_show_count(total=None):
|
||||
|
||||
def item_show_func(item, _it=iter(count())):
|
||||
if item is not None:
|
||||
# starting = False
|
||||
starting = False
|
||||
return maybe_show_total(next(_it))
|
||||
return 'DONE'
|
||||
|
||||
return item_show_func
|
||||
|
||||
|
||||
def maybe_show_progress(it,
|
||||
show_progress,
|
||||
empty_char=DEFAULT_EMPTY_CHAR,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user