wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-22 12:40:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: wassname/catalyst:pandas0.2
Branches Tags
wassname/catalyst:master wassname/catalyst:pandas_0.22.0 wassname/catalyst:pandas0.2 wassname/catalyst:gh-pages wassname/catalyst:develop wassname/catalyst:server-0.3.10 wassname/catalyst:cloud wassname/catalyst:new_exchange_config wassname/catalyst:cloud_conn wassname/catalyst:unit-tests wassname/catalyst:revert-180-jupyter-allow-root-docker wassname/catalyst:abnera-patch-2 wassname/catalyst:damo1884-talib_example wassname/catalyst:concurrent-exchanges wassname/catalyst:live-trading wassname/catalyst:5-minute
wassname/catalyst:0.5.8 wassname/catalyst:0.5.7 wassname/catalyst:0.5.6 wassname/catalyst:0.5.5 wassname/catalyst:0.5.4 wassname/catalyst:0.5.3 wassname/catalyst:0.5.2 wassname/catalyst:0.5.1 wassname/catalyst:0.5.0 wassname/catalyst:0.4.7 wassname/catalyst:0.4.6 wassname/catalyst:0.4.4 wassname/catalyst:0.4.3 wassname/catalyst:0.4.2 wassname/catalyst:0.4.1 wassname/catalyst:0.4.0 wassname/catalyst:0.3.10 wassname/catalyst:0.3.9 wassname/catalyst:0.3.8 wassname/catalyst:0.3.7 wassname/catalyst:0.3.6 wassname/catalyst:0.3.5 wassname/catalyst:0.3.4 wassname/catalyst:0.3.3 wassname/catalyst:0.3.2 wassname/catalyst:0.3.1 wassname/catalyst:0.3 wassname/catalyst:0.2.dev5 wassname/catalyst:0.2.dev4 wassname/catalyst:0.2.dev3 wassname/catalyst:0.2.dev2 wassname/catalyst:0.2.dev1 wassname/catalyst:0.1.dev9 wassname/catalyst:0.1.dev8 wassname/catalyst:0.1.dev7 wassname/catalyst:0.1.dev6 wassname/catalyst:0.1.dev5 wassname/catalyst:0.1.dev4 wassname/catalyst:0.1.dev3
..
compare: wassname/catalyst:unit-tests
Branches Tags
wassname/catalyst:pandas_0.22.0 wassname/catalyst:pandas0.2 wassname/catalyst:gh-pages wassname/catalyst:develop wassname/catalyst:master wassname/catalyst:server-0.3.10 wassname/catalyst:cloud wassname/catalyst:new_exchange_config wassname/catalyst:cloud_conn wassname/catalyst:unit-tests wassname/catalyst:revert-180-jupyter-allow-root-docker wassname/catalyst:abnera-patch-2 wassname/catalyst:damo1884-talib_example wassname/catalyst:concurrent-exchanges wassname/catalyst:live-trading wassname/catalyst:5-minute
wassname/catalyst:0.5.8 wassname/catalyst:0.5.7 wassname/catalyst:0.5.6 wassname/catalyst:0.5.5 wassname/catalyst:0.5.4 wassname/catalyst:0.5.3 wassname/catalyst:0.5.2 wassname/catalyst:0.5.1 wassname/catalyst:0.5.0 wassname/catalyst:0.4.7 wassname/catalyst:0.4.6 wassname/catalyst:0.4.4 wassname/catalyst:0.4.3 wassname/catalyst:0.4.2 wassname/catalyst:0.4.1 wassname/catalyst:0.4.0 wassname/catalyst:0.3.10 wassname/catalyst:0.3.9 wassname/catalyst:0.3.8 wassname/catalyst:0.3.7 wassname/catalyst:0.3.6 wassname/catalyst:0.3.5 wassname/catalyst:0.3.4 wassname/catalyst:0.3.3 wassname/catalyst:0.3.2 wassname/catalyst:0.3.1 wassname/catalyst:0.3 wassname/catalyst:0.2.dev5 wassname/catalyst:0.2.dev4 wassname/catalyst:0.2.dev3 wassname/catalyst:0.2.dev2 wassname/catalyst:0.2.dev1 wassname/catalyst:0.1.dev9 wassname/catalyst:0.1.dev8 wassname/catalyst:0.1.dev7 wassname/catalyst:0.1.dev6 wassname/catalyst:0.1.dev5 wassname/catalyst:0.1.dev4 wassname/catalyst:0.1.dev3
2 Commits
pandas0.2 .. unit-tests
Author SHA1 Message Date
AvishaiW c571c1c2ae TST: fixing zipline tests (WIP) 2018-02-13 11:53:05 +02:00
Victor Grau Serrat f279101393 TST: adapting zipline tests to catalyst: risk, events + data (WIP) 2017-12-04 23:06:06 -07:00
229 changed files with 24013 additions and 30272 deletions
Expand all files Collapse all files
.gitignore
-1
View File
@@ -40,7 +40,6 @@ develop-eggs
coverage.xml coverage.xml
htmlcov htmlcov
nosetests.xml nosetests.xml
.python-version
# C Extensions # C Extensions
*.o *.o
Dockerfile
+5 -5
View File
@@ -1,23 +1,23 @@
# #
# Dockerfile for an image with the currently checked out version of catalyst installed. To build: # Dockerfile for an image with the currently checked out version of catalyst installed. To build:
# #
# docker build -t enigmampc/catalyst . # docker build -t quantopian/catalyst .
# #
# To run the container: # To run the container:
# #
# docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalyst -it enigmampc/catalyst # docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalyst -it quantopian/catalyst
# #
# To access Jupyter when running docker locally (you may need to add NAT rules): # To access Jupyter when running docker locally (you may need to add NAT rules):
# #
# https://127.0.0.1 # https://127.0.0.1
# #
# Default password is 'jupyter'. To provide another, see: # default password is jupyter. to provide another, see:
# http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password # http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
# #
# Once generated, you can pass the new value via `docker run --env` the first time # once generated, you can pass the new value via `docker run --env` the first time
# you start the container. # you start the container.
# #
# You can also run an algo using the docker exec command. For example: # You can also run an algo using the docker exec command. For example:
# #
# docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle # docker exec -it catalyst catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
# #
Dockerfile-dev
+8 -8
View File
@@ -1,31 +1,31 @@
# #
# Dockerfile for an image with the currently checked out version of catalyst installed. To build: # Dockerfile for an image with the currently checked out version of catalyst installed. To build:
# #
# docker build -t enigmampc/catalystdev -f Dockerfile-dev . # docker build -t quantopian/catalystdev -f Dockerfile-dev .
# #
# Note: the dev build requires a enigmampc/catalyst image, which you can build as follows: # Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
# #
# docker build -t enigmampc/catalyst -f Dockerfile . # docker build -t quantopian/catalyst -f Dockerfile
# #
# To run the container: # To run the container:
# #
# docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalystdev -it enigmampc/catalystdev # docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalystdev -it quantopian/catalystdev
# #
# To access Jupyter when running docker locally (you may need to add NAT rules): # To access Jupyter when running docker locally (you may need to add NAT rules):
# #
# https://127.0.0.1 # https://127.0.0.1
# #
# Default password is 'jupyter'. To provide another, see: # default password is jupyter. to provide another, see:
# http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password # http://jupyter-notebook.readthedocs.org/en/latest/public_server.html#preparing-a-hashed-password
# #
# Once generated, you can pass the new value via `docker run --env` the first time # once generated, you can pass the new value via `docker run --env` the first time
# you start the container. # you start the container.
# #
# You can also run an algo using the docker exec command. For example: # You can also run an algo using the docker exec command. For example:
# #
# docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle # docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
# #
FROM enigmampc/catalyst FROM quantopian/catalyst
WORKDIR /catalyst WORKDIR /catalyst
README.rst
+3 -76
View File
@@ -1,76 +1,3 @@
.. image:: https://s3.amazonaws.com/enigmaco-docs/enigma-catalyst.png All the documentation for `Catalyst <https://github.com/enigmampc/catalyst>`_
:target: https://enigmampc.github.io/catalyst can be found in the
:align: center `documentation website <https://enigmampc.github.io/catalyst>`_.
:alt: Enigma | Catalyst
|version tag|
|version status|
|forum|
|discord|
|twitter|
|
Catalyst is an algorithmic trading library for crypto-assets written in Python.
It allows trading strategies to be easily expressed and backtested against
historical data (with daily and minute resolution), providing analytics and
insights regarding a particular strategy's performance. Catalyst also supports
live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex,
and Poloniex) with more being added over time. Catalyst empowers users to share
and curate data and build profitable, data-driven investment strategies. Please
visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
Catalyst builds on top of the well-established
`Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to
minimize structural changes to the general API to maximize compatibility with
existing trading algorithms, developer knowledge, and tutorials. Join us on the
`Catalyst Forum <https://catalyst.enigma.co/>`_ for questions around Catalyst,
algorithmic trading and technical support. We also have a
`Discord <https://discord.gg/SJK32GY>`_ group with the *#catalyst_dev* and
*#catalyst_setup* dedicated channels.
Overview
========
- Ease of use: Catalyst tries to get out of your way so that you can
focus on algorithm development. See
`examples of trading strategies <https://github.com/enigmampc/catalyst/tree/master/catalyst/examples>`_
provided.
- Support for several of the top crypto-exchanges by trading volume:
`Bitfinex <https://www.bitfinex.com>`_, `Bittrex <http://www.bittrex.com>`_,
and `Poloniex <https://www.poloniex.com>`_.
- Secure: You and only you have access to each exchange API keys for your accounts.
- Input of historical pricing data of all crypto-assets by exchange,
with daily and minute resolution. See
`Catalyst Market Coverage Overview <https://www.enigma.co/catalyst/status>`_.
- Backtesting and live-trading functionality, with a seamless transition
between the two modes.
- Output of performance statistics are based on Pandas DataFrames to
integrate nicely into the existing PyData eco-system.
- Statistic and machine learning libraries like matplotlib, scipy,
statsmodels, and sklearn support development, analysis, and
visualization of state-of-the-art trading systems.
- Addition of Bitcoin price (btc_usdt) as a benchmark for comparing
performance across trading algorithms.
Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
.. |version tag| image:: https://img.shields.io/pypi/v/enigma-catalyst.svg
:target: https://pypi.python.org/pypi/enigma-catalyst
.. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
:target: https://pypi.python.org/pypi/enigma-catalyst
.. |forum| image:: https://img.shields.io/badge/forum-join-green.svg
:target: https://catalyst.enigma.co/
.. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
:target: https://discordapp.com/invite/SJK32GY
.. |twitter| image:: https://img.shields.io/twitter/follow/enigmampc.svg?style=social&label=Follow&style=flat-square
:target: https://twitter.com/enigmampc
catalyst/__init__.py
+10 -4
View File
@@ -29,14 +29,11 @@ from ._version import get_versions
from . algorithm import TradingAlgorithm from . algorithm import TradingAlgorithm
from . import api from . import api
from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
__version__ = get_versions()['version']
del get_versions
# PERF: Fire a warning if calendars were instantiated during catalyst import. # PERF: Fire a warning if calendars were instantiated during catalyst import.
# Having calendars doesn't break anything per-se, but it makes catalyst imports # Having calendars doesn't break anything per-se, but it makes catalyst imports
# noticeably slower, which becomes particularly noticeable in the Zipline CLI. # noticeably slower, which becomes particularly noticeable in the Zipline CLI.
from catalyst.utils.calendars.calendar_utils import global_calendar_dispatcher
if global_calendar_dispatcher._calendars: if global_calendar_dispatcher._calendars:
import warnings import warnings
warnings.warn( warnings.warn(
@@ -47,6 +44,10 @@ if global_calendar_dispatcher._calendars:
del global_calendar_dispatcher del global_calendar_dispatcher
__version__ = get_versions()['version']
del get_versions
def load_ipython_extension(ipython): def load_ipython_extension(ipython):
from .__main__ import catalyst_magic from .__main__ import catalyst_magic
ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst') ipython.register_magic_function(catalyst_magic, 'line_cell', 'catalyst')
@@ -68,6 +69,7 @@ if os.name == 'nt':
_() _()
del _ del _
__all__ = [ __all__ = [
'TradingAlgorithm', 'TradingAlgorithm',
'api', 'api',
@@ -78,3 +80,7 @@ __all__ = [
'run_algorithm', 'run_algorithm',
'utils', 'utils',
] ]
from ._version import get_versions
__version__ = get_versions()['version']
del get_versions
catalyst/__main__.py
+33 -205
View File
@@ -3,15 +3,14 @@ import os
from functools import wraps from functools import wraps
import click import click
import sys
import logbook import logbook
import pandas as pd import pandas as pd
from catalyst.marketplace.marketplace import Marketplace
from six import text_type from six import text_type
from catalyst.data import bundles as bundles_module from catalyst.data import bundles as bundles_module
from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.utils.exchange_utils import delete_algo_folder from catalyst.exchange.exchange_utils import delete_algo_folder
from catalyst.exchange.factory import get_exchange
from catalyst.utils.cli import Date, Timestamp from catalyst.utils.cli import Date, Timestamp
from catalyst.utils.run_algo import _run, load_extensions from catalyst.utils.run_algo import _run, load_extensions
@@ -195,7 +194,9 @@ def ipython_only(option):
@click.option( @click.option(
'-x', '-x',
'--exchange-name', '--exchange-name',
help='The name of the targeted exchange.', type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the targeted exchange (supported: bitfinex,'
' bittrex, poloniex).',
) )
@click.option( @click.option(
'-n', '-n',
@@ -257,9 +258,8 @@ def run(ctx,
ctx.fail("must specify a base currency with '-c' in backtest mode") ctx.fail("must specify a base currency with '-c' in backtest mode")
if capital_base is None: if capital_base is None:
ctx.fail("must specify a capital base with '--capital-base'") ctx.fail("must specify a capital base with '--capital-base'"
" in backtest mode")
click.echo('Running in backtesting mode.', sys.stdout)
perf = _run( perf = _run(
initialize=None, initialize=None,
@@ -284,15 +284,11 @@ def run(ctx,
exchange=exchange_name, exchange=exchange_name,
algo_namespace=algo_namespace, algo_namespace=algo_namespace,
base_currency=base_currency, base_currency=base_currency,
analyze_live=None, live_graph=False
live_graph=False,
simulate_orders=True,
auth_aliases=None,
stats_output=None,
) )
if output == '-': if output == '-':
click.echo(str(perf), sys.stdout) click.echo(str(perf))
elif output != os.devnull: # make the catalyst magic not write any data elif output != os.devnull: # make the catalyst magic not write any data
perf.to_pickle(output) perf.to_pickle(output)
@@ -340,12 +336,6 @@ def catalyst_magic(line, cell=None):
type=click.File('r'), type=click.File('r'),
help='The file that contains the algorithm to run.', help='The file that contains the algorithm to run.',
) )
@click.option(
'--capital-base',
type=float,
show_default=True,
help='The amount of capital (in base_currency) allocated to trading.',
)
@click.option( @click.option(
'-t', '-t',
'--algotext', '--algotext',
@@ -384,7 +374,9 @@ def catalyst_magic(line, cell=None):
@click.option( @click.option(
'-x', '-x',
'--exchange-name', '--exchange-name',
help='The name of the targeted exchange.', type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the targeted exchange (supported: bitfinex,'
' bittrex, poloniex).',
) )
@click.option( @click.option(
'-n', '-n',
@@ -397,38 +389,15 @@ def catalyst_magic(line, cell=None):
help='The base currency used to calculate statistics ' help='The base currency used to calculate statistics '
'(e.g. usd, btc, eth).', '(e.g. usd, btc, eth).',
) )
@click.option(
'-e',
'--end',
type=Date(tz='utc', as_timestamp=True),
help='An optional end date at which to stop the execution.',
)
@click.option( @click.option(
'--live-graph/--no-live-graph', '--live-graph/--no-live-graph',
is_flag=True, is_flag=True,
default=False, default=False,
help='Display live graph.', help='Display live graph.',
) )
@click.option(
'--simulate-orders/--no-simulate-orders',
is_flag=True,
default=True,
help='Simulating orders enable the paper trading mode. No orders will be '
'sent to the exchange unless set to false.',
)
@click.option(
'--auth-aliases',
default=None,
help='Authentication file aliases for the specified exchanges. By default,'
'each exchange uses the "auth.json" file in the exchange folder. '
'Specifying an "auth2" alias would use "auth2.json". It should be '
'specified like this: "[exchange_name],[alias],..." For example, '
'"binance,auth2" or "binance,auth2,bittrex,auth2".',
)
@click.pass_context @click.pass_context
def live(ctx, def live(ctx,
algofile, algofile,
capital_base,
algotext, algotext,
define, define,
output, output,
@@ -437,10 +406,7 @@ def live(ctx,
exchange_name, exchange_name,
algo_namespace, algo_namespace,
base_currency, base_currency,
end, live_graph):
live_graph,
auth_aliases,
simulate_orders):
"""Trade live with the given algorithm. """Trade live with the given algorithm.
""" """
if (algotext is not None) == (algofile is not None): if (algotext is not None) == (algofile is not None):
@@ -451,22 +417,11 @@ def live(ctx,
if exchange_name is None: if exchange_name is None:
ctx.fail("must specify an exchange name '-x'") ctx.fail("must specify an exchange name '-x'")
if algo_namespace is None: if algo_namespace is None:
ctx.fail("must specify an algorithm name '-n' in live execution mode") ctx.fail("must specify an algorithm name '-n' in live execution mode")
if base_currency is None: if base_currency is None:
ctx.fail("must specify a base currency '-c' in live execution mode") ctx.fail("must specify a base currency '-c' in live execution mode")
if capital_base is None:
ctx.fail("must specify a capital base with '--capital-base'")
if simulate_orders:
click.echo('Running in paper trading mode.', sys.stdout)
else:
click.echo('Running in live trading mode.', sys.stdout)
perf = _run( perf = _run(
initialize=None, initialize=None,
handle_data=None, handle_data=None,
@@ -476,12 +431,12 @@ def live(ctx,
algotext=algotext, algotext=algotext,
defines=define, defines=define,
data_frequency=None, data_frequency=None,
capital_base=capital_base, capital_base=None,
data=None, data=None,
bundle=None, bundle=None,
bundle_timestamp=None, bundle_timestamp=None,
start=None, start=None,
end=end, end=None,
output=output, output=output,
print_algo=print_algo, print_algo=print_algo,
local_namespace=local_namespace, local_namespace=local_namespace,
@@ -490,15 +445,11 @@ def live(ctx,
exchange=exchange_name, exchange=exchange_name,
algo_namespace=algo_namespace, algo_namespace=algo_namespace,
base_currency=base_currency, base_currency=base_currency,
live_graph=live_graph, live_graph=live_graph
analyze_live=None,
simulate_orders=simulate_orders,
auth_aliases=auth_aliases,
stats_output=None,
) )
if output == '-': if output == '-':
click.echo(str(perf), sys.stdout) click.echo(str(perf))
elif output != os.devnull: # make the catalyst magic not write any data elif output != os.devnull: # make the catalyst magic not write any data
perf.to_pickle(output) perf.to_pickle(output)
@@ -509,7 +460,9 @@ def live(ctx,
@click.option( @click.option(
'-x', '-x',
'--exchange-name', '--exchange-name',
help='The name of the exchange bundle to ingest.', type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the exchange bundle to ingest (supported: bitfinex,'
' bittrex, poloniex).',
) )
@click.option( @click.option(
'-f', '-f',
@@ -567,8 +520,7 @@ def live(ctx,
default=False, default=False,
help='Report potential anomalies found in data bundles.' help='Report potential anomalies found in data bundles.'
) )
@click.pass_context def ingest_exchange(exchange_name, data_frequency, start, end,
def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
include_symbols, exclude_symbols, csv, show_progress, include_symbols, exclude_symbols, csv, show_progress,
verbose, validate): verbose, validate):
""" """
@@ -580,8 +532,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
exchange_bundle = ExchangeBundle(exchange_name) exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name), click.echo('Ingesting exchange bundle {}...'.format(exchange_name))
sys.stdout)
exchange_bundle.ingest( exchange_bundle.ingest(
data_frequency=data_frequency, data_frequency=data_frequency,
include_symbols=include_symbols, include_symbols=include_symbols,
@@ -604,18 +555,19 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
@click.pass_context @click.pass_context
def clean_algo(ctx, algo_namespace): def clean_algo(ctx, algo_namespace):
click.echo( click.echo(
'Cleaning algo state: {}'.format(algo_namespace), 'Cleaning algo state: {}'.format(algo_namespace)
sys.stdout
) )
delete_algo_folder(algo_namespace) delete_algo_folder(algo_namespace)
click.echo('Done', sys.stdout) click.echo('Done')
@main.command(name='clean-exchange') @main.command(name='clean-exchange')
@click.option( @click.option(
'-x', '-x',
'--exchange-name', '--exchange-name',
help='The name of the exchange bundle to ingest.', type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the exchange bundle to ingest (supported: bitfinex,'
' bittrex, poloniex).',
) )
@click.option( @click.option(
'-f', '-f',
@@ -635,12 +587,11 @@ def clean_exchange(ctx, exchange_name, data_frequency):
exchange_bundle = ExchangeBundle(exchange_name) exchange_bundle = ExchangeBundle(exchange_name)
click.echo('Cleaning exchange bundle {}...'.format(exchange_name), click.echo('Cleaning exchange bundle {}...'.format(exchange_name))
sys.stdout)
exchange_bundle.clean( exchange_bundle.clean(
data_frequency=data_frequency, data_frequency=data_frequency,
) )
click.echo('Done', sys.stdout) click.echo('Done')
@main.command() @main.command()
@@ -655,7 +606,9 @@ def clean_exchange(ctx, exchange_name, data_frequency):
@click.option( @click.option(
'-x', '-x',
'--exchange-name', '--exchange-name',
help='The name of the exchange bundle to ingest.', type=click.Choice({'bitfinex', 'bittrex', 'poloniex'}),
help='The name of the exchange bundle to ingest (supported: bitfinex,'
' bittrex, poloniex).',
) )
@click.option( @click.option(
'-c', '-c',
@@ -761,132 +714,7 @@ def bundles():
# because there were no entries, print a single message indicating that # because there were no entries, print a single message indicating that
# no ingestions have yet been made. # no ingestions have yet been made.
for timestamp in ingestions or ["<no ingestions>"]: for timestamp in ingestions or ["<no ingestions>"]:
click.echo("%s %s" % (bundle, timestamp), sys.stdout) click.echo("%s %s" % (bundle, timestamp))
@main.group()
@click.pass_context
def marketplace(ctx):
"""Access the Enigma Data Marketplace to:\n
- Register and Publish new datasets (seller-side)\n
- Subscribe and Ingest premium datasets (buyer-side)\n
"""
pass
@marketplace.command()
@click.pass_context
def ls(ctx):
"""List all available datasets.
"""
click.echo('Listing of available data sources on the marketplace:',
sys.stdout)
marketplace = Marketplace()
marketplace.list()
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.pass_context
def subscribe(ctx, dataset):
"""Subscribe to an existing dataset.
"""
marketplace = Marketplace()
marketplace.subscribe(dataset)
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.option(
'-f',
'--data-frequency',
type=click.Choice({'daily', 'minute', 'daily,minute', 'minute,daily'}),
default='daily',
show_default=True,
help='The data frequency of the desired OHLCV bars.',
)
@click.option(
'-s',
'--start',
default=None,
type=Date(tz='utc', as_timestamp=True),
help='The start date of the data range. (default: one year from end date)',
)
@click.option(
'-e',
'--end',
default=None,
type=Date(tz='utc', as_timestamp=True),
help='The end date of the data range. (default: today)',
)
@click.pass_context
def ingest(ctx, dataset, data_frequency, start, end):
"""Ingest a dataset (requires subscription).
"""
marketplace = Marketplace()
marketplace.ingest(dataset, data_frequency, start, end)
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the dataset to ingest from the Data Marketplace.',
)
@click.pass_context
def clean(ctx, dataset):
"""Clean/Remove local data for a given dataset.
"""
marketplace = Marketplace()
marketplace.clean(dataset)
@marketplace.command()
@click.pass_context
def register(ctx):
"""Register a new dataset.
"""
marketplace = Marketplace()
marketplace.register()
@marketplace.command()
@click.option(
'--dataset',
default=None,
help='The name of the Marketplace dataset to publish data for.',
)
@click.option(
'--datadir',
default=None,
help='The folder that contains the CSV data files to publish.',
)
@click.option(
'--watch/--no-watch',
is_flag=True,
default=False,
help='Whether to watch the datadir for live data.',
)
@click.pass_context
def publish(ctx, dataset, datadir, watch):
"""Publish data for a registered dataset.
"""
marketplace = Marketplace()
if dataset is None:
ctx.fail("must specify a dataset to publish data for "
" with '--dataset'\n")
if datadir is None:
ctx.fail("must specify a datadir where to find the files to publish "
" with '--datadir'\n")
marketplace.publish(dataset, datadir, watch)
if __name__ == '__main__': if __name__ == '__main__':
catalyst/_protocol.pyx
+2 -1
View File
@@ -16,6 +16,7 @@ import warnings
from contextlib import contextmanager from contextlib import contextmanager
from functools import wraps from functools import wraps
from pandas.tslib import normalize_date
import pandas as pd import pandas as pd
import numpy as np import numpy as np
@@ -563,7 +564,7 @@ cdef class BarData:
}) })
cdef bool _is_stale_for_asset(self, asset, dt, adjusted_dt, data_portal): cdef bool _is_stale_for_asset(self, asset, dt, adjusted_dt, data_portal):
session_label = dt.normalize_date() # FIXME session_label = normalize_date(dt) # FIXME
if not asset.is_alive_for_session(session_label): if not asset.is_alive_for_session(session_label):
return False return False
catalyst/algorithm.py
+11 -9
View File
@@ -21,6 +21,7 @@ import logbook
import pytz import pytz
import pandas as pd import pandas as pd
from contextlib2 import ExitStack from contextlib2 import ExitStack
from pandas.tseries.tools import normalize_date
import numpy as np import numpy as np
from itertools import chain, repeat from itertools import chain, repeat
@@ -123,6 +124,7 @@ from catalyst.utils.events import (
from catalyst.utils.factory import create_simulation_parameters from catalyst.utils.factory import create_simulation_parameters
from catalyst.utils.math_utils import ( from catalyst.utils.math_utils import (
tolerant_equals, tolerant_equals,
round_if_near_integer,
round_nearest round_nearest
) )
from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches from catalyst.utils.pandas_utils import clear_dataframe_indexer_caches
@@ -938,7 +940,7 @@ class TradingAlgorithm(object):
The field to query. The options have the following meanings: The field to query. The options have the following meanings:
arena : str arena : str
The arena from the simulation parameters. This will normally The arena from the simulation parameters. This will normally
be ``backtest`` but some systems may use this distinguish be ``'backtest'`` but some systems may use this distinguish
live trading from backtesting. live trading from backtesting.
data_frequency : {'daily', 'minute'} data_frequency : {'daily', 'minute'}
data_frequency tells the algorithm if it is running with data_frequency tells the algorithm if it is running with
@@ -953,7 +955,7 @@ class TradingAlgorithm(object):
The platform that the code is running on. By default this The platform that the code is running on. By default this
will be the string 'catalyst'. This can allow algorithms to will be the string 'catalyst'. This can allow algorithms to
know if they are running on the Quantopian platform instead. know if they are running on the Quantopian platform instead.
\* : dict[str -> any] * : dict[str -> any]
Returns all of the fields in a dictionary. Returns all of the fields in a dictionary.
Returns Returns
@@ -1031,7 +1033,7 @@ class TradingAlgorithm(object):
argument is the name of the column in the preprocessed dataframe argument is the name of the column in the preprocessed dataframe
containing the symbols. This will be used along with the date containing the symbols. This will be used along with the date
information to map the sids in the asset finder. information to map the sids in the asset finder.
\*\*kwargs **kwargs
Forwarded to :func:`pandas.read_csv`. Forwarded to :func:`pandas.read_csv`.
Returns Returns
@@ -1155,7 +1157,7 @@ class TradingAlgorithm(object):
Parameters Parameters
---------- ----------
\*\*kwargs **kwargs
The names and values to record. The names and values to record.
Notes Notes
@@ -1272,7 +1274,7 @@ class TradingAlgorithm(object):
Parameters Parameters
---------- ----------
\*args : iterable[str] *args : iterable[str]
The ticker symbols to lookup. The ticker symbols to lookup.
Returns Returns
@@ -1344,7 +1346,7 @@ class TradingAlgorithm(object):
# Make sure the asset exists, and that there is a last price for it. # Make sure the asset exists, and that there is a last price for it.
# FIXME: we should use BarData's can_trade logic here, but I haven't # FIXME: we should use BarData's can_trade logic here, but I haven't
# yet found a good way to do that. # yet found a good way to do that.
normalized_date = self.datetime.normalize() normalized_date = normalize_date(self.datetime)
if normalized_date < asset.start_date: if normalized_date < asset.start_date:
raise CannotOrderDelistedAsset( raise CannotOrderDelistedAsset(
@@ -1391,7 +1393,7 @@ class TradingAlgorithm(object):
) )
if asset.auto_close_date: if asset.auto_close_date:
day = self.get_datetime().normalize() day = normalize_date(self.get_datetime())
if day > min(asset.end_date, asset.auto_close_date): if day > min(asset.end_date, asset.auto_close_date):
# If we are after the asset's end date or auto close date, warn # If we are after the asset's end date or auto close date, warn
@@ -1483,6 +1485,7 @@ class TradingAlgorithm(object):
""" """
Converts the number of shares to the smallest tradable lot size for Converts the number of shares to the smallest tradable lot size for
the asset being ordered. the asset being ordered.
""" """
return round_nearest(amount, asset.min_trade_size) return round_nearest(amount, asset.min_trade_size)
@@ -1520,7 +1523,6 @@ class TradingAlgorithm(object):
self.updated_portfolio(), self.updated_portfolio(),
self.get_datetime(), self.get_datetime(),
self.trading_client.current_data) self.trading_client.current_data)
@staticmethod @staticmethod
def __convert_order_params_for_blotter(limit_price, stop_price, style): def __convert_order_params_for_blotter(limit_price, stop_price, style):
""" """
@@ -2474,7 +2476,7 @@ class TradingAlgorithm(object):
""" """
Internal implementation of `pipeline_output`. Internal implementation of `pipeline_output`.
""" """
today = self.get_datetime().normalize() today = normalize_date(self.get_datetime())
data = NO_DATA = object() data = NO_DATA = object()
try: try:
data = self._pipeline_cache.unwrap(today) data = self._pipeline_cache.unwrap(today)
catalyst/api.pyi
+8 -65
View File
@@ -34,7 +34,6 @@ def attach_pipeline(pipeline, name, chunks=None):
:func:`catalyst.api.pipeline_output` :func:`catalyst.api.pipeline_output`
""" """
def batch_market_order(share_counts): def batch_market_order(share_counts):
"""Place a batch market order for multiple assets. """Place a batch market order for multiple assets.
@@ -49,7 +48,6 @@ def batch_market_order(share_counts):
Index of ids for newly-created orders. Index of ids for newly-created orders.
""" """
def cancel_order(order_param): def cancel_order(order_param):
"""Cancel an open order. """Cancel an open order.
@@ -59,9 +57,7 @@ def cancel_order(order_param):
The order_id or order object to cancel. The order_id or order object to cancel.
""" """
def continuous_future(root_symbol_str, offset=0, roll='volume', adjustment='mul'):
def continuous_future(root_symbol_str, offset=0, roll='volume',
adjustment='mul'):
"""Create a specifier for a continuous contract. """Create a specifier for a continuous contract.
Parameters Parameters
@@ -85,10 +81,7 @@ def continuous_future(root_symbol_str, offset=0, roll='volume',
The continuous future specifier. The continuous future specifier.
""" """
def fetch_csv(url, pre_func=None, post_func=None, date_column='date', date_format=None, timezone='UTC', symbol=None, mask=True, symbol_column=None, special_params_checker=None, **kwargs):
def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
date_format=None, timezone='UTC', symbol=None, mask=True,
symbol_column=None, special_params_checker=None, **kwargs):
"""Fetch a csv from a remote url and register the data so that it is """Fetch a csv from a remote url and register the data so that it is
queryable from the ``data`` object. queryable from the ``data`` object.
@@ -132,7 +125,6 @@ def fetch_csv(url, pre_func=None, post_func=None, date_column='date',
A requests source that will pull data from the url specified. A requests source that will pull data from the url specified.
""" """
def future_symbol(symbol): def future_symbol(symbol):
"""Lookup a futures contract with a given symbol. """Lookup a futures contract with a given symbol.
@@ -152,7 +144,6 @@ def future_symbol(symbol):
Raised when no contract named 'symbol' is found. Raised when no contract named 'symbol' is found.
""" """
def get_datetime(tz=None): def get_datetime(tz=None):
""" """
Returns the current simulation datetime. Returns the current simulation datetime.
@@ -168,7 +159,6 @@ dt : datetime
The current simulation datetime converted to ``tz``. The current simulation datetime converted to ``tz``.
""" """
def get_environment(field='platform'): def get_environment(field='platform'):
"""Query the execution environment. """Query the execution environment.
@@ -208,7 +198,6 @@ def get_environment(field='platform'):
Raised when ``field`` is not a valid option. Raised when ``field`` is not a valid option.
""" """
def get_order(order_id): def get_order(order_id):
"""Lookup an order based on the order id returned from one of the """Lookup an order based on the order id returned from one of the
order functions. order functions.
@@ -224,12 +213,10 @@ def get_order(order_id):
The order object. The order object.
""" """
def history(bar_count, frequency, field, ffill=True): def history(bar_count, frequency, field, ffill=True):
"""DEPRECATED: use ``data.history`` instead. """DEPRECATED: use ``data.history`` instead.
""" """
def order(asset, amount, limit_price=None, stop_price=None, style=None): def order(asset, amount, limit_price=None, stop_price=None, style=None):
"""Place an order. """Place an order.
@@ -271,9 +258,7 @@ def order(asset, amount, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_percent` :func:`catalyst.api.order_percent`
""" """
def order_percent(asset, percent, limit_price=None, stop_price=None, style=None):
def order_percent(asset, percent, limit_price=None, stop_price=None,
style=None):
"""Place an order in the specified asset corresponding to the given """Place an order in the specified asset corresponding to the given
percent of the current portfolio value. percent of the current portfolio value.
@@ -308,7 +293,6 @@ def order_percent(asset, percent, limit_price=None, stop_price=None,
:func:`catalyst.api.order_value` :func:`catalyst.api.order_value`
""" """
def order_target(asset, target, limit_price=None, stop_price=None, style=None): def order_target(asset, target, limit_price=None, stop_price=None, style=None):
"""Place an order to adjust a position to a target number of shares. If """Place an order to adjust a position to a target number of shares. If
the position doesn't already exist, this is equivalent to placing a new the position doesn't already exist, this is equivalent to placing a new
@@ -360,9 +344,7 @@ def order_target(asset, target, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_target_value` :func:`catalyst.api.order_target_value`
""" """
def order_target_percent(asset, target, limit_price=None, stop_price=None, style=None):
def order_target_percent(asset, target, limit_price=None, stop_price=None,
style=None):
"""Place an order to adjust a position to a target percent of the """Place an order to adjust a position to a target percent of the
current portfolio value. If the position doesn't already exist, this is current portfolio value. If the position doesn't already exist, this is
equivalent to placing a new order. If the position does exist, this is equivalent to placing a new order. If the position does exist, this is
@@ -414,9 +396,7 @@ def order_target_percent(asset, target, limit_price=None, stop_price=None,
:func:`catalyst.api.order_target_value` :func:`catalyst.api.order_target_value`
""" """
def order_target_value(asset, target, limit_price=None, stop_price=None, style=None):
def order_target_value(asset, target, limit_price=None, stop_price=None,
style=None):
"""Place an order to adjust a position to a target value. If """Place an order to adjust a position to a target value. If
the position doesn't already exist, this is equivalent to placing a new the position doesn't already exist, this is equivalent to placing a new
order. If the position does exist, this is equivalent to placing an order. If the position does exist, this is equivalent to placing an
@@ -468,7 +448,6 @@ def order_target_value(asset, target, limit_price=None, stop_price=None,
:func:`catalyst.api.order_target_percent` :func:`catalyst.api.order_target_percent`
""" """
def order_value(asset, value, limit_price=None, stop_price=None, style=None): def order_value(asset, value, limit_price=None, stop_price=None, style=None):
"""Place an order by desired value rather than desired number of """Place an order by desired value rather than desired number of
shares. shares.
@@ -509,7 +488,6 @@ def order_value(asset, value, limit_price=None, stop_price=None, style=None):
:func:`catalyst.api.order_percent` :func:`catalyst.api.order_percent`
""" """
def pipeline_output(name): def pipeline_output(name):
"""Get the results of the pipeline that was attached with the name: """Get the results of the pipeline that was attached with the name:
``name``. ``name``.
@@ -536,7 +514,6 @@ def pipeline_output(name):
:meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline` :meth:`catalyst.pipeline.engine.PipelineEngine.run_pipeline`
""" """
def record(*args, **kwargs): def record(*args, **kwargs):
"""Track and record values each day. """Track and record values each day.
@@ -552,9 +529,7 @@ def record(*args, **kwargs):
:func:`~catalyst.run_algorithm`. :func:`~catalyst.run_algorithm`.
""" """
def schedule_function(func, date_rule=None, time_rule=None, half_days=True, calendar=None):
def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
calendar=None):
"""Schedules a function to be called according to some timed rules. """Schedules a function to be called according to some timed rules.
Parameters Parameters
@@ -574,7 +549,6 @@ def schedule_function(func, date_rule=None, time_rule=None, half_days=True,
:class:`catalyst.api.time_rules` :class:`catalyst.api.time_rules`
""" """
def set_asset_restrictions(restrictions, on_error='fail'): def set_asset_restrictions(restrictions, on_error='fail'):
"""Set a restriction on which assets can be ordered. """Set a restriction on which assets can be ordered.
@@ -588,7 +562,6 @@ def set_asset_restrictions(restrictions, on_error='fail'):
catalyst.finance.asset_restrictions.Restrictions catalyst.finance.asset_restrictions.Restrictions
""" """
def set_benchmark(benchmark): def set_benchmark(benchmark):
"""Set the benchmark asset. """Set the benchmark asset.
@@ -603,7 +576,6 @@ def set_benchmark(benchmark):
automatically reinvested. automatically reinvested.
""" """
def set_cancel_policy(cancel_policy): def set_cancel_policy(cancel_policy):
"""Sets the order cancellation policy for the simulation. """Sets the order cancellation policy for the simulation.
@@ -618,7 +590,6 @@ def set_cancel_policy(cancel_policy):
:class:`catalyst.api.NeverCancel` :class:`catalyst.api.NeverCancel`
""" """
def set_commission(commission): def set_commission(commission):
"""Sets the commission model for the simulation. """Sets the commission model for the simulation.
@@ -634,7 +605,6 @@ def set_commission(commission):
:class:`catalyst.finance.commission.PerDollar` :class:`catalyst.finance.commission.PerDollar`
""" """
def set_do_not_order_list(restricted_list, on_error='fail'): def set_do_not_order_list(restricted_list, on_error='fail'):
"""Set a restriction on which assets can be ordered. """Set a restriction on which assets can be ordered.
@@ -644,13 +614,11 @@ def set_do_not_order_list(restricted_list, on_error='fail'):
The assets that cannot be ordered. The assets that cannot be ordered.
""" """
def set_long_only(on_error='fail'): def set_long_only(on_error='fail'):
"""Set a rule specifying that this algorithm cannot take short """Set a rule specifying that this algorithm cannot take short
positions. positions.
""" """
def set_max_leverage(max_leverage): def set_max_leverage(max_leverage):
"""Set a limit on the maximum leverage of the algorithm. """Set a limit on the maximum leverage of the algorithm.
@@ -661,7 +629,6 @@ def set_max_leverage(max_leverage):
be no maximum. be no maximum.
""" """
def set_max_order_count(max_count, on_error='fail'): def set_max_order_count(max_count, on_error='fail'):
"""Set a limit on the number of orders that can be placed in a single """Set a limit on the number of orders that can be placed in a single
day. day.
@@ -672,9 +639,7 @@ def set_max_order_count(max_count, on_error='fail'):
The maximum number of orders that can be placed on any single day. The maximum number of orders that can be placed on any single day.
""" """
def set_max_order_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
def set_max_order_size(asset=None, max_shares=None, max_notional=None,
on_error='fail'):
"""Set a limit on the number of shares and/or dollar value of any single """Set a limit on the number of shares and/or dollar value of any single
order placed for sid. Limits are treated as absolute values and are order placed for sid. Limits are treated as absolute values and are
enforced at the time that the algo attempts to place an order for sid. enforced at the time that the algo attempts to place an order for sid.
@@ -693,9 +658,7 @@ def set_max_order_size(asset=None, max_shares=None, max_notional=None,
The maximum value that can be ordered at one time. The maximum value that can be ordered at one time.
""" """
def set_max_position_size(asset=None, max_shares=None, max_notional=None, on_error='fail'):
def set_max_position_size(asset=None, max_shares=None, max_notional=None,
on_error='fail'):
"""Set a limit on the number of shares and/or dollar value held for the """Set a limit on the number of shares and/or dollar value held for the
given sid. Limits are treated as absolute values and are enforced at given sid. Limits are treated as absolute values and are enforced at
the time that the algo attempts to place an order for sid. This means the time that the algo attempts to place an order for sid. This means
@@ -718,7 +681,6 @@ def set_max_position_size(asset=None, max_shares=None, max_notional=None,
The maximum value to hold for an asset. The maximum value to hold for an asset.
""" """
def set_slippage(slippage): def set_slippage(slippage):
"""Set the slippage model for the simulation. """Set the slippage model for the simulation.
@@ -732,7 +694,6 @@ def set_slippage(slippage):
:class:`catalyst.finance.slippage.SlippageModel` :class:`catalyst.finance.slippage.SlippageModel`
""" """
def set_symbol_lookup_date(dt): def set_symbol_lookup_date(dt):
"""Set the date for which symbols will be resolved to their assets """Set the date for which symbols will be resolved to their assets
(symbols may map to different firms or underlying assets at (symbols may map to different firms or underlying assets at
@@ -744,7 +705,6 @@ def set_symbol_lookup_date(dt):
The new symbol lookup date. The new symbol lookup date.
""" """
def sid(sid): def sid(sid):
"""Lookup an Asset by its unique asset identifier. """Lookup an Asset by its unique asset identifier.
@@ -764,7 +724,6 @@ def sid(sid):
When a requested ``sid`` does not map to any asset. When a requested ``sid`` does not map to any asset.
""" """
def symbol(symbol_str): def symbol(symbol_str):
"""Lookup an Equity by its ticker symbol. """Lookup an Equity by its ticker symbol.
@@ -789,7 +748,6 @@ def symbol(symbol_str):
:func:`catalyst.api.set_symbol_lookup_date` :func:`catalyst.api.set_symbol_lookup_date`
""" """
def symbols(*args): def symbols(*args):
"""Lookup multuple Equities as a list. """Lookup multuple Equities as a list.
@@ -815,18 +773,3 @@ def symbols(*args):
:func:`catalyst.api.set_symbol_lookup_date` :func:`catalyst.api.set_symbol_lookup_date`
""" """
def get_dataset(ds_name, start=None, end=None):
"""
Lookup a data source from the marketplace
Parameters
----------
ds_name: str
start: pd.Timestamp
end: pd.Timestamp
Returns
-------
"""
catalyst/assets/_assets.pyx
+3 -34
View File
@@ -17,7 +17,6 @@
""" """
Cythonized Asset object. Cythonized Asset object.
""" """
import hashlib import hashlib
cimport cython cimport cython
@@ -39,7 +38,7 @@ from numpy cimport int64_t
import warnings import warnings
cimport numpy as np cimport numpy as np
from catalyst.exchange.utils.exchange_utils import get_sid from catalyst.exchange.exchange_utils import get_sid
from catalyst.utils.calendars import get_calendar from catalyst.utils.calendars import get_calendar
from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
@@ -406,9 +405,6 @@ cdef class TradingPair(Asset):
cdef readonly float taker cdef readonly float taker
cdef readonly int trading_state cdef readonly int trading_state
cdef readonly object data_source cdef readonly object data_source
cdef readonly float max_trade_size
cdef readonly float lot
cdef readonly int decimals
_kwargnames = frozenset({ _kwargnames = frozenset({
'sid', 'sid',
@@ -427,13 +423,10 @@ cdef class TradingPair(Asset):
'end_minute', 'end_minute',
'exchange_symbol', 'exchange_symbol',
'min_trade_size', 'min_trade_size',
'max_trade_size',
'lot',
'maker', 'maker',
'taker', 'taker',
'trading_state', 'trading_state',
'data_source', 'data_source'
'decimals'
}) })
def __init__(self, def __init__(self,
object symbol, object symbol,
@@ -450,11 +443,8 @@ cdef class TradingPair(Asset):
object auto_close_date=None, object auto_close_date=None,
object exchange_full=None, object exchange_full=None,
float min_trade_size=0.0001, float min_trade_size=0.0001,
float max_trade_size=1000000,
float maker=0.0015, float maker=0.0015,
float taker=0.0025, float taker=0.0025,
float lot=0,
int decimals = 8,
int trading_state=0, int trading_state=0,
object data_source='catalyst'): object data_source='catalyst'):
""" """
@@ -519,12 +509,9 @@ cdef class TradingPair(Asset):
:param auto_close_date: :param auto_close_date:
:param exchange_full: :param exchange_full:
:param min_trade_size: :param min_trade_size:
:param max_trade_size:
:param maker: :param maker:
:param taker: :param taker:
:param data_source :param data_source
:param decimals
:param lot
""" """
symbol = symbol.lower() symbol = symbol.lower()
@@ -548,9 +535,6 @@ cdef class TradingPair(Asset):
if end_date is None: if end_date is None:
end_date = pd.Timestamp.utcnow() + timedelta(days=365) end_date = pd.Timestamp.utcnow() + timedelta(days=365)
if lot == 0 and min_trade_size > 0:
lot = min_trade_size
super().__init__( super().__init__(
sid, sid,
exchange, exchange,
@@ -572,9 +556,6 @@ cdef class TradingPair(Asset):
self.exchange_symbol = exchange_symbol self.exchange_symbol = exchange_symbol
self.trading_state = trading_state self.trading_state = trading_state
self.data_source = data_source self.data_source = data_source
self.max_trade_size = max_trade_size
self.lot = lot
self.decimals = decimals
def __repr__(self): def __repr__(self):
return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \ return 'Trading Pair {symbol}({sid}) Exchange: {exchange}, ' \
@@ -601,7 +582,6 @@ cdef class TradingPair(Asset):
""" """
Convert to a python dict. Convert to a python dict.
""" """
#TODO: missing fields
super_dict = super(TradingPair, self).to_dict() super_dict = super(TradingPair, self).to_dict()
super_dict['end_daily'] = self.end_daily super_dict['end_daily'] = self.end_daily
super_dict['end_minute'] = self.end_minute super_dict['end_minute'] = self.end_minute
@@ -630,28 +610,17 @@ cdef class TradingPair(Asset):
and whose second element is a tuple of all the attributes that should and whose second element is a tuple of all the attributes that should
be serialized/deserialized during pickling. be serialized/deserialized during pickling.
""" """
# added arguments for catalyst
return (self.__class__, (self.symbol, return (self.__class__, (self.symbol,
self.exchange, self.exchange,
self.start_date, self.start_date,
self.asset_name, self.asset_name,
self.sid, self.sid,
self.leverage, self.leverage,
self.end_daily,
self.end_minute,
self.end_date, self.end_date,
self.exchange_symbol,
self.first_traded, self.first_traded,
self.auto_close_date, self.auto_close_date,
self.exchange_full, self.exchange_full,
self.min_trade_size, self.min_trade_size))
self.max_trade_size,
self.maker,
self.taker,
self.lot,
self.decimals,
self.trading_state,
self.data_source))
def make_asset_array(int size, Asset asset): def make_asset_array(int size, Asset asset):
cdef np.ndarray out = np.empty([size], dtype=object) cdef np.ndarray out = np.empty([size], dtype=object)
catalyst/constants.py
+3 -30
View File
@@ -7,7 +7,8 @@ import logbook
For example, if you want to see the DEBUG messages, run: For example, if you want to see the DEBUG messages, run:
$ export CATALYST_LOG_LEVEL=10 $ export CATALYST_LOG_LEVEL=10
''' '''
LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO)) # LOG_LEVEL = int(os.environ.get('CATALYST_LOG_LEVEL', logbook.INFO))
LOG_LEVEL = logbook.DEBUG
SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
'{exchange}/symbols.json' '{exchange}/symbols.json'
@@ -15,32 +16,4 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
DATE_FORMAT = '%Y-%m-%d' DATE_FORMAT = '%Y-%m-%d'
try: AUTO_INGEST = False
ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
except Exception as e:
print('unable to get catalyst path: {}'.format(e))
AUTO_INGEST = False
AUTH_SERVER = 'https://data.enigma.co'
ETH_REMOTE_NODE = 'https://mainnet.infura.io'
MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_marketplace_address.txt'
MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_marketplace_abi.json'
ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_enigma_address.txt'
ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
'catalyst/master/catalyst/marketplace/' \
'contract_enigma_abi.json'
SUPPORTED_WALLETS = ['metamask', 'ledger', 'trezor', 'bitbox', 'keystore',
'key']
catalyst/curate/poloniex.py
+137 -143
View File
@@ -1,33 +1,25 @@
import csv import json, time, csv
import json
import os
import shutil
import time
from datetime import datetime from datetime import datetime
import logbook
import pandas as pd import pandas as pd
import requests import os, time, shutil, requests, logbook
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
from catalyst.exchange.utils.exchange_utils import \
get_exchange_symbols_filename
DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())) DT_START = int(time.mktime(datetime(2010, 1, 1, 0, 0).timetuple()))
DT_END = pd.to_datetime('today').value // 10 ** 9 DT_END = pd.to_datetime('today').value // 10 ** 9
CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/') CSV_OUT_FOLDER = os.environ.get('CSV_OUT_FOLDER', '/efs/exchanges/poloniex/')
CONN_RETRIES = 2 CONN_RETRIES = 2
logbook.StderrHandler().push_application() logbook.StderrHandler().push_application()
log = logbook.Logger(__name__) log = logbook.Logger(__name__)
class PoloniexCurator(object): class PoloniexCurator(object):
''' '''
OHLCV data feed generator for crypto data. Based on Poloniex market data OHLCV data feed generator for crypto data. Based on Poloniex market data
''' '''
_api_path = 'https://poloniex.com/public?' _api_path = 'https://poloniex.com/public?'
currency_pairs = [] currency_pairs = []
def __init__(self): def __init__(self):
if not os.path.exists(CSV_OUT_FOLDER): if not os.path.exists(CSV_OUT_FOLDER):
@@ -38,9 +30,10 @@ class PoloniexCurator(object):
CSV_OUT_FOLDER)) CSV_OUT_FOLDER))
log.exception(e) log.exception(e)
def get_currency_pairs(self): def get_currency_pairs(self):
''' '''
Retrieves and returns all currency pairs from the exchange Retrieves and returns all currency pairs from the exchange
''' '''
url = self._api_path + 'command=returnTicker' url = self._api_path + 'command=returnTicker'
@@ -52,7 +45,7 @@ class PoloniexCurator(object):
return None return None
data = response.json() data = response.json()
self.currency_pairs = [] self.currency_pairs = []
for ticker in data: for ticker in data:
self.currency_pairs.append(ticker) self.currency_pairs.append(ticker)
self.currency_pairs.sort() self.currency_pairs.sort()
@@ -61,60 +54,54 @@ class PoloniexCurator(object):
len(self.currency_pairs) len(self.currency_pairs)
)) ))
def _retrieve_tradeID_date(self, row): def _retrieve_tradeID_date(self, row):
''' '''
Helper function that reads tradeID and date fields from CSV readline Helper function that reads tradeID and date fields from CSV readline
''' '''
tId = int(row.split(',')[0]) tId = int(row.split(',')[0])
d = pd.to_datetime(row.split(',')[1], d = pd.to_datetime(row.split(',')[1],
infer_datetime_format=True).value // 10 ** 9 infer_datetime_format=True).value // 10 ** 9
return tId, d return tId, d
def retrieve_trade_history(self, currencyPair, start=DT_START,
def retrieve_trade_history(self, currencyPair, start=DT_START,
end=DT_END, temp=None): end=DT_END, temp=None):
''' '''
Retrieves TradeHistory from exchange for a given currencyPair Retrieves TradeHistory from exchange for a given currencyPair
between start and end dates. If no start date is provided, uses between start and end dates. If no start date is provided, uses
a system-wide one (beginning of time for cryptotrading). a system-wide one (beginning of time for cryptotrading).
If no end date is provided, 'now' is used. If no end date is provided, 'now' is used.
Stores results in CSV file on disk. Stores results in CSV file on disk.
This function is called recursively to work around the This function is called recursively to work around the
limitations imposed by the provider API. limitations imposed by the provider API.
''' '''
csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' csv_fn = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
''' '''
Check what data we already have on disk, reading first and last Check what data we already have on disk, reading first and last
lines from file. Data is stored on file from NEWEST to OLDEST. lines from file. Data is stored on file from NEWEST to OLDEST.
''' '''
try: try:
with open(csv_fn, 'ab+') as f: with open(csv_fn, 'ab+') as f:
f.seek(0, os.SEEK_END) f.seek(0, os.SEEK_END)
if(f.tell() > 2): # Check file size is not 0 if(f.tell() > 2): # Check file size is not 0
f.seek(0) # Go to start to read f.seek(0) # Go to start to read
last_tradeID, end_file = self._retrieve_tradeID_date( last_tradeID, end_file = self._retrieve_tradeID_date(f.readline())
f.readline())
f.seek(-2, os.SEEK_END) # Jump to the 2nd last byte f.seek(-2, os.SEEK_END) # Jump to the 2nd last byte
while f.read(1) != b"\n": # Until EOL is found... while f.read(1) != b"\n": # Until EOL is found...
# ...jump back the read byte plus one more. f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
f.seek(-2, os.SEEK_CUR) first_tradeID, start_file = self._retrieve_tradeID_date(f.readline())
first_tradeID, start_file = self._retrieve_tradeID_date(
f.readline())
if(end_file + 3600 * 6 > DT_END if( end_file + 3600 * 6 > DT_END and ( first_tradeID == 1
and (first_tradeID == 1 or (currencyPair == 'BTC_HUC' and first_tradeID == 2)
or (currencyPair == 'BTC_HUC' or (currencyPair == 'BTC_RIC' and first_tradeID == 2)
and first_tradeID == 2) or (currencyPair == 'BTC_XCP' and first_tradeID == 2)
or (currencyPair == 'BTC_RIC' or (currencyPair == 'BTC_NAV' and first_tradeID == 4569)
and first_tradeID == 2) or (currencyPair == 'BTC_POT' and first_tradeID == 23511) ) ):
or (currencyPair == 'BTC_XCP'
and first_tradeID == 2)
or (currencyPair == 'BTC_NAV'
and first_tradeID == 4569)
or (currencyPair == 'BTC_POT'
and first_tradeID == 23511))):
return return
except Exception as e: except Exception as e:
@@ -122,11 +109,11 @@ class PoloniexCurator(object):
log.exception(e) log.exception(e)
''' '''
Poloniex API limits querying TradeHistory to intervals smaller Poloniex API limits querying TradeHistory to intervals smaller
than 1 month, so we make sure that start date is never more than than 1 month, so we make sure that start date is never more than
1 month apart from end date 1 month apart from end date
''' '''
if(end - start > 2419200): # 60s/min * 60min/hr * 24hr/day * 28days if( end - start > 2419200 ): # 60s/min * 60min/hr * 24hr/day * 28days
newstart = end - 2419200 newstart = end - 2419200
else: else:
newstart = start newstart = start
@@ -137,11 +124,12 @@ class PoloniexCurator(object):
url = '{path}command=returnTradeHistory&currencyPair={pair}' \ url = '{path}command=returnTradeHistory&currencyPair={pair}' \
'&start={start}&end={end}'.format( '&start={start}&end={end}'.format(
path=self._api_path, path = self._api_path,
pair=currencyPair, pair = currencyPair,
start=str(newstart), start = str(newstart),
end=str(end) end = str(end)
) )
print url
attempts = 0 attempts = 0
success = 0 success = 0
@@ -149,14 +137,14 @@ class PoloniexCurator(object):
try: try:
response = requests.get(url) response = requests.get(url)
except Exception as e: except Exception as e:
log.error('Failed to retrieve trade history data' log.error('Failed to retrieve trade history data for {}'.format(
'for {}'.format(currencyPair)) currencyPair
))
log.exception(e) log.exception(e)
attempts += 1 attempts += 1
else: else:
try: try:
if(isinstance(response.json(), dict) if isinstance(response.json(), dict) and response.json()['error']:
and response.json()['error']):
log.error('Failed to to retrieve trade history data ' log.error('Failed to to retrieve trade history data '
'for {}: {}'.format( 'for {}: {}'.format(
currencyPair, currencyPair,
@@ -173,33 +161,33 @@ class PoloniexCurator(object):
if not success: if not success:
return None return None
''' '''
If we get to transactionId == 1, and we already have that on If we get to transactionId == 1, and we already have that on
disk, we got to the end of TradeHistory for this coin. disk, we got to the end of TradeHistory for this coin.
''' '''
if('first_tradeID' in locals() if('first_tradeID' in locals()
and response.json()[-1]['tradeID'] == first_tradeID): and response.json()[-1]['tradeID'] == first_tradeID):
return return
''' '''
There are primarily two scenarios: There are primarily two scenarios:
a) There is newer data available that we need to add at a) There is newer data available that we need to add at
the beginning of the file. We'll retrieve all what we the beginning of the file. We'll retrieve all what we
need until we get to what we already have, writing it need until we get to what we already have, writing it
to a temporary file; and we will write that at the to a temporary file; and we will write that at the
beginning of our existing file. beginning of our existing file.
b) We are going back in time, appending at the end of b) We are going back in time, appending at the end of
our existing TradeHistory until the first transaction our existing TradeHistory until the first transaction
for this currencyPair for this currencyPair
''' '''
try: try:
if(temp is not None if( 'end_file' in locals() and end_file + 3600 < end):
or ('end_file' in locals() and end_file + 3600 < end)):
if (temp is None): if (temp is None):
temp = os.tmpfile() temp = os.tmpfile()
tempcsv = csv.writer(temp) tempcsv = csv.writer(temp)
for item in response.json(): for item in response.json():
if(item['tradeID'] <= last_tradeID): if( item['tradeID'] <= last_tradeID ):
continue continue
tempcsv.writerow([ tempcsv.writerow([
item['tradeID'], item['tradeID'],
@@ -208,28 +196,27 @@ class PoloniexCurator(object):
item['rate'], item['rate'],
item['amount'], item['amount'],
item['total'], item['total'],
item['globalTradeID'], item['globalTradeID']
]) ])
if(response.json()[-1]['tradeID'] > last_tradeID): if( response.json()[-1]['tradeID'] > last_tradeID ):
end = pd.to_datetime(response.json()[-1]['date'], end = pd.to_datetime( response.json()[-1]['date'],
infer_datetime_format=True infer_datetime_format=True).value // 10 ** 9
).value // 10**9 self.retrieve_trade_history(currencyPair, start,
self.retrieve_trade_history(currencyPair, start, end, temp=temp)
end, temp=temp)
else: else:
with open(csv_fn, 'rb+') as f: with open(csv_fn,'rb+') as f:
shutil.copyfileobj(f, temp) shutil.copyfileobj(f,temp)
f.seek(0) f.seek(0)
temp.seek(0) temp.seek(0)
shutil.copyfileobj(temp, f) shutil.copyfileobj(temp,f)
temp.close() temp.close()
end = start_file end = start_file
else: else:
with open(csv_fn, 'ab') as csvfile: with open(csv_fn, 'ab') as csvfile:
csvwriter = csv.writer(csvfile) csvwriter = csv.writer(csvfile)
for item in response.json(): for item in response.json():
if('first_tradeID' in locals() if( 'first_tradeID' in locals()
and item['tradeID'] >= first_tradeID): and item['tradeID'] >= first_tradeID ):
continue continue
csvwriter.writerow([ csvwriter.writerow([
item['tradeID'], item['tradeID'],
@@ -240,66 +227,70 @@ class PoloniexCurator(object):
item['total'], item['total'],
item['globalTradeID'] item['globalTradeID']
]) ])
end = pd.to_datetime(response.json()[-1]['date'], end = pd.to_datetime(response.json()[-1]['date'],
infer_datetime_format=True).value//10**9 infer_datetime_format=True).value // 10 ** 9
except Exception as e: except Exception as e:
log.error('Error opening {}'.format(csv_fn)) log.error('Error opening {}'.format(csv_fn))
log.exception(e) log.exception(e)
''' '''
If we got here, we aren't done yet. Call recursively with If we got here, we aren't done yet. Call recursively with
'end' times that go sequentially back in time. 'end' times that go sequentially back in time.
''' '''
self.retrieve_trade_history(currencyPair, start, end) self.retrieve_trade_history(currencyPair, start, end)
def generate_ohlcv(self, df): def generate_ohlcv(self, df):
''' '''
Generates OHLCV dataframe from a dataframe containing all TradeHistory Generates OHLCV dataframe from a dataframe containing all TradeHistory
by resampling with 1-minute period by resampling with 1-minute period
''' '''
df.set_index('date', inplace=True) # Index by date df.set_index('date', inplace=True) # Index by date
vol = df['total'].to_frame('volume') # set Vol aside vol = df['total'].to_frame('volume') # set Vol aside
df.drop('total', axis=1, inplace=True) # Drop volume data df.drop('total', axis=1, inplace=True) # Drop volume data
ohlc = df.resample('T').ohlc() # Resample OHLC 1min ohlc = df.resample('T').ohlc() # Resample OHLC 1min
ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Rename cols ohlc.columns = ohlc.columns.map(lambda t: t[1]) # Raname columns by dropping 'rate'
closes = ohlc['close'].fillna(method='pad') # Pad fwd missing close closes = ohlc['close'].fillna(method='pad') # Pad fwd missing 'close'
ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill NA w/ last close ohlc = ohlc.apply(lambda x: x.fillna(closes)) # Fill N/A with last close
vol = vol.resample('T').sum().fillna(0) # Add volumes by bin vol = vol.resample('T').sum().fillna(0) # Add volumes by bin
ohlcv = pd.concat([ohlc, vol], axis=1) # Concat OHLC + Vol ohlcv = pd.concat([ohlc,vol], axis=1) # Concatenate OHLC + Vol
return ohlcv return ohlcv
def write_ohlcv_file(self, currencyPair):
def write_ohlcv_file(self, currencyPair):
''' '''
Generates OHLCV data file with 1minute bars from TradeHistory on disk Generates OHLCV data file with 1minute bars from TradeHistory on disk
''' '''
csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv' csv_trades = CSV_OUT_FOLDER + 'crypto_trades-' + currencyPair + '.csv'
csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' csv_1min = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
if(os.path.getmtime(csv_1min) > time.time() - 7200): if( os.path.getmtime(csv_1min) > time.time() - 7200 ):
log.debug(currencyPair+': 1min data file already up to date. ' log.debug(currencyPair+': 1min data file already up to date. '
'Delete the file if you want to rebuild it.') 'Delete the file if you want to rebuild it.')
else: else:
df = pd.read_csv(csv_trades, df = pd.read_csv(csv_trades,
names=['tradeID', names=['tradeID',
'date', 'date',
'type', 'type',
'rate', 'rate',
'amount', 'amount',
'total', 'total',
'globalTradeID'], 'globalTradeID'],
dtype={'tradeID': int, dtype = {'tradeID': int,
'date': str, 'date': str,
'type': str, 'type': str,
'rate': float, 'rate': float,
'amount': float, 'amount': float,
'total': float, 'total': float,
'globalTradeID': int} 'globalTradeID': int }
) )
df.drop(['tradeID', 'type', 'amount', 'globalTradeID'], df.drop(['tradeID','type','amount','globalTradeID'],
axis=1, inplace=True) axis=1, inplace=True)
df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True) df['date'] = pd.to_datetime(df['date'], infer_datetime_format=True)
ohlcv = self.generate_ohlcv(df) ohlcv = self.generate_ohlcv(df)
try: try:
with open(csv_1min, 'w') as csvfile: with open(csv_1min, 'w') as csvfile:
csvwriter = csv.writer(csvfile) csvwriter = csv.writer(csvfile)
for item in ohlcv.itertuples(): for item in ohlcv.itertuples():
@@ -314,28 +305,32 @@ class PoloniexCurator(object):
item.volume, item.volume,
]) ])
except Exception as e: except Exception as e:
log.error('Error opening {}'.format(csv_1min)) log.error('Error opening {}'.format(csv_fn))
log.exception(e) log.exception(e)
log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair)) log.debug('{}: Generated 1min OHLCV data.'.format(currencyPair))
def onemin_to_dataframe(self, currencyPair, start, end): def onemin_to_dataframe(self, currencyPair, start, end):
''' '''
Returns a data frame for a given currencyPair from data on disk Returns a data frame for a given currencyPair from data on disk
''' '''
csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv' csv_fn = CSV_OUT_FOLDER + 'crypto_1min-' + currencyPair + '.csv'
df = pd.read_csv(csv_fn, names=['date', df = pd.read_csv(csv_fn, names=['date',
'open', 'open',
'high', 'high',
'low', 'low',
'close', 'close',
'volume']) 'volume']
df['date'] = pd.to_datetime(df['date'], unit='s') )
df['date'] = pd.to_datetime(df['date'],unit='s')
df.set_index('date', inplace=True) df.set_index('date', inplace=True)
return df[start:end] return df[start : end]
def generate_symbols_json(self, filename=None): def generate_symbols_json(self, filename=None):
''' '''
Generates a symbols.json file with corresponding start_date Generates a symbols.json file with corresponding start_date
for each currencyPair for each currencyPair
''' '''
symbol_map = {} symbol_map = {}
@@ -346,37 +341,36 @@ class PoloniexCurator(object):
with open(filename, 'w') as symbols: with open(filename, 'w') as symbols:
for currencyPair in self.currency_pairs: for currencyPair in self.currency_pairs:
start = None start = None
csv_fn = '{}crypto_trades-{}.csv'.format( csv_fn = '{}crypto_trades-{}.csv'.format(
CSV_OUT_FOLDER, CSV_OUT_FOLDER, currencyPair)
currencyPair) with open(csv_fn, 'r') as f:
with open(csv_fn, 'r') as f:
f.seek(0, os.SEEK_END) f.seek(0, os.SEEK_END)
if(f.tell() > 2): # Check file size is not 0 if(f.tell() > 2): # Check file size is not 0
f.seek(-2, os.SEEK_END) # Jump to 2nd last byte f.seek(-2, os.SEEK_END) # Jump to 2nd last byte
while f.read(1) != b"\n": # Until EOL is found... while f.read(1) != b"\n": # Until EOL is found...
# ...jump back the read byte plus one more. f.seek(-2, os.SEEK_CUR) # ...jump back the read byte plus one more.
f.seek(-2, os.SEEK_CUR) start = pd.to_datetime( f.readline().split(',')[1],
start = pd.to_datetime(f.readline().split(',')[1], infer_datetime_format=True)
infer_datetime_format=True)
if(start is None): if(start is None):
start = time.gmtime() start = time.gmtime()
base, market = currencyPair.lower().split('_') base, market = currencyPair.lower().split('_')
symbol = '{market}_{base}'.format(market=market, base=base) symbol = '{market}_{base}'.format( market=market, base=base )
symbol_map[currencyPair] = dict( symbol_map[currencyPair] = dict(
symbol=symbol, symbol = symbol,
start_date=start.strftime("%Y-%m-%d") start_date = start.strftime("%Y-%m-%d")
) )
json.dump(symbol_map, symbols, sort_keys=True, indent=2, json.dump(symbol_map, symbols, sort_keys=True, indent=2,
separators=(',', ':')) separators=(',',':'))
if __name__ == '__main__': if __name__ == '__main__':
pc = PoloniexCurator() pc = PoloniexCurator()
pc.get_currency_pairs() pc.get_currency_pairs()
# pc.generate_symbols_json() #pc.generate_symbols_json()
for currencyPair in pc.currency_pairs: for currencyPair in pc.currency_pairs:
pc.retrieve_trade_history(currencyPair) pc.retrieve_trade_history(currencyPair)
log.debug('{} up to date.'.format(currencyPair)) log.debug('{} up to date.'.format(currencyPair))
pc.write_ohlcv_file(currencyPair) pc.write_ohlcv_file(currencyPair)
catalyst/data/bundles/__init__.py
+1
View File
@@ -1,5 +1,6 @@
# These imports are necessary to force module-scope register calls to happen. # These imports are necessary to force module-scope register calls to happen.
from . import quandl # noqa from . import quandl # noqa
from . import poloniex
from .core import ( from .core import (
UnknownBundle, UnknownBundle,
bundles, bundles,
catalyst/data/bundles/base.py
+36 -35
View File
@@ -13,9 +13,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from itertools import count from itertools import count
import tarfile import tarfile
from time import sleep from time import time, sleep
from abc import abstractmethod, abstractproperty from abc import abstractmethod, abstractproperty
import logbook import logbook
@@ -36,7 +37,6 @@ log = logbook.Logger(__name__, level=LOG_LEVEL)
DEFAULT_RETRIES = 5 DEFAULT_RETRIES = 5
class BaseBundle(object): class BaseBundle(object):
def __init__(self, asset_filter=[]): def __init__(self, asset_filter=[]):
self._asset_filter = asset_filter self._asset_filter = asset_filter
@@ -104,11 +104,11 @@ class BaseBundle(object):
def post_process_symbol_metadata(self, metadata, data): def post_process_symbol_metadata(self, metadata, data):
return metadata return metadata
@abstractmethod @abstractmethod
def fetch_raw_symbol_frame(self, api_key, symbol, start_date, end_date): def fetch_raw_symbol_frame(self, api_key, symbol, start_date, end_date):
raise NotImplementedError() raise NotImplementedError()
def ingest(self, def ingest(self,
environ, environ,
asset_db_writer, asset_db_writer,
@@ -128,7 +128,7 @@ class BaseBundle(object):
retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5) retries = environ.get('CATALYST_DOWNLOAD_ATTEMPTS', 5)
if is_compile: if is_compile:
# User has instructed local compilation & ingestion of bundle. # User has instructed local compilation and ingestion of bundle.
# Fetch raw metadata for all symbols. # Fetch raw metadata for all symbols.
raw_metadata = self._fetch_metadata_frame( raw_metadata = self._fetch_metadata_frame(
api_key, api_key,
@@ -157,9 +157,9 @@ class BaseBundle(object):
show_progress=show_progress, show_progress=show_progress,
) )
# Post-process metadata using cached symbol frames, and write # Post-process metadata using cached symbol frames, and write to
# to disk. This metadata must be written before any attempt # disk. This metadata must be written before any attempt to write
# to write minute data. # minute data.
metadata = self._post_process_metadata( metadata = self._post_process_metadata(
raw_metadata, raw_metadata,
cache, cache,
@@ -184,11 +184,10 @@ class BaseBundle(object):
show_progress=show_progress, show_progress=show_progress,
) )
# For legacy purposes, this call is required to ensure the # For legacy purposes, this call is required to ensure the database
# database contains an appropriately initialized file # contains an appropriately initialized file structure. We don't
# structure. We don't forsee a usecase for adjustments at # forsee a usecase for adjustments at this time, but may later
# this time, but may later choose to expose this functionality # choose to expose this functionality in the future.
# in the future.
adjustment_writer.write( adjustment_writer.write(
splits=( splits=(
pd.concat(self.splits, ignore_index=True) pd.concat(self.splits, ignore_index=True)
@@ -233,12 +232,12 @@ class BaseBundle(object):
tar.extractall(output_dir) tar.extractall(output_dir)
def _fetch_metadata_frame(self, def _fetch_metadata_frame(self,
api_key, api_key,
cache, cache,
retries=DEFAULT_RETRIES, retries=DEFAULT_RETRIES,
environ=None, environ=None,
show_progress=False): show_progress=False):
# Setup raw metadata iterator to fetch pages if necessary. # Setup raw metadata iterator to fetch pages if necessary.
raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ) raw_iter = self._fetch_metadata_iter(api_key, cache, retries, environ)
@@ -252,7 +251,7 @@ class BaseBundle(object):
show_percent=False, show_percent=False,
) as blocks: ) as blocks:
metadata = pd.concat(blocks, ignore_index=True) metadata = pd.concat(blocks, ignore_index=True)
return metadata return metadata
def _fetch_metadata_iter(self, api_key, cache, retries, environ): def _fetch_metadata_iter(self, api_key, cache, retries, environ):
@@ -270,20 +269,21 @@ class BaseBundle(object):
page_number, page_number,
) )
break break
except ValueError: except ValueError as e:
raw = pd.DataFrame([]) raw = pd.DataFrame([])
break break
except Exception: except Exception as e:
log.exception( log.exception(
'Failed to load metadata from {}. ' 'Failed to load metadata from {}. '
'Retrying.'.format(self.name) 'Retrying.'.format(self.name)
) )
else: else:
raise ValueError( raise ValueError(
'Failed to download metadata page {} after {} ' 'Failed to download metadata page {} after {} '
'attempts.'.format(page_number, retries) 'attempts.'.format(page_number, retries)
) )
if raw.empty: if raw.empty:
# Empty DataFrame signals completion. # Empty DataFrame signals completion.
break break
@@ -305,7 +305,7 @@ class BaseBundle(object):
columns=self.md_column_names, columns=self.md_column_names,
index=metadata.index, index=metadata.index,
) )
# Iterate over the available symbols, loading the asset's raw symbol # Iterate over the available symbols, loading the asset's raw symbol
# data from the cache. The final metadata is computed and recorded in # data from the cache. The final metadata is computed and recorded in
# the appropriate row depending on the asset's id. # the appropriate row depending on the asset's id.
@@ -318,22 +318,22 @@ class BaseBundle(object):
show_percent=False, show_percent=False,
) as symbols_map: ) as symbols_map:
for asset_id, symbol in symbols_map: for asset_id, symbol in symbols_map:
# Attempt to load data from disk, the cache should have an # Attempt to load data from disk, the cache should have an entry
# entry for each symbol at this point of the execution. If one # for each symbol at this point of the execution. If one does
# does not exist, we should fail. # not exist, we should fail.
key = '{sym}.daily.frame'.format(sym=symbol) key = '{sym}.daily.frame'.format(sym=symbol)
try: try:
raw_data = cache[key] raw_data = cache[key]
except KeyError: except KeyError:
raise ValueError( raise ValueError(
'Unable to find cached data for symbol:' 'Unable to find cached data for symbol: {0}'.format(symbol)
' {0}'.format(symbol)) )
# Perform and require post-processing of metadata. # Perform and require post-processing of metadata.
final_symbol_metadata = self.post_process_symbol_metadata( final_symbol_metadata = self.post_process_symbol_metadata(
asset_id, asset_id,
metadata.iloc[asset_id], metadata.iloc[asset_id],
raw_data, raw_data,
) )
# Record symbol's final metadata. # Record symbol's final metadata.
@@ -363,8 +363,8 @@ class BaseBundle(object):
# returns the cached data unaltered. The `should_sleep` flag # returns the cached data unaltered. The `should_sleep` flag
# indicates that an API call was attempted, and that we should be # indicates that an API call was attempted, and that we should be
# ensure aren't exceeding our rate limit before proceeding to the # ensure aren't exceeding our rate limit before proceeding to the
# next symbol. If the raw_data is updated, it is cached before # next symbol. If the raw_data is updated, it is cached before being
# being returned. # returned.
raw_data, should_sleep = self._maybe_update_symbol_frame( raw_data, should_sleep = self._maybe_update_symbol_frame(
start_time, start_time,
api_key, api_key,
@@ -414,7 +414,7 @@ class BaseBundle(object):
last = start_session last = start_session
if raw_data is not None and len(raw_data) > 0: if raw_data is not None and len(raw_data) > 0:
last = raw_data.index[-1].tz_localize('UTC') last = raw_data.index[-1].tz_localize('UTC')
should_sleep = False should_sleep = False
# Determine time at which cached data will be considered stale. # Determine time at which cached data will be considered stale.
@@ -455,7 +455,7 @@ class BaseBundle(object):
retries=DEFAULT_RETRIES): retries=DEFAULT_RETRIES):
# Data for symbol is old enough to attempt an update or is not # Data for symbol is old enough to attempt an update or is not
# present in the cache. Fetch raw data for a single symbol # present in the cache. Fetch raw data for a single symbol
# with requested intervals and frequency. Retry as necessary. # with requested intervals and frequency. Retry as necessary.
for _ in range(retries): for _ in range(retries):
try: try:
@@ -468,6 +468,7 @@ class BaseBundle(object):
data_frequency, data_frequency,
) )
raw_data.index = pd.to_datetime(raw_data.index, utc=True) raw_data.index = pd.to_datetime(raw_data.index, utc=True)
#raw_data.index = raw_data.index.tz_localize('UTC')
# Filter incoming data to fit start and end sessions. # Filter incoming data to fit start and end sessions.
raw_data = raw_data[ raw_data = raw_data[
@@ -481,7 +482,7 @@ class BaseBundle(object):
return raw_data return raw_data
except Exception: except Exception as e:
log.exception( log.exception(
'Exception raised fetching {name} data. Retrying.' 'Exception raised fetching {name} data. Retrying.'
.format(name=self.name) .format(name=self.name)
catalyst/data/bundles/base_pricing.py
-3
View File
@@ -16,7 +16,6 @@
from catalyst.data.bundles.base import BaseBundle from catalyst.data.bundles.base import BaseBundle
from catalyst.utils.memoize import lazyval from catalyst.utils.memoize import lazyval
class BasePricingBundle(BaseBundle): class BasePricingBundle(BaseBundle):
@lazyval @lazyval
def md_dtypes(self): def md_dtypes(self):
@@ -39,7 +38,6 @@ class BasePricingBundle(BaseBundle):
('volume', 'float64'), ('volume', 'float64'),
] ]
class BaseCryptoPricingBundle(BasePricingBundle): class BaseCryptoPricingBundle(BasePricingBundle):
@lazyval @lazyval
def calendar_name(self): def calendar_name(self):
@@ -57,7 +55,6 @@ class BaseCryptoPricingBundle(BasePricingBundle):
def dividends(self): def dividends(self):
return [] return []
class BaseEquityPricingBundle(BasePricingBundle): class BaseEquityPricingBundle(BasePricingBundle):
@lazyval @lazyval
def calendar_name(self): def calendar_name(self):
catalyst/data/bundles/core.py
+1 -4
View File
@@ -37,7 +37,6 @@ from catalyst.utils.cli import maybe_show_progress
ONE_MEGABYTE = 1024 * 1024 ONE_MEGABYTE = 1024 * 1024
def asset_db_path(bundle_name, timestr, environ=None, db_version=None): def asset_db_path(bundle_name, timestr, environ=None, db_version=None):
return pth.data_path( return pth.data_path(
asset_db_relative(bundle_name, timestr, environ, db_version), asset_db_relative(bundle_name, timestr, environ, db_version),
@@ -136,7 +135,6 @@ def ingestions_for_bundle(bundle, environ=None):
reverse=True, reverse=True,
) )
def download_with_progress(url, chunk_size, **progress_kwargs): def download_with_progress(url, chunk_size, **progress_kwargs):
""" """
Download streaming data from a URL, printing progress information to the Download streaming data from a URL, printing progress information to the
@@ -707,5 +705,4 @@ def _make_bundle_core():
) )
bundles, register_bundle, register, unregister, ingest, load, clean = \ bundles, register_bundle, register, unregister, ingest, load, clean = _make_bundle_core()
_make_bundle_core()
catalyst/data/bundles/poloniex.py
+18 -16
View File
@@ -14,17 +14,19 @@
# limitations under the License. # limitations under the License.
import sys import sys
from six.moves.urllib.parse import urlencode
from datetime import datetime
import pandas as pd import pandas as pd
from six.moves.urllib.parse import urlencode
from catalyst.data.bundles.core import register_bundle from catalyst.data.bundles.core import register_bundle
from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
from catalyst.utils.memoize import lazyval from catalyst.utils.memoize import lazyval
from catalyst.curate.poloniex import PoloniexCurator from catalyst.curate.poloniex import PoloniexCurator
class PoloniexBundle(BaseCryptoPricingBundle): class PoloniexBundle(BaseCryptoPricingBundle):
@lazyval @lazyval
def name(self): def name(self):
@@ -44,8 +46,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
@lazyval @lazyval
def tar_url(self): def tar_url(self):
return ( return (
'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
'poloniex/poloniex-bundle.tar.gz'
) )
@lazyval @lazyval
@@ -66,11 +67,12 @@ class PoloniexBundle(BaseCryptoPricingBundle):
raw = raw.sort_index().reset_index() raw = raw.sort_index().reset_index()
raw.rename( raw.rename(
columns={'index': 'symbol'}, columns={'index':'symbol'},
inplace=True, inplace=True,
) )
raw = raw[raw['isFrozen'] == 0] raw = raw[raw['isFrozen'] == 0]
return raw return raw
def post_process_symbol_metadata(self, asset_id, sym_md, sym_data): def post_process_symbol_metadata(self, asset_id, sym_md, sym_data):
@@ -96,8 +98,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
frequency): frequency):
# TODO: replace this with direct exchange call # TODO: replace this with direct exchange call
# The end date and frequency should be used to # The end date and frequency should be used to calculate the number of bars
# calculate the number of bars
if(frequency == 'minute'): if(frequency == 'minute'):
pc = PoloniexCurator() pc = PoloniexCurator()
raw = pc.onemin_to_dataframe(symbol, start_date, end_date) raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
@@ -115,9 +116,8 @@ class PoloniexBundle(BaseCryptoPricingBundle):
) )
raw.set_index('date', inplace=True) raw.set_index('date', inplace=True)
# BcolzDailyBarReader introduces a 1/1000 factor in the way # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
# pricing is stored on disk, which we compensate here to get # on disk, which we compensate here to get the right pricing amounts
# the right pricing amounts
# ref: data/us_equity_pricing.py # ref: data/us_equity_pricing.py
scale = 1 scale = 1
raw.loc[:, 'open'] /= scale raw.loc[:, 'open'] /= scale
@@ -139,6 +139,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
return self._format_polo_query(query_params) return self._format_polo_query(query_params)
def _format_data_url(self, def _format_data_url(self,
api_key, api_key,
symbol, symbol,
@@ -161,26 +162,27 @@ class PoloniexBundle(BaseCryptoPricingBundle):
('end', end_date.value / 10**9), ('end', end_date.value / 10**9),
('period', period), ('period', period),
] ]
return self._format_polo_query(query_params) return self._format_polo_query(query_params)
def _format_polo_query(self, query_params): def _format_polo_query(self, query_params):
# TODO: got against the exchange object # TODO: got against the exchange object
return 'https://poloniex.com/public?{query}'.format( return 'https://poloniex.com/public?{query}'.format(
query=urlencode(query_params), query=urlencode(query_params),
) )
'''
''' As a second parameter, you can pass an array of currency pairs
As a second parameter, you can pass an array of currency pairs that will be processed as an asset_filter to only process that
that will be processed as an asset_filter to only process that
subset of assets in the bundle, such as: subset of assets in the bundle, such as:
register_bundle(PoloniexBundle, ['USDT_BTC',]) register_bundle(PoloniexBundle, ['USDT_BTC',])
For a production environment make sure to use (to bundle all pairs): For a production environment make sure to use (to bundle all pairs):
register_bundle(PoloniexBundle) register_bundle(PoloniexBundle)
''' '''
if 'ingest' in sys.argv and '-c' in sys.argv: if 'ingest' in sys.argv and '-c' in sys.argv:
register_bundle(PoloniexBundle) register_bundle(PoloniexBundle)
else: else:
register_bundle(PoloniexBundle, create_writers=False) register_bundle(PoloniexBundle, create_writers=False)
catalyst/data/bundles/quandl.py
+19 -8
View File
@@ -16,6 +16,7 @@
from datetime import datetime from datetime import datetime
import pandas as pd import pandas as pd
from six.moves.urllib.parse import urlencode from six.moves.urllib.parse import urlencode
from catalyst.data.bundles.core import register_bundle from catalyst.data.bundles.core import register_bundle
@@ -25,16 +26,25 @@ from catalyst.utils.memoize import lazyval
""" """
Module for building a complete daily dataset from Quandl's WIKI dataset. Module for building a complete daily dataset from Quandl's WIKI dataset.
""" """
from itertools import count
import tarfile
from time import time, sleep
from datetime import datetime
from logbook import Logger from logbook import Logger
import pandas as pd
from six.moves.urllib.parse import urlencode
from catalyst.utils.calendars import register_calendar_alias
from catalyst.utils.cli import maybe_show_progress
from . import core as bundles
from catalyst.constants import LOG_LEVEL from catalyst.constants import LOG_LEVEL
from catalyst.utils.calendars import register_calendar_alias
log = Logger(__name__, level=LOG_LEVEL) log = Logger(__name__, level=LOG_LEVEL)
seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds() seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
class QuandlBundle(BaseEquityPricingBundle): class QuandlBundle(BaseEquityPricingBundle):
@lazyval @lazyval
def name(self): def name(self):
@@ -99,8 +109,8 @@ class QuandlBundle(BaseEquityPricingBundle):
# Filter out invalid symbols # Filter out invalid symbols
raw = raw[~raw.symbol.isin(self._excluded_symbols)] raw = raw[~raw.symbol.isin(self._excluded_symbols)]
# cut out all the other stuff in the name column. We need to # cut out all the other stuff in the name column
# escape the paren because it is actually splitting on a regex # we need to escape the paren because it is actually splitting on a regex
raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0) raw.asset_name = raw.asset_name.str.split(r' \(', 1).str.get(0)
return raw return raw
@@ -165,6 +175,7 @@ class QuandlBundle(BaseEquityPricingBundle):
df['sid'] = asset_id df['sid'] = asset_id
self.splits.append(df) self.splits.append(df)
def _update_dividends(self, asset_id, raw_data): def _update_dividends(self, asset_id, raw_data):
divs = raw_data.ex_dividend divs = raw_data.ex_dividend
df = pd.DataFrame({'amount': divs[divs != 0]}) df = pd.DataFrame({'amount': divs[divs != 0]})
@@ -175,6 +186,7 @@ class QuandlBundle(BaseEquityPricingBundle):
df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT df['record_date'] = df['declared_date'] = df['pay_date'] = pd.NaT
self.dividends.append(df) self.dividends.append(df)
def _format_metadata_url(self, api_key, page_number): def _format_metadata_url(self, api_key, page_number):
"""Build the query RL for the quandl WIKI metadata. """Build the query RL for the quandl WIKI metadata.
""" """
@@ -188,10 +200,10 @@ class QuandlBundle(BaseEquityPricingBundle):
query_params = [('api_key', api_key)] + query_params query_params = [('api_key', api_key)] + query_params
return ( return (
'https://www.quandl.com/api/v3/datasets.csv?' 'https://www.quandl.com/api/v3/datasets.csv?' + urlencode(query_params)
+ urlencode(query_params)
) )
def _format_wiki_url(self, def _format_wiki_url(self,
api_key, api_key,
symbol, symbol,
@@ -217,6 +229,5 @@ class QuandlBundle(BaseEquityPricingBundle):
) )
) )
register_calendar_alias('QUANDL', 'NYSE') register_calendar_alias('QUANDL', 'NYSE')
register_bundle(QuandlBundle) register_bundle(QuandlBundle)
catalyst/data/data_portal.py
+11 -10
View File
@@ -20,6 +20,7 @@ import numpy as np
from numpy import float64, int64, nan from numpy import float64, int64, nan
import pandas as pd import pandas as pd
from pandas import isnull from pandas import isnull
from pandas.tslib import normalize_date
from six import iteritems from six import iteritems
from six.moves import reduce from six.moves import reduce
@@ -438,7 +439,7 @@ class DataPortal(object):
(isinstance(asset, (Asset, ContinuousFuture)))) (isinstance(asset, (Asset, ContinuousFuture))))
def _get_fetcher_value(self, asset, field, dt): def _get_fetcher_value(self, asset, field, dt):
day = dt.normalize() day = normalize_date(dt)
try: try:
return \ return \
@@ -655,11 +656,11 @@ class DataPortal(object):
return spot_value return spot_value
def _get_minutely_spot_value(self, def _get_minutely_spot_value(self,
asset, asset,
column, column,
dt, dt,
data_frequency, data_frequency,
ffill=False): ffill=False):
reader = self._get_pricing_reader(data_frequency) reader = self._get_pricing_reader(data_frequency)
@@ -705,7 +706,7 @@ class DataPortal(object):
asset, asset,
column, column,
dt, dt,
ffill, ffill,
'minute', 'minute',
) )
@@ -1129,7 +1130,7 @@ class DataPortal(object):
if self._asset_start_dates[sid] > dt: if self._asset_start_dates[sid] > dt:
raise NoTradeDataAvailableTooEarly( raise NoTradeDataAvailableTooEarly(
sid=sid, sid=sid,
dt=dt.normalize(), dt=normalize_date(dt),
start_dt=start_date start_dt=start_date
) )
@@ -1137,7 +1138,7 @@ class DataPortal(object):
if self._asset_end_dates[sid] < dt: if self._asset_end_dates[sid] < dt:
raise NoTradeDataAvailableTooLate( raise NoTradeDataAvailableTooLate(
sid=sid, sid=sid,
dt=dt.normalize(), dt=normalize_date(dt),
end_dt=end_date end_dt=end_date
) )
@@ -1261,7 +1262,7 @@ class DataPortal(object):
if self._extra_source_df is None: if self._extra_source_df is None:
return [] return []
day = dt.normalize() day = normalize_date(dt)
if day in self._extra_source_df.index: if day in self._extra_source_df.index:
assets = self._extra_source_df.loc[day]['sid'] assets = self._extra_source_df.loc[day]['sid']
catalyst/data/dispatch_bar_reader.py
+2 -4
View File
@@ -88,11 +88,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
if self._last_available_dt is not None: if self._last_available_dt is not None:
return self._last_available_dt return self._last_available_dt
else: else:
return min(r.last_available_dt for r in list(self._readers.values())) return min(r.last_available_dt for r in self._readers.values())
@lazyval @lazyval
def first_trading_day(self): def first_trading_day(self):
return max(r.first_trading_day for r in list(self._readers.values())) return max(r.first_trading_day for r in self._readers.values())
def get_value(self, sid, dt, field): def get_value(self, sid, dt, field):
asset = self._asset_finder.retrieve_asset(sid) asset = self._asset_finder.retrieve_asset(sid)
@@ -133,13 +133,11 @@ class AssetDispatchBarReader(with_metaclass(ABCMeta)):
return results return results
class AssetDispatchMinuteBarReader(AssetDispatchBarReader): class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
def _dt_window_size(self, start_dt, end_dt): def _dt_window_size(self, start_dt, end_dt):
return len(self.trading_calendar.minutes_in_range(start_dt, end_dt)) return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
class AssetDispatchSessionBarReader(AssetDispatchBarReader): class AssetDispatchSessionBarReader(AssetDispatchBarReader):
def _dt_window_size(self, start_dt, end_dt): def _dt_window_size(self, start_dt, end_dt):
catalyst/data/history_loader.py
+3 -2
View File
@@ -21,6 +21,7 @@ from abc import (
from numpy import concatenate from numpy import concatenate
from lru import LRU from lru import LRU
from pandas import isnull from pandas import isnull
from pandas.tslib import normalize_date
from toolz import sliding_window from toolz import sliding_window
from six import with_metaclass from six import with_metaclass
@@ -92,8 +93,8 @@ class HistoryCompatibleUSEquityAdjustmentReader(object):
The adjustments as a dict of loc -> Float64Multiply The adjustments as a dict of loc -> Float64Multiply
""" """
sid = int(asset) sid = int(asset)
start = dts[0].normalize() start = normalize_date(dts[0])
end = dts[-1].normalize() end = normalize_date(dts[-1])
adjs = {} adjs = {}
if field != 'volume': if field != 'volume':
mergers = self._adjustments_reader.get_adjustments_for_sid( mergers = self._adjustments_reader.get_adjustments_for_sid(
catalyst/data/loader.py
+83 -24
View File
@@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import datetime
import os import os
from collections import OrderedDict from collections import OrderedDict
@@ -22,7 +23,6 @@ from pandas_datareader.data import DataReader
from six import iteritems from six import iteritems
from six.moves.urllib_error import HTTPError from six.moves.urllib_error import HTTPError
from catalyst.constants import LOG_LEVEL
from catalyst.utils.calendars import get_calendar from catalyst.utils.calendars import get_calendar
from . import treasuries, treasuries_can from . import treasuries, treasuries_can
from .benchmarks import get_benchmark_returns from .benchmarks import get_benchmark_returns
@@ -32,6 +32,8 @@ from ..utils.paths import (
data_root, data_root,
) )
from catalyst.constants import LOG_LEVEL
logger = logbook.Logger('Loader', level=LOG_LEVEL) logger = logbook.Logger('Loader', level=LOG_LEVEL)
# Mapping from index symbol to appropriate bond data # Mapping from index symbol to appropriate bond data
@@ -101,7 +103,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
trading_day = get_calendar('OPEN').trading_day trading_day = get_calendar('OPEN').trading_day
# TODO: consider making configurable # TODO: consider making configurable
bm_symbol = 'btc_usd' bm_symbol = 'btc_usdt'
# if trading_days is None: # if trading_days is None:
# trading_days = get_calendar('OPEN').schedule # trading_days = get_calendar('OPEN').schedule
@@ -127,13 +129,11 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
# before this date. # before this date.
''' '''
if(bundle_data): if(bundle_data):
# If we are using the bundle to retrieve the cryptobenchmark, find # If we are using the bundle to retrieve the cryptobenchmark, find the last
# the last date for which there is trading data in the bundle # date for which there is trading data in the bundle
asset = bundle_data.asset_finder.lookup_symbol( asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
symbol=bm_symbol,as_of_date=None)
ix = bundle_data.daily_bar_reader._last_rows[asset.sid] ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
last_date = pd.to_datetime( last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
else: else:
last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2] last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
''' '''
@@ -142,11 +142,10 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
if exchange is None: if exchange is None:
# This is exceptional, since placing the import at the module scope # This is exceptional, since placing the import at the module scope
# breaks things and it's only needed here # breaks things and it's only needed here
from catalyst.exchange.utils.factory import get_exchange from catalyst.exchange.factory import get_exchange
exchange = get_exchange( exchange = get_exchange(
exchange_name='bitfinex', base_currency='usd' exchange_name='poloniex', base_currency='usdt'
) )
exchange.init()
benchmark_asset = exchange.get_asset(bm_symbol) benchmark_asset = exchange.get_asset(bm_symbol)
@@ -165,8 +164,8 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
br.loc[start_dt] = 0 br.loc[start_dt] = 0
br = br.sort_index() br = br.sort_index()
# Override first_date for treasury data since we have it for many more # Override first_date for treasury data since we have it for many more years
# years and is independent of crypto data # and is independent of crypto data
first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC') first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC')
tc = ensure_treasury_data( tc = ensure_treasury_data(
bm_symbol, bm_symbol,
@@ -302,14 +301,14 @@ def ensure_crypto_benchmark_data(symbol,
if (bundle == 'poloniex'): if (bundle == 'poloniex'):
''' '''
If we're using the Poloniex bundle, we'll get the benchmark from the If we're using the Poloniex bundle, we'll get the benchmark from the bundle
bundle instead of downloading it from Poloniex every time we need it. instead of downloading it from Poloniex every time we need it.
Poloniex has a captcha for API queries originating from outside the US Poloniex has a captcha for API queries originating from outside the US that
that prevents users abroad from getting Catalyst to work prevents users abroad from getting Catalyst to work
''' '''
logger.info( logger.info(
('Retrieving benchmark data from bundle for {symbol!r}' (
' from {first_date} to {last_date}'), 'Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
symbol=symbol, first_date=first_date, last_date=last_date) symbol=symbol, first_date=first_date, last_date=last_date)
asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol, asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,
@@ -331,12 +330,11 @@ def ensure_crypto_benchmark_data(symbol,
last_date)] last_date)]
else: else:
# This is how it used to be: downloading the benchmark everytime. # This is how it used to be: downloading the benchmark everytime.
# Leaving this code here to be repurposed in the future for # Leaving this code here to be repurposed in the future for other bundles.
# other bundles.
logger.info( logger.info(
('Downloading benchmark data for {symbol!r}' (
' from {first_date} to {last_date}'), 'Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
symbol=symbol, first_date=first_date, last_date=last_date) symbol=symbol, first_date=first_date, last_date=last_date)
raise DeprecationWarning('poloniex bundle deprecated') raise DeprecationWarning('poloniex bundle deprecated')
@@ -433,6 +431,67 @@ def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
return data return data
def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,
environ=None):
"""
Ensure we have benchmark data for `symbol` from `first_date` to `last_date`
Parameters
----------
symbol : str
The symbol for the benchmark to load.
first_date : pd.Timestamp
First required date for the cache.
last_date : pd.Timestamp
Last required date for the cache.
now : pd.Timestamp
The current time. This is used to prevent repeated attempts to
re-download data that isn't available due to scheduling quirks or other
failures.
trading_day : pd.CustomBusinessDay
A trading day delta. Used to find the day before first_date so we can
get the close of the day prior to first_date.
We attempt to download data unless we already have data stored at the data
cache for `symbol` whose first entry is before or on `first_date` and whose
last entry is on or after `last_date`.
If we perform a download and the cache criteria are not satisfied, we wait
at least one hour before attempting a redownload. This is determined by
comparing the current time to the result of os.path.getmtime on the cache
path.
"""
filename = get_benchmark_filename(symbol)
data = _load_cached_data(filename, first_date, last_date, now, 'benchmark',
environ)
if data is not None:
return data
# If no cached data was found or it was missing any dates then download the
# necessary data.
logger.info(
('Downloading benchmark data for {symbol!r} '
'from {first_date} to {last_date}'),
symbol=symbol,
first_date=first_date - trading_day,
last_date=last_date
)
try:
data = get_benchmark_returns(
symbol,
first_date - trading_day,
last_date,
)
data.to_csv(get_data_filepath(filename, environ))
except (OSError, IOError, HTTPError):
logger.exception('Failed to cache the new benchmark returns')
raise
if not has_data_for_dates(data, first_date, last_date):
logger.warn("Still don't have expected data after redownload!")
return data
def ensure_treasury_data(symbol, first_date, last_date, now, environ=None): def ensure_treasury_data(symbol, first_date, last_date, now, environ=None):
""" """
Ensure we have treasury data from treasury module associated with Ensure we have treasury data from treasury module associated with
catalyst/data/minute_bars.py
+11 -8
View File
@@ -341,10 +341,12 @@ class BcolzMinuteBarMetadata(object):
'end_session': str(self.end_session.date()), 'end_session': str(self.end_session.date()),
# Write these values for backwards compatibility # Write these values for backwards compatibility
'first_trading_day': str(self.start_session.date()), 'first_trading_day': str(self.start_session.date()),
'market_opens': (market_opens.values.astype('datetime64[m]'). 'market_opens': (
astype(np.int64).tolist()), market_opens.values.astype('datetime64[m]').
'market_closes': (market_closes.values.astype('datetime64[m]'). astype(np.int64).tolist()),
astype(np.int64).tolist()), 'market_closes': (
market_closes.values.astype('datetime64[m]').
astype(np.int64).tolist()),
} }
with open(self.metadata_path(rootdir), 'w+') as fp: with open(self.metadata_path(rootdir), 'w+') as fp:
json.dump(metadata, fp) json.dump(metadata, fp)
@@ -1254,8 +1256,8 @@ class BcolzMinuteBarReader(MinuteBarReader):
values = carray[start_idx:end_idx + 1] values = carray[start_idx:end_idx + 1]
if indices_to_exclude is not None: if indices_to_exclude is not None:
for excl_start, excl_stop in indices_to_exclude[::-1]: for excl_start, excl_stop in indices_to_exclude[::-1]:
excl_slice = np.s_[excl_start - start_idx:excl_stop excl_slice = np.s_[
- start_idx + 1] excl_start - start_idx:excl_stop - start_idx + 1]
values = np.delete(values, excl_slice) values = np.delete(values, excl_slice)
where = values != 0 where = values != 0
@@ -1318,8 +1320,9 @@ class H5MinuteBarUpdateWriter(object):
def __init__(self, path, complevel=None, complib=None): def __init__(self, path, complevel=None, complib=None):
self._complevel = complevel if complevel \ self._complevel = complevel if complevel \
is not None else self._COMPLEVEL is not None else self._COMPLEVEL
self._complib = complib if complib is not None else self._COMPLIB self._complib = complib if complib \
is not None else self._COMPLIB
self._path = path self._path = path
def write(self, frames): def write(self, frames):
catalyst/data/us_equity_pricing.py
+9 -12
View File
@@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from __future__ import division # Python2 req for division of ints yield float from __future__ import division # Python2 req to have division of ints yield float
from errno import ENOENT from errno import ENOENT
from functools import partial from functools import partial
@@ -49,6 +49,7 @@ from pandas import (
to_datetime, to_datetime,
Timestamp, Timestamp,
) )
from pandas.tslib import iNaT
from six import ( from six import (
iteritems, iteritems,
string_types, string_types,
@@ -119,8 +120,7 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
UINT32_MAX = iinfo(uint32).max UINT32_MAX = iinfo(uint32).max
UINT64_MAX = iinfo(uint64).max UINT64_MAX = iinfo(uint64).max
# Provides 9 decimals resolution. Also affects _equities.pyx L220 PRICE_ADJUSTMENT_FACTOR = 1000000000 # Provides 9 decimals resolution. Also affects _equities.pyx L220
PRICE_ADJUSTMENT_FACTOR = 1000000000
def check_uint32_safe(value, colname): def check_uint32_safe(value, colname):
@@ -130,7 +130,6 @@ def check_uint32_safe(value, colname):
"for uint32" % (value, colname) "for uint32" % (value, colname)
) )
def check_uint64_safe(value, colname): def check_uint64_safe(value, colname):
if value >= UINT64_MAX: if value >= UINT64_MAX:
raise ValueError( raise ValueError(
@@ -323,8 +322,8 @@ class BcolzDailyBarWriter(object):
# Maps column name -> output carray. # Maps column name -> output carray.
columns = { columns = {
k: carray(array([], dtype=uint64)) k: carray(array([], dtype=uint64))
if k in OHLCV if k in OHLCV
else carray(array([], dtype=uint32)) else carray(array([], dtype=uint32))
for k in US_EQUITY_PRICING_BCOLZ_COLUMNS for k in US_EQUITY_PRICING_BCOLZ_COLUMNS
} }
@@ -421,7 +420,7 @@ class BcolzDailyBarWriter(object):
) )
full_table.attrs['first_trading_day'] = ( full_table.attrs['first_trading_day'] = (
earliest_date if earliest_date is not None else NaT earliest_date if earliest_date is not None else iNaT
) )
full_table.attrs['first_row'] = first_row full_table.attrs['first_row'] = first_row
@@ -440,13 +439,11 @@ class BcolzDailyBarWriter(object):
return raw_data return raw_data
winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC) winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
processed = (raw_data[list(OHLC)] processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
* PRICE_ADJUSTMENT_FACTOR).astype('uint64')
dates = raw_data.index.values.astype('datetime64[s]') dates = raw_data.index.values.astype('datetime64[s]')
check_uint32_safe(dates.max().view(np.int64), 'day') check_uint32_safe(dates.max().view(np.int64), 'day')
processed['day'] = dates.astype('uint32') processed['day'] = dates.astype('uint32')
processed['volume'] = (raw_data.volume processed['volume'] = (raw_data.volume * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
* PRICE_ADJUSTMENT_FACTOR).astype('uint64')
return ctable.fromdataframe(processed) return ctable.fromdataframe(processed)
@@ -499,7 +496,7 @@ class BcolzDailyBarReader(SessionBarReader):
The data in these columns is interpreted as follows: The data in these columns is interpreted as follows:
- Price columns ('open', 'high', 'low', 'close') and Volume are interpreted - Price columns ('open', 'high', 'low', 'close') and Volume are interpreted
as 10^9 * as-traded dollar value. as 10^9 * as-traded dollar value.
- Day is interpreted as seconds since midnight UTC, Jan 1, 1970. - Day is interpreted as seconds since midnight UTC, Jan 1, 1970.
- Id is the asset id of the row. - Id is the asset id of the row.
catalyst/examples/arbitrage_with_interface.py
+11 -18
View File
@@ -6,7 +6,7 @@ from catalyst.api import (
symbol, symbol,
get_open_orders get_open_orders
) )
from catalyst.exchange.utils.stats_utils import get_pretty_stats from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm from catalyst.utils.run_algo import run_algorithm
algo_namespace = 'arbitrage_eth_btc' algo_namespace = 'arbitrage_eth_btc'
@@ -263,20 +263,13 @@ def analyze(context, stats):
pass pass
if __name__ == '__main__': run_algorithm(
# The execution mode: backtest or live initialize=initialize,
MODE = 'live' handle_data=handle_data,
if MODE == 'live': analyze=analyze,
run_algorithm( exchange_name='poloniex,bitfinex',
capital_base=0.1, live=True,
initialize=initialize, algo_namespace=algo_namespace,
handle_data=handle_data, quote_currency='btc',
analyze=analyze, live_graph=False
exchange_name='poloniex,bitfinex', )
live=True,
algo_namespace=algo_namespace,
base_currency='btc',
live_graph=False,
simulate_orders=True,
stats_output=None,
)
catalyst/examples/buy_and_hodl.py
+5 -4
View File
@@ -19,11 +19,11 @@ import matplotlib.pyplot as plt
from catalyst import run_algorithm from catalyst import run_algorithm
from catalyst.api import (order_target_value, symbol, record, from catalyst.api import (order_target_value, symbol, record,
cancel_order, get_open_orders, ) cancel_order, get_open_orders, )
def initialize(context): def initialize(context):
context.ASSET_NAME = 'btc_usdt' context.ASSET_NAME = 'btc_usd'
context.TARGET_HODL_RATIO = 0.8 context.TARGET_HODL_RATIO = 0.8
context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
@@ -61,6 +61,7 @@ def handle_data(context, data):
context.asset, context.asset,
target_hodl_value, target_hodl_value,
limit_price=price * 1.1, limit_price=price * 1.1,
stop_price=price * 0.9,
) )
record( record(
@@ -140,9 +141,9 @@ if __name__ == '__main__':
initialize=initialize, initialize=initialize,
handle_data=handle_data, handle_data=handle_data,
analyze=analyze, analyze=analyze,
exchange_name='poloniex', exchange_name='bitfinex',
algo_namespace='buy_and_hodl', algo_namespace='buy_and_hodl',
base_currency='usdt', base_currency='usd',
start=pd.to_datetime('2015-03-01', utc=True), start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True), end=pd.to_datetime('2017-10-31', utc=True),
) )
catalyst/examples/buy_btc_simple.py
+17 -36
View File
@@ -1,49 +1,30 @@
''' '''
This is a very simple example referenced in the beginner's tutorial: This is a very simple example referenced in the beginner's tutorial:
https://enigmampc.github.io/catalyst/beginner-tutorial.html https://enigmampc.github.io/catalyst/beginner-tutorial.html
Run this example, by executing the following from your terminal: Run this example, by executing the following from your terminal:
catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 \ catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
--end 2017-9-30 -o buy_btc_simple_out.pickle
If you want to run this code using another exchange, make sure that If you want to run this code using another exchange, make sure that
the asset is available on that exchange. For example, if you were to run the asset is available on that exchange. For example, if you were to run
it for exchange Poloniex, you would need to edit the following line: it for exchange Poloniex, you would need to edit the following line:
context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd' context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd'
and specify exchange poloniex as follows: and specify exchange poloniex as follows:
catalyst ingest-exchange -x poloniex -f daily -i btc_usdt catalyst ingest-exchange -x poloniex -f daily -i btc_usdt
catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 \ catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
--end 2017-9-30 -o buy_btc_simple_out.pickle
To see which assets are available on each exchange, visit: To see which assets are available on each exchange, visit:
https://www.enigma.co/catalyst/status https://www.enigma.co/catalyst/status
''' '''
from catalyst import run_algorithm
from catalyst.api import order, record, symbol from catalyst.api import order, record, symbol
import pandas as pd
def initialize(context): def initialize(context):
context.asset = symbol('btc_usdt') context.asset = symbol('btc_usd')
def handle_data(context, data): def handle_data(context, data):
order(context.asset, 1) order(context.asset, 1)
record(btc=data.current(context.asset, 'price')) record(btc = data.current(context.asset, 'price'))
if __name__ == '__main__':
run_algorithm(
capital_base=10000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
exchange_name='poloniex',
algo_namespace='buy_and_hodl',
base_currency='usdt',
start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True),
)
catalyst/examples/buy_low_sell_high.py
+41 -53
View File
@@ -1,5 +1,15 @@
'''
This algorithm requires an additional library (ta-lib) beyond those required by catalyst.
Install it first by running:
$ pip install TA-Lib
If you get build errors like "fatal error: ta-lib/ta_libc.h: No such file or directory"
it typically means that it can't find the underlying TA-Lib library and needs to be installed.
See https://mrjbq7.github.io/ta-lib/install.html for instructions on how to install
the required dependencies.
'''
import talib import talib
import pandas as pd
from logbook import Logger from logbook import Logger
from catalyst.api import ( from catalyst.api import (
@@ -7,59 +17,66 @@ from catalyst.api import (
order_target_percent, order_target_percent,
symbol, symbol,
record, record,
get_open_orders,
) )
from catalyst.exchange.utils.stats_utils import get_pretty_stats from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm
algo_namespace = 'buy_the_dip_live' algo_namespace = 'buy_low_sell_high_xrp'
log = Logger('buy low sell high') log = Logger(algo_namespace)
def initialize(context): def initialize(context):
log.info('initializing algo') log.info('initializing algo')
context.ASSET_NAME = 'btc_usdt' context.ASSET_NAME = 'XRP_USDT'
context.asset = symbol(context.ASSET_NAME) context.asset = symbol(context.ASSET_NAME)
context.TARGET_POSITIONS = 30 context.TARGET_POSITIONS = 5000
context.PROFIT_TARGET = 0.1 context.PROFIT_TARGET = 0.1
context.SLIPPAGE_ALLOWED = 0.02 context.SLIPPAGE_ALLOWED = 0.05
context.retry_check_open_orders = 10
context.retry_update_portfolio = 10
context.retry_order = 5
context.swallow_errors = True
context.errors = [] context.errors = []
pass pass
def _handle_data(context, data): def _handle_data(context, data):
price = data.current(context.asset, 'price')
log.info('got price {price}'.format(price=price))
prices = data.history( prices = data.history(
context.asset, context.asset,
fields='price', fields='price',
bar_count=20, bar_count=20,
frequency='1D' frequency='15m'
) )
rsi = talib.RSI(prices.values, timeperiod=14)[-1] rsi = talib.RSI(prices.values, timeperiod=14)[-1]
log.info('got rsi: {}'.format(rsi)) log.info('got rsi: {}'.format(rsi))
# Buying more when RSI is low, this should lower our cost basis # Buying more when RSI is low, this should lower our cost basis
if rsi <= 30: if rsi <= 30:
buy_increment = 1 buy_increment = 50
elif rsi <= 40: elif rsi <= 40:
buy_increment = 0.5 buy_increment = 20
elif rsi <= 70: elif rsi <= 70:
buy_increment = 0.2 buy_increment = 5
else: else:
buy_increment = 0.1 buy_increment = None
cash = context.portfolio.cash cash = context.portfolio.cash
log.info('base currency available: {cash}'.format(cash=cash)) log.info('base currency available: {cash}'.format(cash=cash))
price = data.current(context.asset, 'price')
log.info('got price {price}'.format(price=price))
record( record(
price=price, price=price,
rsi=rsi, rsi=rsi,
) )
orders = context.blotter.open_orders orders = get_open_orders(context.asset)
if orders: if orders:
log.info('skipping bar until all open orders execute') log.info('skipping bar until all open orders execute')
return return
@@ -83,8 +100,8 @@ def _handle_data(context, data):
if price < cost_basis: if price < cost_basis:
is_buy = True is_buy = True
elif (position.amount > 0 elif position.amount > 0 and \
and price > cost_basis * (1 + context.PROFIT_TARGET)): price > cost_basis * (1 + context.PROFIT_TARGET):
profit = (price * position.amount) - (cost_basis * position.amount) profit = (price * position.amount) - (cost_basis * position.amount)
log.info('closing position, taking profit: {}'.format(profit)) log.info('closing position, taking profit: {}'.format(profit))
order_target_percent( order_target_percent(
@@ -121,11 +138,11 @@ def _handle_data(context, data):
def handle_data(context, data): def handle_data(context, data):
log.info('handling bar {}'.format(data.current_dt)) log.info('handling bar {}'.format(data.current_dt))
# try: try:
_handle_data(context, data) _handle_data(context, data)
# except Exception as e: except Exception as e:
# log.warn('aborting the bar on error {}'.format(e)) log.warn('aborting the bar on error {}'.format(e))
# context.errors.append(e) context.errors.append(e)
log.info('completed bar {}, total execution errors {}'.format( log.info('completed bar {}, total execution errors {}'.format(
data.current_dt, data.current_dt,
@@ -139,32 +156,3 @@ def handle_data(context, data):
def analyze(context, stats): def analyze(context, stats):
log.info('the daily stats:\n{}'.format(get_pretty_stats(stats))) log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
pass pass
if __name__ == '__main__':
live = True
if live:
run_algorithm(
capital_base=1000,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
live=True,
algo_namespace=algo_namespace,
base_currency='btc',
simulate_orders=True,
)
else:
run_algorithm(
capital_base=10000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace='buy_and_hodl',
base_currency='usdt',
start=pd.to_datetime('2015-03-01', utc=True),
end=pd.to_datetime('2017-10-31', utc=True),
)
catalyst/examples/buy_low_sell_high_live.py
+168
View File
@@ -0,0 +1,168 @@
import talib
from logbook import Logger
import pandas as pd
from catalyst.api import (
order,
order_target_percent,
symbol,
record,
get_open_orders,
)
from catalyst.exchange.stats_utils import get_pretty_stats
from catalyst.utils.run_algo import run_algorithm
algo_namespace = 'buy_the_dip_live'
log = Logger('buy low sell high')
def initialize(context):
log.info('initializing algo')
context.ASSET_NAME = 'btc_usdt'
context.asset = symbol(context.ASSET_NAME)
context.TARGET_POSITIONS = 30
context.PROFIT_TARGET = 0.1
context.SLIPPAGE_ALLOWED = 0.02
context.retry_check_open_orders = 10
context.retry_update_portfolio = 10
context.retry_order = 5
context.errors = []
pass
def _handle_data(context, data):
price = data.current(context.asset, 'price')
log.info('got price {price}'.format(price=price))
prices = data.history(
context.asset,
fields='price',
bar_count=20,
frequency='1d'
)
rsi = talib.RSI(prices.values, timeperiod=14)[-1]
log.info('got rsi: {}'.format(rsi))
# Buying more when RSI is low, this should lower our cost basis
if rsi <= 30:
buy_increment = 1
elif rsi <= 40:
buy_increment = 0.5
elif rsi <= 70:
buy_increment = 0.2
else:
buy_increment = 0.1
cash = context.portfolio.cash
log.info('base currency available: {cash}'.format(cash=cash))
record(
price=price,
rsi=rsi,
)
orders = get_open_orders(context.asset)
if orders:
log.info('skipping bar until all open orders execute')
return
is_buy = False
cost_basis = None
if context.asset in context.portfolio.positions:
position = context.portfolio.positions[context.asset]
cost_basis = position.cost_basis
log.info(
'found {amount} positions with cost basis {cost_basis}'.format(
amount=position.amount,
cost_basis=cost_basis
)
)
if position.amount >= context.TARGET_POSITIONS:
log.info('reached positions target: {}'.format(position.amount))
return
if price < cost_basis:
is_buy = True
elif position.amount > 0 and \
price > cost_basis * (1 + context.PROFIT_TARGET):
profit = (price * position.amount) - (cost_basis * position.amount)
log.info('closing position, taking profit: {}'.format(profit))
order_target_percent(
asset=context.asset,
target=0,
limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
)
else:
log.info('no buy or sell opportunity found')
else:
is_buy = True
if is_buy:
if buy_increment is None:
log.info('the rsi is too high to consider buying {}'.format(rsi))
return
if price * buy_increment > cash:
log.info('not enough base currency to consider buying')
return
log.info(
'buying position cheaper than cost basis {} < {}'.format(
price,
cost_basis
)
)
order(
asset=context.asset,
amount=buy_increment,
limit_price=price * (1 + context.SLIPPAGE_ALLOWED)
)
def handle_data(context, data):
log.info('handling bar {}'.format(data.current_dt))
# try:
_handle_data(context, data)
# except Exception as e:
# log.warn('aborting the bar on error {}'.format(e))
# context.errors.append(e)
log.info('completed bar {}, total execution errors {}'.format(
data.current_dt,
len(context.errors)
))
if len(context.errors) > 0:
log.info('the errors:\n{}'.format(context.errors))
def analyze(context, stats):
log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
pass
run_algorithm(
capital_base=100000,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
start=pd.to_datetime('2017-5-01', utc=True),
end=pd.to_datetime('2017-10-16', utc=True),
base_currency='usdt',
data_frequency='daily'
)
# run_algorithm(
# initialize=initialize,
# handle_data=handle_data,
# analyze=analyze,
# exchange_name='poloniex',
# live=True,
# algo_namespace=algo_namespace,
# base_currency='btc'
# )
catalyst/examples/dual_moving_average.py
+25 -36
View File
@@ -1,16 +1,16 @@
import matplotlib.pyplot as plt
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from logbook import Logger from logbook import Logger
import matplotlib.pyplot as plt
from catalyst import run_algorithm from catalyst import run_algorithm
from catalyst.api import (record, symbol, order_target_percent,) from catalyst.api import (order, record, symbol, order_target_percent,
from catalyst.exchange.utils.stats_utils import extract_transactions get_open_orders)
from catalyst.exchange.stats_utils import extract_transactions
NAMESPACE = 'dual_moving_average' NAMESPACE = 'dual_moving_average'
log = Logger(NAMESPACE) log = Logger(NAMESPACE)
def initialize(context): def initialize(context):
context.i = 0 context.i = 0
context.asset = symbol('ltc_usd') context.asset = symbol('ltc_usd')
@@ -25,24 +25,16 @@ def handle_data(context, data):
# Skip as many bars as long_window to properly compute the average # Skip as many bars as long_window to properly compute the average
context.i += 1 context.i += 1
if context.i < long_window: if context.i < long_window:
return return
# Compute moving averages calling data.history() for each # Compute moving averages calling data.history() for each
# moving average with the appropriate parameters. We choose to use # moving average with the appropriate parameters. We choose to use
# minute bars for this simulation -> freq="1m" # minute bars for this simulation -> freq="1m"
# Returns a pandas dataframe. # Returns a pandas dataframe.
short_data = data.history(context.asset, short_mavg = data.history(context.asset, 'price',
'price', bar_count=short_window, frequency="1m").mean()
bar_count=short_window, long_mavg = data.history(context.asset, 'price',
frequency="1T", bar_count=long_window, frequency="1m").mean()
)
short_mavg = short_data.mean()
long_data = data.history(context.asset,
'price',
bar_count=long_window,
frequency="1T",
)
long_mavg = long_data.mean()
# Let's keep the price of our asset in a more handy variable # Let's keep the price of our asset in a more handy variable
price = data.current(context.asset, 'price') price = data.current(context.asset, 'price')
@@ -62,7 +54,7 @@ def handle_data(context, data):
# Since we are using limit orders, some orders may not execute immediately # Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades. # we wait until all orders are executed before considering more trades.
orders = context.blotter.open_orders orders = get_open_orders(context.asset)
if len(orders) > 0: if len(orders) > 0:
return return
@@ -75,17 +67,17 @@ def handle_data(context, data):
# Trading logic # Trading logic
if short_mavg > long_mavg and pos_amount == 0: if short_mavg > long_mavg and pos_amount == 0:
# we buy 100% of our portfolio for this asset # we buy 100% of our portfolio for this asset
order_target_percent(context.asset, 1) order_target_percent(context.asset, 1)
elif short_mavg < long_mavg and pos_amount > 0: elif short_mavg < long_mavg and pos_amount > 0:
# we sell all our positions for this asset # we sell all our positions for this asset
order_target_percent(context.asset, 0) order_target_percent(context.asset, 0)
def analyze(context, perf): def analyze(context, perf):
# Get the base_currency that was passed as a parameter to the simulation # Get the base_currency that was passed as a parameter to the simulation
exchange = list(context.exchanges.values())[0] base_currency = context.exchanges.values()[0].base_currency.upper()
base_currency = exchange.base_currency.upper()
# First chart: Plot portfolio value using base_currency # First chart: Plot portfolio value using base_currency
ax1 = plt.subplot(411) ax1 = plt.subplot(411)
@@ -93,20 +85,18 @@ def analyze(context, perf):
ax1.legend_.remove() ax1.legend_.remove()
ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency))
start, end = ax1.get_ylim() start, end = ax1.get_ylim()
ax1.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Second chart: Plot asset price, moving averages and buys/sells # Second chart: Plot asset price, moving averages and buys/sells
ax2 = plt.subplot(412, sharex=ax1) ax2 = plt.subplot(412, sharex=ax1)
perf.loc[:, ['price', 'short_mavg', 'long_mavg']].plot( perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price')
ax=ax2,
label='Price')
ax2.legend_.remove() ax2.legend_.remove()
ax2.set_ylabel('{asset}\n({base})'.format( ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.asset.symbol, asset = context.asset.symbol,
base=base_currency base = base_currency
)) ))
start, end = ax2.get_ylim() start, end = ax2.get_ylim()
ax2.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
transaction_df = extract_transactions(perf) transaction_df = extract_transactions(perf)
if not transaction_df.empty: if not transaction_df.empty:
@@ -136,20 +126,19 @@ def analyze(context, perf):
ax3.legend_.remove() ax3.legend_.remove()
ax3.set_ylabel('Percent Change') ax3.set_ylabel('Percent Change')
start, end = ax3.get_ylim() start, end = ax3.get_ylim()
ax3.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5))
# Fourth chart: Plot our cash # Fourth chart: Plot our cash
ax4 = plt.subplot(414, sharex=ax1) ax4 = plt.subplot(414, sharex=ax1)
perf.cash.plot(ax=ax4) perf.cash.plot(ax=ax4)
ax4.set_ylabel('Cash\n({})'.format(base_currency)) ax4.set_ylabel('Cash\n({})'.format(base_currency))
start, end = ax4.get_ylim() start, end = ax4.get_ylim()
ax4.yaxis.set_ticks(np.arange(0, end, end / 5)) ax4.yaxis.set_ticks(np.arange(0, end, end/5))
plt.show() plt.show()
if __name__ == '__main__': if __name__ == '__main__':
run_algorithm( run_algorithm(
capital_base=1000, capital_base=1000,
data_frequency='minute', data_frequency='minute',
@@ -161,4 +150,4 @@ if __name__ == '__main__':
base_currency='usd', base_currency='usd',
start=pd.to_datetime('2017-9-22', utc=True), start=pd.to_datetime('2017-9-22', utc=True),
end=pd.to_datetime('2017-9-23', utc=True), end=pd.to_datetime('2017-9-23', utc=True),
) )
catalyst/examples/dual_vwap.py
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python
#
# Copyright 2017 Enigma MPC, Inc.
# Copyright 2014 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from catalyst.api import (
order_target_percent,
record,
symbol,
get_open_orders,
set_max_leverage,
schedule_function,
date_rules,
attach_pipeline,
pipeline_output,
)
from catalyst.pipeline import Pipeline
from catalyst.pipeline.data import CryptoPricing
from catalyst.pipeline.factors.crypto import VWAP
def initialize(context):
context.ASSET_NAME = 'USDT_BTC'
context.TARGET_INVESTMENT_RATIO = 0.8
context.SHORT_WINDOW = 30
context.LONG_WINDOW = 100
# For all trading pairs in the poloniex bundle, the default denomination
# currently supported by Catalyst is 1/1000th of a full coin. Use this
# constant to scale the price of up to that of a full coin if desired.
context.TICK_SIZE = 1000.0
context.i = 0
context.asset = symbol(context.ASSET_NAME)
set_max_leverage(1.0)
attach_pipeline(make_pipeline(context), 'vwap_pipeline')
schedule_function(
rebalance,
time_rules=times_rules.every_minute(),
)
def before_trading_start(context, data):
context.pipeline_data = pipeline_output('vwap_pipeline')
def make_pipeline(context):
return Pipeline(
columns={
'price': CryptoPricing.open.latest,
'volume': CryptoPricing.volume.latest,
'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
'long_mavg': VWAP(window_length=context.LONG_WINDOW),
}
)
def rebalance(context, data):
context.i += 1
# skip first LONG_WINDOW bars to fill windows
if context.i < context.LONG_WINDOW:
return
# get pipeline data for asset of interest
pipeline_data = context.pipeline_data
pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0]
# retrieve long and short moving averages from pipeline
short_mavg = pipeline_data.short_mavg
long_mavg = pipeline_data.long_mavg
price = pipeline_data.price
volume = pipeline_data.volume
# check that order has not already been placed
open_orders = get_open_orders()
if context.asset not in open_orders:
# check that the asset of interest can currently be traded
if data.can_trade(context.asset):
# adjust portfolio based on comparison of long and short vwap
if short_mavg > long_mavg:
order_target_percent(
context.asset,
context.TARGET_INVESTMENT_RATIO,
)
elif short_mavg < long_mavg:
order_target_percent(
context.asset,
0.0,
)
record(
price=price,
cash=context.portfolio.cash,
leverage=context.account.leverage,
short_mavg=short_mavg,
long_mavg=long_mavg,
volume=volume,
)
def analyze(context=None, results=None):
import matplotlib.pyplot as plt
# Plot the portfolio and asset data.
ax1 = plt.subplot(611)
results[['portfolio_value']].plot(ax=ax1)
ax1.set_ylabel('Portfolio value (USD)')
ax2 = plt.subplot(612, sharex=ax1)
ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
(context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
trans = results.ix[[t != [] for t in results.transactions]]
amounts = [t[0]['amount'] for t in trans.transactions]
buys = trans.ix[
[t[0]['amount'] > 0 for t in trans.transactions]
]
sells = trans.ix[
[t[0]['amount'] < 0 for t in trans.transactions]
]
ax2.plot(
buys.index,
context.TICK_SIZE * results.price[buys.index],
'^',
markersize=10,
color='g',
)
ax2.plot(
sells.index,
context.TICK_SIZE * results.price[sells.index],
'v',
markersize=10,
color='r',
)
ax3 = plt.subplot(613, sharex=ax1)
results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
ax3.set_ylabel('Leverage (USD)')
ax4 = plt.subplot(614, sharex=ax1)
results[['cash']].plot(ax=ax4)
ax4.set_ylabel('Cash (USD)')
results[[
'treasury',
'algorithm',
'benchmark',
]] = results[[
'treasury_period_return',
'algorithm_period_return',
'benchmark_period_return',
]]
ax5 = plt.subplot(615, sharex=ax1)
results[[
'treasury',
'algorithm',
'benchmark',
]].plot(ax=ax5)
ax5.set_ylabel('Percent Change')
ax6 = plt.subplot(616, sharex=ax1)
results[['volume']].plot(ax=ax6)
ax6.set_ylabel('Volume (mBTC/day)')
plt.legend(loc=3)
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
catalyst/examples/marketplace/github-research.py
-70
View File
@@ -1,70 +0,0 @@
import pandas as pd
import matplotlib.pyplot as plt
from catalyst import run_algorithm
from catalyst.api import symbol, get_dataset
START = '2017-01-01'
END = '2017-12-31'
def initialize(context):
pass
def handle_data(context, data):
context.github = get_dataset('github')
context.github.sort_index(level=0, inplace=True)
context.zec = data.history(symbol('zec_usdt'),
['price', ],
bar_count=365,
frequency="1d")
context.xmr = data.history(symbol('xmr_usdt'),
['price', ],
bar_count=365,
frequency="1d")
def analyze(context=None, results=None):
ax1 = plt.subplot(211)
idx = pd.IndexSlice
df = context.github.loc[START:END].loc[
idx[:, [b'ZEC']], ['commits']].reset_index(
level='symbol', drop=True)
df.plot(ax=ax1, color='blue')
ax1.legend(loc=2)
ax1.set_title('Zcash')
ax2 = ax1.twinx()
context.zec['price'].loc[START:END].plot(ax=ax2, color='green')
ax2.legend(loc=1)
ax3 = plt.subplot(212)
idx = pd.IndexSlice
df = context.github.loc[START:END].loc[
idx[:, [b'XMR']], ['commits']].reset_index(
level='symbol', drop=True)
df.plot(ax=ax3, color='blue')
ax3.legend(loc=2)
ax3.set_title('Monero')
ax4 = ax3.twinx()
context.xmr['price'].loc[START:END].plot(ax=ax4, color='green')
ax4.legend(loc=1)
plt.show()
if __name__ == '__main__':
run_algorithm(
capital_base=1000,
data_frequency='daily',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace='algo-github',
base_currency='usdt',
live=False,
start=pd.to_datetime(END, utc=True),
end=pd.to_datetime(END, utc=True),
)
catalyst/examples/marketplace/mean_reversion_by_marketcap.py
-237
View File
@@ -1,237 +0,0 @@
# For this example, we're going to write a simple momentum script. When the
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
# going to sell. Hopefully we'll ride the waves.
import os
import tempfile
import time
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_dataset
from catalyst.exchange.utils.stats_utils import set_print_settings, \
get_pretty_stats
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
df = get_dataset('testmarketcap2') # type: pd.DataFrame
# Picking a specific date in our DataFrame
first_dt = df.index.get_level_values(0)[0]
# Since we use a MultiIndex with date / symbol, picking a date will
# result in a new DataFrame for the selected date with a single
# symbol index
df = df.xs(first_dt, level=0)
# Keep only the top coins by market cap
df = df.loc[df['market_cap_usd'].isin(df['market_cap_usd'].nlargest(100))]
set_print_settings()
df.sort_values(by=['market_cap_usd'], ascending=True, inplace=True)
print('the marketplace data:\n{}'.format(df))
# Pick the 5 assets with the lowest market cap for trading
quote_currency = 'eth'
exchange = context.exchanges[next(iter(context.exchanges))]
symbols = [a.symbol for a in exchange.assets
if a.start_date < context.datetime]
context.assets = []
for currency, price in df['market_cap_usd'].iteritems():
if len(context.assets) >= 5:
break
s = '{}_{}'.format(currency.decode('utf-8'), quote_currency)
if s in symbols:
context.assets.append(symbol(s))
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = dict()
context.current_day = today
# Preparing dictionaries for asset-level data points
volumes = dict()
rsis = dict()
price_values = dict()
cash = context.portfolio.cash
for asset in context.assets:
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.assets variable. For this example,
# we're using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this
# convention for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
asset,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare
# to the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(asset, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
# if asset not in context.base_price:
# context.base_price[asset] = price
#
# base_price = context.base_price[asset]
# price_change = (price - base_price) / base_price
# Tracking the relevant data
volumes[asset] = current['volume']
rsis[asset] = rsi[-1]
price_values[asset] = price
# price_changes[asset] = price_change
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if asset in context.traded_today:
continue
# Exit if we cannot trade
if not data.can_trade(asset):
continue
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we
# calculate how long or short our position is at this minute.
pos_amount = context.portfolio.positions[asset].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
target = 1.0 / len(context.assets)
order_target_percent(
asset, target, limit_price=limit_price
)
context.traded_today[asset] = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
asset, 0, limit_price=limit_price
)
context.traded_today[asset] = True
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
current_price=price_values,
volume=volumes,
rsi=rsis,
cash=cash,
)
def analyze(context=None, perf=None):
stats = get_pretty_stats(perf)
print('the algo stats:\n{}'.format(stats))
pass
if __name__ == '__main__':
# The execution mode: backtest or live
live = False
if live:
run_algorithm(
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
live=True,
algo_namespace=NAMESPACE,
base_currency='btc',
live_graph=False,
simulate_orders=False,
stats_output=None,
)
else:
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=100,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace=NAMESPACE,
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-10-15', utc=True),
)
log.info('saved perf stats: {}'.format(out))
catalyst/examples/mean_reversion_simple.py
+38 -48
View File
@@ -1,4 +1,4 @@
# For this example, we're going to write a simple momentum script. When the # For this example, we're going to write a simple momentum script. When the
# stock goes up quickly, we're going to buy; when it goes down quickly, we're # stock goes up quickly, we're going to buy; when it goes down quickly, we're
# going to sell. Hopefully we'll ride the waves. # going to sell. Hopefully we'll ride the waves.
import os import os
@@ -12,7 +12,7 @@ from logbook import Logger
from catalyst import run_algorithm from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.utils.stats_utils import extract_transactions from catalyst.exchange.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state. # We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos` # In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its # directory. If we stop and start the algorithm, Catalyst will resume its
@@ -33,19 +33,16 @@ def initialize(context):
# parameters or values you're going to use. # parameters or values you're going to use.
# In our example, we're looking at Neo in Ether. # In our example, we're looking at Neo in Ether.
context.market = symbol('bnb_eth') context.neo_eth = symbol('neo_eth')
context.base_price = None context.base_price = None
context.current_day = None context.current_day = None
context.RSI_OVERSOLD = 60 context.RSI_OVERSOLD = 55
context.RSI_OVERBOUGHT = 70 context.RSI_OVERBOUGHT = 82
context.CANDLE_SIZE = '15T' context.CANDLE_SIZE = '5T'
context.start_time = time.time() context.start_time = time.time()
context.set_commission(maker=0.001, taker=0.002)
context.set_slippage(spread=0.001)
def handle_data(context, data): def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is # This handle_data function is where the real work is done. Our data is
@@ -62,14 +59,14 @@ def handle_data(context, data):
context.current_day = today context.current_day = today
# We're computing the volume-weighted-average-price of the security # We're computing the volume-weighted-average-price of the security
# defined above, in the context.market variable. For this example, we're # defined above, in the context.neo_eth variable. For this example, we're
# using three bars on the 15 min bars. # using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention # The frequency attribute determine the bar size. We use this convention
# for the frequency alias: # for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history( prices = data.history(
context.market, context.neo_eth,
fields='close', fields='close',
bar_count=50, bar_count=50,
frequency=context.CANDLE_SIZE frequency=context.CANDLE_SIZE
@@ -84,7 +81,7 @@ def handle_data(context, data):
# We need a variable for the current price of the security to compare to # We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current() # the average. Since we are requesting two fields, data.current()
# returns a DataFrame with # returns a DataFrame with
current = data.current(context.market, fields=['close', 'volume']) current = data.current(context.neo_eth, fields=['close', 'volume'])
price = current['close'] price = current['close']
# If base_price is not set, we use the current value. This is the # If base_price is not set, we use the current value. This is the
@@ -98,36 +95,34 @@ def handle_data(context, data):
# Now that we've collected all current data for this frame, we use # Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as # the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis. # a parameter of the analyze() function for further analysis.
record( record(
volume=current['volume'],
price=price, price=price,
volume=current['volume'],
price_change=price_change, price_change=price_change,
rsi=rsi[-1], rsi=rsi[-1],
cash=cash cash=cash
) )
# We are trying to avoid over-trading by limiting our trades to # We are trying to avoid over-trading by limiting our trades to
# one per day. # one per day.
if context.traded_today: if context.traded_today:
return return
# TODO: retest with open orders
# Since we are using limit orders, some orders may not execute immediately # Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades. # we wait until all orders are executed before considering more trades.
orders = context.blotter.open_orders orders = get_open_orders(context.neo_eth)
if len(orders) > 0: if len(orders) > 0:
log.info('exiting because orders are open: {}'.format(orders))
return return
# Exit if we cannot trade # Exit if we cannot trade
if not data.can_trade(context.market): if not data.can_trade(context.neo_eth):
return return
# Another powerful built-in feature of the Catalyst backtester is the # Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash, # portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate # cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute. # how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.market].amount pos_amount = context.portfolio.positions[context.neo_eth].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info( log.info(
@@ -138,7 +133,7 @@ def handle_data(context, data):
# Set a style for limit orders, # Set a style for limit orders,
limit_price = price * 1.005 limit_price = price * 1.005
order_target_percent( order_target_percent(
context.market, 1, limit_price=limit_price context.neo_eth, 1, limit_price=limit_price
) )
context.traded_today = True context.traded_today = True
@@ -150,7 +145,7 @@ def handle_data(context, data):
) )
limit_price = price * 0.995 limit_price = price * 0.995
order_target_percent( order_target_percent(
context.market, 0, limit_price=limit_price context.neo_eth, 0, limit_price=limit_price
) )
context.traded_today = True context.traded_today = True
@@ -161,7 +156,7 @@ def analyze(context=None, perf=None):
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
# The base currency of the algo exchange # The base currency of the algo exchange
base_currency = list(context.exchanges.values())[0].base_currency.upper() base_currency = context.exchanges.values()[0].base_currency.upper()
# Plot the portfolio value over time. # Plot the portfolio value over time.
ax1 = plt.subplot(611) ax1 = plt.subplot(611)
@@ -173,7 +168,7 @@ def analyze(context=None, perf=None):
perf.loc[:, 'price'].plot(ax=ax2, label='Price') perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format( ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.market.symbol, base=base_currency asset=context.neo_eth.symbol, base=base_currency
)) ))
transaction_df = extract_transactions(perf) transaction_df = extract_transactions(perf)
@@ -234,7 +229,7 @@ def analyze(context=None, perf=None):
) )
plt.legend(loc=3) plt.legend(loc=3)
start, end = ax6.get_ylim() start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end / 5)) ax6.yaxis.set_ticks(np.arange(0, end, end/5))
# Show the plot. # Show the plot.
plt.gcf().set_size_inches(18, 8) plt.gcf().set_size_inches(18, 8)
@@ -244,25 +239,9 @@ def analyze(context=None, perf=None):
if __name__ == '__main__': if __name__ == '__main__':
# The execution mode: backtest or live # The execution mode: backtest or live
live = True MODE = 'live'
if live: if MODE == 'backtest':
run_algorithm(
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='binance',
live=True,
algo_namespace=NAMESPACE,
base_currency='eth',
live_graph=False,
simulate_orders=False,
stats_output=None,
# auth_aliases=dict(poloniex='auth2')
)
else:
folder = os.path.join( folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE tempfile.gettempdir(), 'catalyst', NAMESPACE
) )
@@ -270,20 +249,31 @@ if __name__ == '__main__':
timestr = time.strftime('%Y%m%d-%H%M%S') timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr)) out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py \ # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm( run_algorithm(
capital_base=0.035, capital_base=0.1,
data_frequency='minute', data_frequency='minute',
initialize=initialize, initialize=initialize,
handle_data=handle_data, handle_data=handle_data,
analyze=analyze, analyze=analyze,
exchange_name='bitfinex', exchange_name='bitfinex',
algo_namespace=NAMESPACE, algo_namespace=NAMESPACE,
base_currency='btc', base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True), start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True), end=pd.to_datetime('2017-11-10', utc=True),
output=out output=out
) )
log.info('saved perf stats: {}'.format(out)) log.info('saved perf stats: {}'.format(out))
elif MODE == 'live':
run_algorithm(
capital_base=0.1,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bittrex',
live=True,
algo_namespace=NAMESPACE,
base_currency='eth',
live_graph=False
)
catalyst/examples/mean_reversion_simple_custom_fees.py
-288
View File
@@ -1,288 +0,0 @@
# For this example, we're going to write a simple momentum script. When the
# stock goes up quickly, we're going to buy; when it goes down quickly, we're
# going to sell. Hopefully we'll ride the waves.
import os
import tempfile
import time
import numpy as np
import pandas as pd
import talib
from logbook import Logger
from catalyst import run_algorithm
from catalyst.api import symbol, record, order_target_percent, get_open_orders
from catalyst.exchange.utils.stats_utils import extract_transactions
# We give a name to the algorithm which Catalyst will use to persist its state.
# In this example, Catalyst will create the `.catalyst/data/live_algos`
# directory. If we stop and start the algorithm, Catalyst will resume its
# state using the files included in the folder.
from catalyst.utils.paths import ensure_directory
NAMESPACE = 'mean_reversion_simple'
log = Logger(NAMESPACE)
# To run an algorithm in Catalyst, you need two functions: initialize and
# handle_data.
def initialize(context):
# This initialize function sets any data or variables that you'll use in
# your algorithm. For instance, you'll want to define the trading pair (or
# trading pairs) you want to backtest. You'll also want to define any
# parameters or values you're going to use.
# In our example, we're looking at Neo in Ether.
context.market = symbol('eth_btc')
context.base_price = None
context.current_day = None
context.RSI_OVERSOLD = 50
context.RSI_OVERBOUGHT = 60
context.CANDLE_SIZE = '5T'
context.start_time = time.time()
context.set_commission(maker=0.001, taker=0.002)
# context.set_slippage(spread=0.001)
def handle_data(context, data):
# This handle_data function is where the real work is done. Our data is
# minute-level tick data, and each minute is called a frame. This function
# runs on each frame of the data.
# We flag the first period of each day.
# Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
# would only execute once. This method works with minute and daily
# frequencies.
today = data.current_dt.floor('1D')
if today != context.current_day:
context.traded_today = False
context.current_day = today
# We're computing the volume-weighted-average-price of the security
# defined above, in the context.market variable. For this example, we're
# using three bars on the 15 min bars.
# The frequency attribute determine the bar size. We use this convention
# for the frequency alias:
# http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
prices = data.history(
context.market,
fields='close',
bar_count=50,
frequency=context.CANDLE_SIZE
)
# Ta-lib calculates various technical indicator based on price and
# volume arrays.
# In this example, we are comp
rsi = talib.RSI(prices.values, timeperiod=14)
# We need a variable for the current price of the security to compare to
# the average. Since we are requesting two fields, data.current()
# returns a DataFrame with
current = data.current(context.market, fields=['close', 'volume'])
price = current['close']
# If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change.
if context.base_price is None:
context.base_price = price
price_change = (price - context.base_price) / context.base_price
cash = context.portfolio.cash
# Now that we've collected all current data for this frame, we use
# the record() method to save it. This data will be available as
# a parameter of the analyze() function for further analysis.
record(
volume=current['volume'],
price=price,
price_change=price_change,
rsi=rsi[-1],
cash=cash
)
# We are trying to avoid over-trading by limiting our trades to
# one per day.
if context.traded_today:
return
# TODO: retest with open orders
# Since we are using limit orders, some orders may not execute immediately
# we wait until all orders are executed before considering more trades.
orders = get_open_orders(context.market)
if len(orders) > 0:
log.info('exiting because orders are open: {}'.format(orders))
return
# Exit if we cannot trade
if not data.can_trade(context.market):
return
# Another powerful built-in feature of the Catalyst backtester is the
# portfolio object. The portfolio object tracks your positions, cash,
# cost basis of specific holdings, and more. In this line, we calculate
# how long or short our position is at this minute.
pos_amount = context.portfolio.positions[context.market].amount
if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
log.info(
'{}: buying - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
# Set a style for limit orders,
limit_price = price * 1.005
order_target_percent(
context.market, 1, limit_price=limit_price
)
context.traded_today = True
elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
log.info(
'{}: selling - price: {}, rsi: {}'.format(
data.current_dt, price, rsi[-1]
)
)
limit_price = price * 0.995
order_target_percent(
context.market, 0, limit_price=limit_price
)
context.traded_today = True
def analyze(context=None, perf=None):
end = time.time()
log.info('elapsed time: {}'.format(end - context.start_time))
import matplotlib.pyplot as plt
# The base currency of the algo exchange
base_currency = list(context.exchanges.values())[0].base_currency.upper()
# Plot the portfolio value over time.
ax1 = plt.subplot(611)
perf.loc[:, 'portfolio_value'].plot(ax=ax1)
ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
# Plot the price increase or decrease over time.
ax2 = plt.subplot(612, sharex=ax1)
perf.loc[:, 'price'].plot(ax=ax2, label='Price')
ax2.set_ylabel('{asset}\n({base})'.format(
asset=context.market.symbol, base=base_currency
))
transaction_df = extract_transactions(perf)
if not transaction_df.empty:
buy_df = transaction_df[transaction_df['amount'] > 0]
sell_df = transaction_df[transaction_df['amount'] < 0]
ax2.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'price'],
marker='^',
s=100,
c='green',
label=''
)
ax2.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'price'],
marker='v',
s=100,
c='red',
label=''
)
ax4 = plt.subplot(613, sharex=ax1)
perf.loc[:, 'cash'].plot(
ax=ax4, label='Base Currency ({})'.format(base_currency)
)
ax4.set_ylabel('Cash\n({})'.format(base_currency))
perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
ax5 = plt.subplot(614, sharex=ax1)
perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
ax5.set_ylabel('Percent\nChange')
ax6 = plt.subplot(615, sharex=ax1)
perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
ax6.set_ylabel('RSI')
ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
if not transaction_df.empty:
ax6.scatter(
buy_df.index.to_pydatetime(),
perf.loc[buy_df.index.floor('1 min'), 'rsi'],
marker='^',
s=100,
c='green',
label=''
)
ax6.scatter(
sell_df.index.to_pydatetime(),
perf.loc[sell_df.index.floor('1 min'), 'rsi'],
marker='v',
s=100,
c='red',
label=''
)
plt.legend(loc=3)
start, end = ax6.get_ylim()
ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
# Show the plot.
plt.gcf().set_size_inches(18, 8)
plt.show()
pass
if __name__ == '__main__':
# The execution mode: backtest or live
live = False
if live:
run_algorithm(
capital_base=0.025,
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
live=True,
algo_namespace=NAMESPACE,
base_currency='btc',
live_graph=False,
simulate_orders=False,
stats_output=None,
)
else:
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', NAMESPACE
)
ensure_directory(folder)
timestr = time.strftime('%Y%m%d-%H%M%S')
out = os.path.join(folder, '{}.p'.format(timestr))
# catalyst run -f catalyst/examples/mean_reversion_simple.py \
# -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
# --data-frequency minute --capital-base 10000
run_algorithm(
capital_base=0.1,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='bitfinex',
algo_namespace=NAMESPACE,
base_currency='eth',
start=pd.to_datetime('2017-10-01', utc=True),
end=pd.to_datetime('2017-11-10', utc=True),
output=out
)
log.info('saved perf stats: {}'.format(out))
catalyst/examples/portfolio_optimization.py
+105 -122
View File
@@ -1,7 +1,7 @@
'''Use this code to execute a portfolio optimization model. This code '''Use this code to execute a portfolio optimization model. This code
will select the portfolio with the maximum Sharpe Ratio. The parameters will select the portfolio with the maximum Sharpe Ratio. The parameters
are set to use 180 days of historical data and rebalance every 30 days. are set to use 180 days of historical data and rebalance every 30 days.
This is the code used in the following article: This is the code used in the following article:
https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556 https://blog.enigma.co/markowitz-portfolio-optimization-for-cryptocurrencies-in-catalyst-b23c38652556
@@ -15,136 +15,119 @@ import os
import pytz import pytz
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from scipy.optimize import minimize
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from datetime import datetime from datetime import datetime
from catalyst.api import record, symbols, order_target_percent from catalyst.api import record, symbol, symbols, order_target_percent
from catalyst.utils.run_algo import run_algorithm from catalyst.utils.run_algo import run_algorithm
np.set_printoptions(threshold='nan', suppress=True) np.set_printoptions(threshold='nan', suppress=True)
def initialize(context): def initialize(context):
# Portfolio assets list # Portfolio assets list
context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt', context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt',
'xmr_usdt') 'xmr_usdt')
context.nassets = len(context.assets) context.nassets = len(context.assets)
# Set the time window that will be used to compute expected return # Set the time window that will be used to compute expected return
# and asset correlations # and asset correlations
context.window = 180 context.window = 180
# Set the number of days between each portfolio rebalancing # Set the number of days between each portfolio rebalancing
context.rebalance_period = 30 context.rebalance_period = 30
context.i = 0 context.i = 0
def handle_data(context, data): def handle_data(context, data):
# Only rebalance at the beggining of the algorithm execution and # Only rebalance at the beggining of the algorithm execution and
# every multiple of the rebalance period # every multiple of the rebalance period
if context.i == 0 or context.i % context.rebalance_period == 0: if context.i == 0 or context.i%context.rebalance_period == 0:
n = context.window n = context.window
prices = data.history(context.assets, fields='price', prices = data.history(context.assets, fields='price',
bar_count=n + 1, frequency='1d') bar_count=n+1, frequency='1d')
pr = np.asmatrix(prices) pr = np.asmatrix(prices)
t_prices = prices.iloc[1:n + 1] t_prices = prices.iloc[1:n+1]
t_val = t_prices.values t_val = t_prices.values
tminus_prices = prices.iloc[0:n] tminus_prices = prices.iloc[0:n]
tminus_val = tminus_prices.values tminus_val = tminus_prices.values
# Compute daily returns (r) # Compute daily returns (r)
r = np.asmatrix(t_val / tminus_val - 1) r = np.asmatrix(t_val/tminus_val-1)
# Compute the expected returns of each asset with the average # Compute the expected returns of each asset with the average
# daily return for the selected time window # daily return for the selected time window
m = np.asmatrix(np.mean(r, axis=0)) m = np.asmatrix(np.mean(r, axis=0))
# ### # ###
stds = np.std(r, axis=0) stds = np.std(r, axis=0)
# Compute excess returns matrix (xr) # Compute excess returns matrix (xr)
xr = r - m xr = r - m
# Matrix algebra to get variance-covariance matrix # Matrix algebra to get variance-covariance matrix
cov_m = np.dot(np.transpose(xr), xr) / n cov_m = np.dot(np.transpose(xr),xr)/n
# Compute asset correlation matrix (informative only) # Compute asset correlation matrix (informative only)
corr_m = cov_m / np.dot(np.transpose(stds), stds) corr_m = cov_m/np.dot(np.transpose(stds),stds)
# Define portfolio optimization parameters # Define portfolio optimization parameters
n_portfolios = 50000 n_portfolios = 50000
results_array = np.zeros((3 + context.nassets, n_portfolios)) results_array = np.zeros((3+context.nassets,n_portfolios))
for p in range(n_portfolios): for p in xrange(n_portfolios):
weights = np.random.random(context.nassets) weights = np.random.random(context.nassets)
weights /= np.sum(weights) weights /= np.sum(weights)
w = np.asmatrix(weights) w = np.asmatrix(weights)
p_r = np.sum(np.dot(w, np.transpose(m))) * 365 p_r = np.sum(np.dot(w,np.transpose(m)))*365
p_std = np.sqrt(np.dot(np.dot(w, cov_m), p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365)
np.transpose(w))) * np.sqrt(365)
#store results in results array
# store results in results array results_array[0,p] = p_r
results_array[0, p] = p_r results_array[1,p] = p_std
results_array[1, p] = p_std #store Sharpe Ratio (return / volatility) - risk free rate element
# store Sharpe Ratio (return / volatility) - risk free rate element #excluded for simplicity
# excluded for simplicity results_array[2,p] = results_array[0,p] / results_array[1,p]
results_array[2, p] = results_array[0, p] / results_array[1, p] i = 0
i = 0 for iw in weights:
for iw in weights: results_array[3+i,p] = weights[i]
results_array[3 + i, p] = weights[i] i += 1
i += 1
#convert results array to Pandas DataFrame
# convert results array to Pandas DataFrame results_frame = pd.DataFrame(np.transpose(results_array),
results_frame = pd.DataFrame(np.transpose(results_array), columns=['r','stdev','sharpe']+context.assets)
columns=['r', 'stdev', 'sharpe'] #locate position of portfolio with highest Sharpe Ratio
+ context.assets) max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()]
# locate position of portfolio with highest Sharpe Ratio #locate positon of portfolio with minimum standard deviation
max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()] min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()]
# locate positon of portfolio with minimum standard deviation
# min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()] #order optimal weights for each asset
for asset in context.assets:
# order optimal weights for each asset if data.can_trade(asset):
for asset in context.assets: order_target_percent(asset, max_sharpe_port[asset])
if data.can_trade(asset):
order_target_percent(asset, max_sharpe_port[asset]) #create scatter plot coloured by Sharpe Ratio
plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn')
# create scatter plot coloured by Sharpe Ratio plt.xlabel('Volatility')
plt.scatter(results_frame.stdev, plt.ylabel('Returns')
results_frame.r, plt.colorbar()
c=results_frame.sharpe, #plot red star to highlight position of portfolio with highest Sharpe Ratio
cmap='RdYlGn') plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200)
plt.xlabel('Volatility') #plot green star to highlight position of minimum variance portfolio
plt.ylabel('Returns') plt.show()
plt.colorbar() print(max_sharpe_port)
# plot red star to highlight position of portfolio record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m)
# with highest Sharpe Ratio context.i += 1
plt.scatter(max_sharpe_port[1],
max_sharpe_port[0],
marker='o',
color='b',
s=200)
# plot green star to highlight position of minimum variance portfolio
plt.show()
print(max_sharpe_port)
record(pr=pr,
r=r,
m=m,
stds=stds,
max_sharpe_port=max_sharpe_port,
corr_m=corr_m)
context.i += 1
def analyze(context=None, results=None): def analyze(context=None, results=None):
# Form DataFrame with selected data # Form DataFrame with selected data
data = results[['pr', 'r', 'm', 'stds', 'max_sharpe_port', 'corr_m', data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']]
'portfolio_value']]
# Save results in CSV file
# Save results in CSV file filename = os.path.splitext(os.path.basename(__file__))[0]
filename = os.path.splitext(os.path.basename(__file__))[0] data.to_csv(filename + '.csv')
data.to_csv(filename + '.csv')
if __name__ == '__main__': # Bitcoin data is available from 2015-3-2. Dates vary for other tokens.
# Bitcoin data is available from 2015-3-2. Dates vary for other tokens. start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc)
start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc) end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc)
end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) results = run_algorithm(initialize=initialize,
results = run_algorithm(initialize=initialize, handle_data=handle_data,
handle_data=handle_data, analyze=analyze,
analyze=analyze, start=start,
start=start, end=end,
end=end, exchange_name='poloniex',
exchange_name='poloniex', capital_base=100000, )
capital_base=100000,
base_currency='usdt', )
catalyst/examples/rsi_profit_target.py
+31 -20
View File
@@ -11,6 +11,7 @@ from catalyst.api import (
record, record,
get_open_orders, get_open_orders,
) )
from catalyst.exchange.stats_utils import crossover, crossunder
from catalyst.utils.run_algo import run_algorithm from catalyst.utils.run_algo import run_algorithm
algo_namespace = 'rsi' algo_namespace = 'rsi'
@@ -54,7 +55,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
stop=None stop=None
) )
# action = None action = None
if context.position is not None: if context.position is not None:
cost_basis = context.position['cost_basis'] cost_basis = context.position['cost_basis']
amount = context.position['amount'] amount = context.position['amount']
@@ -79,7 +80,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
amount=-amount, amount=-amount,
limit_price=price * (1 - context.SLIPPAGE_ALLOWED), limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
) )
# action = 0 action = 0
context.position = None context.position = None
else: else:
@@ -96,7 +97,7 @@ def _handle_buy_sell_decision(context, data, signal, price):
amount=buy_amount, amount=buy_amount,
stop=None stop=None
) )
# action = 0 action = 0
def _handle_data_rsi_only(context, data): def _handle_data_rsi_only(context, data):
@@ -114,7 +115,7 @@ def _handle_data_rsi_only(context, data):
prices = data.history( prices = data.history(
context.asset, context.asset,
fields='price', fields='price',
bar_count=20, bar_count=17,
frequency='30T' frequency='30T'
) )
except Exception as e: except Exception as e:
@@ -156,7 +157,7 @@ def handle_data(context, data):
dt = data.current_dt dt = data.current_dt
if context.last_bar is None or ( if context.last_bar is None or (
context.last_bar + timedelta(minutes=15)) <= dt: context.last_bar + timedelta(minutes=15)) <= dt:
context.last_bar = dt context.last_bar = dt
else: else:
return return
@@ -175,7 +176,7 @@ def handle_data(context, data):
def analyze(context=None, results=None): def analyze(context=None, results=None):
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
base_currency = list(context.exchanges.values())[0].base_currency.upper() base_currency = context.exchanges.values()[0].base_currency.upper()
# Plot the portfolio and asset data. # Plot the portfolio and asset data.
ax1 = plt.subplot(611) ax1 = plt.subplot(611)
results.loc[:, 'portfolio_value'].plot(ax=ax1) results.loc[:, 'portfolio_value'].plot(ax=ax1)
@@ -249,17 +250,27 @@ def analyze(context=None, results=None):
pass pass
if __name__ == '__main__': # run_algorithm(
# Backtest # initialize=initialize,
run_algorithm( # handle_data=handle_data,
capital_base=0.5, # analyze=analyze,
data_frequency='minute', # exchange_name='bittrex',
initialize=initialize, # live=True,
handle_data=handle_data, # algo_namespace=algo_namespace,
analyze=analyze, # base_currency='btc',
exchange_name='poloniex', # live_graph=False
algo_namespace=algo_namespace, # )
base_currency='btc',
start=pd.to_datetime('2017-9-1', utc=True), # Backtest
end=pd.to_datetime('2017-10-1', utc=True), run_algorithm(
) capital_base=0.5,
data_frequency='minute',
initialize=initialize,
handle_data=handle_data,
analyze=analyze,
exchange_name='poloniex',
algo_namespace=algo_namespace,
base_currency='btc',
start=pd.to_datetime('2017-9-1', utc=True),
end=pd.to_datetime('2017-10-1', utc=True),
)
catalyst/examples/running_catalyst_in_jupyter_notebook.ipynb
-2493
View File
File diff suppressed because one or more lines are too long
catalyst/examples/simple_loop.py
+32 -45
View File
@@ -1,38 +1,32 @@
import pandas as pd
import talib import talib
from logbook import Logger, INFO import pandas as pd
from catalyst import run_algorithm from catalyst import run_algorithm
from catalyst.api import symbol, record from catalyst.api import symbol, record
from catalyst.exchange.utils.stats_utils import get_pretty_stats, \ from catalyst.exchange.stats_utils import get_pretty_stats, \
extract_transactions extract_transactions
log = Logger('simple_loop', level=INFO)
def initialize(context): def initialize(context):
log.info('initializing') print('initializing')
context.asset = symbol('eth_btc') context.asset = symbol('neo_eth')
context.base_price = None context.base_price = None
def handle_data(context, data): def handle_data(context, data):
log.info('handling bar: {}'.format(data.current_dt)) print('handling bar: {}'.format(data.current_dt))
price = data.current(context.asset, 'close') price = data.current(context.asset, 'close')
log.info('got price {price}'.format(price=price)) print('got price {price}'.format(price=price))
prices = data.history( prices = data.history(
context.asset, context.asset,
fields='price', fields='price',
bar_count=20, bar_count=20,
frequency='30T' frequency='15T'
) )
last_traded = prices.index[-1]
log.info('last candle date: {}'.format(last_traded))
rsi = talib.RSI(prices.values, timeperiod=14)[-1] rsi = talib.RSI(prices.values, timeperiod=14)[-1]
log.info('got rsi: {}'.format(rsi)) print('got rsi: {}'.format(rsi))
# If base_price is not set, we use the current value. This is the # If base_price is not set, we use the current value. This is the
# price at the first bar which we reference to calculate price_change. # price at the first bar which we reference to calculate price_change.
@@ -54,10 +48,10 @@ def handle_data(context, data):
def analyze(context, perf): def analyze(context, perf):
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
log.info('the stats: {}'.format(get_pretty_stats(perf))) print('the stats: {}'.format(get_pretty_stats(perf)))
# The base currency of the algo exchange # The base currency of the algo exchange
base_currency = list(context.exchanges.values())[0].base_currency.upper() base_currency = context.exchanges.values()[0].base_currency.upper()
# Plot the portfolio value over time. # Plot the portfolio value over time.
ax1 = plt.subplot(611) ax1 = plt.subplot(611)
@@ -113,32 +107,25 @@ def analyze(context, perf):
pass pass
if __name__ == '__main__': run_algorithm(
mode = 'live' capital_base=250,
start=pd.to_datetime('2017-11-1 0:00', utc=True),
if mode == 'backtest': end=pd.to_datetime('2017-11-10 23:59', utc=True),
run_algorithm( data_frequency='daily',
capital_base=1, initialize=initialize,
initialize=initialize, handle_data=handle_data,
handle_data=handle_data, analyze=analyze,
analyze=None, exchange_name='bitfinex',
exchange_name='poloniex', algo_namespace='simple_loop',
algo_namespace='simple_loop', base_currency='usd'
base_currency='eth', )
data_frequency='minute', # run_algorithm(
start=pd.to_datetime('2017-9-1', utc=True), # initialize=initialize,
end=pd.to_datetime('2017-12-1', utc=True), # handle_data=handle_data,
) # analyze=None,
else: # exchange_name='binance',
run_algorithm( # live=True,
capital_base=1, # algo_namespace='simple_loop',
initialize=initialize, # base_currency='eth',
handle_data=handle_data, # live_graph=False,
analyze=None, # )
exchange_name='binance',
live=True,
algo_namespace='simple_loop',
base_currency='eth',
live_graph=False,
simulate_orders=True
)
catalyst/examples/simple_universe.py
+60 -102
View File
@@ -2,117 +2,73 @@
Requires Catalyst version 0.3.0 or above Requires Catalyst version 0.3.0 or above
Tested on Catalyst version 0.3.3 Tested on Catalyst version 0.3.3
This example aims to provide an easy way for users to learn how to These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
collect data from any given exchange and select a subset of the available You simply need to specify the exchange and the market that you want to focus on.
currency pairs for trading. You simply need to specify the exchange and You will all see how to create a universe and filter it base on the exchange and the market you desire.
the market (base_currency) that you want to focus on. You will then see
how to create a universe of assets, and filter it based the market you
desire.
The example prints out the closing price of all the pairs for a given The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
market in a given exchange every 30 minutes. The example also contains The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
the OHLCV data with minute-resolution for the past seven days which Use this as the backbone to create your own trading strategies.
could be used to create indicators. Use this code as the backbone to
create your own trading strategy.
The lookback_date variable is used to ensure data for a coin existed on
the lookback period specified.
To run, execute the following two commands in a terminal (inside catalyst
environment). The first one retrieves all the pricing data needed for this
script to run (only needs to be run once), and the second one executes this
script with the parameters specified in the run_algorithm() call at the end
of the file:
catalyst ingest-exchange -x bitfinex -f minute
python simple_universe.py
Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
""" """
from datetime import timedelta
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from datetime import timedelta
from catalyst import run_algorithm from catalyst import run_algorithm
from catalyst.api import (symbols, ) from catalyst.exchange.exchange_utils import get_exchange_symbols
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols
from catalyst.api import (
symbols,
)
def initialize(context): def initialize(context):
context.i = -1 # minute counter context.i = -1 # counts the minutes
context.exchange = list(context.exchanges.values())[0].name.lower() context.exchange = context.exchanges.values()[0].name.lower() # exchange name
context.base_currency = list(context.exchanges.values())[0].base_currency.lower() context.base_currency = context.exchanges.values()[0].base_currency.lower() # market base currency
def handle_data(context, data): def handle_data(context, data):
context.i += 1 context.i += 1
lookback_days = 7 # 7 days lookback_days = 7 # 7 days
# current date & time in each iteration formatted into a string # current date formatted into a string
now = data.current_dt today = data.current_dt
date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ') date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
lookback_date = now - timedelta(days=lookback_days) lookback_date = today - timedelta(days=lookback_days) # subtract the amount of days specified in lookback
# keep only the date as a string, discard the time lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] # get only the date as a string
lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]
one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute' # update universe everyday
# update universe everyday at midnight new_day = 60 * 24 # assuming data_frequency='minute'
if not context.i % one_day_in_minutes: if not context.i % new_day:
context.universe = universe(context, lookback_date, date) context.universe = universe(context, lookback_date, date)
# get data every 30 minutes # get data every 30 minutes
minutes = 30 minutes = 30
one_day_in_minutes = 1440 # 1440 assumes data_frequency='minute'
# get lookback_days of history data: that is 'lookback' number of bins lookback = one_day_in_minutes / minutes * lookback_days # get N lookback_days of history data
lookback = int(one_day_in_minutes / minutes * lookback_days)
if not context.i % minutes and context.universe: if not context.i % minutes and context.universe:
# we iterate for every pair in the current universe # we iterate for every pair in the current universe
for coin in context.coins: for coin in context.coins:
pair = str(coin.symbol) pair = str(coin.symbol)
# Get 30 minute interval OHLCV data. This is the standard data # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
# required for candlestick or indicators/signals. Return Pandas # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
# DataFrames. 30T means 30-minute re-sampling of one minute data. opened = fill(data.history(coin, 'open', bar_count=lookback, frequency='30T')).values
# Adjust it to your desired time interval as needed. high = fill(data.history(coin, 'high', bar_count=lookback, frequency='30T')).values
opened = fill(data.history(coin, low = fill(data.history(coin, 'low', bar_count=lookback, frequency='30T')).values
'open', close = fill(data.history(coin, 'price', bar_count=lookback, frequency='30T')).values
bar_count=lookback, volume = fill(data.history(coin, 'volume', bar_count=lookback, frequency='30T')).values
frequency='30T')).values
high = fill(data.history(coin,
'high',
bar_count=lookback,
frequency='30T')).values
low = fill(data.history(coin,
'low',
bar_count=lookback,
frequency='30T')).values
close = fill(data.history(coin,
'price',
bar_count=lookback,
frequency='30T')).values
volume = fill(data.history(coin,
'volume',
bar_count=lookback,
frequency='30T')).values
# close[-1] is the last value in the set, which is the equivalent # close[-1] is the equivalent to current price
# to current price (as in the most recent value)
# displays the minute price for each pair every 30 minutes # displays the minute price for each pair every 30 minutes
print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},' print(today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
'\tV:{v}'.format(
now=now,
pair=pair,
o=opened[-1],
h=high[-1],
l=low[-1],
c=close[-1],
v=volume[-1],
))
# ------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------
# --------------- Insert Your Strategy Here ------------------- # -------------------------------------- Insert Your Strategy Here -----------------------------------------
# ------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------
def analyze(context=None, results=None): def analyze(context=None, results=None):
@@ -122,24 +78,23 @@ def analyze(context=None, results=None):
# Get the universe for a given exchange and a given base_currency market # Get the universe for a given exchange and a given base_currency market
# Example: Poloniex BTC Market # Example: Poloniex BTC Market
def universe(context, lookback_date, current_date): def universe(context, lookback_date, current_date):
# get all the pairs for the given exchange json_symbols = get_exchange_symbols(context.exchange) # get all the pairs for the exchange
json_symbols = get_exchange_symbols(context.exchange) universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) # convert into a dataframe
# convert into a DataFrame for easier processing universe_df['base_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[1],
df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) axis=1)
df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1], universe_df['market_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[0],
axis=1) axis=1)
df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],
axis=1)
# Filter all the pairs to get only the ones for a given base_currency # Filter all the exchange pairs to only the ones for a give base currency
df = df[df['base_currency'] == context.base_currency] universe_df = universe_df[universe_df['base_currency'] == context.base_currency]
# Filter all pairs to ensure that pair existed in the current date range # Filter all the pairs to ensure that pair existed in the current date range
df = df[df.start_date < lookback_date] universe_df = universe_df[universe_df.start_date < lookback_date]
df = df[df.end_daily >= current_date] universe_df = universe_df[universe_df.end_daily >= current_date]
context.coins = symbols(*df.symbol) # convert all the pairs to symbols context.coins = symbols(*universe_df.symbol) # convert all the pairs to symbols
return df.symbol.tolist() # print(universe_df.symbol.tolist())
return universe_df.symbol.tolist()
# Replace all NA, NAN or infinite values with its nearest value # Replace all NA, NAN or infinite values with its nearest value
@@ -147,9 +102,7 @@ def fill(series):
if isinstance(series, pd.Series): if isinstance(series, pd.Series):
return series.replace([np.inf, -np.inf], np.nan).ffill().bfill() return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
elif isinstance(series, np.ndarray): elif isinstance(series, np.ndarray):
return pd.Series(series).replace( return pd.Series(series).replace([np.inf, -np.inf], np.nan).ffill().bfill().values
[np.inf, -np.inf], np.nan
).ffill().bfill().values
else: else:
return series return series
@@ -159,13 +112,18 @@ if __name__ == '__main__':
end_date = pd.to_datetime('2017-11-13', utc=True) end_date = pd.to_datetime('2017-11-13', utc=True)
performance = run_algorithm(start=start_date, end=end_date, performance = run_algorithm(start=start_date, end=end_date,
capital_base=100.0, # amount of base_currency capital_base=100.0, # amount of base_currency, not always in dollars unless usd
initialize=initialize, initialize=initialize,
handle_data=handle_data, handle_data=handle_data,
analyze=analyze, analyze=analyze,
exchange_name='poloniex', exchange_name='bitfinex',
data_frequency='minute', data_frequency='minute',
base_currency='btc', base_currency='btc',
live=False, live=False,
live_graph=False, live_graph=False,
algo_namespace='simple_universe') algo_namespace='simple_universe')
"""
Run in Terminal (inside catalyst environment):
python simple_universe.py
"""
catalyst/examples/talib_simple.py
+6 -8
View File
@@ -1,11 +1,9 @@
# Run Command # Run Command
# catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle \ # catalyst run --start 2017-1-1 --end 2017-11-1 -o talib_simple.pickle -f talib_simple.py -x poloniex
# -f talib_simple.py -x poloniex #
#
# Description # Description
# Simple TALib Example showing how to use various indicators # Simple TALib Example showing how to use various indicators in you strategy
# in you strategy. Based loosly on # Based loosly on https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
# https://github.com/mellertson/talib-macd-example/blob/master/talib-macd-matplotlib-example.py
import os import os
@@ -23,7 +21,7 @@ from catalyst.api import (
order_target_percent, order_target_percent,
symbol, symbol,
) )
from catalyst.exchange.utils.stats_utils import get_pretty_stats from catalyst.exchange.stats_utils import get_pretty_stats
algo_namespace = 'talib_sample' algo_namespace = 'talib_sample'
log = Logger(algo_namespace) log = Logger(algo_namespace)
@@ -90,7 +88,7 @@ def _handle_data(context, data):
prices.close.as_matrix(), fastperiod=context.MACD_FAST, prices.close.as_matrix(), fastperiod=context.MACD_FAST,
slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL) slowperiod=context.MACD_SLOW, signalperiod=context.MACD_SIGNAL)
# Stochastics %K %D # Stochastics %K %D
# %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100 # %K = (Current Close - Lowest Low)/(Highest High - Lowest Low) * 100
# %D = 3-day SMA of %K # %D = 3-day SMA of %K
analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH( analysis['stoch_k'], analysis['stoch_d'] = ta.STOCH(
catalyst/exchange/asset_finder_exchange.py
+99
View File
@@ -0,0 +1,99 @@
from logbook import Logger
from catalyst.constants import LOG_LEVEL
log = Logger('AssetFinderExchange', level=LOG_LEVEL)
class AssetFinderExchange(object):
def __init__(self):
self._asset_cache = {}
@property
def sids(self):
"""
This seems to be used to pre-fetch assets.
I don't think that we need this for live-trading.
Leaving the list empty.
"""
return list()
def retrieve_all(self, sids, default_none=False):
"""
Retrieve all assets in `sids`.
Parameters
----------
sids : iterable of int
Assets to retrieve.
default_none : bool
If True, return None for failed lookups.
If False, raise `SidsNotFound`.
Returns
-------
assets : list[Asset or None]
A list of the same length as `sids` containing Assets (or Nones)
corresponding to the requested sids.
Raises
------
SidsNotFound
When a requested sid is not found and default_none=False.
"""
# for sid in sids:
# if sid in self._asset_cache:
# log.debug('got asset from cache: {}'.format(sid))
# else:
# log.debug('fetching asset: {}'.format(sid))
return list()
def lookup_symbol(self, symbol, exchange, data_frequency=None,
as_of_date=None, fuzzy=False):
"""Lookup an asset by symbol.
Parameters
----------
symbol : str
The ticker symbol to resolve.
as_of_date : datetime or None
Look up the last owner of this symbol as of this datetime.
If ``as_of_date`` is None, then this can only resolve the equity
if exactly one equity has ever owned the ticker.
fuzzy : bool, optional
Should fuzzy symbol matching be used? Fuzzy symbol matching
attempts to resolve differences in representations for
shareclasses. For example, some people may represent the ``A``
shareclass of ``BRK`` as ``BRK.A``, where others could write
``BRK_A``.
Returns
-------
equity : Asset
The equity that held ``symbol`` on the given ``as_of_date``, or the
only equity to hold ``symbol`` if ``as_of_date`` is None.
Raises
------
SymbolNotFound
Raised when no equity has ever held the given symbol.
MultipleSymbolsFound
Raised when no ``as_of_date`` is given and more than one equity
has held ``symbol``. This is also raised when ``fuzzy=True`` and
there are multiple candidates for the given ``symbol`` on the
``as_of_date``.
"""
log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
if data_frequency is not None:
key = ','.join([exchange.name, symbol, data_frequency])
else:
key = ','.join([exchange.name, symbol])
if key in self._asset_cache:
return self._asset_cache[key]
else:
asset = exchange.get_asset(symbol, data_frequency)
self._asset_cache[key] = asset
return asset
catalyst/exchange/utils/__init__.py → catalyst/exchange/bitfinex/__init__.py
View File
catalyst/exchange/bitfinex/bitfinex.py
+705
View File
@@ -0,0 +1,705 @@
import base64
import datetime
import hashlib
import hmac
import json
import re
import time
import numpy as np
import pandas as pd
import pytz
import requests
import six
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
InvalidHistoryFrequencyError,
InvalidOrderStyle, OrderCancelError)
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
ExchangeStopLimitOrder, ExchangeStopOrder
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
download_exchange_symbols, get_symbols_string
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.protocol import Account
# Trying to account for REST api instability
# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
requests.adapters.DEFAULT_RETRIES = 20
BITFINEX_URL = 'https://api.bitfinex.com'
from catalyst.constants import LOG_LEVEL
log = Logger('Bitfinex', level=LOG_LEVEL)
warning_logger = Logger('AlgoWarning')
class Bitfinex(Exchange):
def __init__(self, key, secret, base_currency, portfolio=None):
self.url = BITFINEX_URL
self.key = key
self.secret = secret.encode('UTF-8')
self.name = 'bitfinex'
self.color = 'green'
self.assets = dict()
self.load_assets()
self.local_assets = dict()
self.load_assets(is_local=True)
self.base_currency = base_currency
self._portfolio = portfolio
self.minute_writer = None
self.minute_reader = None
# The candle limit for each request
self.num_candles_limit = 1000
# Max is 90 but playing it safe
# https://www.bitfinex.com/posts/188
self.max_requests_per_minute = 80
self.request_cpt = dict()
self.bundle = ExchangeBundle(self.name)
def _request(self, operation, data, version='v1'):
payload_object = {
'request': '/{}/{}'.format(version, operation),
'nonce': '{0:f}'.format(time.time() * 1000000),
# convert to string
'options': {}
}
if data is None:
payload_dict = payload_object
else:
payload_dict = payload_object.copy()
payload_dict.update(data)
payload_json = json.dumps(payload_dict)
if six.PY3:
payload = base64.b64encode(bytes(payload_json, 'utf-8'))
else:
payload = base64.b64encode(payload_json)
m = hmac.new(self.secret, payload, hashlib.sha384)
m = m.hexdigest()
# headers
headers = {
'X-BFX-APIKEY': self.key,
'X-BFX-PAYLOAD': payload,
'X-BFX-SIGNATURE': m
}
if data is None:
request = requests.get(
'{url}/{version}/{operation}'.format(
url=self.url,
version=version,
operation=operation
), data={},
headers=headers)
else:
request = requests.post(
'{url}/{version}/{operation}'.format(
url=self.url,
version=version,
operation=operation
),
headers=headers)
return request
def _get_v2_symbol(self, asset):
pair = asset.symbol.split('_')
symbol = 't' + pair[0].upper() + pair[1].upper()
return symbol
def _get_v2_symbols(self, assets):
"""
Workaround to support Bitfinex v2
TODO: Might require a separate asset dictionary
:param assets:
:return:
"""
v2_symbols = []
for asset in assets:
v2_symbols.append(self._get_v2_symbol(asset))
return v2_symbols
def _create_order(self, order_status):
"""
Create a Catalyst order object from a Bitfinex order dictionary
:param order_status:
:return: Order
"""
if order_status['is_cancelled']:
status = ORDER_STATUS.CANCELLED
elif not order_status['is_live']:
log.info('found executed order {}'.format(order_status))
status = ORDER_STATUS.FILLED
else:
status = ORDER_STATUS.OPEN
amount = float(order_status['original_amount'])
filled = float(order_status['executed_amount'])
if order_status['side'] == 'sell':
amount = -amount
filled = -filled
price = float(order_status['price'])
order_type = order_status['type']
stop_price = None
limit_price = None
# TODO: is this comprehensive enough?
if order_type.endswith('limit'):
limit_price = price
elif order_type.endswith('stop'):
stop_price = price
executed_price = float(order_status['avg_execution_price'])
# TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
commission = None
date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
date = pytz.utc.localize(date)
order = Order(
dt=date,
asset=self.assets[order_status['symbol']],
amount=amount,
stop=stop_price,
limit=limit_price,
filled=filled,
id=str(order_status['id']),
commission=commission
)
order.status = status
return order, executed_price
def get_balances(self):
log.debug('retrieving wallets balances')
try:
self.ask_request()
response = self._request('balances', None)
balances = response.json()
except Exception as e:
raise ExchangeRequestError(error=e)
if 'message' in balances:
raise ExchangeRequestError(
error='unable to fetch balance {}'.format(balances['message'])
)
std_balances = dict()
for balance in balances:
currency = balance['currency'].lower()
std_balances[currency] = float(balance['available'])
return std_balances
@property
def account(self):
account = Account()
account.settled_cash = None
account.accrued_interest = None
account.buying_power = None
account.equity_with_loan = None
account.total_positions_value = None
account.total_positions_exposure = None
account.regt_equity = None
account.regt_margin = None
account.initial_margin_requirement = None
account.maintenance_margin_requirement = None
account.available_funds = None
account.excess_liquidity = None
account.cushion = None
account.day_trades_remaining = None
account.leverage = None
account.net_leverage = None
account.net_liquidation = None
return account
@property
def time_skew(self):
# TODO: research the time skew conditions
return pd.Timedelta('0s')
def get_account(self):
# TODO: fetch account data and keep in cache
return None
def get_candles(self, freq, assets, bar_count=None,
start_dt=None, end_dt=None):
"""
Retrieve OHLVC candles from Bitfinex
:param data_frequency:
:param assets:
:param bar_count:
:return:
Available Frequencies
---------------------
'1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
'1M'
"""
log.debug(
'retrieving {bars} {freq} candles on {exchange} from '
'{end_dt} for markets {symbols}, '.format(
bars=bar_count,
freq=freq,
exchange=self.name,
end_dt=end_dt,
symbols=get_symbols_string(assets)
)
)
allowed_frequencies = ['1T', '5T', '15T', '30T', '60T', '180T',
'360T', '720T', '1D', '7D', '14D', '30D']
if freq not in allowed_frequencies:
raise InvalidHistoryFrequencyError(frequency=freq)
freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
if freq_match:
number = int(freq_match.group(1))
unit = freq_match.group(2)
if unit == 'T':
if number in [60, 180, 360, 720]:
number = number / 60
converted_unit = 'h'
else:
converted_unit = 'm'
else:
converted_unit = unit
frequency = '{}{}'.format(number, converted_unit)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
symbol = self._get_v2_symbol(asset)
url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
url=self.url,
frequency=frequency,
symbol=symbol
)
if bar_count:
is_list = True
url += '/hist?limit={}'.format(int(bar_count))
def get_ms(date):
epoch = datetime.datetime.utcfromtimestamp(0)
epoch = epoch.replace(tzinfo=pytz.UTC)
return (date - epoch).total_seconds() * 1000.0
if start_dt is not None:
start_ms = get_ms(start_dt)
url += '&start={0:f}'.format(start_ms)
if end_dt is not None:
end_ms = get_ms(end_dt)
url += '&end={0:f}'.format(end_ms)
else:
is_list = False
url += '/last'
try:
self.ask_request()
response = requests.get(url)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response.content:
raise ExchangeRequestError(
error='Unable to retrieve candles: {}'.format(
response.content)
)
candles = response.json()
def ohlc_from_candle(candle):
last_traded = pd.Timestamp.utcfromtimestamp(
candle[0] / 1000.0)
last_traded = last_traded.replace(tzinfo=pytz.UTC)
ohlc = dict(
open=np.float64(candle[1]),
high=np.float64(candle[3]),
low=np.float64(candle[4]),
close=np.float64(candle[2]),
volume=np.float64(candle[5]),
price=np.float64(candle[2]),
last_traded=last_traded
)
return ohlc
if is_list:
ohlc_bars = []
# We can to list candles from old to new
for candle in reversed(candles):
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
else:
ohlc = ohlc_from_candle(candles)
ohlc_map[asset] = ohlc
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def create_order(self, asset, amount, is_buy, style):
"""
Creating order on the exchange.
:param asset:
:param amount:
:param is_buy:
:param style:
:return:
"""
exchange_symbol = self.get_symbol(asset)
if isinstance(style, ExchangeLimitOrder) \
or isinstance(style, ExchangeStopLimitOrder):
price = style.get_limit_price(is_buy)
order_type = 'limit'
elif isinstance(style, ExchangeStopOrder):
price = style.get_stop_price(is_buy)
order_type = 'stop'
else:
raise InvalidOrderStyle(exchange=self.name,
style=style.__class__.__name__)
req = dict(
symbol=exchange_symbol,
amount=str(float(abs(amount))),
price="{:.20f}".format(float(price)),
side='buy' if is_buy else 'sell',
type='exchange ' + order_type, # TODO: support margin trades
exchange=self.name,
is_hidden=False,
is_postonly=False,
use_all_available=0,
ocoorder=False,
buy_price_oco=0,
sell_price_oco=0
)
date = pd.Timestamp.utcnow()
try:
self.ask_request()
response = self._request('order/new', req)
order_status = response.json()
except Exception as e:
raise ExchangeRequestError(error=e)
if 'message' in order_status:
raise ExchangeRequestError(
error='unable to create Bitfinex order {}'.format(
order_status['message'])
)
order_id = str(order_status['id'])
order = Order(
dt=date,
asset=asset,
amount=amount,
stop=style.get_stop_price(is_buy),
limit=style.get_limit_price(is_buy),
id=order_id
)
return order
def get_open_orders(self, asset=None):
"""Retrieve all of the current open orders.
Parameters
----------
asset : Asset
If passed and not None, return only the open orders for the given
asset instead of all open orders.
Returns
-------
open_orders : dict[list[Order]] or list[Order]
If no asset is passed this will return a dict mapping Assets
to a list containing all the open orders for the asset.
If an asset is passed then this will return a list of the open
orders for this asset.
"""
try:
self.ask_request()
response = self._request('orders', None)
order_statuses = response.json()
except Exception as e:
raise ExchangeRequestError(error=e)
if 'message' in order_statuses:
raise ExchangeRequestError(
error='Unable to retrieve open orders: {}'.format(
order_statuses['message'])
)
orders = []
for order_status in order_statuses:
order, executed_price = self._create_order(order_status)
if asset is None or asset == order.sid:
orders.append(order)
return orders
def get_order(self, order_id):
"""Lookup an order based on the order id returned from one of the
order functions.
Parameters
----------
order_id : str
The unique identifier for the order.
Returns
-------
order : Order
The order object.
"""
try:
self.ask_request()
response = self._request(
'order/status', {'order_id': int(order_id)})
order_status = response.json()
except Exception as e:
raise ExchangeRequestError(error=e)
if 'message' in order_status:
raise ExchangeRequestError(
error='Unable to retrieve order status: {}'.format(
order_status['message'])
)
return self._create_order(order_status)
def cancel_order(self, order_param):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
"""
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
try:
self.ask_request()
response = self._request('order/cancel', {'order_id': order_id})
status = response.json()
except Exception as e:
raise ExchangeRequestError(error=e)
if 'message' in status:
raise OrderCancelError(
order_id=order_id,
exchange=self.name,
error=status['message']
)
def tickers(self, assets):
"""
Fetch ticket data for assets
https://docs.bitfinex.com/v2/reference#rest-public-tickers
:param assets:
:return:
"""
symbols = self._get_v2_symbols(assets)
log.debug('fetching tickers {}'.format(symbols))
try:
self.ask_request()
response = requests.get(
'{url}/v2/tickers?symbols={symbols}'.format(
url=self.url,
symbols=','.join(symbols),
)
)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response.content:
raise ExchangeRequestError(
error='Unable to retrieve tickers: {}'.format(
response.content)
)
try:
tickers = response.json()
except Exception as e:
raise ExchangeRequestError(error=e)
ticks = dict()
for index, ticker in enumerate(tickers):
if not len(ticker) == 11:
raise ExchangeRequestError(
error='Invalid ticker in response: {}'.format(ticker)
)
ticks[assets[index]] = dict(
timestamp=pd.Timestamp.utcnow(),
bid=ticker[1],
ask=ticker[3],
last_price=ticker[7],
low=ticker[10],
high=ticker[9],
volume=ticker[8],
)
log.debug('got tickers {}'.format(ticks))
return ticks
def generate_symbols_json(self, filename=None, source_dates=False):
symbol_map = {}
if not source_dates:
fn, r = download_exchange_symbols(self.name)
with open(fn) as data_file:
cached_symbols = json.load(data_file)
response = self._request('symbols', None)
for symbol in response.json():
if (source_dates):
start_date = self.get_symbol_start_date(symbol)
else:
try:
start_date = cached_symbols[symbol]['start_date']
except KeyError as e:
start_date = time.strftime('%Y-%m-%d')
try:
end_daily = cached_symbols[symbol]['end_daily']
except KeyError as e:
end_daily = 'N/A'
try:
end_minute = cached_symbols[symbol]['end_minute']
except KeyError as e:
end_minute = 'N/A'
symbol_map[symbol] = dict(
symbol=symbol[:-3] + '_' + symbol[-3:],
start_date=start_date,
end_daily=end_daily,
end_minute=end_minute,
)
if (filename is None):
filename = get_exchange_symbols_filename(self.name)
with open(filename, 'w') as f:
json.dump(symbol_map, f, sort_keys=True, indent=2,
separators=(',', ':'))
def get_symbol_start_date(self, symbol):
print(symbol)
symbol_v2 = 't' + symbol.upper()
"""
For each symbol we retrieve candles with Monhtly resolution
We get the first month, and query again with daily resolution
around that date, and we get the first date
"""
url = '{url}/v2/candles/trade:1M:{symbol}/hist'.format(
url=self.url,
symbol=symbol_v2
)
try:
self.ask_request()
response = requests.get(url)
except Exception as e:
raise ExchangeRequestError(error=e)
"""
If we don't get any data back for our monthly-resolution query
it means that symbol started trading less than a month ago, so
arbitrarily set the ref. date to 15 days ago to be safe with
+/- 31 days
"""
if (len(response.json())):
startmonth = response.json()[-1][0]
else:
startmonth = int((time.time() - 15 * 24 * 3600) * 1000)
"""
Query again with daily resolution setting the start and end around
the startmonth we got above. Avoid end dates greater than now: time.time()
"""
url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format(
url=self.url,
symbol=symbol_v2,
start=startmonth - 3600 * 24 * 31 * 1000,
end=min(startmonth + 3600 * 24 * 31 * 1000,
int(time.time() * 1000))
)
try:
self.ask_request()
response = requests.get(url)
except Exception as e:
raise ExchangeRequestError(error=e)
return time.strftime('%Y-%m-%d',
time.gmtime(int(response.json()[-1][0] / 1000)))
def get_orderbook(self, asset, order_type='all', limit=100):
exchange_symbol = asset.exchange_symbol
try:
self.ask_request()
# TODO: implement limit
response = self._request(
'book/{}'.format(exchange_symbol), None)
data = response.json()
except Exception as e:
raise ExchangeRequestError(error=e)
# TODO: filter by type
result = dict()
for order_type in data:
result[order_type] = []
for entry in data[order_type]:
result[order_type].append(dict(
rate=float(entry['price']),
quantity=float(entry['amount'])
))
return result
catalyst/exchange/bitfinex/symbols.json
+127
View File
@@ -0,0 +1,127 @@
{
"neobtc": {
"symbol": "neo_btc",
"start_date": "2017-09-07",
"precision": 5
},
"neousd": {
"symbol": "neo_usd",
"start_date": "2017-09-07"
},
"neoeth": {
"symbol": "neo_eth",
"start_date": "2017-09-07"
},
"btcusd": {
"symbol": "btc_usd",
"start_date": "2010-01-01"
},
"bchusd": {
"symbol": "bch_usd",
"start_date": "2010-01-01"
},
"ltcusd": {
"symbol": "ltc_usd",
"start_date": "2010-01-01"
},
"ltcbtc": {
"symbol": "ltc_btc",
"start_date": "2010-01-01"
},
"ethusd": {
"symbol": "eth_usd",
"start_date": "2017-01-01"
},
"ethbtc": {
"symbol": "eth_btc",
"start_date": "2017-01-01"
},
"etcbtc": {
"symbol": "etc_btc",
"start_date": "2017-01-01"
},
"etcusd": {
"symbol": "etc_usd",
"start_date": "2017-01-01"
},
"rrtusd": {
"symbol": "rrt_usd",
"start_date": "2010-01-01"
},
"rrtbtc": {
"symbol": "rrt_btc",
"start_date": "2010-01-01"
},
"zecusd": {
"symbol": "zec_usd",
"start_date": "2010-01-01"
},
"zecbtc": {
"symbol": "zec_btc",
"start_date": "2010-01-01"
},
"xmrusd": {
"symbol": "xmr_usd",
"start_date": "2010-01-01"
},
"xmrbtc": {
"symbol": "xmr_btc",
"start_date": "2010-01-01"
},
"dshusd": {
"symbol": "dsh_usd",
"start_date": "2010-01-01"
},
"dshbtc": {
"symbol": "dsh_btc",
"start_date": "2010-01-01"
},
"bccbtc": {
"symbol": "bcc_btc",
"start_date": "2010-01-01"
},
"bcubtc": {
"symbol": "bcu_btc",
"start_date": "2010-01-01"
},
"bccusd": {
"symbol": "bcc_usd",
"start_date": "2010-01-01"
},
"bcuusd": {
"symbol": "bcu_usd",
"start_date": "2010-01-01"
},
"xrpusd": {
"symbol": "xrp_usd",
"start_date": "2010-01-01"
},
"xrpbtc": {
"symbol": "xrp_btc",
"start_date": "2010-01-01"
},
"iotusd": {
"symbol": "iot_usd",
"start_date": "2010-01-01"
},
"iotbtc": {
"symbol": "iot_btc",
"start_date": "2010-01-01"
},
"ioteth": {
"symbol": "iot_eth",
"start_date": "2010-01-01"
},
"eosusd": {
"symbol": "eos_usd",
"start_date": "2010-01-01"
},
"eosbtc": {
"symbol": "eos_btc",
"start_date": "2010-01-01"
},
"eoseth": {
"symbol": "eos_eth",
"start_date": "2010-01-01"
}
}
catalyst/marketplace/__init__.py → catalyst/exchange/bittrex/__init__.py
View File
catalyst/exchange/bittrex/bittrex.py
+416
View File
@@ -0,0 +1,416 @@
import json
import time
import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from six.moves import urllib
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
CreateOrderError
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
download_exchange_symbols, get_symbols_string
from catalyst.finance.execution import LimitOrder, StopLimitOrder
from catalyst.finance.order import Order, ORDER_STATUS
# TODO: consider using this: https://github.com/mondeja/bittrex_v2
log = Logger('Bittrex', level=LOG_LEVEL)
URL2 = 'https://bittrex.com/Api/v2.0'
class Bittrex(Exchange):
def __init__(self, key, secret, base_currency, portfolio=None):
self.api = Bittrex_api(key=key, secret=secret)
self.name = 'bittrex'
self.color = 'blue'
self.base_currency = base_currency
self._portfolio = portfolio
self.num_candles_limit = 2000
# Not sure what the rate limit is but trying to play it safe
# https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit
self.max_requests_per_minute = 60
self.request_cpt = dict()
self.minute_writer = None
self.minute_reader = None
self.assets = dict()
self.load_assets()
self.local_assets = dict()
self.load_assets(is_local=True)
self.bundle = ExchangeBundle(self.name)
@property
def account(self):
pass
@property
def time_skew(self):
# TODO: research the time skew conditions
return pd.Timedelta('0s')
def sanitize_curency_symbol(self, exchange_symbol):
"""
Helper method used to build the universal pair.
Include any symbol mapping here if appropriate.
:param exchange_symbol:
:return universal_symbol:
"""
return exchange_symbol.lower()
def get_balances(self):
balances = self.api.getbalances()
try:
log.debug('retrieving wallet balances')
self.ask_request()
except Exception as e:
raise ExchangeRequestError(error=e)
std_balances = dict()
try:
for balance in balances:
currency = balance['Currency'].lower()
std_balances[currency] = balance['Available']
except TypeError:
raise ExchangeRequestError(error=balances)
return std_balances
def create_order(self, asset, amount, is_buy, style):
log.info('creating {} order'.format('buy' if is_buy else 'sell'))
exchange_symbol = self.get_symbol(asset)
if isinstance(style, LimitOrder) or isinstance(style, StopLimitOrder):
if isinstance(style, StopLimitOrder):
log.warn('{} will ignore the stop price'.format(self.name))
price = style.get_limit_price(is_buy)
try:
self.ask_request()
if is_buy:
order_status = self.api.buylimit(exchange_symbol, amount,
price)
else:
order_status = self.api.selllimit(exchange_symbol,
abs(amount), price)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'uuid' in order_status:
order_id = order_status['uuid']
order = Order(
dt=pd.Timestamp.utcnow(),
asset=asset,
amount=amount,
stop=style.get_stop_price(is_buy),
limit=style.get_limit_price(is_buy),
id=order_id
)
return order
else:
if order_status == 'INSUFFICIENT_FUNDS':
log.warn('not enough funds to create order')
return None
elif order_status == 'DUST_TRADE_DISALLOWED_MIN_VALUE_50K_SAT':
log.warn('Your order is too small, order at least 50K'
' Satoshi')
return None
else:
raise CreateOrderError(
exchange=self.name,
error=order_status
)
else:
raise InvalidOrderStyle(exchange=self.name,
style=style.__class__.__name__)
def get_open_orders(self, asset):
symbol = self.get_symbol(asset)
try:
self.ask_request()
open_orders = self.api.getopenorders(symbol)
except Exception as e:
raise ExchangeRequestError(error=e)
orders = list()
for order_status in open_orders:
order = self._create_order(order_status)
orders.append(order)
return orders
def _create_order(self, order_status):
log.info(
'creating catalyst order from Bittrex {}'.format(order_status))
if order_status['CancelInitiated']:
status = ORDER_STATUS.CANCELLED
elif order_status['Closed'] is not None:
status = ORDER_STATUS.FILLED
else:
status = ORDER_STATUS.OPEN
date = pd.to_datetime(order_status['Opened'], utc=True)
amount = order_status['Quantity']
filled = amount - order_status['QuantityRemaining']
order = Order(
dt=date,
asset=self.assets[order_status['Exchange']],
amount=amount,
stop=None, # Not yet supported by Bittrex
limit=order_status['Limit'],
filled=filled,
id=order_status['OrderUuid'],
commission=order_status['CommissionPaid']
)
order.status = status
executed_price = order_status['PricePerUnit']
return order, executed_price
def get_order(self, order_id):
log.info('retrieving order {}'.format(order_id))
try:
self.ask_request()
order_status = self.api.getorder(order_id)
except Exception as e:
raise ExchangeRequestError(error=e)
if order_status is None:
raise OrderNotFound(order_id=order_id, exchange=self.name)
return self._create_order(order_status)
def cancel_order(self, order_param):
order_id = order_param.id \
if isinstance(order_param, Order) else order_param
log.info('cancelling order {}'.format(order_id))
try:
self.ask_request()
status = self.api.cancel(order_id)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'message' in status:
raise OrderCancelError(
order_id=order_id,
exchange=self.name,
error=status['message']
)
def get_candles(self, freq, assets, bar_count=None,
start_dt=None, end_dt=None):
"""
Supported Intervals
-------------------
day, oneMin, fiveMin, thirtyMin, hour
:param freq:
:param assets:
:param bar_count:
:param start_dt
:param end_dt
:return:
"""
# TODO: this has no effect at the moment
if end_dt is None:
end_dt = pd.Timestamp.utcnow()
log.debug(
'retrieving {bars} {freq} candles on {exchange} from '
'{end_dt} for markets {symbols}, '.format(
bars=bar_count,
freq=freq,
exchange=self.name,
end_dt=end_dt,
symbols=get_symbols_string(assets)
)
)
if freq == '1T':
frequency = 'oneMin'
elif freq == '5T':
frequency = 'fiveMin'
elif freq == '30T':
frequency = 'thirtyMin'
elif freq == '60T':
frequency = 'hour'
elif freq == '1D':
frequency = 'day'
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
for asset in asset_list:
end = int(time.mktime(end_dt.timetuple()))
url = '{url}/pub/market/GetTicks?marketName={symbol}' \
'&tickInterval={frequency}&_={end}'.format(
url=URL2,
symbol=self.get_symbol(asset),
frequency=frequency,
end=end
)
try:
data = json.loads(urllib.request.urlopen(url).read().decode())
except Exception as e:
raise ExchangeRequestError(error=e)
if data['message']:
raise ExchangeRequestError(
error='Unable to fetch candles {}'.format(data['message'])
)
candles = data['result']
def ohlc_from_candle(candle):
ohlc = dict(
open=candle['O'],
high=candle['H'],
low=candle['L'],
close=candle['C'],
volume=candle['V'],
price=candle['C'],
last_traded=pd.to_datetime(candle['T'], utc=True)
)
return ohlc
ordered_candles = list(reversed(candles))
ohlc_map = dict()
if bar_count is None:
ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
else:
# TODO: optimize
ohlc_bars = []
for candle in ordered_candles[:bar_count]:
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def tickers(self, assets):
"""
As of v1.1, Bittrex only allows one ticker at the time.
So we have to make multiple calls to fetch multiple assets.
:param assets:
:return:
"""
log.info('retrieving tickers')
ticks = dict()
for asset in assets:
symbol = self.get_symbol(asset)
try:
self.ask_request()
ticker = self.api.getticker(symbol)
except Exception as e:
raise ExchangeRequestError(error=e)
# TODO: catch invalid ticker
ticks[asset] = dict(
timestamp=pd.Timestamp.utcnow(),
bid=ticker['Bid'],
ask=ticker['Ask'],
last_price=ticker['Last']
)
log.debug('got tickers {}'.format(ticks))
return ticks
def get_account(self):
log.info('retrieving account data')
pass
def generate_symbols_json(self, filename=None):
symbol_map = {}
fn, r = download_exchange_symbols(self.name)
with open(fn) as data_file:
cached_symbols = json.load(data_file)
markets = self.api.getmarkets()
for market in markets:
exchange_symbol = market['MarketName']
symbol = '{market}_{base}'.format(
market=self.sanitize_curency_symbol(market['MarketCurrency']),
base=self.sanitize_curency_symbol(market['BaseCurrency'])
)
try:
end_daily = cached_symbols[exchange_symbol]['end_daily']
except KeyError as e:
end_daily = 'N/A'
try:
end_minute = cached_symbols[exchange_symbol]['end_minute']
except KeyError as e:
end_minute = 'N/A'
symbol_map[exchange_symbol] = dict(
symbol=symbol,
start_date=pd.to_datetime(market['Created'],
utc=True).strftime("%Y-%m-%d"),
end_daily=end_daily,
end_minute=end_minute,
)
if (filename is None):
filename = get_exchange_symbols_filename(self.name)
with open(filename, 'w') as f:
json.dump(symbol_map, f, sort_keys=True, indent=2,
separators=(',', ':'))
def get_orderbook(self, asset, order_type='all', limit=100):
if order_type == 'all':
order_type = 'both'
elif order_type == 'bid':
order_type = 'buy'
elif order_type == 'ask':
order_type = 'sell'
else:
raise ValueError('invalid type')
exchange_symbol = asset.exchange_symbol
data = self.api.getorderbook(
market=exchange_symbol,
type=order_type,
depth=100
)
result = dict()
for exchange_type in data:
if exchange_type == 'buy':
order_type = 'bids'
elif exchange_type == 'sell':
order_type = 'asks'
result[order_type] = []
for entry in data[exchange_type]:
result[order_type].append(dict(
rate=entry['Rate'],
quantity=entry['Quantity']
))
return result
catalyst/exchange/bittrex/bittrex_api.py
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python
import json
import time
import hmac
import hashlib
import ssl
# Workaround for backwards compatibility
# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
from six.moves import urllib
urlopen = urllib.request.urlopen
class Bittrex_api(object):
def __init__(self, key, secret):
self.key = key
self.secret = secret
self.public = ['getmarkets', 'getcurrencies', 'getticker',
'getmarketsummaries', 'getmarketsummary',
'getorderbook', 'getmarkethistory']
self.market = ['buylimit', 'buymarket', 'selllimit', 'sellmarket',
'cancel', 'getopenorders']
self.account = ['getbalances', 'getbalance', 'getdepositaddress',
'withdraw', 'getorder', 'getorderhistory',
'getwithdrawalhistory', 'getdeposithistory']
def query(self, method, values={}):
if method in self.public:
url = 'https://bittrex.com/api/v1.1/public/'
elif method in self.market:
url = 'https://bittrex.com/api/v1.1/market/'
elif method in self.account:
url = 'https://bittrex.com/api/v1.1/account/'
else:
return 'Something went wrong, sorry.'
url += method + '?' + urllib.parse.urlencode(values)
if method not in self.public:
url += '&apikey=' + self.key
url += '&nonce=' + str(int(time.time()))
signature = hmac.new(self.secret.encode('utf-8'),
url.encode('utf-8'),
hashlib.sha512).hexdigest()
headers = {'apisign': signature}
else:
headers = {}
req = urllib.request.Request(url, headers=headers)
response = json.loads(urlopen(
req, context=ssl._create_unverified_context()).read())
if response["result"]:
return response["result"]
else:
return response["message"]
def getmarkets(self):
return self.query('getmarkets')
def getcurrencies(self):
return self.query('getcurrencies')
def getticker(self, market):
return self.query('getticker', {'market': market})
def getmarketsummaries(self):
return self.query('getmarketsummaries')
def getmarketsummary(self, market):
return self.query('getmarketsummary', {'market': market})
def getorderbook(self, market, type, depth=20):
return self.query('getorderbook',
{'market': market, 'type': type, 'depth': depth})
def getmarkethistory(self, market, count=20):
return self.query('getmarkethistory',
{'market': market, 'count': count})
def buylimit(self, market, quantity, rate):
return self.query('buylimit', {'market': market, 'quantity': quantity,
'rate': rate})
def buymarket(self, market, quantity):
return self.query('buymarket',
{'market': market, 'quantity': quantity})
def selllimit(self, market, quantity, rate):
return self.query('selllimit', {'market': market, 'quantity': quantity,
'rate': rate})
def sellmarket(self, market, quantity):
return self.query('sellmarket',
{'market': market, 'quantity': quantity})
def cancel(self, uuid):
return self.query('cancel', {'uuid': uuid})
def getopenorders(self, market):
return self.query('getopenorders', {'market': market})
def getbalances(self):
return self.query('getbalances')
def getbalance(self, currency):
return self.query('getbalance', {'currency': currency})
def getdepositaddress(self, currency):
return self.query('getdepositaddress', {'currency': currency})
def withdraw(self, currency, quantity, address):
return self.query('withdraw',
{'currency': currency, 'quantity': quantity,
'address': address})
def getorder(self, uuid):
return self.query('getorder', {'uuid': uuid})
def getorderhistory(self, market, count):
return self.query('getorderhistory',
{'market': market, 'count': count})
def getwithdrawalhistory(self, currency, count):
return self.query('getwithdrawalhistory',
{'currency': currency, 'count': count})
def getdeposithistory(self, currency, count):
return self.query('getdeposithistory',
{'currency': currency, 'count': count})
catalyst/exchange/bittrex/extensions-example.py
+7
View File
@@ -0,0 +1,7 @@
from catalyst.data.bundles import register
from catalyst.exchange.exchange_bundle import exchange_bundle
symbols = (
'neo_btc',
)
register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))
catalyst/exchange/bundle_utils.py
+359
View File
@@ -0,0 +1,359 @@
import calendar
import os
import tarfile
from datetime import timedelta, datetime, date
import numpy as np
import pandas as pd
import pytz
from catalyst.assets._assets import TradingPair
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.exchange_utils import get_exchange_bundles_folder, \
get_exchange_symbols
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
API_URL = 'http://data.enigma.co/api/v1'
def get_date_from_ms(ms):
"""
The date from the number of miliseconds from the epoch.
Parameters
----------
ms: int
Returns
-------
datetime
"""
return datetime.fromtimestamp(ms / 1000.0)
def get_seconds_from_date(date):
"""
The number of seconds from the epoch.
Parameters
----------
date: datetime
Returns
-------
int
"""
epoch = datetime.utcfromtimestamp(0)
epoch = epoch.replace(tzinfo=pytz.UTC)
return int((date - epoch).total_seconds())
def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
"""
Download and extract a bcolz bundle.
Parameters
----------
exchange_name: str
symbol: str
data_frequency: str
period: str
Returns
-------
str
Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
"""
root = get_exchange_bundles_folder(exchange_name)
name = '{exchange}-{frequency}-{symbol}-{period}'.format(
exchange=exchange_name,
frequency=data_frequency,
symbol=symbol,
period=period
)
path = os.path.join(root, name)
if not os.path.isdir(path):
url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
'exchange-{exchange}/{name}.tar.gz'.format(
exchange=exchange_name,
name=name
)
bytes = download_without_progress(url)
with tarfile.open('r', fileobj=bytes) as tar:
tar.extractall(path)
return path
def get_delta(periods, data_frequency):
"""
Get a time delta based on the specified data frequency.
Parameters
----------
periods: int
data_frequency: str
Returns
-------
timedelta
"""
return timedelta(minutes=periods) \
if data_frequency == 'minute' else timedelta(days=periods)
def get_periods_range(start_dt, end_dt, freq):
"""
Get a date range for the specified parameters.
Parameters
----------
start_dt: datetime
end_dt: datetime
freq: str
Returns
-------
DateTimeIndex
"""
if freq == 'minute':
freq = 'T'
elif freq == 'daily':
freq = 'D'
return pd.date_range(start_dt, end_dt, freq=freq)
def get_periods(start_dt, end_dt, freq):
"""
The number of periods in the specified range.
Parameters
----------
start_dt: datetime
end_dt: datetime
freq: str
Returns
-------
int
"""
return len(get_periods_range(start_dt, end_dt, freq))
def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
"""
The start date based on specified end date and data frequency.
Parameters
----------
end_dt: datetime
bar_count: int
data_frequency: str
Returns
-------
datetime
"""
periods = bar_count
if periods > 1:
delta = get_delta(periods, data_frequency)
start_dt = end_dt - delta
if not include_first:
start_dt += get_delta(1, data_frequency)
else:
start_dt = end_dt
return start_dt
def get_period_label(dt, data_frequency):
"""
The period label for the specified date and frequency.
Parameters
----------
dt: datetime
data_frequency: str
Returns
-------
str
"""
return '{}-{:02d}'.format(dt.year, dt.month) if data_frequency == 'minute' \
else '{}'.format(dt.year)
def get_month_start_end(dt, first_day=None, last_day=None):
"""
The first and last day of the month for the specified date.
Parameters
----------
dt: datetime
first_day: datetime
last_day: datetime
Returns
-------
datetime, datetime
"""
month_range = calendar.monthrange(dt.year, dt.month)
if first_day:
month_start = first_day
else:
month_start = pd.to_datetime(datetime(
dt.year, dt.month, 1, 0, 0, 0, 0
), utc=True)
if last_day:
month_end = last_day
else:
month_end = pd.to_datetime(datetime(
dt.year, dt.month, month_range[1], 23, 59, 0, 0
), utc=True)
if month_end > pd.Timestamp.utcnow():
month_end = pd.Timestamp.utcnow().floor('1D')
return month_start, month_end
def get_year_start_end(dt, first_day=None, last_day=None):
"""
The first and last day of the year for the specified date.
Parameters
----------
dt: datetime
first_day: datetime
last_day: datetime
Returns
-------
datetime, datetime
"""
year_start = first_day if first_day \
else pd.to_datetime(date(dt.year, 1, 1), utc=True)
year_end = last_day if last_day \
else pd.to_datetime(date(dt.year, 12, 31), utc=True)
if year_end > pd.Timestamp.utcnow():
year_end = pd.Timestamp.utcnow().floor('1D')
return year_start, year_end
def get_df_from_arrays(arrays, periods):
"""
A DataFrame from the specified OHCLV arrays.
Parameters
----------
arrays: Object
periods: DateTimeIndex
Returns
-------
DataFrame
"""
ohlcv = dict()
for index, field in enumerate(
['open', 'high', 'low', 'close', 'volume']):
ohlcv[field] = arrays[index].flatten()
df = pd.DataFrame(
data=ohlcv,
index=periods
)
return df
def range_in_bundle(asset, start_dt, end_dt, reader):
"""
Evaluate whether price data of an asset is included has been ingested in
the exchange bundle for the given date range.
Parameters
----------
asset: TradingPair
start_dt: datetime
end_dt: datetime
reader: BcolzBarMinuteReader
Returns
-------
bool
"""
has_data = True
dates = [start_dt, end_dt]
while dates and has_data:
try:
dt = dates.pop(0)
close = reader.get_value(asset.sid, dt, 'close')
if np.isnan(close):
has_data = False
except Exception as e:
has_data = False
return has_data
def get_assets(exchange, include_symbols, exclude_symbols):
"""
Get assets from an exchange, including or excluding the specified
symbols.
Parameters
----------
exchange: Exchange
include_symbols: str
exclude_symbols: str
Returns
-------
list[TradingPair]
"""
if include_symbols is not None:
include_symbols_list = include_symbols.split(',')
return exchange.get_assets(include_symbols_list)
else:
all_assets = exchange.get_assets()
if exclude_symbols is not None:
exclude_symbols_list = exclude_symbols.split(',')
assets = []
for asset in all_assets:
if asset.symbol not in exclude_symbols_list:
assets.append(asset)
return assets
else:
return all_assets
catalyst/exchange/ccxt/ccxt_exchange.py
+115 -654
View File
File diff suppressed because it is too large Load Diff
catalyst/exchange/exchange.py
+185 -256
View File
@@ -5,22 +5,26 @@ from time import sleep
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.algorithm import MarketOrder
from catalyst.constants import LOG_LEVEL from catalyst.constants import LOG_LEVEL
from catalyst.data.data_portal import BASE_FIELDS from catalyst.data.data_portal import BASE_FIELDS
from catalyst.exchange.bundle_utils import get_start_dt, \
get_delta, get_periods, get_periods_range
from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
SymbolNotFoundOnExchange, \ BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
PricingDataNotLoadedError, \ PricingDataNotLoadedError, \
NoDataAvailableOnExchange, NoValueForField, \ NoDataAvailableOnExchange, NoValueForField
NoCandlesReceivedFromExchange, \ from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
TickerNotFoundError, NotEnoughCashError ExchangeLimitOrder, ExchangeStopOrder
from catalyst.exchange.utils.datetime_utils import get_delta, \ from catalyst.exchange.exchange_portfolio import ExchangePortfolio
get_periods_range, \ from catalyst.exchange.exchange_utils import get_exchange_symbols, \
get_periods, get_start_dt, get_frequency, \ get_frequency, resample_history_df
get_candles_number_from_minutes from catalyst.finance.order import ORDER_STATUS
from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ from catalyst.finance.transaction import Transaction
resample_history_df, has_bundle, get_candles_df
from logbook import Logger
log = Logger('Exchange', level=LOG_LEVEL) log = Logger('Exchange', level=LOG_LEVEL)
@@ -32,6 +36,7 @@ class Exchange:
self.name = None self.name = None
self.assets = [] self.assets = []
self._symbol_maps = [None, None] self._symbol_maps = [None, None]
self._portfolio = None
self.minute_writer = None self.minute_writer = None
self.minute_reader = None self.minute_reader = None
self.base_currency = None self.base_currency = None
@@ -41,7 +46,26 @@ class Exchange:
self.request_cpt = None self.request_cpt = None
self.bundle = ExchangeBundle(self.name) self.bundle = ExchangeBundle(self.name)
self.low_balance_threshold = None @property
def positions(self):
return self.portfolio.positions
@property
def portfolio(self):
"""
The exchange portfolio
Returns
-------
ExchangePortfolio
"""
if self._portfolio is None:
self._portfolio = ExchangePortfolio(
start_date=pd.Timestamp.utcnow()
)
self.synchronize_portfolio()
return self._portfolio
@abstractproperty @abstractproperty
def account(self): def account(self):
@@ -51,9 +75,6 @@ class Exchange:
def time_skew(self): def time_skew(self):
pass pass
def has_bundle(self, data_frequency):
return has_bundle(self.name, data_frequency)
def is_open(self, dt): def is_open(self, dt):
""" """
Is the exchange open Is the exchange open
@@ -156,7 +177,7 @@ class Exchange:
def get_assets(self, symbols=None, data_frequency=None, def get_assets(self, symbols=None, data_frequency=None,
is_exchange_symbol=False, is_exchange_symbol=False,
is_local=None, quote_currency=None): is_local=None):
""" """
The list of markets for the specified symbols. The list of markets for the specified symbols.
@@ -180,27 +201,14 @@ class Exchange:
if symbols is None: if symbols is None:
# Make a distinct list of all symbols # Make a distinct list of all symbols
symbols = list(set([asset.symbol for asset in self.assets])) symbols = list(set([asset.symbol for asset in self.assets]))
symbols.sort()
if quote_currency is not None:
for symbol in symbols[:]:
suffix = '_{}'.format(quote_currency.lower())
if not symbol.endswith(suffix):
symbols.remove(symbol)
is_exchange_symbol = False is_exchange_symbol = False
assets = [] assets = []
for symbol in symbols: for symbol in symbols:
try: asset = self.get_asset(
asset = self.get_asset( symbol, data_frequency, is_exchange_symbol, is_local
symbol, data_frequency, is_exchange_symbol, is_local )
) assets.append(asset)
assets.append(asset)
except SymbolNotFoundOnExchange as e:
log.warn(e)
return assets return assets
def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False, def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
@@ -233,15 +241,11 @@ class Exchange:
""" """
asset = None asset = None
# TODO: temp mapping, fix to use a single symbol convention
og_symbol = symbol
symbol = self.get_symbol(symbol) if not is_exchange_symbol else symbol
log.debug( log.debug(
'searching assets for: {} {}'.format( 'searching assets for: {} {}'.format(
self.name, symbol self.name, symbol
) )
) )
# TODO: simplify and loose the loop
for a in self.assets: for a in self.assets:
if asset is not None: if asset is not None:
break break
@@ -252,11 +256,8 @@ class Exchange:
elif data_frequency is not None: elif data_frequency is not None:
applies = ( applies = (
( (data_frequency == 'minute' and a.end_minute is not None)
data_frequency == 'minute' and or (data_frequency == 'daily' and a.end_daily is not None)
a.end_minute is not None)
or (
data_frequency == 'daily' and a.end_daily is not None)
) )
else: else:
@@ -264,24 +265,17 @@ class Exchange:
# The symbol provided may use the Catalyst or the exchange # The symbol provided may use the Catalyst or the exchange
# convention # convention
key = a.exchange_symbol if \ key = a.exchange_symbol if is_exchange_symbol else a.symbol
is_exchange_symbol else self.get_symbol(a) if not asset and key.lower() == symbol.lower() and applies:
if not asset and key.lower() == symbol.lower(): asset = a
if applies:
asset = a
else:
raise NoDataAvailableOnExchange(
symbol=key,
exchange=self.name,
data_frequency=data_frequency,
)
if asset is None: if asset is None:
supported_symbols = sorted([a.symbol for a in self.assets]) supported_symbols = sorted([
asset.symbol for asset in self.assets
])
raise SymbolNotFoundOnExchange( raise SymbolNotFoundOnExchange(
symbol=og_symbol, symbol=symbol,
exchange=self.name.title(), exchange=self.name.title(),
supported_symbols=supported_symbols supported_symbols=supported_symbols
) )
@@ -299,16 +293,6 @@ class Exchange:
self._symbol_maps[index] = symbol_map self._symbol_maps[index] = symbol_map
return symbol_map return symbol_map
@abstractmethod
def init(self):
"""
Load the asset list from the network.
Returns
-------
"""
@abstractmethod @abstractmethod
def load_assets(self, is_local=False): def load_assets(self, is_local=False):
""" """
@@ -329,6 +313,54 @@ class Exchange:
""" """
pass pass
def check_open_orders(self):
"""
Loop through the list of open orders in the Portfolio object.
For each executed order found, create a transaction and apply to the
Portfolio.
Returns
-------
list[Transaction]
"""
transactions = list()
if self.portfolio.open_orders:
for order_id in list(self.portfolio.open_orders):
log.debug('found open order: {}'.format(order_id))
order, executed_price = self.get_order(order_id)
log.debug(
'got updated order {} {}'.format(
order, executed_price
)
)
if order.status == ORDER_STATUS.FILLED:
transaction = Transaction(
asset=order.asset,
amount=order.amount,
dt=pd.Timestamp.utcnow(),
price=executed_price,
order_id=order.id,
commission=order.commission
)
transactions.append(transaction)
self.portfolio.execute_order(order, transaction)
elif order.status == ORDER_STATUS.CANCELLED:
self.portfolio.remove_order(order)
else:
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'order {order_id} still open after {delta}'.format(
order_id=order_id,
delta=delta
)
)
return transactions
def get_spot_value(self, assets, field, dt=None, data_frequency='minute'): def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
""" """
Public API method that returns a scalar value representing the value Public API method that returns a scalar value representing the value
@@ -414,7 +446,6 @@ class Exchange:
return value return value
# TODO: replace with catalyst.exchange.exchange_utils.get_candles_df
def get_series_from_candles(self, candles, start_dt, end_dt, def get_series_from_candles(self, candles, start_dt, end_dt,
data_frequency, field, previous_value=None): data_frequency, field, previous_value=None):
""" """
@@ -439,7 +470,7 @@ class Exchange:
series = pd.Series(values, index=dates) series = pd.Series(values, index=dates)
periods = get_periods_range( periods = get_periods_range(
start_dt=start_dt, end_dt=end_dt, freq=data_frequency start_dt, end_dt, data_frequency
) )
# TODO: ensure that this working as expected, if not use fillna # TODO: ensure that this working as expected, if not use fillna
series = series.reindex( series = series.reindex(
@@ -447,7 +478,7 @@ class Exchange:
method='ffill', method='ffill',
fill_value=previous_value, fill_value=previous_value,
) )
series.sort_index(inplace=True)
return series return series
def get_history_window(self, def get_history_window(self,
@@ -457,7 +488,7 @@ class Exchange:
frequency, frequency,
field, field,
data_frequency=None, data_frequency=None,
is_current=False): ffill=True):
""" """
Public API method that returns a dataframe containing the requested Public API method that returns a dataframe containing the requested
@@ -484,15 +515,10 @@ class Exchange:
The frequency of the data to query; i.e. whether the data is The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars. 'daily' or 'minute' bars.
is_current: bool # TODO: fill how?
Skip date filters when current data is requested (last few bars ffill: boolean
until now). Forward-fill missing values. Only has effect if field
is 'price'.
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns Returns
------- -------
@@ -501,62 +527,35 @@ class Exchange:
""" """
freq, candle_size, unit, data_frequency = get_frequency( freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D', 'H'] frequency, data_frequency
) )
adj_bar_count = candle_size * bar_count
# we want to avoid receiving empty candles start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
# so we request more than needed
# TODO: consider defining a const per asset
# and/or some retry mechanism (in each iteration request more data)
kExtra_minutes_candles = 150
requested_bar_count = bar_count + \
get_candles_number_from_minutes(unit,
candle_size,
kExtra_minutes_candles)
# The get_history method supports multiple asset # The get_history method supports multiple asset
candles = self.get_candles( candles = self.get_candles(
freq=freq, freq=freq,
assets=assets, assets=assets,
bar_count=requested_bar_count, bar_count=bar_count,
end_dt=end_dt if not is_current else None, start_dt=start_dt,
end_dt=end_dt
) )
# candles sanity check - verify no empty candles were received: series = dict()
for asset in candles: for asset in candles:
if not candles[asset]: asset_series = self.get_series_from_candles(
raise NoCandlesReceivedFromExchange( candles=candles[asset],
bar_count=requested_bar_count, start_dt=start_dt,
end_dt=end_dt, end_dt=end_dt,
asset=asset, data_frequency=frequency,
exchange=self.name) field=field,
)
# for avoiding unnecessary forward fill end_dt is taken back one second series[asset] = asset_series
forward_fill_till_dt = end_dt - timedelta(seconds=1)
series = get_candles_df(candles=candles,
field=field,
freq=frequency,
bar_count=requested_bar_count,
end_dt=forward_fill_till_dt)
# TODO: consider how to approach this edge case
# delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
# delta = get_delta(delta_candle_size, data_frequency)
# adj_end_dt = end_dt - delta
# last_traded = asset_series.index[-1]
# if last_traded < adj_end_dt:
# raise LastCandleTooEarlyError(
# last_traded=last_traded,
# end_dt=adj_end_dt,
# exchange=self.name,
# )
df = pd.DataFrame(series) df = pd.DataFrame(series)
df.dropna(inplace=True) df.dropna(inplace=True)
return df.tail(bar_count) return df
def get_history_window_with_bundle(self, def get_history_window_with_bundle(self,
assets, assets,
@@ -604,10 +603,8 @@ class Exchange:
A dataframe containing the requested data. A dataframe containing the requested data.
""" """
# TODO: this function needs some work,
# we're currently using it just for benchmark data
freq, candle_size, unit, data_frequency = get_frequency( freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D'] frequency, data_frequency
) )
adj_bar_count = candle_size * bar_count adj_bar_count = candle_size * bar_count
try: try:
@@ -619,7 +616,6 @@ class Exchange:
data_frequency=data_frequency, data_frequency=data_frequency,
force_auto_ingest=force_auto_ingest force_auto_ingest=force_auto_ingest
) )
except (PricingDataNotLoadedError, NoDataAvailableOnExchange): except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
series = dict() series = dict()
@@ -631,19 +627,20 @@ class Exchange:
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
trailing_dt = \ trailing_dt = \
series[asset].index[-1] + get_delta(1, data_frequency) \ series[asset].index[-1] + get_delta(1, data_frequency) \
if asset in series else start_dt if asset in series else start_dt
# The get_history method supports multiple asset # The get_history method supports multiple asset
# Use the original frequency to let each api optimize # Use the original frequency to let each api optimize
# the size of result sets # the size of result sets
trailing_bars = get_periods( trailing_bar_count = get_periods(
trailing_dt, end_dt, freq trailing_dt, end_dt, freq
) )
candles = self.get_candles( candles = self.get_candles(
freq=freq, freq=freq,
assets=asset, assets=asset,
end_dt=end_dt, bar_count=trailing_bar_count,
bar_count=trailing_bars if trailing_bars < 500 else 500, start_dt=start_dt,
end_dt=end_dt
) )
last_value = series[asset].iloc(0) if asset in series \ last_value = series[asset].iloc(0) if asset in series \
@@ -672,101 +669,51 @@ class Exchange:
return df return df
def _check_low_balance(self, currency, balances, amount): def synchronize_portfolio(self):
free = balances[currency]['free'] if currency in balances else 0.0
if free < amount:
return free, True
else:
return free, False
def sync_positions(self, positions, cash=None,
check_balances=False):
""" """
Update the portfolio cash and position balances based on the Update the portfolio cash and position balances based on the
latest ticker prices. latest ticker prices.
Parameters
----------
positions:
The positions to synchronize.
check_balances:
Check balances amounts against the exchange.
""" """
free_cash = 0.0 log.debug('synchronizing portfolio with exchange {}'.format(self.name))
if check_balances: balances = self.get_balances()
log.debug('fetching {} balances'.format(self.name))
balances = self.get_balances()
log.debug(
'got free balances for {} currencies'.format(
len(balances)
)
)
if cash is not None:
free_cash, is_lower = self._check_low_balance(
currency=self.base_currency,
balances=balances,
amount=cash,
)
if is_lower:
raise NotEnoughCashError(
currency=self.base_currency,
exchange=self.name,
free=free_cash,
cash=cash,
)
positions_value = 0.0 base_position_available = balances[self.base_currency]['free'] \
if positions: if self.base_currency in balances else None
assets = list(set([position.asset for position in positions]))
if base_position_available is None:
raise BaseCurrencyNotFoundError(
base_currency=self.base_currency,
exchange=self.name.title()
)
portfolio = self._portfolio
portfolio.cash = base_position_available
log.debug('found base currency balance: {}'.format(portfolio.cash))
if portfolio.starting_cash is None:
portfolio.starting_cash = portfolio.cash
if portfolio.positions:
assets = list(portfolio.positions.keys())
tickers = self.tickers(assets) tickers = self.tickers(assets)
for position in positions: portfolio.positions_value = 0.0
asset = position.asset for asset in tickers:
if asset not in tickers: # TODO: convert if the position is not in the base currency
raise TickerNotFoundError(
symbol=asset.symbol,
exchange=self.name,
)
ticker = tickers[asset] ticker = tickers[asset]
log.debug( position = portfolio.positions[asset]
'updating {symbol} position, last traded on {dt} for '
'{price}{currency}'.format(
symbol=asset.symbol,
dt=ticker['last_traded'],
price=ticker['last_price'],
currency=asset.quote_currency,
)
)
position.last_sale_price = ticker['last_price'] position.last_sale_price = ticker['last_price']
position.last_sale_date = ticker['last_traded'] position.last_sale_date = ticker['last_traded']
positions_value += \ portfolio.positions_value += \
position.amount * position.last_sale_price position.amount * position.last_sale_price
portfolio.portfolio_value = \
portfolio.positions_value + portfolio.cash
if check_balances: def order(self, asset, amount, limit_price=None, stop_price=None,
free, is_lower = self._check_low_balance( style=None):
currency=asset.base_currency,
balances=balances,
amount=position.amount,
)
if is_lower:
log.warn(
'detected lower balance for {} on {}: {} < {}, '
'updating position amount'.format(
asset.symbol, self.name, free, position.amount
)
)
position.amount = free
return free_cash, positions_value
def order(self, asset, amount, style):
"""Place an order. """Place an order.
Parameters Parameters
@@ -825,11 +772,24 @@ class Exchange:
) )
is_buy = (amount > 0) is_buy = (amount > 0)
display_price = style.get_limit_price(is_buy)
if limit_price is not None and stop_price is not None:
style = ExchangeStopLimitOrder(
limit_price, stop_price, exchange=self.name
)
elif limit_price is not None:
style = ExchangeLimitOrder(limit_price, exchange=self.name)
elif stop_price is not None:
style = ExchangeStopOrder(stop_price, exchange=self.name)
else:
style = MarketOrder(exchange=self.name)
display_price = limit_price if limit_price is not None else stop_price
log.debug( log.debug(
'issuing {side} order of {amount} {symbol} for {type}:' 'issuing {side} order of {amount} {symbol} for {type}: {price}'.format(
' {price}'.format(
side='buy' if is_buy else 'sell', side='buy' if is_buy else 'sell',
amount=amount, amount=amount,
symbol=asset.symbol, symbol=asset.symbol,
@@ -838,7 +798,12 @@ class Exchange:
) )
) )
return self.create_order(asset, amount, is_buy, style) order = self.create_order(asset, amount, is_buy, style)
if order:
self._portfolio.create_order(order)
return order.id
else:
return None
# The methods below must be implemented for each exchange. # The methods below must be implemented for each exchange.
@abstractmethod @abstractmethod
@@ -922,24 +887,7 @@ class Exchange:
pass pass
@abstractmethod @abstractmethod
def process_order(self, order): def cancel_order(self, order_param, symbol_or_asset=None):
"""
Similar to get_order but looks only for executed orders.
Parameters
----------
order: Order
Returns
-------
float
Avg execution price
"""
@abstractmethod
def cancel_order(self, order_param,
symbol_or_asset=None, params={}):
"""Cancel an open order. """Cancel an open order.
Parameters Parameters
@@ -948,12 +896,12 @@ class Exchange:
The order_id or order object to cancel. The order_id or order object to cancel.
symbol_or_asset: str|TradingPair symbol_or_asset: str|TradingPair
The catalyst symbol, some exchanges need this The catalyst symbol, some exchanges need this
params:
""" """
pass pass
@abstractmethod @abstractmethod
def get_candles(self, freq, assets, bar_count, start_dt=None, end_dt=None): def get_candles(self, freq, assets, bar_count=None,
start_dt=None, end_dt=None):
""" """
Retrieve OHLCV candles for the given assets Retrieve OHLCV candles for the given assets
@@ -993,15 +941,13 @@ class Exchange:
pass pass
@abc.abstractmethod @abc.abstractmethod
def tickers(self, assets, on_ticker_error='raise'): def tickers(self, assets):
""" """
Retrieve current tick data for the given assets Retrieve current tick data for the given assets
Parameters Parameters
---------- ----------
assets: list[TradingPair] assets: list[TradingPair]
on_ticker_error: str [raise|warn]
How to handle an error when retrieving a single ticker.
Returns Returns
------- -------
@@ -1020,7 +966,7 @@ class Exchange:
@abc.abstractmethod @abc.abstractmethod
def get_orderbook(self, asset, order_type, limit): def get_orderbook(self, asset, order_type, limit):
""" """
Retrieve the orderbook for the given trading pair. Retrieve the the orderbook for the given trading pair.
Parameters Parameters
---------- ----------
@@ -1034,20 +980,3 @@ class Exchange:
list[dict[str, float] list[dict[str, float]
""" """
pass pass
@abc.abstractmethod
def get_trades(self, asset, my_trades, start_dt, limit):
"""
Retrieve a list of trades.
Parameters
----------
my_trades: bool
List only my trades.
start_dt
limit
Returns
-------
"""
catalyst/exchange/exchange_algorithm.py
+366 -621
View File
File diff suppressed because it is too large Load Diff
catalyst/exchange/exchange_asset_finder.py
-179
View File
@@ -1,179 +0,0 @@
import pandas as pd
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.utils.factory import find_exchanges
from logbook import Logger
log = Logger('ExchangeAssetFinder', level=LOG_LEVEL)
class ExchangeAssetFinder(object):
def __init__(self, exchanges):
self.exchanges = exchanges
@property
def sids(self):
"""
This seems to be used to pre-fetch assets.
I don't think that we need this for live-trading.
Leaving the list empty.
"""
all_sids = []
for exchange_name in self.exchanges:
# This is what initializes each exchanges at the beginning
# of an algo
exchange = self.exchanges[exchange_name]
exchange.init()
all_sids += [asset.sid for asset in exchange.assets]
sids = list(set(all_sids))
return sids
def retrieve_asset(self, sid, default_none=False):
"""
Retrieve the first Asset found for a given sid.
"""
asset = None
for exchange_name in self.exchanges:
if asset is not None:
break
exchange = self.exchanges[exchange_name]
assets = [asset for asset in exchange.assets if asset.sid == sid]
if assets:
asset = assets[0]
return asset
def retrieve_all(self, sids, default_none=False):
"""
Retrieve all assets in `sids`.
Parameters
----------
sids : iterable of int
Assets to retrieve.
default_none : bool
If True, return None for failed lookups.
If False, raise `SidsNotFound`.
Returns
-------
assets : list[Asset or None]
A list of the same length as `sids` containing Assets (or Nones)
corresponding to the requested sids.
Raises
------
SidsNotFound
When a requested sid is not found and default_none=False.
"""
assets = []
for exchange_name in self.exchanges:
exchange = self.exchanges[exchange_name]
xas = [asset for asset in exchange.assets if asset.sid in sids]
assets += xas
return assets
def lookup_symbol(self, symbol, exchange, data_frequency=None,
as_of_date=None, fuzzy=False):
"""Lookup an asset by symbol.
Parameters
----------
symbol : str
The ticker symbol to resolve.
as_of_date : datetime or None
Look up the last owner of this symbol as of this datetime.
If ``as_of_date`` is None, then this can only resolve the equity
if exactly one equity has ever owned the ticker.
fuzzy : bool, optional
Should fuzzy symbol matching be used? Fuzzy symbol matching
attempts to resolve differences in representations for
shareclasses. For example, some people may represent the ``A``
shareclass of ``BRK`` as ``BRK.A``, where others could write
``BRK_A``.
Returns
-------
equity : Asset
The equity that held ``symbol`` on the given ``as_of_date``, or the
only equity to hold ``symbol`` if ``as_of_date`` is None.
Raises
------
SymbolNotFound
Raised when no equity has ever held the given symbol.
MultipleSymbolsFound
Raised when no ``as_of_date`` is given and more than one equity
has held ``symbol``. This is also raised when ``fuzzy=True`` and
there are multiple candidates for the given ``symbol`` on the
``as_of_date``.
"""
log.debug('looking up symbol: {} {}'.format(symbol, exchange.name))
return exchange.get_asset(symbol, data_frequency)
def lifetimes(self, dates, include_start_date):
"""
Compute a DataFrame representing asset lifetimes for the specified date
range.
Parameters
----------
dates : pd.DatetimeIndex
The dates for which to compute lifetimes.
include_start_date : bool
Whether or not to count the asset as alive on its start_date.
This is useful in a backtesting context where `lifetimes` is being
used to signify "do I have data for this asset as of the morning of
this date?" For many financial metrics, (e.g. daily close), data
isn't available for an asset until the end of the asset's first
day.
Returns
-------
lifetimes : pd.DataFrame
A frame of dtype bool with `dates` as index and an Int64Index of
assets as columns. The value at `lifetimes.loc[date, asset]` will
be True iff `asset` existed on `date`. If `include_start_date` is
False, then lifetimes.loc[date, asset] will be false when date ==
asset.start_date.
See Also
--------
numpy.putmask
catalyst.pipeline.engine.SimplePipelineEngine._compute_root_mask
"""
exchanges = find_exchanges(features=['minuteBundle'])
if not exchanges:
raise ValueError('exchange with minute bundles not found')
# TODO: find a way to support multiple exchanges
exchange = exchanges[0]
# Using a single exchange for now because are not unique for the
# same asset in different exchanges. I'd like to avoid binding
# pipeline to a single exchange.
exchange.init()
data = []
for dt in dates:
exists = []
for asset in exchange.assets:
if include_start_date:
condition = (asset.start_date <= dt < asset.end_minute)
else:
condition = (asset.start_date < dt < asset.end_minute)
exists.append(condition)
data.append(exists)
sids = [asset.sid for asset in exchange.assets]
df = pd.DataFrame(data, index=dates, columns=exchange.assets)
return df
catalyst/exchange/exchange_blotter.py
+35 -178
View File
@@ -1,20 +1,21 @@
import numpy as np
import pandas as pd
from logbook import Logger
from redo import retry
from catalyst.assets._assets import TradingPair from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.constants import LOG_LEVEL from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange_errors import ExchangeRequestError
from catalyst.finance.blotter import Blotter from catalyst.finance.blotter import Blotter
from catalyst.finance.commission import CommissionModel from catalyst.finance.commission import CommissionModel
from catalyst.finance.order import ORDER_STATUS
from catalyst.finance.slippage import SlippageModel from catalyst.finance.slippage import SlippageModel
from catalyst.finance.transaction import create_transaction, Transaction from catalyst.finance.transaction import create_transaction
from catalyst.utils.input_validation import expect_types
log = Logger('exchange_blotter', level=LOG_LEVEL) log = Logger('exchange_blotter', level=LOG_LEVEL)
# It seems like we need to accept greater slippage risk in cryptos
# Orders won't often close at Equity levels.
# TODO: should work with set_commission and set_slippage
DEFAULT_SLIPPAGE_SPREAD = 0.0001
DEFAULT_MAKER_FEE = 0.0015
DEFAULT_TAKER_FEE = 0.0025
class TradingPairFeeSchedule(CommissionModel): class TradingPairFeeSchedule(CommissionModel):
""" """
@@ -22,55 +23,40 @@ class TradingPairFeeSchedule(CommissionModel):
Parameters Parameters
---------- ----------
maker : float, optional fee : float, optional
The percentage maker fee. The percentage fee.
taker: float, optional
The percentage taker fee.
""" """
def __init__(self, maker=None, taker=None): def __init__(self,
self.maker = maker maker_fee=DEFAULT_MAKER_FEE,
self.taker = taker taker_fee=DEFAULT_TAKER_FEE):
self.maker_fee = maker_fee
self.taker_fee = taker_fee
def __repr__(self): def __repr__(self):
return ( return (
'{class_name}(maker={maker}, ' '{class_name}(maker_fee={maker_fee}, '
'taker={taker})'.format( 'taker_fee={taker_fee})'.format(
class_name=self.__class__.__name__, class_name=self.__class__.__name__,
maker=self.maker, maker_fee=self.maker_fee,
taker=self.taker, taker_fee=self.taker_fee,
) )
) )
def get_maker_taker(self, asset):
maker = self.maker if self.maker is not None else asset.maker
taker = self.taker if self.taker is not None else asset.taker
return maker, taker
def calculate(self, order, transaction): def calculate(self, order, transaction):
""" """
Calculate the final fee based on the order parameters. Calculate the final fee based on the order parameters.
:param order: Order :param order:
:param transaction: Transaction :param transaction:
:return float: :return float:
The total commission. The total commission.
""" """
cost = abs(transaction.amount) * transaction.price cost = abs(transaction.amount) * transaction.price
asset = order.asset # Assuming just the taker fee for now
maker, taker = self.get_maker_taker(asset) fee = cost * self.taker_fee
multiplier = taker
if order.limit is not None:
multiplier = maker \
if ((order.amount > 0 and order.limit < transaction.price)
or (order.amount < 0 and order.limit > transaction.price)) \
and order.limit_reached else taker
fee = cost * multiplier
return fee return fee
@@ -84,7 +70,7 @@ class TradingPairFixedSlippage(SlippageModel):
spread / 2 will be added to buys and subtracted from sells. spread / 2 will be added to buys and subtracted from sells.
""" """
def __init__(self, spread=0.0001): def __init__(self, spread=DEFAULT_SLIPPAGE_SPREAD):
super(TradingPairFixedSlippage, self).__init__() super(TradingPairFixedSlippage, self).__init__()
self.spread = spread self.spread = spread
@@ -95,6 +81,7 @@ class TradingPairFixedSlippage(SlippageModel):
def simulate(self, data, asset, orders_for_asset): def simulate(self, data, asset, orders_for_asset):
self._volume_for_bar = 0 self._volume_for_bar = 0
price = data.current(asset, 'close') price = data.current(asset, 'close')
dt = data.current_dt dt = data.current_dt
@@ -104,20 +91,18 @@ class TradingPairFixedSlippage(SlippageModel):
order.check_triggers(price, dt) order.check_triggers(price, dt)
if not order.triggered: if not order.triggered:
log.info( log.debug('order has not reached the trigger at current '
'order has not reached the trigger at current ' 'price {}'.format(price))
'price {}'.format(price)
)
continue continue
execution_price, execution_volume = self.process_order(data, order) execution_price, execution_volume = self.process_order(data, order)
if execution_price is not None:
transaction = create_transaction(
order, dt, execution_price, execution_volume
)
self._volume_for_bar += abs(transaction.amount) transaction = create_transaction(
yield order, transaction order, dt, execution_price, execution_volume
)
self._volume_for_bar += abs(transaction.amount)
yield order, transaction
def process_order(self, data, order): def process_order(self, data, order):
price = data.current(order.asset, 'close') price = data.current(order.asset, 'close')
@@ -136,15 +121,6 @@ class TradingPairFixedSlippage(SlippageModel):
class ExchangeBlotter(Blotter): class ExchangeBlotter(Blotter):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.simulate_orders = kwargs.pop('simulate_orders', False)
self.attempts = kwargs.pop('attempts', False)
self.exchanges = kwargs.pop('exchanges', None)
if not self.exchanges:
raise ValueError(
'ExchangeBlotter must have an `exchanges` attribute.'
)
super(ExchangeBlotter, self).__init__(*args, **kwargs) super(ExchangeBlotter, self).__init__(*args, **kwargs)
# Using the equity models for now # Using the equity models for now
@@ -156,122 +132,3 @@ class ExchangeBlotter(Blotter):
self.commission_models = { self.commission_models = {
TradingPair: TradingPairFeeSchedule() TradingPair: TradingPairFeeSchedule()
} }
def exchange_order(self, asset, amount, style=None):
exchange = self.exchanges[asset.exchange]
return exchange.order(
asset, amount, style
)
@expect_types(asset=TradingPair)
def order(self, asset, amount, style, order_id=None):
log.debug('ordering {} {}'.format(amount, asset.symbol))
if amount == 0:
log.warn('skipping 0 amount orders')
return None
if self.simulate_orders:
return super(ExchangeBlotter, self).order(
asset, amount, style, order_id
)
else:
order = retry(
action=self.exchange_order,
attempts=self.attempts['order_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('Ordering again.'),
args=(asset, amount, style),
)
self.open_orders[order.asset].append(order)
self.orders[order.id] = order
self.new_orders.append(order)
return order.id
def check_open_orders(self):
"""
Loop through the list of open orders in the Portfolio object.
For each executed order found, create a transaction and apply to the
Portfolio.
Returns
-------
list[Transaction]
"""
for asset in self.open_orders:
exchange = self.exchanges[asset.exchange]
for order in self.open_orders[asset]:
log.debug('found open order: {}'.format(order.id))
transactions = exchange.process_order(order)
# This is a temporary measure, we should really update all
# trades, not just when the order gets filled. I just think
# that this is safer until we have a robust way to track
# the trades already processed by the algo. We can't loose
# them if the algo shuts down.
if transactions and order.status == ORDER_STATUS.FILLED:
avg_price = np.average(
a=[t.price for t in transactions],
weights=[t.amount for t in transactions],
)
ostatus = 'filled' if order.open_amount == 0 else 'partial'
log.info(
'{} order {} / {}: {}, avg price: {}'.format(
ostatus,
order.id,
asset.symbol,
order.filled,
avg_price,
)
)
for transaction in transactions:
yield order, transaction
elif order.status == ORDER_STATUS.CANCELLED:
yield order, None
else:
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'{exchange} order {order_id} for {symbol} still open '
'after {delta}'.format(
exchange=exchange.name,
order_id=order.id,
delta=delta,
symbol=order.asset.symbol,
)
)
def get_exchange_transactions(self):
closed_orders = []
transactions = []
commissions = []
for order, txn in self.check_open_orders():
order.dt = txn.dt
transactions.append(txn)
if not order.open:
closed_orders.append(order)
return transactions, commissions, closed_orders
def get_transactions(self, bar_data):
if self.simulate_orders:
return super(ExchangeBlotter, self).get_transactions(bar_data)
else:
return retry(
action=self.get_exchange_transactions,
attempts=self.attempts['get_transactions_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn(
'Fetching exchange transactions again.'
)
)
catalyst/exchange/exchange_bundle.py
+66 -59
View File
@@ -1,6 +1,6 @@
import os import os
import shutil import shutil
from datetime import timedelta from datetime import datetime, timedelta
from functools import partial from functools import partial
from itertools import chain from itertools import chain
from operator import is_not from operator import is_not
@@ -8,29 +8,30 @@ from operator import is_not
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import pytz import pytz
from catalyst import get_calendar
from catalyst.assets._assets import TradingPair from catalyst.assets._assets import TradingPair
from logbook import Logger
from pytz import UTC
from six import itervalues
from catalyst import get_calendar
from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
from catalyst.constants import LOG_LEVEL from catalyst.constants import LOG_LEVEL
from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
BcolzMinuteBarMetadata BcolzMinuteBarMetadata
from catalyst.exchange.bundle_utils import range_in_bundle, \
get_bcolz_chunk, get_month_start_end, \
get_year_start_end, get_df_from_arrays, get_start_dt, get_period_label, \
get_delta, get_assets
from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \ from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
BcolzExchangeBarWriter BcolzExchangeBarWriter
from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
TempBundleNotFoundError, \ TempBundleNotFoundError, \
NoDataAvailableOnExchange, \ NoDataAvailableOnExchange, \
PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ from catalyst.exchange.exchange_utils import get_exchange_folder, \
get_bcolz_chunk, get_df_from_arrays, get_assets save_exchange_symbols, mixin_market_params
from catalyst.exchange.utils.datetime_utils import get_start_dt, \
get_period_label, get_month_start_end, get_year_start_end
from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
save_exchange_symbols, mixin_market_params, get_catalyst_symbol
from catalyst.utils.cli import maybe_show_progress from catalyst.utils.cli import maybe_show_progress
from catalyst.utils.paths import ensure_directory from catalyst.utils.paths import ensure_directory
from logbook import Logger
from pytz import UTC
from six import itervalues
log = Logger('exchange_bundle', level=LOG_LEVEL) log = Logger('exchange_bundle', level=LOG_LEVEL)
@@ -232,13 +233,11 @@ class ExchangeBundle:
problem = '{name} ({start_dt} to {end_dt}) has empty ' \ problem = '{name} ({start_dt} to {end_dt}) has empty ' \
'periods: {dates}'.format( 'periods: {dates}'.format(
name=asset.symbol, name=asset.symbol,
start_dt=asset.start_date.strftime( start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
DATE_TIME_FORMAT), end_dt=end_dt.strftime(DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT), dates=[date.strftime(DATE_TIME_FORMAT) for date in dates]
dates=[date.strftime( )
DATE_TIME_FORMAT) for date in dates])
if empty_rows_behavior == 'warn': if empty_rows_behavior == 'warn':
log.warn(problem) log.warn(problem)
@@ -246,7 +245,8 @@ class ExchangeBundle:
raise EmptyValuesInBundleError( raise EmptyValuesInBundleError(
name=asset.symbol, name=asset.symbol,
end_minute=end_dt, end_minute=end_dt,
dates=dates, ) dates=dates
)
else: else:
ohlcv_df.dropna(inplace=True) ohlcv_df.dropna(inplace=True)
@@ -286,12 +286,13 @@ class ExchangeBundle:
problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \ problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
'identical close values on: {dates}'.format( 'identical close values on: {dates}'.format(
name=asset.symbol, name=asset.symbol,
start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
end_dt=end_dt.strftime(DATE_TIME_FORMAT), end_dt=end_dt.strftime(DATE_TIME_FORMAT),
threshold=threshold, threshold=threshold,
dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
for date in dates]) for date in dates]
)
problems.append(problem) problems.append(problem)
@@ -461,7 +462,7 @@ class ExchangeBundle:
(earliest_trade is not None and earliest_trade > start): (earliest_trade is not None and earliest_trade > start):
start = earliest_trade start = earliest_trade
if last_entry is not None and (end is None or end > last_entry): if end is None or (last_entry is not None and end > last_entry):
end = last_entry.replace(minute=59, hour=23) \ end = last_entry.replace(minute=59, hour=23) \
if data_frequency == 'minute' else last_entry if data_frequency == 'minute' else last_entry
@@ -598,9 +599,8 @@ class ExchangeBundle:
# we want to give an end_date far in time # we want to give an end_date far in time
writer = self.get_writer(start_dt, end_dt, data_frequency) writer = self.get_writer(start_dt, end_dt, data_frequency)
if show_breakdown: if show_breakdown:
if chunks: for asset in chunks:
for asset in chunks: with maybe_show_progress(
with maybe_show_progress(
chunks[asset], chunks[asset],
show_progress, show_progress,
label='Ingesting {frequency} price data for ' label='Ingesting {frequency} price data for '
@@ -608,30 +608,6 @@ class ExchangeBundle:
exchange=self.exchange_name, exchange=self.exchange_name,
frequency=data_frequency, frequency=data_frequency,
symbol=asset.symbol symbol=asset.symbol
)) as it:
for chunk in it:
problems += self.ingest_ctable(
asset=chunk['asset'],
data_frequency=data_frequency,
period=chunk['period'],
writer=writer,
empty_rows_behavior='strip',
cleanup=True
)
else:
all_chunks = list(chain.from_iterable(itervalues(chunks)))
# We sort the chunks by end date to ingest most recent data first
if all_chunks:
all_chunks.sort(
key=lambda chunk: pd.to_datetime(chunk['period'])
)
with maybe_show_progress(
all_chunks,
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
)) as it: )) as it:
for chunk in it: for chunk in it:
problems += self.ingest_ctable( problems += self.ingest_ctable(
@@ -642,6 +618,30 @@ class ExchangeBundle:
empty_rows_behavior='strip', empty_rows_behavior='strip',
cleanup=True cleanup=True
) )
else:
all_chunks = list(chain.from_iterable(itervalues(chunks)))
# We sort the chunks by end date to ingest most recent data first
all_chunks.sort(
key=lambda chunk: pd.to_datetime(chunk['period'])
)
with maybe_show_progress(
all_chunks,
show_progress,
label='Ingesting {frequency} price data on '
'{exchange}'.format(
exchange=self.exchange_name,
frequency=data_frequency,
)) as it:
for chunk in it:
problems += self.ingest_ctable(
asset=chunk['asset'],
data_frequency=data_frequency,
period=chunk['period'],
writer=writer,
empty_rows_behavior='strip',
cleanup=True
)
if show_report and len(problems) > 0: if show_report and len(problems) > 0:
log.info('problems during ingestion:{}\n'.format( log.info('problems during ingestion:{}\n'.format(
@@ -668,7 +668,7 @@ class ExchangeBundle:
if self.exchange is None: if self.exchange is None:
# Avoid circular dependencies # Avoid circular dependencies
from catalyst.exchange.utils.factory import get_exchange from catalyst.exchange.factory import get_exchange
self.exchange = get_exchange(self.exchange_name) self.exchange = get_exchange(self.exchange_name)
problems = [] problems = []
@@ -681,7 +681,6 @@ class ExchangeBundle:
last_traded=np.object_, last_traded=np.object_,
open=np.float64, open=np.float64,
high=np.float64, high=np.float64,
low=np.float64,
close=np.float64, close=np.float64,
volume=np.float64 volume=np.float64
), ),
@@ -731,7 +730,7 @@ class ExchangeBundle:
if data_frequency == 'minute' else asset_def['end_minute'] if data_frequency == 'minute' else asset_def['end_minute']
else: else:
params['symbol'] = get_catalyst_symbol(market) params['symbol'] = self.exchange.get_catalyst_symbol(market)
params['end_daily'] = end_dt \ params['end_daily'] = end_dt \
if data_frequency == 'daily' else 'N/A' if data_frequency == 'daily' else 'N/A'
@@ -756,10 +755,9 @@ class ExchangeBundle:
) )
for symbol in assets: for symbol in assets:
# here the symbol is the market['id']
asset = assets[symbol] asset = assets[symbol]
ohlcv_df = df.loc[ ohlcv_df = df.loc[
(df.index.get_level_values(0) == asset.symbol) (df.index.get_level_values(0) == symbol)
] # type: pd.DataFrame ] # type: pd.DataFrame
ohlcv_df.index = ohlcv_df.index.droplevel(0) ohlcv_df.index = ohlcv_df.index.droplevel(0)
@@ -807,7 +805,7 @@ class ExchangeBundle:
else: else:
if self.exchange is None: if self.exchange is None:
# Avoid circular dependencies # Avoid circular dependencies
from catalyst.exchange.utils.factory import get_exchange from catalyst.exchange.factory import get_exchange
self.exchange = get_exchange(self.exchange_name) self.exchange = get_exchange(self.exchange_name)
assets = get_assets( assets = get_assets(
@@ -831,6 +829,7 @@ class ExchangeBundle:
field, field,
data_frequency, data_frequency,
algo_end_dt=None, algo_end_dt=None,
trailing_bar_count=None,
force_auto_ingest=False force_auto_ingest=False
): ):
""" """
@@ -858,6 +857,7 @@ class ExchangeBundle:
bar_count=bar_count, bar_count=bar_count,
field=field, field=field,
data_frequency=data_frequency, data_frequency=data_frequency,
trailing_bar_count=trailing_bar_count,
) )
return pd.DataFrame(series) return pd.DataFrame(series)
@@ -886,6 +886,7 @@ class ExchangeBundle:
field=field, field=field,
data_frequency=data_frequency, data_frequency=data_frequency,
reset_reader=True, reset_reader=True,
trailing_bar_count=trailing_bar_count,
) )
return series return series
@@ -896,6 +897,7 @@ class ExchangeBundle:
bar_count=bar_count, bar_count=bar_count,
field=field, field=field,
data_frequency=data_frequency, data_frequency=data_frequency,
trailing_bar_count=trailing_bar_count,
) )
return pd.DataFrame(series) return pd.DataFrame(series)
@@ -959,12 +961,17 @@ class ExchangeBundle:
bar_count, bar_count,
field, field,
data_frequency, data_frequency,
trailing_bar_count=None,
reset_reader=False): reset_reader=False):
start_dt = get_start_dt(end_dt, bar_count, data_frequency, False) start_dt = get_start_dt(end_dt, bar_count, data_frequency, False)
start_dt, _ = self.get_adj_dates( start_dt, _ = self.get_adj_dates(
start_dt, end_dt, assets, data_frequency start_dt, end_dt, assets, data_frequency
) )
if trailing_bar_count:
delta = get_delta(trailing_bar_count, data_frequency)
end_dt += delta
# This is an attempt to resolve some caching with the reader # This is an attempt to resolve some caching with the reader
# when auto-ingesting data. # when auto-ingesting data.
# TODO: needs more work # TODO: needs more work
catalyst/exchange/exchange_data_portal.py
+127 -94
View File
@@ -1,30 +1,30 @@
import abc import abc
from time import sleep
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from catalyst.assets._assets import TradingPair from catalyst.assets._assets import TradingPair
from logbook import Logger
from catalyst.constants import LOG_LEVEL, AUTO_INGEST from catalyst.constants import LOG_LEVEL, AUTO_INGEST
from catalyst.data.data_portal import DataPortal from catalyst.data.data_portal import DataPortal
from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import ( from catalyst.exchange.exchange_errors import (
ExchangeRequestError, ExchangeRequestError,
ExchangeBarDataError,
PricingDataNotLoadedError) PricingDataNotLoadedError)
from catalyst.exchange.utils.exchange_utils import resample_history_df, \ from catalyst.exchange.exchange_utils import get_frequency, resample_history_df
group_assets_by_exchange
from catalyst.exchange.utils.datetime_utils import get_frequency, get_start_dt
from logbook import Logger
from redo import retry
log = Logger('DataPortalExchange', level=LOG_LEVEL) log = Logger('DataPortalExchange', level=LOG_LEVEL)
class DataPortalExchangeBase(DataPortal): class DataPortalExchangeBase(DataPortal):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.attempts = dict(
get_spot_value_attempts=5, # TODO: put somewhere accessible by each algo
get_history_window_attempts=5, self.retry_get_history_window = 5
retry_sleeptime=5, self.retry_get_spot_value = 5
) self.retry_delay = 5
super(DataPortalExchangeBase, self).__init__(*args, **kwargs) super(DataPortalExchangeBase, self).__init__(*args, **kwargs)
@@ -35,14 +35,39 @@ class DataPortalExchangeBase(DataPortal):
frequency, frequency,
field, field,
data_frequency, data_frequency,
ffill=True): ffill=True,
exchange_assets = group_assets_by_exchange(assets) attempt_index=0):
if len(exchange_assets) > 1: try:
df_list = [] exchange_assets = dict()
for exchange_name in exchange_assets: for asset in assets:
assets = exchange_assets[exchange_name] if asset.exchange not in exchange_assets:
exchange_assets[asset.exchange] = list()
df_exchange = self.get_exchange_history_window( exchange_assets[asset.exchange].append(asset)
if len(exchange_assets) > 1:
df_list = []
for exchange_name in exchange_assets:
assets = exchange_assets[exchange_name]
df_exchange = self.get_exchange_history_window(
exchange_name,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency,
ffill)
df_list.append(df_exchange)
# Merging the values values of each exchange
return pd.concat(df_list)
else:
exchange_name = list(exchange_assets.keys())[0]
return self.get_exchange_history_window(
exchange_name, exchange_name,
assets, assets,
end_dt, end_dt,
@@ -52,22 +77,26 @@ class DataPortalExchangeBase(DataPortal):
data_frequency, data_frequency,
ffill) ffill)
df_list.append(df_exchange) except ExchangeRequestError as e:
log.warn(
# Merging the values values of each exchange 'get history attempt {}: {}'.format(attempt_index, e)
return pd.concat(df_list) )
if attempt_index < self.retry_get_history_window:
else: sleep(self.retry_delay)
exchange_name = list(exchange_assets.keys())[0] return self._get_history_window(assets,
return self.get_exchange_history_window( end_dt,
exchange_name, bar_count,
assets, frequency,
end_dt, field,
bar_count, data_frequency,
frequency, ffill,
field, attempt_index + 1)
data_frequency, else:
ffill) raise ExchangeBarDataError(
data_type='history',
attempts=attempt_index,
error=e
)
def get_history_window(self, def get_history_window(self,
assets, assets,
@@ -81,19 +110,13 @@ class DataPortalExchangeBase(DataPortal):
if field == 'price': if field == 'price':
field = 'close' field = 'close'
return retry( return self._get_history_window(assets,
action=self._get_history_window, end_dt,
attempts=self.attempts['get_history_window_attempts'], bar_count,
sleeptime=self.attempts['retry_sleeptime'], frequency,
retry_exceptions=(ExchangeRequestError,), field,
cleanup=lambda: log.warn('fetching history again.'), data_frequency,
args=(assets, ffill)
end_dt,
bar_count,
frequency,
field,
data_frequency,
ffill))
@abc.abstractmethod @abc.abstractmethod
def get_exchange_history_window(self, def get_exchange_history_window(self,
@@ -107,58 +130,69 @@ class DataPortalExchangeBase(DataPortal):
ffill=True): ffill=True):
pass pass
def _get_spot_value(self, assets, field, dt, data_frequency): def _get_spot_value(self, assets, field, dt, data_frequency,
if isinstance(assets, TradingPair): attempt_index=0):
spot_values = self.get_exchange_spot_value( try:
assets.exchange, [assets], field, dt, data_frequency) if isinstance(assets, TradingPair):
spot_values = self.get_exchange_spot_value(
assets.exchange, [assets], field, dt, data_frequency)
if not spot_values: if not spot_values:
return np.nan return np.nan
return spot_values[0] return spot_values[0]
else:
exchange_assets = dict()
for asset in assets:
if asset.exchange not in exchange_assets:
exchange_assets[asset.exchange] = list()
exchange_assets[asset.exchange].append(asset)
if len(list(exchange_assets.keys())) == 1:
exchange_name = list(exchange_assets.keys())[0]
return self.get_exchange_spot_value(
exchange_name, assets, field, dt, data_frequency)
else: else:
spot_values = [] exchange_assets = dict()
for exchange_name in exchange_assets: for asset in assets:
assets = exchange_assets[exchange_name] if asset.exchange not in exchange_assets:
exchange_spot_values = self.get_exchange_spot_value( exchange_assets[asset.exchange] = list()
exchange_name,
assets,
field,
dt,
data_frequency
)
if len(assets) == 1:
spot_values.append(exchange_spot_values)
else:
spot_values += exchange_spot_values
return spot_values exchange_assets[asset.exchange].append(asset)
if len(list(exchange_assets.keys())) == 1:
exchange_name = list(exchange_assets.keys())[0]
return self.get_exchange_spot_value(
exchange_name, assets, field, dt, data_frequency)
else:
spot_values = []
for exchange_name in exchange_assets:
assets = exchange_assets[exchange_name]
exchange_spot_values = self.get_exchange_spot_value(
exchange_name,
assets,
field,
dt,
data_frequency
)
if len(assets) == 1:
spot_values.append(exchange_spot_values)
else:
spot_values += exchange_spot_values
return spot_values
except ExchangeRequestError as e:
log.warn(
'get spot value attempt {}: {}'.format(attempt_index, e)
)
if attempt_index < self.retry_get_spot_value:
sleep(self.retry_delay)
return self._get_spot_value(assets, field, dt, data_frequency,
attempt_index + 1)
else:
raise ExchangeBarDataError(
data_type='spot',
attempts=attempt_index,
error=e
)
def get_spot_value(self, assets, field, dt, data_frequency): def get_spot_value(self, assets, field, dt, data_frequency):
if field == 'price': if field == 'price':
field = 'close' field = 'close'
return retry( return self._get_spot_value(assets, field, dt, data_frequency)
action=self._get_spot_value,
attempts=self.attempts['get_spot_value_attempts'],
sleeptime=self.attempts['retry_sleeptime'],
retry_exceptions=(ExchangeRequestError,),
cleanup=lambda: log.warn('fetching spot value again.'),
args=(assets, field, dt, data_frequency))
@abc.abstractmethod @abc.abstractmethod
def get_exchange_spot_value(self, exchange_name, assets, field, dt, def get_exchange_spot_value(self, exchange_name, assets, field, dt,
@@ -208,7 +242,6 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
""" """
exchange = self.exchanges[exchange_name] exchange = self.exchanges[exchange_name]
df = exchange.get_history_window( df = exchange.get_history_window(
assets, assets,
end_dt, end_dt,
@@ -216,7 +249,7 @@ class DataPortalExchangeLive(DataPortalExchangeBase):
frequency, frequency,
field, field,
data_frequency, data_frequency,
False) ffill)
return df return df
def get_exchange_spot_value(self, exchange_name, assets, field, dt, def get_exchange_spot_value(self, exchange_name, assets, field, dt,
@@ -292,13 +325,13 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
DataFrame DataFrame
""" """
# TODO: verify that the exchange supports the timeframe
bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle
freq, candle_size, unit, adj_data_frequency = get_frequency( freq, candle_size, unit, adj_data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D'] frequency, data_frequency
) )
adj_bar_count = candle_size * bar_count adj_bar_count = candle_size * bar_count
trailing_bar_count = candle_size - 1
if data_frequency == 'minute' and adj_data_frequency == 'daily': if data_frequency == 'minute' and adj_data_frequency == 'daily':
end_dt = end_dt.floor('1D') end_dt = end_dt.floor('1D')
@@ -310,10 +343,10 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
field=field, field=field,
data_frequency=adj_data_frequency, data_frequency=adj_data_frequency,
algo_end_dt=self._last_available_session, algo_end_dt=self._last_available_session,
trailing_bar_count=trailing_bar_count
) )
start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency) df = resample_history_df(pd.DataFrame(series), freq, field)
df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
return df return df
def get_exchange_spot_value(self, def get_exchange_spot_value(self,
catalyst/exchange/exchange_errors.py
+15 -100
View File
@@ -100,19 +100,6 @@ class InvalidHistoryFrequencyError(ZiplineError):
).strip() ).strip()
class UnsupportedHistoryFrequencyError(ZiplineError):
msg = (
'{exchange} does not support candle frequency {freq}, please choose '
'from: {freqs}.'
).strip()
class InvalidHistoryTimeframeError(ZiplineError):
msg = (
'CCXT timeframe {timeframe} not supported by the exchange.'
).strip()
class MismatchingFrequencyError(ZiplineError): class MismatchingFrequencyError(ZiplineError):
msg = ( msg = (
'Bar aggregate frequency {frequency} not compatible with ' 'Bar aggregate frequency {frequency} not compatible with '
@@ -156,8 +143,7 @@ class OrphanOrderError(ZiplineError):
class OrphanOrderReverseError(ZiplineError): class OrphanOrderReverseError(ZiplineError):
msg = ( msg = (
'Order {order_id} tracked by algorithm, but not found in exchange ' 'Order {order_id} tracked by algorithm, but not found in exchange {exchange}.'
'{exchange}.'
).strip() ).strip()
@@ -175,8 +161,8 @@ class SidHashError(ZiplineError):
class BaseCurrencyNotFoundError(ZiplineError): class BaseCurrencyNotFoundError(ZiplineError):
msg = ( msg = (
'Algorithm base currency {base_currency} not found in account ' 'Algorithm base currency {base_currency} not found in exchange '
'balances on {exchange}: {balances}' '{exchange}.'
).strip() ).strip()
@@ -220,9 +206,8 @@ class EmptyValuesInBundleError(ZiplineError):
class PricingDataBeforeTradingError(ZiplineError): class PricingDataBeforeTradingError(ZiplineError):
msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} ' msg = ('Pricing data for trading pairs {symbols} on exchange {exchange} '
'starts on {first_trading_day}, but you are either trying to trade ' 'starts on {first_trading_day}, but you are either trying to trade or '
'or retrieve pricing data on {dt}. Adjust your dates accordingly.' 'retrieve pricing data on {dt}. Adjust your dates accordingly.').strip()
).strip()
class PricingDataNotLoadedError(ZiplineError): class PricingDataNotLoadedError(ZiplineError):
@@ -232,100 +217,30 @@ class PricingDataNotLoadedError(ZiplineError):
'{data_frequency} -i {symbol_list}`. See catalyst documentation ' '{data_frequency} -i {symbol_list}`. See catalyst documentation '
'for details.').strip() 'for details.').strip()
class PricingDataValueError(ZiplineError): class PricingDataValueError(ZiplineError):
msg = ('Unable to retrieve pricing data for {exchange} {symbol} ' msg = ('Unable to retrieve pricing data for {exchange} {symbol} '
'[{start_dt} - {end_dt}]: {error}').strip() '[{start_dt} - {end_dt}]: {error}').strip()
class DataCorruptionError(ZiplineError): class DataCorruptionError(ZiplineError):
msg = ( msg = ('Unable to validate data for {exchange} {symbols} in date range '
'Unable to validate data for {exchange} {symbols} in date range ' '[{start_dt} - {end_dt}]. The data is either corrupted or '
'[{start_dt} - {end_dt}]. The data is either corrupted or ' 'unavailable. Please try deleting this bundle:'
'unavailable. Please try deleting this bundle:' '\n`catalyst clean-exchange -x {exchange}\n'
'\n`catalyst clean-exchange -x {exchange}\n' 'Then, ingest the data again. Please contact the Catalyst team if '
'Then, ingest the data again. Please contact the Catalyst team if ' 'the issue persists.').strip()
'the issue persists.'
).strip()
class ApiCandlesError(ZiplineError): class ApiCandlesError(ZiplineError):
msg = ( msg = ('Unable to fetch candles from the remote API: {error}.').strip()
'Unable to fetch candles from the remote API: {error}.'
).strip()
class NoDataAvailableOnExchange(ZiplineError): class NoDataAvailableOnExchange(ZiplineError):
msg = ( msg = (
'Requested data for trading pair {symbol} is not available on ' 'Requested data for trading pair {symbol} is not available on exchange {exchange} '
'exchange {exchange} '
'in `{data_frequency}` frequency at this time. ' 'in `{data_frequency}` frequency at this time. '
'Check `http://enigma.co/catalyst/status` for market coverage.' 'Check `http://enigma.co/catalyst/status` for market coverage.').strip()
).strip()
class NoValueForField(ZiplineError): class NoValueForField(ZiplineError):
msg = ( msg = ('Value not found for field: {field}.').strip()
'Value not found for field: {field}.'
).strip()
class OrderTypeNotSupported(ZiplineError):
msg = (
'Order type `{order_type}` not currency supported by Catalyst. '
'Please use `limit` or `market` orders only.'
).strip()
class NotEnoughCapitalError(ZiplineError):
msg = (
'Not enough capital on exchange {exchange} for trading. Each '
'exchange should contain at least as much {base_currency} '
'as the specified `capital_base`. The current balance {balance} is '
'lower than the `capital_base`: {capital_base}'
).strip()
class NotEnoughCashError(ZiplineError):
msg = (
'Total {currency} amount on {exchange} is lower than the cash '
'reserved for this algo: {free} < {cash}. While trades can be made on '
'the exchange accounts outside of the algo, exchange must have enough '
'free {currency} to cover the algo cash.'
).strip()
class LastCandleTooEarlyError(ZiplineError):
msg = (
'The trade date of the last candle {last_traded} is before the '
'specified end date minus one candle {end_dt}. Please verify how '
'{exchange} calculates the start date of OHLCV candles.'
).strip()
class TickerNotFoundError(ZiplineError):
msg = (
'Unable to fetch ticker for {symbol} on {exchange}.'
).strip()
class BalanceNotFoundError(ZiplineError):
msg = (
'{currency} not found in account balance on {exchange}: {balances}.'
).strip()
class BalanceTooLowError(ZiplineError):
msg = (
'Balance for {currency} on {exchange} too low: {free} < {amount}. '
'Positions have likely been sold outside of this algorithm. Please '
'add positions to hold a free amount greater than {amount}, or clean '
'the state of this algo and restart.'
).strip()
class NoCandlesReceivedFromExchange(ZiplineError):
msg = (
'Although requesting {bar_count} candles until {end_dt} of asset {asset}, '
'an empty list of candles was received for {exchange}.'
).strip()
catalyst/exchange/exchange_execution.py
+1 -1
View File
@@ -1,4 +1,4 @@
from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder from catalyst.finance.execution import LimitOrder, StopOrder, StopLimitOrder, MarketOrder
class ExchangeLimitOrder(LimitOrder): class ExchangeLimitOrder(LimitOrder):
catalyst/exchange/exchange_portfolio.py
+6 -23
View File
@@ -1,7 +1,8 @@
import numpy as np import numpy as np
from logbook import Logger
from catalyst.constants import LOG_LEVEL from catalyst.constants import LOG_LEVEL
from catalyst.protocol import Portfolio, Positions, Position from catalyst.protocol import Portfolio, Positions, Position
from logbook import Logger
log = Logger('ExchangePortfolio', level=LOG_LEVEL) log = Logger('ExchangePortfolio', level=LOG_LEVEL)
@@ -39,13 +40,7 @@ class ExchangePortfolio(Portfolio):
""" """
log.debug('creating order {}'.format(order.id)) log.debug('creating order {}'.format(order.id))
self.open_orders[order.id] = order
open_orders = self.open_orders[order.asset] \
if order.asset is self.open_orders else []
open_orders.append(order)
self.open_orders[order.asset] = open_orders
order_position = self.positions[order.asset] \ order_position = self.positions[order.asset] \
if order.asset in self.positions else None if order.asset in self.positions else None
@@ -57,17 +52,6 @@ class ExchangePortfolio(Portfolio):
order_position.amount += order.amount order_position.amount += order.amount
log.debug('open order added to portfolio') log.debug('open order added to portfolio')
def _remove_open_order(self, order):
try:
open_orders = self.open_orders[order.asset]
if order in open_orders:
open_orders.remove(order)
except Exception:
raise ValueError(
'unable to clear order not found in open order list.'
)
def execute_order(self, order, transaction): def execute_order(self, order, transaction):
""" """
Update the open orders and positions to apply an executed order. Update the open orders and positions to apply an executed order.
@@ -82,15 +66,14 @@ class ExchangePortfolio(Portfolio):
""" """
log.debug('executing order {}'.format(order.id)) log.debug('executing order {}'.format(order.id))
self._remove_open_order(order) del self.open_orders[order.id]
order_position = self.positions[order.asset] \ order_position = self.positions[order.asset] \
if order.asset in self.positions else None if order.asset in self.positions else None
if order_position is None: if order_position is None:
raise ValueError( raise ValueError(
'Trying to execute order for a position not held:' 'Trying to execute order for a position not held: %s' % order.id
' {}'.format(order.id)
) )
self.capital_used += order.amount * transaction.price self.capital_used += order.amount * transaction.price
@@ -116,7 +99,7 @@ class ExchangePortfolio(Portfolio):
""" """
log.info('removing cancelled order {}'.format(order.id)) log.info('removing cancelled order {}'.format(order.id))
self._remove_open_order(order) del self.open_orders[order.id]
order_position = self.positions[order.asset] \ order_position = self.positions[order.asset] \
if order.asset in self.positions else None if order.asset in self.positions else None
catalyst/exchange/exchange_pricing_loader.py
-177
View File
@@ -1,177 +0,0 @@
# Copyright 2015 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from catalyst.constants import LOG_LEVEL
from catalyst.data.us_equity_pricing import BcolzDailyBarReader
from catalyst.errors import NoFurtherDataError
from catalyst.exchange.utils.factory import get_exchange
from catalyst.lib.adjusted_array import AdjustedArray
from catalyst.pipeline.data import DataSet, Column
from catalyst.pipeline.loaders.base import PipelineLoader
from catalyst.utils.calendars import get_calendar
from catalyst.utils.numpy_utils import float64_dtype
from logbook import Logger
from numpy import (
iinfo,
uint32,
)
UINT32_MAX = iinfo(uint32).max
log = Logger('ExchangePriceLoader', level=LOG_LEVEL)
class TradingPairPricing(DataSet):
"""
Dataset representing daily trading prices and volumes.
"""
open = Column(float64_dtype)
high = Column(float64_dtype)
low = Column(float64_dtype)
close = Column(float64_dtype)
volume = Column(float64_dtype)
class ExchangePricingLoader(PipelineLoader):
"""
PipelineLoader for Crypto Pricing data
Delegates loading of baselines and adjustments.
"""
def __init__(self, data_frequency):
cal = get_calendar('OPEN')
if data_frequency == 'daily':
reader = None
all_sessions = cal.all_sessions
elif data_frequency == 'minute':
reader = None
all_sessions = cal.all_minutes
else:
raise ValueError(
'Invalid data frequency: {}'.format(data_frequency)
)
self.data_frequency = data_frequency
self.raw_price_loader = reader
self._columns = TradingPairPricing.columns
self._all_sessions = all_sessions
@classmethod
def from_files(cls, pricing_path):
"""
Create a loader from a bcolz equity pricing dir and a SQLite
adjustments path.
Parameters
----------
pricing_path : str
Path to a bcolz directory written by a BcolzDailyBarWriter.
"""
return cls(
BcolzDailyBarReader(pricing_path),
)
def load_adjusted_array(self, columns, dates, assets, mask):
# load_adjusted_array is called with dates on which the user's algo
# will be shown data, which means we need to return the data that would
# be known at the start of each date. We assume that the latest data
# known on day N is the data from day (N - 1), so we shift all query
# dates back by a day.
start_date, end_date = _shift_dates(
self._all_sessions, dates[0], dates[-1], shift=1,
)
colnames = [c.name for c in columns]
if len(assets) == 0:
raise ValueError(
'Pipeline cannot load data with eligible assets.'
)
exchange_names = []
for asset in assets:
if asset.exchange not in exchange_names:
exchange_names.append(asset.exchange)
exchange = get_exchange(exchange_names[0])
reader = exchange.bundle.get_reader(self.data_frequency)
raw_arrays = reader.load_raw_arrays(
colnames,
start_date,
end_date,
assets,
)
out = {}
for c, c_raw in zip(columns, raw_arrays):
out[c] = AdjustedArray(
c_raw.astype(c.dtype),
mask,
{},
c.missing_value,
)
return out
@property
def columns(self):
return self._columns
def _shift_dates(dates, start_date, end_date, shift):
try:
start = dates.get_loc(start_date)
except KeyError:
if start_date < dates[0]:
raise NoFurtherDataError(
msg=(
"Pipeline Query requested data starting on {query_start}, "
"but first known date is {calendar_start}"
).format(
query_start=str(start_date),
calendar_start=str(dates[0]),
)
)
else:
raise ValueError("Query start %s not in calendar" % start_date)
# Make sure that shifting doesn't push us out of the calendar.
if start < shift:
raise NoFurtherDataError(
msg=(
"Pipeline Query requested data from {shift}"
" days before {query_start}, but first known date is only "
"{start} days earlier."
).format(shift=shift, query_start=start_date, start=start),
)
try:
end = dates.get_loc(end_date)
except KeyError:
if end_date > dates[-1]:
raise NoFurtherDataError(
msg=(
"Pipeline Query requesting data up to {query_end}, "
"but last known date is {calendar_end}"
).format(
query_end=end_date,
calendar_end=dates[-1],
)
)
else:
raise ValueError("Query end %s not in calendar" % end_date)
return dates[start - shift], dates[end - shift]
catalyst/exchange/utils/exchange_utils.py → catalyst/exchange/exchange_utils.py
+100 -224
View File
@@ -2,6 +2,7 @@ import hashlib
import json import json
import os import os
import pickle import pickle
import re
import shutil import shutil
from datetime import date, datetime from datetime import date, datetime
@@ -11,9 +12,8 @@ from six import string_types
from six.moves.urllib import request from six.moves.urllib import request
from catalyst.constants import DATE_FORMAT, SYMBOLS_URL from catalyst.constants import DATE_FORMAT, SYMBOLS_URL
from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias
ExchangeJSONDecoder
from catalyst.utils.paths import data_root, ensure_directory, \ from catalyst.utils.paths import data_root, ensure_directory, \
last_modified_time last_modified_time
@@ -62,13 +62,6 @@ def get_exchange_folder(exchange_name, environ=None):
return exchange_folder return exchange_folder
def is_blacklist(exchange_name, environ=None):
exchange_folder = get_exchange_folder(exchange_name, environ)
filename = os.path.join(exchange_folder, 'blacklist.txt')
return os.path.exists(filename)
def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None): def get_exchange_symbols_filename(exchange_name, is_local=False, environ=None):
""" """
The absolute path of the exchange's symbol.json file. The absolute path of the exchange's symbol.json file.
@@ -108,6 +101,20 @@ def download_exchange_symbols(exchange_name, environ=None):
return response return response
def symbols_parser(asset_def):
for key, value in asset_def.items():
match = isinstance(value, string_types) \
and re.search(r'(\d{4}-\d{2}-\d{2})', value)
if match:
try:
asset_def[key] = pd.to_datetime(value, utc=True)
except ValueError:
pass
return asset_def
def get_exchange_symbols(exchange_name, is_local=False, environ=None): def get_exchange_symbols(exchange_name, is_local=False, environ=None):
""" """
The de-serialized content of the exchange's symbols.json. The de-serialized content of the exchange's symbols.json.
@@ -126,20 +133,17 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None):
filename = get_exchange_symbols_filename(exchange_name, is_local) filename = get_exchange_symbols_filename(exchange_name, is_local)
if not is_local and (not os.path.isfile(filename) or pd.Timedelta( if not is_local and (not os.path.isfile(filename) or pd.Timedelta(
pd.Timestamp('now', tz='UTC') - last_modified_time( pd.Timestamp('now', tz='UTC') - last_modified_time(
filename)).days > 1): filename)).days > 1):
try: download_exchange_symbols(exchange_name, environ)
download_exchange_symbols(exchange_name, environ)
except Exception:
pass
if os.path.isfile(filename): if os.path.isfile(filename):
with open(filename) as data_file: with open(filename) as data_file:
try: try:
data = json.load(data_file, cls=ExchangeJSONDecoder) data = json.load(data_file, object_hook=symbols_parser)
return data return data
except ValueError: except ValueError as e:
return dict() return dict()
else: else:
raise ExchangeSymbolsNotFound( raise ExchangeSymbolsNotFound(
@@ -191,7 +195,7 @@ def get_symbols_string(assets):
return ', '.join([asset.symbol for asset in array]) return ', '.join([asset.symbol for asset in array])
def get_exchange_auth(exchange_name, alias=None, environ=None): def get_exchange_auth(exchange_name, environ=None):
""" """
The de-serialized contend of the exchange's auth.json file. The de-serialized contend of the exchange's auth.json file.
@@ -206,8 +210,7 @@ def get_exchange_auth(exchange_name, alias=None, environ=None):
""" """
exchange_folder = get_exchange_folder(exchange_name, environ) exchange_folder = get_exchange_folder(exchange_name, environ)
name = 'auth' if alias is None else alias filename = os.path.join(exchange_folder, 'auth.json')
filename = os.path.join(exchange_folder, '{}.json'.format(name))
if os.path.isfile(filename): if os.path.isfile(filename):
with open(filename) as data_file: with open(filename) as data_file:
@@ -263,7 +266,7 @@ def get_algo_folder(algo_name, environ=None):
return algo_folder return algo_folder
def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'): def get_algo_object(algo_name, key, environ=None, rel_path=None):
""" """
The de-serialized object of the algo name and key. The de-serialized object of the algo name and key.
@@ -273,7 +276,6 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
key: str key: str
environ: environ:
rel_path: str rel_path: str
how: str
Returns Returns
------- -------
@@ -288,25 +290,19 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'):
if rel_path is not None: if rel_path is not None:
folder = os.path.join(folder, rel_path) folder = os.path.join(folder, rel_path)
name = '{}.p'.format(key) if how == 'pickle' else '{}.json'.format(key) filename = os.path.join(folder, key + '.p')
filename = os.path.join(folder, name)
if os.path.isfile(filename): if os.path.isfile(filename):
if how == 'pickle': try:
with open(filename, 'rb') as handle: with open(filename, 'rb') as handle:
return pickle.load(handle) return pickle.load(handle)
except Exception as e:
else: return None
with open(filename) as data_file:
data = json.load(data_file, cls=ExchangeJSONDecoder)
return data
else: else:
return None return None
def save_algo_object(algo_name, key, obj, environ=None, rel_path=None, def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
how='pickle'):
""" """
Serialize and save an object by algo name and key. Serialize and save an object by algo name and key.
@@ -317,7 +313,6 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
obj: Object obj: Object
environ: environ:
rel_path: str rel_path: str
how: str
""" """
folder = get_algo_folder(algo_name, environ) folder = get_algo_folder(algo_name, environ)
@@ -326,15 +321,10 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None,
folder = os.path.join(folder, rel_path) folder = os.path.join(folder, rel_path)
ensure_directory(folder) ensure_directory(folder)
if how == 'json': filename = os.path.join(folder, key + '.p')
filename = os.path.join(folder, '{}.json'.format(key))
with open(filename, 'wt') as handle:
json.dump(obj, handle, indent=4, cls=ExchangeJSONEncoder)
else: with open(filename, 'wb') as handle:
filename = os.path.join(folder, '{}.p'.format(key)) pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(filename, 'wb') as handle:
pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
def get_algo_df(algo_name, key, environ=None, rel_path=None): def get_algo_df(algo_name, key, environ=None, rel_path=None):
@@ -394,71 +384,6 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
df.to_csv(handle, encoding='UTF_8') df.to_csv(handle, encoding='UTF_8')
def clear_frame_stats_directory(algo_name):
"""
remove the outdated directory
to avoid overloading the disk
Parameters
----------
algo_name: str
Returns
-------
error: str
"""
error = None
algo_folder = get_algo_folder(algo_name)
folder = os.path.join(algo_folder, 'frame_stats')
if os.path.exists(folder):
try:
shutil.rmtree(folder)
except OSError:
error = 'unable to remove {}, the analyze ' \
'data will be inconsistent'.format(folder)
return error
def remove_old_files(algo_name, today, rel_path, environ=None):
"""
remove old files from a directory
to avoid overloading the disk
Parameters
----------
algo_name: str
today: Timestamp
rel_path: str
environ:
Returns
-------
error: str
"""
error = None
algo_folder = get_algo_folder(algo_name, environ)
folder = os.path.join(algo_folder, rel_path)
ensure_directory(folder)
# run on all files in the folder
for f in os.listdir(folder):
try:
file_path = os.path.join(folder, f)
creation_unix = os.path.getctime(file_path)
creation_time = pd.to_datetime(creation_unix, unit='s', utc=True)
# if the file is older than 30 days erase it
if today - pd.DateOffset(30) > creation_time:
os.unlink(file_path)
except OSError:
error = 'unable to erase files in {}'.format(folder)
return error
def get_exchange_minute_writer_root(exchange_name, environ=None): def get_exchange_minute_writer_root(exchange_name, environ=None):
""" """
The minute writer folder for the exchange. The minute writer folder for the exchange.
@@ -503,15 +428,6 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
return temp_bundles return temp_bundles
def has_bundle(exchange_name, data_frequency, environ=None):
exchange_folder = get_exchange_folder(exchange_name, environ)
folder_name = '{}_bundle'.format(data_frequency.lower())
folder = os.path.join(exchange_folder, folder_name)
return os.path.isdir(folder)
def symbols_serial(obj): def symbols_serial(obj):
""" """
JSON serializer for objects not serializable by default json code JSON serializer for objects not serializable by default json code
@@ -579,7 +495,68 @@ def get_common_assets(exchanges):
return assets return assets
def resample_history_df(df, freq, field, start_dt=None): def get_frequency(freq, data_frequency):
"""
Get the frequency parameters.
Notes
-----
We're trying to use Pandas convention for frequency aliases.
Parameters
----------
freq: str
data_frequency: str
Returns
-------
str, int, str, str
"""
if freq == 'minute':
unit = 'T'
candle_size = 1
elif freq == 'daily':
unit = 'D'
candle_size = 1
else:
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
if unit.lower() == 'd':
alias = '{}D'.format(candle_size)
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm' or unit == 'T':
alias = '{}T'.format(candle_size)
if data_frequency == 'daily':
data_frequency = 'minute'
# elif unit.lower() == 'h':
# candle_size = candle_size * 60
#
# alias = '{}T'.format(candle_size)
# if data_frequency == 'daily':
# data_frequency = 'minute'
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
return alias, candle_size, unit, data_frequency
def resample_history_df(df, freq, field):
""" """
Resample the OHCLV DataFrame using the specified frequency. Resample the OHCLV DataFrame using the specified frequency.
@@ -607,16 +584,7 @@ def resample_history_df(df, freq, field, start_dt=None):
else: else:
raise ValueError('Invalid field.') raise ValueError('Invalid field.')
resampled_df = df.resample( resampled_df = df.resample(freq).agg(agg)
freq, closed='left', label='left'
).agg(agg) # type: pd.DataFrame
# Because the samples are closed left, we get one more candle at
# the beginning then the requested number for bars. Removing this
# candle to avoid confusion.
if start_dt and not resampled_df.empty:
resampled_df = resampled_df[resampled_df.index >= start_dt]
return resampled_df return resampled_df
@@ -636,15 +604,13 @@ def mixin_market_params(exchange_name, params, market):
# TODO: make this more externalized / configurable # TODO: make this more externalized / configurable
if 'lot' in market: if 'lot' in market:
params['min_trade_size'] = market['lot'] params['min_trade_size'] = market['lot']
params['lot'] = market['lot']
if exchange_name == 'bitfinex': if exchange_name == 'bitfinex':
params['maker'] = 0.001 params['maker'] = 0.001
params['taker'] = 0.002 params['taker'] = 0.002
elif 'maker' in market and 'taker' in market and \ elif 'maker' in market and 'taker' in market \
market['maker'] is not None and market['taker'] is not None: and market['maker'] is not None and market['taker'] is not None:
params['maker'] = market['maker'] params['maker'] = market['maker']
params['taker'] = market['taker'] params['taker'] = market['taker']
@@ -658,96 +624,6 @@ def mixin_market_params(exchange_name, params, market):
if 'minimum_order_size' in info: if 'minimum_order_size' in info:
params['min_trade_size'] = float(info['minimum_order_size']) params['min_trade_size'] = float(info['minimum_order_size'])
if 'lot' not in params:
params['lot'] = params['min_trade_size']
def from_ms_timestamp(ms):
def group_assets_by_exchange(assets): return pd.to_datetime(ms, unit='ms', utc=True)
exchange_assets = dict()
for asset in assets:
if asset.exchange not in exchange_assets:
exchange_assets[asset.exchange] = list()
exchange_assets[asset.exchange].append(asset)
return exchange_assets
def get_catalyst_symbol(market_or_symbol):
"""
The Catalyst symbol.
Parameters
----------
market_or_symbol
Returns
-------
"""
if isinstance(market_or_symbol, string_types):
parts = market_or_symbol.split('/')
return '{}_{}'.format(parts[0].lower(), parts[1].lower())
else:
return '{}_{}'.format(
market_or_symbol['base'].lower(),
market_or_symbol['quote'].lower(),
)
def save_asset_data(folder, df, decimals=8):
symbols = df.index.get_level_values('symbol')
for symbol in symbols:
symbol_df = df.loc[(symbols == symbol)] # Type: pd.DataFrame
filename = os.path.join(folder, '{}.csv'.format(symbol))
if os.path.exists(filename):
print_headers = False
else:
print_headers = True
with open(filename, 'a') as f:
symbol_df.to_csv(
path_or_buf=f,
header=print_headers,
float_format='%.{}f'.format(decimals),
)
def forward_fill_df_if_needed(df, periods):
df = df.reindex(periods)
# volume should always be 0 (if there were no trades in this interval)
df['volume'] = df['volume'].fillna(0.0)
# ie pull the last close into this close
df['close'] = df.fillna(method='pad')
# now copy the close that was pulled down from the last timestep
# into this row, across into o/h/l
df['open'] = df['open'].fillna(df['close'])
df['low'] = df['low'].fillna(df['close'])
df['high'] = df['high'].fillna(df['close'])
return df
def transform_candles_to_df(candles):
return pd.DataFrame(candles).set_index('last_traded')
def get_candles_df(candles, field, freq, bar_count, end_dt):
all_series = dict()
for asset in candles:
asset_df = transform_candles_to_df(candles[asset])
rounded_end_dt = end_dt.floor(freq)
periods = pd.date_range(end=rounded_end_dt,
periods=bar_count,
freq=freq)
asset_df = forward_fill_df_if_needed(asset_df, periods)
all_series[asset] = pd.Series(asset_df[field])
df = pd.DataFrame(all_series)
df.dropna(inplace=True)
return df
catalyst/exchange/factory.py
+36
View File
@@ -0,0 +1,36 @@
import os
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.exchange_utils import get_exchange_auth, \
get_exchange_folder
def get_exchange(exchange_name, base_currency=None, portfolio=None,
must_authenticate=False):
exchange_auth = get_exchange_auth(exchange_name)
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name), 'auth.json'
)
)
return CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
base_currency=base_currency,
portfolio=portfolio
)
def get_exchanges(exchange_names):
exchanges = dict()
for exchange_name in exchange_names:
exchanges[exchange_name] = get_exchange(exchange_name)
return exchanges
catalyst/exchange/live_graph_clock.py
+171 -12
View File
@@ -1,12 +1,14 @@
import pandas as pd import pandas as pd
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.utils.stats_utils import prepare_stats
from catalyst.gens.sim_engine import ( from catalyst.gens.sim_engine import (
BAR, BAR,
SESSION_START SESSION_START
) )
from logbook import Logger from logbook import Logger
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.exchange_errors import \
MismatchingBaseCurrenciesExchanges
log = Logger('LiveGraphClock', level=LOG_LEVEL) log = Logger('LiveGraphClock', level=LOG_LEVEL)
@@ -36,23 +38,177 @@ class LiveGraphClock(object):
the exchange and the live trading machine's clock. It's not used currently. the exchange and the live trading machine's clock. It's not used currently.
""" """
def __init__(self, sessions, context, callback=None, def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
time_skew=pd.Timedelta('0s')):
global mdates, plt # TODO: Could be cleaner
import matplotlib.dates as mdates
from matplotlib import pyplot as plt
from matplotlib import style
self.sessions = sessions self.sessions = sessions
self.time_skew = time_skew self.time_skew = time_skew
self._last_emit = None self._last_emit = None
self._before_trading_start_bar_yielded = True self._before_trading_start_bar_yielded = True
self.context = context self.context = context
self.callback = callback self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
style.use('dark_background')
fig = plt.figure()
fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
self.context.algo_namespace))
self.ax_pnl = fig.add_subplot(311)
self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl)
self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl)
if len(context.minute_stats) > 0:
self.draw_pnl()
self.draw_custom_signals()
self.draw_exposure()
# rotates and right aligns the x labels, and moves the bottom of the
# axes up to make room for them
fig.autofmt_xdate()
fig.subplots_adjust(hspace=0.5)
plt.tight_layout()
plt.ion()
plt.show()
def format_ax(self, ax):
"""
Trying to assign reasonable parameters to the time axis.
Parameters
----------
ax:
"""
# TODO: room for improvement
ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
ax.xaxis.set_major_formatter(self.fmt)
locator = mdates.HourLocator(interval=4)
locator.MAXTICKS = 5000
ax.xaxis.set_minor_locator(locator)
datemin = pd.Timestamp.utcnow()
ax.set_xlim(datemin)
ax.grid(True)
def set_legend(self, ax):
"""
Set legend on the chart.
Parameters
----------
ax
"""
ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
def draw_pnl(self):
"""
Draw p&l line on the chart.
"""
ax = self.ax_pnl
df = self.context.pnl_stats
ax.clear()
ax.set_title('Performance')
ax.plot(df.index, df['performance'], '-',
color='green',
linewidth=1.0,
label='Performance'
)
def perc(val):
return '{:2f}'.format(val)
ax.format_ydata = perc
self.set_legend(ax)
self.format_ax(ax)
def draw_custom_signals(self):
"""
Draw custom signals on the chart.
"""
ax = self.ax_custom_signals
df = self.context.custom_signals_stats
colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
ax.clear()
ax.set_title('Custom Signals')
for index, column in enumerate(df.columns.values.tolist()):
ax.plot(df.index, df[column], '-',
color=colors[index],
linewidth=1.0,
label=column
)
self.set_legend(ax)
self.format_ax(ax)
def draw_exposure(self):
"""
Draw exposure line on the chart.
"""
ax = self.ax_exposure
context = self.context
df = context.exposure_stats
# TODO: list exchanges in graph
base_currency = None
positions = []
for exchange_name in context.exchanges:
exchange = context.exchanges[exchange_name]
if not base_currency:
base_currency = exchange.base_currency
elif base_currency != exchange.base_currency:
raise MismatchingBaseCurrenciesExchanges(
base_currency=base_currency,
exchange_name=exchange.name,
exchange_currency=exchange.base_currency
)
positions += exchange.portfolio.positions
ax.clear()
ax.set_title('Exposure')
ax.plot(df.index, df['base_currency'], '-',
color='green',
linewidth=1.0,
label='Base Currency: {}'.format(base_currency.upper())
)
symbols = []
for position in positions:
symbols.append(position.symbol)
ax.plot(df.index, df['long_exposure'], '-',
color='blue',
linewidth=1.0,
label='Long Exposure: {}'.format(', '.join(symbols).upper()))
self.set_legend(ax)
self.format_ax(ax)
def __iter__(self): def __iter__(self):
from matplotlib import pyplot as plt
yield pd.Timestamp.utcnow(), SESSION_START yield pd.Timestamp.utcnow(), SESSION_START
while True: while True:
current_time = pd.Timestamp.utcnow() current_time = pd.Timestamp.utcnow()
current_minute = current_time.floor('1T') current_minute = current_time.floor('1 min')
if self._last_emit is None or current_minute > self._last_emit: if self._last_emit is None or current_minute > self._last_emit:
log.debug('emitting minutely bar: {}'.format(current_minute)) log.debug('emitting minutely bar: {}'.format(current_minute))
@@ -60,11 +216,14 @@ class LiveGraphClock(object):
self._last_emit = current_minute self._last_emit = current_minute
yield current_minute, BAR yield current_minute, BAR
recorded_cols = list(self.context.recorded_vars.keys()) try:
df, _ = prepare_stats( self.draw_pnl()
self.context.frame_stats, recorded_cols=recorded_cols self.draw_custom_signals()
) self.draw_exposure()
self.callback(self.context, df)
plt.draw()
except Exception as e:
log.warn('Unable to update the graph: {}'.format(e))
else: else:
# I can't use the "animate" reactive approach here because # I can't use the "animate" reactive approach here because
catalyst/marketplace/utils/__init__.py → catalyst/exchange/poloniex/__init__.py
View File
catalyst/exchange/poloniex/poloniex.py
+655
View File
@@ -0,0 +1,655 @@
import json
import json
import time
from collections import defaultdict
import numpy as np
import pandas as pd
import pytz
from catalyst.assets._assets import TradingPair
from logbook import Logger
# import six
from six import iteritems
from catalyst.constants import LOG_LEVEL
# from websocket import create_connection
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_bundle import ExchangeBundle
from catalyst.exchange.exchange_errors import (
ExchangeRequestError,
InvalidHistoryFrequencyError,
InvalidOrderStyle, OrphanOrderReverseError)
from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
ExchangeStopLimitOrder
from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
download_exchange_symbols, get_symbols_string
from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
from catalyst.finance.order import Order, ORDER_STATUS
from catalyst.finance.transaction import Transaction
from catalyst.protocol import Account
log = Logger('Poloniex', level=LOG_LEVEL)
class Poloniex(Exchange):
def __init__(self, key, secret, base_currency, portfolio=None):
self.api = Poloniex_api(key=key, secret=secret)
self.name = 'poloniex'
self.assets = dict()
self.load_assets()
self.local_assets = dict()
self.load_assets(is_local=True)
self.base_currency = base_currency
self._portfolio = portfolio
self.minute_writer = None
self.minute_reader = None
self.transactions = defaultdict(list)
self.num_candles_limit = 2000
self.max_requests_per_minute = 60
self.request_cpt = dict()
self.bundle = ExchangeBundle(self.name)
def sanitize_curency_symbol(self, exchange_symbol):
"""
Helper method used to build the universal pair.
Include any symbol mapping here if appropriate.
:param exchange_symbol:
:return universal_symbol:
"""
return exchange_symbol.lower()
def _create_order(self, order_status):
"""
Create a Catalyst order object from the Exchange order dictionary
:param order_status:
:return: Order
"""
# if order_status['is_cancelled']:
# status = ORDER_STATUS.CANCELLED
# elif not order_status['is_live']:
# log.info('found executed order {}'.format(order_status))
# status = ORDER_STATUS.FILLED
# else:
status = ORDER_STATUS.OPEN
amount = float(order_status['amount'])
# filled = float(order_status['executed_amount'])
filled = None
if order_status['type'] == 'sell':
amount = -amount
# filled = -filled
price = float(order_status['rate'])
order_type = order_status['type']
stop_price = None
limit_price = None
# TODO: is this comprehensive enough?
# if order_type.endswith('limit'):
# limit_price = price
# elif order_type.endswith('stop'):
# stop_price = price
# executed_price = float(order_status['avg_execution_price'])
executed_price = price
# TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
commission = None
# date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
# date = pytz.utc.localize(date)
date = None
order = Order(
dt=date,
asset=self.assets[order_status['symbol']],
# No such field in Poloniex
amount=amount,
stop=stop_price,
limit=limit_price,
filled=filled,
id=str(order_status['orderNumber']),
commission=commission
)
order.status = status
return order, executed_price
def get_balances(self):
balances = self.api.returnbalances()
try:
log.debug('retrieving wallets balances')
except Exception as e:
log.debug(e)
raise ExchangeRequestError(error=e)
if 'error' in balances:
raise ExchangeRequestError(
error='unable to fetch balance {}'.format(balances['error'])
)
std_balances = dict()
for (key, value) in iteritems(balances):
currency = key.lower()
std_balances[currency] = float(value)
return std_balances
@property
def account(self):
account = Account()
account.settled_cash = None
account.accrued_interest = None
account.buying_power = None
account.equity_with_loan = None
account.total_positions_value = None
account.total_positions_exposure = None
account.regt_equity = None
account.regt_margin = None
account.initial_margin_requirement = None
account.maintenance_margin_requirement = None
account.available_funds = None
account.excess_liquidity = None
account.cushion = None
account.day_trades_remaining = None
account.leverage = None
account.net_leverage = None
account.net_liquidation = None
return account
@property
def time_skew(self):
# TODO: research the time skew conditions
return pd.Timedelta('0s')
def get_account(self):
# TODO: fetch account data and keep in cache
return None
def get_candles(self, freq, assets, bar_count=None,
start_dt=None, end_dt=None):
"""
Retrieve OHLVC candles from Poloniex
:param freq:
:param assets:
:param bar_count:
:return:
Available Frequencies
---------------------
'5m', '15m', '30m', '2h', '4h', '1D'
"""
if end_dt is None:
end_dt = pd.Timestamp.utcnow()
log.debug(
'retrieving {bars} {freq} candles on {exchange} from '
'{end_dt} for markets {symbols}, '.format(
bars=bar_count,
freq=freq,
exchange=self.name,
end_dt=end_dt,
symbols=get_symbols_string(assets)
)
)
if freq == '1T' and (bar_count == 1 or bar_count is None):
# TODO: use the order book instead
# We use the 5m to fetch the last bar
frequency = 300
elif freq == '5T':
frequency = 300
elif freq == '15T':
frequency = 900
elif freq == '30T':
frequency = 1800
elif freq == '120T':
frequency = 7200
elif freq == '240T':
frequency = 14400
elif freq == '1D':
frequency = 86400
else:
# Poloniex does not offer 1m data candles
# It is likely to error out there frequently
raise InvalidHistoryFrequencyError(frequency=freq)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
end = int(delta.total_seconds())
if bar_count is None:
start = end - 2 * frequency
else:
start = end - bar_count * frequency
try:
response = self.api.returnchartdata(
self.get_symbol(asset), frequency, start, end
)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve candles: {}'.format(
response.content)
)
def ohlc_from_candle(candle):
last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
last_traded = last_traded.replace(tzinfo=pytz.UTC)
ohlc = dict(
open=np.float64(candle['open']),
high=np.float64(candle['high']),
low=np.float64(candle['low']),
close=np.float64(candle['close']),
volume=np.float64(candle['volume']),
price=np.float64(candle['close']),
last_traded=last_traded
)
return ohlc
if bar_count is None:
ohlc_map[asset] = ohlc_from_candle(response[0])
else:
ohlc_bars = []
for candle in response:
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def create_order(self, asset, amount, is_buy, style):
"""
Creating order on the exchange.
:param asset:
:param amount:
:param is_buy:
:param style:
:return:
"""
exchange_symbol = self.get_symbol(asset)
if isinstance(style, ExchangeLimitOrder) or isinstance(style,
ExchangeStopLimitOrder):
if isinstance(style, ExchangeStopLimitOrder):
log.warn('{} will ignore the stop price'.format(self.name))
price = style.get_limit_price(is_buy)
try:
if (is_buy):
response = self.api.buy(exchange_symbol, amount, price)
else:
response = self.api.sell(exchange_symbol, -amount, price)
except Exception as e:
raise ExchangeRequestError(error=e)
date = pd.Timestamp.utcnow()
if ('orderNumber' in response):
order_id = str(response['orderNumber'])
order = Order(
dt=date,
asset=asset,
amount=amount,
stop=style.get_stop_price(is_buy),
limit=style.get_limit_price(is_buy),
id=order_id
)
return order
else:
log.warn(
'{} order failed: {}'.format('buy' if is_buy else 'sell',
response['error']))
return None
else:
raise InvalidOrderStyle(exchange=self.name,
style=style.__class__.__name__)
def get_open_orders(self, asset='all'):
"""Retrieve all of the current open orders.
Parameters
----------
asset : Asset
If passed and not 'all', return only the open orders for the given
asset instead of all open orders.
Returns
-------
open_orders : dict[list[Order]] or list[Order]
If 'all' is passed this will return a dict mapping Assets
to a list containing all the open orders for the asset.
If an asset is passed then this will return a list of the open
orders for this asset.
"""
return self.portfolio.open_orders
"""
TODO: Why going to the exchange if we already have this info locally?
And why creating all these Orders if we later discard them?
"""
try:
if (asset == 'all'):
response = self.api.returnopenorders('all')
else:
response = self.api.returnopenorders(self.get_symbol(asset))
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve open orders: {}'.format(
order_statuses['message'])
)
print(self.portfolio.open_orders)
# TODO: Need to handle openOrders for 'all'
orders = list()
for order_status in response:
order, executed_price = self._create_order(
order_status) # will Throw error b/c Polo doesn't track order['symbol']
if asset is None or asset == order.sid:
orders.append(order)
return orders
def get_order(self, order_id):
"""Lookup an order based on the order id returned from one of the
order functions.
Parameters
----------
order_id : str
The unique identifier for the order.
Returns
-------
order : Order
The order object.
"""
try:
order = self._portfolio.open_orders[order_id]
except Exception as e:
raise OrphanOrderError(order_id=order_id, exchange=self.name)
return order
# TODO: Need to decide whether we fetch orders locally or from exchnage
# The code below is ignored
try:
response = self.api.returnopenorders(self.get_symbol(order.sid))
except Exception as e:
raise ExchangeRequestError(error=e)
for o in response:
if (int(o['orderNumber']) == int(order_id)):
return order
return None
def cancel_order(self, order_param):
"""Cancel an open order.
Parameters
----------
order_param : str or Order
The order_id or order object to cancel.
"""
if (isinstance(order_param, Order)):
order = order_param
else:
order = self._portfolio.open_orders[order_param]
try:
response = self.api.cancelorder(order.id)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
log.info(
'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format(
order_id=order.id,
exchange=self.name,
error=response['error']
))
# raise OrderCancelError(
# order_id=order.id,
# exchange=self.name,
# error=response['error']
# )
self.portfolio.remove_order(order)
def tickers(self, assets):
"""
Fetch ticket data for assets
https://docs.bitfinex.com/v2/reference#rest-public-tickers
:param assets:
:return:
"""
symbols = self.get_symbols(assets)
log.debug('fetching tickers {}'.format(symbols))
try:
response = self.api.returnticker()
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve tickers: {}'.format(
response['error'])
)
ticks = dict()
for index, symbol in enumerate(symbols):
ticks[assets[index]] = dict(
timestamp=pd.Timestamp.utcnow(),
bid=float(response[symbol]['highestBid']),
ask=float(response[symbol]['lowestAsk']),
last_price=float(response[symbol]['last']),
low=float(response[symbol]['lowestAsk']),
# TODO: Polo does not provide low
high=float(response[symbol]['highestBid']),
# TODO: Polo does not provide high
volume=float(response[symbol]['baseVolume']),
)
log.debug('got tickers {}'.format(ticks))
return ticks
def generate_symbols_json(self, filename=None, source_dates=False):
symbol_map = {}
if not source_dates:
fn, r = download_exchange_symbols(self.name)
with open(fn) as data_file:
cached_symbols = json.load(data_file)
response = self.api.returnticker()
for exchange_symbol in response:
base, market = self.sanitize_curency_symbol(exchange_symbol).split(
'_')
symbol = '{market}_{base}'.format(market=market, base=base)
if (source_dates):
start_date = self.get_symbol_start_date(exchange_symbol)
else:
try:
start_date = cached_symbols[exchange_symbol]['start_date']
except KeyError as e:
start_date = time.strftime('%Y-%m-%d')
try:
end_daily = cached_symbols[exchange_symbol]['end_daily']
except KeyError as e:
end_daily = 'N/A'
try:
end_minute = cached_symbols[exchange_symbol]['end_minute']
except KeyError as e:
end_minute = 'N/A'
symbol_map[exchange_symbol] = dict(
symbol=symbol,
start_date=start_date,
end_daily=end_daily,
end_minute=end_minute,
)
if (filename is None):
filename = get_exchange_symbols_filename(self.name)
with open(filename, 'w') as f:
json.dump(symbol_map, f, sort_keys=True, indent=2,
separators=(',', ':'))
def get_symbol_start_date(self, symbol):
try:
r = self.api.returnchartdata(symbol, 86400, pd.to_datetime(
'2010-1-1').value // 10 ** 9)
except Exception as e:
raise ExchangeRequestError(error=e)
return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date'])))
def check_open_orders(self):
"""
Need to override this function for Poloniex:
Loop through the list of open orders in the Portfolio object.
Check if any transactions have been executed:
If so, create a transaction and apply to the Portfolio.
Check if the order is still open:
If not, remove it from open orders
:return:
transactions: Transaction[]
"""
transactions = list()
if self.portfolio.open_orders:
for order_id in list(self.portfolio.open_orders):
order = self._portfolio.open_orders[order_id]
log.debug('found open order: {}'.format(order_id))
try:
order_open = self.get_order(order_id)
except Exception as e:
raise ExchangeRequestError(error=e)
if (order_open):
delta = pd.Timestamp.utcnow() - order.dt
log.info(
'order {order_id} still open after {delta}'.format(
order_id=order_id,
delta=delta)
)
try:
response = self.api.returnordertrades(order_id)
except Exception as e:
raise ExchangeRequestError(error=e)
if ('error' in response):
if (not order_open):
raise OrphanOrderReverseError(order_id=order_id,
exchange=self.name)
else:
for tx in response:
"""
We maintain a list of dictionaries of transactions that correspond to
partially filled orders, indexed by order_id. Every time we query
executed transactions from the exchange, we check if we had that
transaction for that order already. If not, we process it.
When an order if fully filled, we flush the dict of transactions
associated with that order.
"""
if (not filter(
lambda item: item['order_id'] == tx['tradeID'],
self.transactions[order_id])):
log.debug(
'Got new transaction for order {}: amount {}, price {}'.format(
order_id, tx['amount'], tx['rate']))
tx['amount'] = float(tx['amount'])
if (tx['type'] == 'sell'):
tx['amount'] = -tx['amount']
transaction = Transaction(
asset=order.asset,
amount=tx['amount'],
dt=pd.to_datetime(tx['date'], utc=True),
price=float(tx['rate']),
order_id=tx['tradeID'],
# it's a misnomer, but keeping it for compatibility
commission=float(tx['fee'])
)
self.transactions[order_id].append(transaction)
self.portfolio.execute_transaction(transaction)
transactions.append(transaction)
if (not order_open):
"""
Since transactions have been executed individually
the only thing left to do is remove them from list of open_orders
"""
del self.portfolio.open_orders[order_id]
del self.transactions[order_id]
return transactions
def get_orderbook(self, asset, order_type='all'):
exchange_symbol = asset.exchange_symbol
data = self.api.returnOrderBook(market=exchange_symbol)
result = dict()
for order_type in data:
# TODO: filter by type
if order_type != 'asks' and order_type != 'bids':
continue
result[order_type] = []
for entry in data[order_type]:
if len(entry) == 2:
result[order_type].append(
dict(
rate=float(entry[0]),
quantity=float(entry[1])
)
)
return result
catalyst/exchange/poloniex/poloniex_api.py
+215
View File
@@ -0,0 +1,215 @@
#!/usr/bin/env python
import json
import time
import hmac
import hashlib
import ssl
from six.moves import urllib
# Workaround for backwards compatibility
# https://stackoverflow.com/questions/3745771/urllib-request-in-python-2-7
urlopen = urllib.request.urlopen
class Poloniex_api(object):
def __init__(self, key, secret):
self.key = key
self.secret = secret
self.max_requests_per_second = 6
self.request_cpt = dict()
self.public = ['returnTicker', 'return24Volume', 'returnOrderBook',
'returnTradeHistory', 'returnChartData',
'returnCurrencies', 'returnLoanOrders']
self.trading = ['returnBalances', 'returnCompleteBalances',
'returnDepositAddresses',
'generateNewAddress', 'returnDepositsWithdrawals',
'returnOpenOrders',
'returnTradeHistory', 'returnOrderTrades',
'buy', 'sell', 'cancelOrder', 'moveOrder',
'withdraw', 'returnFeeInfo',
'returnAvailableAccountBalances',
'returnTradableBalances', 'transferBalance',
'returnMarginAccountSummary', 'marginBuy',
'marginSell',
'getMarginPosition', 'closeMarginPosition',
'createLoanOffer',
'cancelLoanOffer', 'returnOpenLoanOffers',
'returnActiveLoans',
'returnLendingHistory', 'toggleAutoRenew']
def ask_request(self):
"""
Asks permission to issue a request to the exchange.
The primary purpose is to avoid hitting rate limits.
The application will pause if the maximum requests per minute
permitted by the exchange is exceeded.
:return boolean:
"""
now = time.time()
if not self.request_cpt:
self.request_cpt = dict()
self.request_cpt[now] = 0
return True
cpt_date = list(self.request_cpt.keys())[0]
cpt = self.request_cpt[cpt_date]
if now > cpt_date + 1:
self.request_cpt = dict()
self.request_cpt[now] = 0
return True
if cpt >= self.max_requests_per_second:
time.sleep(1)
now = time.time()
self.request_cpt = dict()
self.request_cpt[now] = 0
return True
else:
self.request_cpt[cpt_date] += 1
def query(self, method, req={}):
if method in self.public:
url = 'https://poloniex.com/public?command=' + method + '&' + \
urllib.parse.urlencode(req)
headers = {}
post_data = None
elif method in self.trading:
url = 'https://poloniex.com/tradingApi'
req['command'] = method
req['nonce'] = int(time.time() * 1000)
post_data = urllib.parse.urlencode(req)
signature = hmac.new(self.secret.encode('utf-8'),
post_data.encode('utf-8'),
hashlib.sha512).hexdigest()
headers = {'Sign': signature, 'Key': self.key}
post_data = post_data.encode('utf-8')
else:
raise ValueError(
'Method "' + method + '" not found in neither the Public API '
'or Trading API endpoints'
)
self.ask_request()
req = urllib.request.Request(
url,
data=post_data,
headers=headers,
)
return json.loads(
urlopen(req, context=ssl._create_unverified_context()).read())
def returnticker(self):
return self.query('returnTicker', {})
def return24volume(self):
return self.query('return24Volume', {})
def returnOrderBook(self, market='all'):
return self.query('returnOrderBook', {'currencyPair': market})
def returntradehistory(self, market, start=None, end=None):
if (start is not None and end is not None):
return self.query('returntradehistory',
{'currencyPair': market, 'start': start,
'end': end})
else:
return self.query('returntradehistory', {'currencyPair': market})
def returnchartdata(self, market, period, start, end=9999999999):
return self.query('returnChartData',
{'currencyPair': market, 'period': period,
'start': start, 'end': end})
def returncurrencies(self):
return self.query('returnCurrencies', {})
def returnloadorders(self, market):
return self.query('returnLoanOrders', {'currency': market})
def returnbalances(self):
return self.query('returnBalances')
def returncompletebalances(self, account):
if (account):
return self.query('returnCompleteBalances', {'account': account})
else:
return self.query('returnCompleteBalances')
def returndepositaddresses(self):
return self.query('returnDepositAddresses')
def generatenewaddress(self, currency):
return self.query('generateNewAddress', {'currency': currency})
def returnDepositsWithdrawals(self, start, end):
return self.query('returnDepositsWithdrawals',
{'start': start, 'end': end})
def returnopenorders(self, market):
return self.query('returnOpenOrders', {'currencyPair': market})
def returntradehistory(self, market):
# TODO: optional start and/or end and limit
return self.query('returnTradeHistory', {'currencyPair': market})
def returnordertrades(self, ordernumber):
return self.query('returnOrderTrades', {'orderNumber': ordernumber})
def buy(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
postonly=0):
if (fillorkill):
return self.query('buy', {'currencyPair': market, 'rate': rate,
'amount': amount,
'fillOrKill': fillorkill, })
elif (immediateorcancel):
return self.query('buy', {'currencyPair': market, 'rate': rate,
'amount': amount,
'immediateOrCancel': immediateorcancel, })
elif (postonly):
return self.query('buy', {'currencyPair': market, 'rate': rate,
'amount': amount,
'postOnly': postonly, })
else:
return self.query('buy', {'currencyPair': market, 'rate': rate,
'amount': amount, })
def sell(self, market, amount, rate, fillorkill=0, immediateorcancel=0,
postonly=0):
if (fillorkill):
return self.query('sell', {'currencyPair': market, 'rate': rate,
'amount': amount,
'fillOrKill': fillorkill, })
elif (immediateorcancel):
return self.query('sell', {'currencyPair': market, 'rate': rate,
'amount': amount,
'immediateOrCancel': immediateorcancel, })
elif (postonly):
return self.query('sell', {'currencyPair': market, 'rate': rate,
'amount': amount,
'postOnly': postonly, })
else:
return self.query('sell', {'currencyPair': market, 'rate': rate,
'amount': amount, })
def cancelorder(self, ordernumber):
return self.query('cancelOrder', {'orderNumber': ordernumber})
def withdraw(self, currency, quantity, address):
return self.query('withdraw',
{'currency': currency, 'amount': quantity,
'address': address})
def returnfeeinfo(self):
return self.query('returnFeeInfo')
catalyst/exchange/simple_clock.py
+3 -3
View File
@@ -14,13 +14,14 @@
from time import sleep from time import sleep
import pandas as pd import pandas as pd
from catalyst.constants import LOG_LEVEL
from catalyst.gens.sim_engine import ( from catalyst.gens.sim_engine import (
BAR, BAR,
SESSION_START SESSION_START
) )
from logbook import Logger from logbook import Logger
from catalyst.constants import LOG_LEVEL
log = Logger('ExchangeClock', level=LOG_LEVEL) log = Logger('ExchangeClock', level=LOG_LEVEL)
@@ -30,8 +31,7 @@ class SimpleClock(object):
This class is a drop-in replacement for This class is a drop-in replacement for
:class:`zipline.gens.sim_engine.MinuteSimulationClock`. :class:`zipline.gens.sim_engine.MinuteSimulationClock`.
This is a stripped down version because crypto exchanges run This is a stripped down version because crypto exchanges run around the clock.
around the clock.
The :param:`time_skew` parameter represents the time difference between The :param:`time_skew` parameter represents the time difference between
the Broker and the live trading machine's clock. the Broker and the live trading machine's clock.
catalyst/exchange/stats_utils.py
+221
View File
@@ -0,0 +1,221 @@
import numbers
import numpy as np
import pandas as pd
def trend_direction(series):
if series[-1] is np.nan or series[-1] is np.nan:
return None
if series[-1] > series[-2]:
return 'up'
else:
return 'down'
def crossover(source, target):
"""
The `x`-series is defined as having crossed over `y`-series if the value
of `x` is greater than the value of `y` and the value of `x` was less than
the value of `y` on the bar immediately preceding the current bar.
Parameters
----------
source: Series
target: Series
Returns
-------
bool
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
return False
if source[-1] >= target > source[-2]:
return True
else:
return False
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] > target[-1] and source[-2] < target[-2]:
return True
else:
return False
def crossunder(source, target):
"""
The `x`-series is defined as having crossed under `y`-series if the value
of `x` is less than the value of `y` and the value of `x` was greater than
the value of `y` on the bar immediately preceding the current bar.
Parameters
----------
source: Series
target: Series
Returns
-------
bool
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
return False
if source[-1] < target <= source[-2]:
return True
else:
return False
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] < target[-1] and source[-2] >= target[-2]:
return True
else:
return False
def vwap(df):
"""
Volume-weighted average price (VWAP) is a ratio generally used by
institutional investors and mutual funds to make buys and sells so as not
to disturb the market prices with large orders. It is the average share
price of a stock weighted against its trading volume within a particular
time frame, generally one day.
Read more: Volume Weighted Average Price - VWAP
https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
Parameters
----------
df: pd.DataFrame
Returns
-------
"""
if 'close' not in df.columns or 'volume' not in df.columns:
raise ValueError('price data must include `volume` and `close`')
vol_sum = np.nansum(df['volume'].values)
try:
ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
except ZeroDivisionError:
ret = np.nan
return ret
def get_pretty_stats(stats_df, recorded_cols=None, num_rows=10):
"""
Format and print the last few rows of a statistics DataFrame.
See the pyfolio project for the data structure.
Parameters
----------
stats_df: DataFrame
num_rows: int
Returns
-------
str
"""
stats_df.set_index('period_close', drop=True, inplace=True)
stats_df.dropna(axis=1, how='all', inplace=True)
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 3)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
columns = ['starting_cash', 'ending_cash', 'portfolio_value',
'pnl', 'long_exposure', 'short_exposure', 'orders',
'transactions', 'positions']
if recorded_cols is not None:
for column in recorded_cols:
columns.append(column)
def format_positions(positions):
parts = []
for position in positions:
msg = '{amount:.2f}{base} cost basis {cost_basis:.4f}{quote}'.format(
amount=position['amount'],
base=position['sid'].base_currency,
cost_basis=position['cost_basis'],
quote=position['sid'].quote_currency
)
parts.append(msg)
return ', '.join(parts)
formatters = {
'orders': lambda orders: len(orders),
'transactions': lambda transactions: len(transactions),
'returns': lambda returns: "{0:.4f}".format(returns),
'positions': format_positions
}
return stats_df.tail(num_rows).to_string(
columns=columns,
formatters=formatters
)
def df_to_string(df):
"""
Create a formatted str representation of the DataFrame.
Parameters
----------
df: DataFrame
Returns
-------
str
"""
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 8)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
return df.to_string()
def extract_transactions(perf):
"""
Compute indexes for buy and sell transactions
Parameters
----------
perf: DataFrame
The algo performance DataFrame.
Returns
-------
DataFrame
A DataFrame of transactions.
"""
trans_list = perf.transactions.values
all_trans = [t for sublist in trans_list for t in sublist]
all_trans.sort(key=lambda t: t['dt'])
transactions = pd.DataFrame(all_trans)
if not transactions.empty:
transactions.set_index('dt', inplace=True, drop=True)
return transactions
catalyst/exchange/utils/bundle_utils.py
-158
View File
@@ -1,158 +0,0 @@
import os
import tarfile
from datetime import datetime
import numpy as np
import pandas as pd
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
import os
import tarfile
from datetime import datetime
import numpy as np
import pandas as pd
from catalyst.data.bundles.core import download_without_progress
from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder
EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
API_URL = 'http://data.enigma.co/api/v1'
def get_bcolz_chunk(exchange_name, symbol, data_frequency, period):
"""
Download and extract a bcolz bundle.
Parameters
----------
exchange_name: str
symbol: str
data_frequency: str
period: str
Returns
-------
str
Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
"""
root = get_exchange_bundles_folder(exchange_name)
name = '{exchange}-{frequency}-{symbol}-{period}'.format(
exchange=exchange_name,
frequency=data_frequency,
symbol=symbol,
period=period
)
path = os.path.join(root, name)
if not os.path.isdir(path):
url = 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/' \
'exchange-{exchange}/{name}.tar.gz'.format(
exchange=exchange_name,
name=name)
bytes = download_without_progress(url)
with tarfile.open('r', fileobj=bytes) as tar:
tar.extractall(path)
return path
def get_df_from_arrays(arrays, periods):
"""
A DataFrame from the specified OHCLV arrays.
Parameters
----------
arrays: Object
periods: DateTimeIndex
Returns
-------
DataFrame
"""
ohlcv = dict()
for index, field in enumerate(
['open', 'high', 'low', 'close', 'volume']):
ohlcv[field] = arrays[index].flatten()
df = pd.DataFrame(
data=ohlcv,
index=periods
)
return df
def range_in_bundle(asset, start_dt, end_dt, reader):
"""
Evaluate whether price data of an asset is included has been ingested in
the exchange bundle for the given date range.
Parameters
----------
asset: TradingPair
start_dt: datetime
end_dt: datetime
reader: BcolzBarMinuteReader
Returns
-------
bool
"""
has_data = True
dates = [start_dt, end_dt]
while dates and has_data:
try:
dt = dates.pop(0)
close = reader.get_value(asset.sid, dt, 'close')
if np.isnan(close):
has_data = False
except Exception:
has_data = False
return has_data
def get_assets(exchange, include_symbols, exclude_symbols):
"""
Get assets from an exchange, including or excluding the specified
symbols.
Parameters
----------
exchange: Exchange
include_symbols: str
exclude_symbols: str
Returns
-------
list[TradingPair]
"""
if include_symbols is not None:
include_symbols_list = include_symbols.split(',')
return exchange.get_assets(include_symbols_list)
else:
all_assets = exchange.get_assets()
if exclude_symbols is not None:
exclude_symbols_list = exclude_symbols.split(',')
assets = []
for asset in all_assets:
if asset.symbol not in exclude_symbols_list:
assets.append(asset)
return assets
else:
return all_assets
catalyst/exchange/utils/datetime_utils.py
-362
View File
@@ -1,362 +0,0 @@
import calendar
import math
import re
from datetime import datetime, timedelta, date
import pandas as pd
import pytz
from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
InvalidHistoryFrequencyAlias
def get_date_from_ms(ms):
"""
The date from the number of miliseconds from the epoch.
Parameters
----------
ms: int
Returns
-------
datetime
"""
return datetime.fromtimestamp(ms / 1000.0)
def get_seconds_from_date(date):
"""
The number of seconds from the epoch.
Parameters
----------
date: datetime
Returns
-------
int
"""
epoch = datetime.utcfromtimestamp(0)
epoch = epoch.replace(tzinfo=pytz.UTC)
return int((date - epoch).total_seconds())
def get_delta(periods, data_frequency):
"""
Get a time delta based on the specified data frequency.
Parameters
----------
periods: int
data_frequency: str
Returns
-------
timedelta
"""
return timedelta(minutes=periods) \
if data_frequency == 'minute' else timedelta(days=periods)
def get_periods_range(freq, start_dt=None, end_dt=None, periods=None):
"""
Get a date range for the specified parameters.
Parameters
----------
start_dt: datetime
end_dt: datetime
freq: str
Returns
-------
DateTimeIndex
"""
if freq == 'minute':
freq = 'T'
elif freq == 'daily':
freq = 'D'
if start_dt is not None and end_dt is not None and periods is None:
return pd.date_range(start_dt, end_dt, freq=freq)
elif periods is not None and (start_dt is not None or end_dt is not None):
_, unit_periods, unit, _ = get_frequency(freq)
adj_periods = periods * unit_periods
# TODO: standardize time aliases to avoid any mapping
unit = 'd' if unit == 'D' else 'h' if unit == 'H' else 'm'
delta = pd.Timedelta(adj_periods, unit)
if start_dt is not None:
return pd.date_range(
start=start_dt,
end=start_dt + delta,
freq=freq,
closed='left',
)
else:
return pd.date_range(
start=end_dt - delta,
end=end_dt,
freq=freq,
)
else:
raise ValueError(
'Choose only two parameters between start_dt, end_dt '
'and periods.'
)
def get_periods(start_dt, end_dt, freq):
"""
The number of periods in the specified range.
Parameters
----------
start_dt: datetime
end_dt: datetime
freq: str
Returns
-------
int
"""
return len(get_periods_range(start_dt=start_dt, end_dt=end_dt, freq=freq))
def get_start_dt(end_dt, bar_count, data_frequency, include_first=True):
"""
The start date based on specified end date and data frequency.
Parameters
----------
end_dt: datetime
bar_count: int
data_frequency: str
include_first
Returns
-------
datetime
"""
periods = bar_count
if periods > 1:
delta = get_delta(periods, data_frequency)
start_dt = end_dt - delta
if not include_first:
start_dt += get_delta(1, data_frequency)
else:
start_dt = end_dt
return start_dt
def get_period_label(dt, data_frequency):
"""
The period label for the specified date and frequency.
Parameters
----------
dt: datetime
data_frequency: str
Returns
-------
str
"""
if data_frequency == 'minute':
return '{}-{:02d}'.format(dt.year, dt.month)
else:
return '{}'.format(dt.year)
def get_month_start_end(dt, first_day=None, last_day=None):
"""
The first and last day of the month for the specified date.
Parameters
----------
dt: datetime
first_day: datetime
last_day: datetime
Returns
-------
datetime, datetime
"""
month_range = calendar.monthrange(dt.year, dt.month)
if first_day:
month_start = first_day
else:
month_start = pd.to_datetime(datetime(
dt.year, dt.month, 1, 0, 0, 0, 0
), utc=True)
if last_day:
month_end = last_day
else:
month_end = pd.to_datetime(datetime(
dt.year, dt.month, month_range[1], 23, 59, 0, 0
), utc=True)
if month_end > pd.Timestamp.utcnow():
month_end = pd.Timestamp.utcnow().floor('1D')
return month_start, month_end
def get_year_start_end(dt, first_day=None, last_day=None):
"""
The first and last day of the year for the specified date.
Parameters
----------
dt: datetime
first_day: datetime
last_day: datetime
Returns
-------
datetime, datetime
"""
year_start = first_day if first_day \
else pd.to_datetime(date(dt.year, 1, 1), utc=True)
year_end = last_day if last_day \
else pd.to_datetime(date(dt.year, 12, 31), utc=True)
if year_end > pd.Timestamp.utcnow():
year_end = pd.Timestamp.utcnow().floor('1D')
return year_start, year_end
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']):
"""
Takes an arbitrary candle size (e.g. 15T) and converts to the lowest
common denominator supported by the data bundles (e.g. 1T). The data
bundles only support 1T and 1D frequencies. If another frequency
is requested, Catalyst must request the underlying data and resample.
Notes
-----
We're trying to use Pandas convention for frequency aliases.
Parameters
----------
freq: str
data_frequency: str
Returns
-------
str, int, str, str
"""
if data_frequency is None:
data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
if freq == 'minute':
unit = 'T'
candle_size = 1
elif freq == 'daily':
unit = 'D'
candle_size = 1
else:
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# TODO: some exchanges support H and W frequencies but not bundles
# Find a way to pass-through these parameters to exchanges
# but resample from minute or daily in backtest mode
# see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
# Pandas offet aliases (used by Catalyst) and the CCXT timeframes
if unit.lower() == 'd':
unit = 'D'
alias = '{}D'.format(candle_size)
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm' or unit == 'T':
unit = 'T'
alias = '{}T'.format(candle_size)
data_frequency = 'minute'
elif unit.lower() == 'h':
data_frequency = 'minute'
if 'H' in supported_freqs:
unit = 'H'
alias = '{}H'.format(candle_size)
else:
candle_size = candle_size * 60
alias = '{}T'.format(candle_size)
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
return alias, candle_size, unit, data_frequency
def from_ms_timestamp(ms):
return pd.to_datetime(ms, unit='ms', utc=True)
def get_epoch():
return pd.to_datetime('1970-1-1', utc=True)
def get_candles_number_from_minutes(unit, candle_size, minutes):
"""
Get the number of bars needed for the given time interval
in minutes.
Notes
-----
Supports only "T", "D" and "H" units
Parameters
----------
unit: str
candle_size : int
minutes: int
Returns
-------
int
"""
if unit == "T":
res = (float(minutes) / candle_size)
elif unit == "H":
res = (minutes / 60.0) / candle_size
else: # unit == "D"
res = (minutes / 1440.0) / candle_size
return int(math.ceil(res))
catalyst/exchange/utils/factory.py
-99
View File
@@ -1,99 +0,0 @@
import os
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.ccxt.ccxt_exchange import CCXT
from catalyst.exchange.exchange import Exchange
from catalyst.exchange.exchange_errors import ExchangeAuthEmpty
from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \
get_exchange_folder, is_blacklist
from logbook import Logger
log = Logger('factory', level=LOG_LEVEL)
exchange_cache = dict()
def get_exchange(exchange_name, base_currency=None, must_authenticate=False,
skip_init=False, auth_alias=None):
key = (exchange_name, base_currency)
if key in exchange_cache:
return exchange_cache[key]
exchange_auth = get_exchange_auth(exchange_name, alias=auth_alias)
has_auth = (exchange_auth['key'] != '' and exchange_auth['secret'] != '')
if must_authenticate and not has_auth:
raise ExchangeAuthEmpty(
exchange=exchange_name.title(),
filename=os.path.join(
get_exchange_folder(exchange_name), 'auth.json'
)
)
exchange = CCXT(
exchange_name=exchange_name,
key=exchange_auth['key'],
secret=exchange_auth['secret'],
password=exchange_auth['password'] if 'password'
in exchange_auth.keys() else '',
base_currency=base_currency,
)
exchange_cache[key] = exchange
if not skip_init:
exchange.init()
return exchange
def get_exchanges(exchange_names):
exchanges = dict()
for exchange_name in exchange_names:
exchanges[exchange_name] = get_exchange(exchange_name)
return exchanges
def find_exchanges(features=None, skip_blacklist=True, is_authenticated=False,
base_currency=None):
"""
Find exchanges filtered by a list of feature.
Parameters
----------
features: str
The list of features.
skip_blacklist: bool
is_authenticated: bool
base_currency: bool
Returns
-------
list[Exchange]
"""
exchange_names = CCXT.find_exchanges(features, is_authenticated)
exchanges = []
for exchange_name in exchange_names:
if skip_blacklist and is_blacklist(exchange_name):
continue
exchange = get_exchange(
exchange_name=exchange_name,
skip_init=True,
base_currency=base_currency,
)
if features is not None:
if 'dailyBundle' in features \
and not exchange.has_bundle('daily'):
continue
elif 'minuteBundle' in features \
and not exchange.has_bundle('minute'):
continue
exchanges.append(exchange)
return exchanges
catalyst/exchange/utils/live_chart_utils.py
-131
View File
@@ -1,131 +0,0 @@
import matplotlib.dates as mdates
import pandas as pd
from catalyst.exchange.exchange_errors import \
MismatchingBaseCurrenciesExchanges
fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
def format_ax(ax):
"""
Trying to assign reasonable parameters to the time axis.
Parameters
----------
ax:
"""
# TODO: room for improvement
ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
ax.xaxis.set_major_formatter(fmt)
locator = mdates.HourLocator(interval=4)
locator.MAXTICKS = 5000
ax.xaxis.set_minor_locator(locator)
datemin = pd.Timestamp.utcnow()
ax.set_xlim(datemin)
ax.grid(True)
def set_legend(ax):
"""
Set legend on the chart.
Parameters
----------
ax
"""
ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
def draw_pnl(ax, df):
"""
Draw p&l line on the chart.
"""
ax.clear()
ax.set_title('Performance')
index = df.index.unique()
dt = index.get_level_values(level=0)
pnl = index.get_level_values(level=4)
ax.plot(
dt, pnl, '-',
color='green',
linewidth=1.0,
label='Performance'
)
def perc(val):
return '{:2f}'.format(val)
ax.format_ydata = perc
set_legend(ax)
format_ax(ax)
def draw_custom_signals(ax, df):
"""
Draw custom signals on the chart.
"""
colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
ax.clear()
ax.set_title('Custom Signals')
for index, column in enumerate(df.columns.values.tolist()):
ax.plot(df.index, df[column], '-',
color=colors[index],
linewidth=1.0,
label=column
)
set_legend(ax)
format_ax(ax)
def draw_exposure(ax, df, context):
"""
Draw exposure line on the chart.
"""
# TODO: list exchanges in graph
base_currency = None
positions = []
for exchange_name in context.exchanges:
exchange = context.exchanges[exchange_name]
if not base_currency:
base_currency = exchange.base_currency
elif base_currency != exchange.base_currency:
raise MismatchingBaseCurrenciesExchanges(
base_currency=base_currency,
exchange_name=exchange.name,
exchange_currency=exchange.base_currency
)
positions += exchange.portfolio.positions
ax.clear()
ax.set_title('Exposure')
ax.plot(df.index, df['base_currency'], '-',
color='green',
linewidth=1.0,
label='Base Currency: {}'.format(base_currency.upper())
)
symbols = []
for position in positions:
symbols.append(position.symbol)
ax.plot(df.index, df['long_exposure'], '-',
color='blue',
linewidth=1.0,
label='Long Exposure: {}'.format(', '.join(symbols).upper()))
set_legend(ax)
format_ax(ax)
catalyst/exchange/utils/serialization_utils.py
-69
View File
@@ -1,69 +0,0 @@
import json
import re
from json import JSONEncoder
import pandas as pd
from catalyst.constants import DATE_TIME_FORMAT
from six import string_types
class ExchangeJSONEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, pd.Timestamp):
return obj.strftime(DATE_TIME_FORMAT)
# Let the base class default method raise the TypeError
return JSONEncoder.default(self, obj)
class ExchangeJSONDecoder(json.JSONDecoder):
def __init__(self, *args, **kwargs):
json.JSONDecoder.__init__(
self, object_hook=self.object_hook, *args, **kwargs
)
def recursive_iter(self, obj):
if isinstance(obj, dict):
for key, value in obj.items():
match = isinstance(value, string_types) and re.search(
r'(\d{4}-\d{2}-\d{2}).*', value
)
if match:
try:
obj[key] = pd.to_datetime(value, utc=True)
except ValueError:
pass
elif any(isinstance(obj, t) for t in (list, tuple)):
for item in obj:
self.recursive_iter(item)
def object_hook(self, obj):
self.recursive_iter(obj)
return obj
def portfolio_to_dict(portfolio):
positions = []
for asset in portfolio.positions:
p = portfolio.positions[asset] # Type: Position
position = dict(
symbol=asset.symbol,
exchange=asset.exchange,
amount=p.amount,
cost_basis=p.cost_basis,
last_sale_price=p.last_sale_price,
last_sale_date=p.last_sale_date,
)
positions.append(position)
portfolio_dict = vars(portfolio)
portfolio_dict['positions'] = positions
return portfolio_dict
def portfolio_from_dict(self, portfolio_data):
from catalyst.protocol import Portfolio
return Portfolio()
catalyst/exchange/utils/stats_utils.py
-486
View File
@@ -1,486 +0,0 @@
import copy
import csv
import json
import numbers
import os
import time
import numpy as np
import pandas as pd
from catalyst.assets._assets import TradingPair
from catalyst.exchange.utils.exchange_utils import get_algo_folder
from catalyst.utils.paths import data_root, ensure_directory
from operator import itemgetter
s3_conn = []
mailgun = []
def trend_direction(series):
if series[-1] is np.nan or series[-1] is np.nan:
return None
if series[-1] > series[-2]:
return 'up'
else:
return 'down'
def crossover(source, target):
"""
The `x`-series is defined as having crossed over `y`-series if the value
of `x` is greater than the value of `y` and the value of `x` was less than
the value of `y` on the bar immediately preceding the current bar.
Parameters
----------
source: Series
target: Series
Returns
-------
bool
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
return False
if source[-1] >= target > source[-2]:
return True
else:
return False
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] > target[-1] and source[-2] < target[-2]:
return True
else:
return False
def crossunder(source, target):
"""
The `x`-series is defined as having crossed under `y`-series if the value
of `x` is less than the value of `y` and the value of `x` was greater than
the value of `y` on the bar immediately preceding the current bar.
Parameters
----------
source: Series
target: Series
Returns
-------
bool
"""
if isinstance(target, numbers.Number):
if source[-1] is np.nan or source[-2] is np.nan \
or target is np.nan:
return False
if source[-1] < target <= source[-2]:
return True
else:
return False
else:
if source[-1] is np.nan or source[-2] is np.nan \
or target[-1] is np.nan or target[-2] is np.nan:
return False
if source[-1] < target[-1] and source[-2] >= target[-2]:
return True
else:
return False
def vwap(df):
"""
Volume-weighted average price (VWAP) is a ratio generally used by
institutional investors and mutual funds to make buys and sells so as not
to disturb the market prices with large orders. It is the average share
price of a stock weighted against its trading volume within a particular
time frame, generally one day.
Read more: Volume Weighted Average Price - VWAP
https://www.investopedia.com/terms/v/vwap.asp#ixzz4xt922daE
Parameters
----------
df: pd.DataFrame
Returns
-------
"""
if 'close' not in df.columns or 'volume' not in df.columns:
raise ValueError('price data must include `volume` and `close`')
vol_sum = np.nansum(df['volume'].values)
try:
ret = np.nansum(df['close'].values * df['volume'].values) / vol_sum
except ZeroDivisionError:
ret = np.nan
return ret
def set_position_row(row, asset, asset_values=list()):
"""
Apply the position data as individual columns.
Parameters
----------
row: dict[str, Object]
asset: TradingPair
asset_values: list[str]
If a recorded_col contains a tuple which first value is an asset
matching a position, its value will be displayed with the
position and not in the index.
Returns
-------
"""
asset_cols = ['symbol']
row['symbol'] = asset.symbol
position = next((p for p in row['positions'] if p['sid'] == asset), None)
columns = ['amount', 'cost_basis', 'last_sale_price']
for column in columns:
if position is not None:
row[column] = position[column]
else:
row[column] = 0
asset_cols.append(column)
values = asset_values[asset] if asset in asset_values else list()
for column in values:
row[column] = values[column]
asset_cols.append(column)
return asset_cols
def prepare_stats(stats, recorded_cols=list()):
"""
Prepare the stats DataFrame for user-friendly output.
Parameters
----------
stats: list[Object]
recorded_cols: list[str]
Returns
-------
"""
asset_cols = list()
stats = copy.deepcopy(stats)
# Using a copy since we are adding rows inside the loop.
for row_index, row_data in enumerate(list(stats)):
assets = [p['sid'] for p in row_data['positions']]
asset_values = dict()
if recorded_cols is not None:
for column in recorded_cols[:]:
value = row_data[column]
if isinstance(value, pd.Series):
value = value.to_dict()
if type(value) is dict:
for asset in value:
if not isinstance(asset, TradingPair):
break
if asset not in assets:
assets.append(asset)
if asset not in asset_values:
asset_values[asset] = dict()
asset_values[asset][column] = value[asset]
if len(assets) == 1:
row = stats[row_index]
asset_cols = set_position_row(row, assets[0], asset_values)
elif len(assets) > 1:
for asset_index, asset in enumerate(assets):
if asset_index > 0:
row = copy.deepcopy(row_data)
stats.append(row)
else:
row = stats[row_index]
asset_cols = set_position_row(row, assets[asset_index],
asset_values)
df = pd.DataFrame(stats)
df['orders'] = df['orders'].apply(lambda orders: len(orders))
df['transactions'] = df['transactions'].apply(
lambda transactions: len(transactions)
)
index_cols = [
'period_close', 'starting_cash', 'ending_cash', 'portfolio_value',
'pnl', 'long_exposure', 'short_exposure', 'orders', 'transactions',
]
# Removing the asset specific entries
if recorded_cols is not None:
recorded_cols = [x for x in recorded_cols if x not in asset_cols]
for column in recorded_cols:
index_cols.append(column)
if asset_cols:
columns = asset_cols
df.set_index(index_cols, drop=True, inplace=True)
else:
columns = index_cols
columns.remove('period_close')
df.set_index('period_close', drop=False, inplace=True)
df.dropna(axis=1, how='all', inplace=True)
df.sort_index(axis=0, level=0, inplace=True)
return df, columns
def set_print_settings():
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 8)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
"""
Format and print the last few rows of a statistics DataFrame.
See the pyfolio project for the data structure.
Parameters
----------
stats: list[Object]
An array of statistics for the period.
num_rows: int
The number of rows to display on the screen.
Returns
-------
str
"""
if isinstance(stats, pd.DataFrame):
stats = list(stats.T.to_dict().values())
stats.sort(key=itemgetter('period_close'))
if len(stats) > num_rows:
display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
else:
display_stats = stats
df, columns = prepare_stats(
display_stats, recorded_cols=recorded_cols
)
set_print_settings()
return df.to_string(columns=columns)
def get_csv_stats(stats, recorded_cols=None):
"""
Create a CSV buffer from the stats DataFrame.
Parameters
----------
path: str
stats: list[Object]
recorded_cols: list[str]
Returns
-------
"""
df, columns = prepare_stats(stats, recorded_cols=recorded_cols)
return df.to_csv(
None,
columns=columns,
# encoding='utf-8',
quoting=csv.QUOTE_NONNUMERIC
).encode()
def stats_to_s3(uri, stats, algo_namespace, recorded_cols=None,
folder='catalyst/stats', bytes_to_write=None):
"""
Uploads the performance stats to a S3 bucket.
Parameters
----------
uri: str
stats: list[Object]
algo_namespace: str
recorded_cols: list[str]
folder: str
bytes_to_write: str
Option to reuse bytes instead of re-computing the csv
Returns
-------
"""
if not s3_conn:
import boto3
s3_conn.append(boto3.resource('s3'))
s3 = s3_conn[0]
if bytes_to_write is None:
bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
now = pd.Timestamp.utcnow()
timestr = now.strftime('%Y%m%d')
pid = os.getpid()
parts = uri.split('//')
path = '{folder}/{algo}/{time}-{algo}-{pid}.csv'.format(
folder=folder,
algo=algo_namespace,
time=timestr,
pid=pid,
)
obj = s3.Object(parts[1], path)
obj.put(Body=bytes_to_write)
def email_error(algo_name, dt, e, environ=None):
import requests
import traceback
if not mailgun:
root = data_root(environ)
filename = os.path.join(root, 'mailgun.json')
if not os.path.exists(filename):
raise ValueError(
'mailgun.json not found in the catalyst data folder'
)
with open(filename) as data_file:
mailgun.append(json.load(data_file))
mg = mailgun[0]
return requests.post(
mg['url'],
auth=("api", mg['api']),
data={
"from": mg['from'],
"to": mg['to'],
"subject": 'Error: {}'.format(algo_name),
"text": '{}\n\n{}\n{}'.format(
dt, e, traceback.format_exc()
)})
def stats_to_algo_folder(stats, algo_namespace,
folder_name, recorded_cols=None):
"""
Saves the performance stats to the algo local folder.
Parameters
----------
stats: list[Object]
algo_namespace: str
folder_name: str
recorded_cols: list[str]
Returns
-------
str
"""
bytes_to_write = get_csv_stats(stats, recorded_cols=recorded_cols)
timestr = time.strftime('%Y%m%d')
folder = get_algo_folder(algo_namespace)
stats_folder = os.path.join(folder, folder_name)
ensure_directory(stats_folder)
filename = os.path.join(stats_folder, '{}.csv'.format(timestr))
with open(filename, 'wb') as handle:
handle.write(bytes_to_write)
return bytes_to_write
def df_to_string(df):
"""
Create a formatted str representation of the DataFrame.
Parameters
----------
df: DataFrame
Returns
-------
str
"""
pd.set_option('display.expand_frame_repr', False)
pd.set_option('precision', 8)
pd.set_option('display.width', 1000)
pd.set_option('display.max_colwidth', 1000)
return df.to_string()
def extract_orders(perf):
order_list = perf.orders.values
all_orders = [t for sublist in order_list for t in sublist]
all_orders.sort(key=lambda o: o['dt'])
orders = pd.DataFrame(all_orders)
if not orders.empty:
orders.set_index('dt', inplace=True, drop=True)
return orders
def extract_transactions(perf):
"""
Compute indexes for buy and sell transactions
Parameters
----------
perf: DataFrame
The algo performance DataFrame.
Returns
-------
DataFrame
A DataFrame of transactions.
"""
trans_list = perf.transactions.values
all_trans = [t for sublist in trans_list for t in sublist]
all_trans.sort(key=lambda t: t['dt'])
transactions = pd.DataFrame(all_trans)
if not transactions.empty:
transactions.set_index('dt', inplace=True, drop=True)
return transactions
catalyst/exchange/utils/test_utils.py
-82
View File
@@ -1,82 +0,0 @@
import os
import random
import tempfile
from catalyst.assets._assets import TradingPair
from catalyst.exchange.utils.exchange_utils import get_exchange_folder
from catalyst.exchange.utils.factory import find_exchanges
from catalyst.utils.paths import ensure_directory
def handle_exchange_error(exchange, e):
try:
message = '{}: {}'.format(
e.__class__, e.message.decode('ascii', 'ignore')
)
except Exception:
message = 'unexpected error'
folder = get_exchange_folder(exchange.name)
filename = os.path.join(folder, 'blacklist.txt')
with open(filename, 'wt') as handle:
handle.write(message)
def select_random_exchanges(population=3, features=None,
is_authenticated=False, base_currency=None):
all_exchanges = find_exchanges(
features=features,
is_authenticated=is_authenticated,
base_currency=base_currency,
)
if population is not None:
if len(all_exchanges) < population:
population = len(all_exchanges)
exchanges = random.sample(all_exchanges, population)
else:
exchanges = all_exchanges
return exchanges
def select_random_assets(all_assets, population=3):
assets = random.sample(all_assets, population)
return assets
def output_df(df, assets, name=None):
"""
Outputs a price DataFrame to a temp folder.
Parameters
----------
df: pd.DataFrame
assets
name
Returns
-------
"""
if isinstance(assets, TradingPair):
asset_folder = '{}_{}'.format(assets.exchange, assets.symbol)
else:
asset_folder = ','.join(
['{}_{}'.format(a.exchange, a.symbol) for a in assets]
)
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', asset_folder
)
ensure_directory(folder)
if name is None:
name = 'output'
path = os.path.join(folder, '{}.csv'.format(name))
df.to_csv(path)
return path, folder
catalyst/exchange/validator.py
+142
View File
@@ -0,0 +1,142 @@
import os
import tempfile
import pandas as pd
import six
from catalyst.assets._assets import TradingPair, get_calendar
from logbook import Logger
from pandas.util.testing import assert_frame_equal
from catalyst.constants import LOG_LEVEL
from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest
from catalyst.exchange.factory import get_exchanges
from catalyst.utils.paths import ensure_directory
log = Logger('Validator', level=LOG_LEVEL)
def output_df(df, assets, name=None):
"""
Outputs a price DataFrame to a temp folder.
Parameters
----------
df: pd.DataFrame
assets
name
Returns
-------
"""
if isinstance(assets, TradingPair):
exchange_folder = assets.exchange
asset_folder = assets.symbol
else:
exchange_folder = ','.join([asset.exchange for asset in assets])
asset_folder = ','.join([asset.symbol for asset in assets])
folder = os.path.join(
tempfile.gettempdir(), 'catalyst', exchange_folder, asset_folder
)
ensure_directory(folder)
if name is None:
name = 'output'
path = os.path.join(folder, '{}.csv'.format(name))
df.to_csv(path)
return path
class Validator(object):
def __init__(self, data_portal):
self.data_portal = data_portal
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
sample_minutes):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
sample_minutes
Returns
-------
"""
freq = '{}T'.format(sample_minutes)
log.info('creating data sample from bundle')
df1 = self.data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field='close',
data_frequency='minute'
)
path = output_df(df1, assets, '{}_resampled'.format(freq))
log.info('saved resampled bundle candles: {}\n{}'.format(
path, df1.tail(10))
)
log.info('creating data sample from exchange api')
candles = exchange.get_candles(
end_dt=end_dt,
freq='{}T'.format(sample_minutes),
assets=assets,
bar_count=bar_count
)
series = dict()
for asset in assets:
series[asset] = pd.Series(
data=[candle['close'] for candle in candles[asset]],
index=[candle['last_traded'] for candle in candles[asset]]
)
df2 = pd.DataFrame(series)
path = output_df(df2, assets, '{}_api'.format(freq))
log.info('saved exchange api candles: {}\n{}'.format(
path, df2.tail(10))
)
try:
assert_frame_equal(df1, df2)
return True
except:
log.warn('differences found in dataframes')
return False
if __name__ == '__main__':
exchanges = get_exchanges(['poloniex'])
exchange = six.next(six.itervalues(exchanges))
assets = exchange.get_assets(symbols=['eth_btc'])
open_calendar = get_calendar('OPEN')
asset_finder = AssetFinderExchange()
data_portal = DataPortalExchangeBacktest(
exchanges=exchanges,
asset_finder=asset_finder,
trading_calendar=open_calendar,
first_trading_day=None # will set dynamically based on assets
)
validator = Validator(data_portal=data_portal)
validator.compare_bundle_with_exchange(
exchange=exchange,
assets=assets,
end_dt=pd.to_datetime('2017-11-10 1:00', utc=True),
bar_count=200,
sample_minutes=30
)
catalyst/finance/execution.py
+6 -1
View File
@@ -15,9 +15,14 @@
import abc import abc
from numpy import isfinite from sys import float_info
from six import with_metaclass from six import with_metaclass
import catalyst.utils.math_utils as zp_math
from numpy import isfinite
from catalyst.errors import BadOrderParameters from catalyst.errors import BadOrderParameters
catalyst/finance/performance/tracker.py
+2 -1
View File
@@ -62,6 +62,7 @@ from __future__ import division
import logbook import logbook
import pandas as pd import pandas as pd
from pandas.tseries.tools import normalize_date
from catalyst.finance.performance.period import PerformancePeriod from catalyst.finance.performance.period import PerformancePeriod
from catalyst.errors import NoFurtherDataError from catalyst.errors import NoFurtherDataError
@@ -343,7 +344,7 @@ class PerformanceTracker(object):
""" """
self.position_tracker.sync_last_sale_prices(dt, False, data_portal) self.position_tracker.sync_last_sale_prices(dt, False, data_portal)
self.update_performance() self.update_performance()
todays_date = dt.normalize() todays_date = normalize_date(dt)
account = self.get_account(False) account = self.get_account(False)
bench_returns = self.all_benchmark_returns.loc[todays_date:dt] bench_returns = self.all_benchmark_returns.loc[todays_date:dt]
catalyst/finance/risk/cumulative.py
+18 -40
View File
@@ -18,6 +18,7 @@ import logbook
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from pandas.tseries.tools import normalize_date
from six import iteritems from six import iteritems
@@ -26,15 +27,15 @@ from .risk import (
choose_treasury choose_treasury
) )
from catalyst.patches.stats import ( from empyrical import (
alpha_beta_aligned, alpha_beta_aligned,
annual_volatility, annual_volatility,
cum_returns,
downside_risk, downside_risk,
information_ratio, information_ratio,
max_drawdown, max_drawdown,
sharpe_ratio, sharpe_ratio,
sortino_ratio, sortino_ratio,
cum_returns,
) )
import warnings import warnings
from catalyst.constants import LOG_LEVEL from catalyst.constants import LOG_LEVEL
@@ -79,7 +80,7 @@ class RiskMetricsCumulative(object):
# on the first day. # on the first day.
self.day_before_start = self.start_session - self.sessions.freq self.day_before_start = self.start_session - self.sessions.freq
last_day = sim_params.end_session.normalize() last_day = normalize_date(sim_params.end_session)
if last_day not in self.sessions: if last_day not in self.sessions:
last_day = pd.tseries.index.DatetimeIndex( last_day = pd.tseries.index.DatetimeIndex(
[last_day] [last_day]
@@ -160,13 +161,9 @@ class RiskMetricsCumulative(object):
if len(self.algorithm_returns) == 1: if len(self.algorithm_returns) == 1:
self.algorithm_returns = np.append(0.0, self.algorithm_returns) self.algorithm_returns = np.append(0.0, self.algorithm_returns)
try: self.algorithm_cumulative_returns[dt_loc] = cum_returns(
self.algorithm_cumulative_returns[dt_loc] = cum_returns( self.algorithm_returns
self.algorithm_returns )[-1]
)[-1]
except Exception as e:
log.debug('unable to calculate cum returns: {}'.format(e))
self.algorithm_cumulative_returns[dt_loc] = np.nan
algo_cumulative_returns_to_date = \ algo_cumulative_returns_to_date = \
self.algorithm_cumulative_returns[:dt_loc + 1] self.algorithm_cumulative_returns[:dt_loc + 1]
@@ -199,11 +196,8 @@ class RiskMetricsCumulative(object):
self.benchmark_cumulative_returns[dt_loc] = cum_returns( self.benchmark_cumulative_returns[dt_loc] = cum_returns(
self.benchmark_returns self.benchmark_returns
)[-1] )[-1]
except Exception as e: except Exception:
log.debug( self.benchmark_cumulative_returns[dt_loc] = 0
'unable to calculate benchmark cum returns: {}'.format(e)
)
self.benchmark_cumulative_returns[dt_loc] = np.nan
benchmark_cumulative_returns_to_date = \ benchmark_cumulative_returns_to_date = \
self.benchmark_cumulative_returns[:dt_loc + 1] self.benchmark_cumulative_returns[:dt_loc + 1]
@@ -275,16 +269,9 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
self.sharpe[dt_loc] = sharpe_ratio( self.sharpe[dt_loc] = sharpe_ratio(
self.algorithm_returns, self.algorithm_returns,
) )
self.downside_risk[dt_loc] = downside_risk(
try: self.algorithm_returns
self.downside_risk[dt_loc] = downside_risk( )
self.algorithm_returns
)
except Exception as e:
log.debug(
'unable to calculate downside risk returns: {}'.format(e)
)
self.downside_risk[dt_loc] = np.nan
try: try:
risk = self.downside_risk[dt_loc] risk = self.downside_risk[dt_loc]
@@ -292,26 +279,17 @@ algorithm_returns ({algo_count}) in range {start} : {end} on {dt}"
self.algorithm_returns, self.algorithm_returns,
_downside_risk=risk _downside_risk=risk
) )
except Exception as e: except Exception:
log.debug( # TODO: what causes it to error out?
'unable to calculate benchmark cum returns: {}'.format(e) self.sortino[dt_loc] = 0
)
self.sortino[dt_loc] = np.nan
self.information[dt_loc] = information_ratio( self.information[dt_loc] = information_ratio(
self.algorithm_returns, self.algorithm_returns,
self.benchmark_returns, self.benchmark_returns,
) )
try: self.max_drawdown = max_drawdown(
self.max_drawdown = max_drawdown( self.algorithm_returns
self.algorithm_returns )
)
except Exception as e:
log.debug(
'unable to calculate max drawdown: {}'.format(e)
)
self.max_drawdown = np.nan
self.max_drawdowns[dt_loc] = self.max_drawdown self.max_drawdowns[dt_loc] = self.max_drawdown
self.max_leverage = self.calculate_max_leverage() self.max_leverage = self.calculate_max_leverage()
self.max_leverages[dt_loc] = self.max_leverage self.max_leverages[dt_loc] = self.max_leverage
catalyst/finance/risk/period.py
+4 -6
View File
@@ -29,15 +29,13 @@ from .risk import check_entry
from empyrical import ( from empyrical import (
alpha_beta_aligned, alpha_beta_aligned,
annual_volatility, annual_volatility,
cum_returns,
downside_risk, downside_risk,
information_ratio, information_ratio,
max_drawdown,
sharpe_ratio, sharpe_ratio,
sortino_ratio sortino_ratio
) )
from catalyst.patches.stats import (
max_drawdown,
cum_returns,
)
from catalyst.constants import LOG_LEVEL from catalyst.constants import LOG_LEVEL
@@ -156,8 +154,8 @@ class RiskMetricsPeriod(object):
self.algorithm_returns.values, self.algorithm_returns.values,
self.benchmark_returns.values, self.benchmark_returns.values,
) )
self.excess_return = self.algorithm_period_returns \ self.excess_return = self.algorithm_period_returns - \
- self.treasury_period_return self.treasury_period_return
self.max_drawdown = max_drawdown(self.algorithm_returns.values) self.max_drawdown = max_drawdown(self.algorithm_returns.values)
self.max_leverage = self.calculate_max_leverage() self.max_leverage = self.calculate_max_leverage()
catalyst/finance/risk/risk.py
+1 -2
View File
@@ -160,8 +160,7 @@ def choose_treasury(select_treasury, treasury_curves, start_session,
) )
break break
# Supress warning for 'OPEN' calendar if search_day and trading_calendar.name != 'OPEN': # Supress warning for 'OPEN' calendar
if search_day and trading_calendar.name != 'OPEN':
if (search_dist is None or search_dist > 1) and \ if (search_dist is None or search_dist > 1) and \
search_days[0] <= end_session <= search_days[-1]: search_days[0] <= end_session <= search_days[-1]:
message = "No rate within 1 trading day of end date = \ message = "No rate within 1 trading day of end date = \
catalyst/finance/slippage.py
+1
View File
@@ -41,6 +41,7 @@ DEFAULT_EQUITY_VOLUME_SLIPPAGE_BAR_LIMIT = 0.025
DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05 DEFAULT_FUTURE_VOLUME_SLIPPAGE_BAR_LIMIT = 0.05
class LiquidityExceeded(Exception): class LiquidityExceeded(Exception):
pass pass
catalyst/finance/trading.py
+8 -20
View File
@@ -16,6 +16,7 @@ from functools import partial
import logbook import logbook
import pandas as pd import pandas as pd
from pandas.tslib import normalize_date
from six import string_types from six import string_types
from sqlalchemy import create_engine from sqlalchemy import create_engine
@@ -94,24 +95,11 @@ class TradingEnvironment(object):
if not trading_calendar: if not trading_calendar:
trading_calendar = get_calendar("NYSE") trading_calendar = get_calendar("NYSE")
# todo: uncomment and add a well defined benchmark self.benchmark_returns, self.treasury_curves = load(
# self.benchmark_returns, self.treasury_curves = load( trading_calendar.day,
# trading_calendar.day, trading_calendar.schedule.index,
# trading_calendar.schedule.index, self.bm_symbol,
# self.bm_symbol, )
# exchange=exchange,
# )
start_data = get_calendar('OPEN').first_trading_session
end_data = pd.Timestamp.utcnow()
treasure_cols = ['1month', '3month', '6month', '1year', '2year',
'3year', '5year', '7year', '10year', '20year', '30year']
self.benchmark_returns = pd.DataFrame(data=0.001,
index=pd.date_range(start_data, end_data),
columns=['close'])
self.treasury_curves = pd.DataFrame(data=0.001,
index=pd.date_range(start_data, end_data),
columns=treasure_cols)
self.exchange_tz = exchange_tz self.exchange_tz = exchange_tz
@@ -163,8 +151,8 @@ class SimulationParameters(object):
# chop off any minutes or hours on the given start and end dates, # chop off any minutes or hours on the given start and end dates,
# as we only support session labels here (and we represent session # as we only support session labels here (and we represent session
# labels as midnight UTC). # labels as midnight UTC).
self._start_session = start_session.normalize() self._start_session = normalize_date(start_session)
self._end_session = end_session.normalize() self._end_session = normalize_date(end_session)
self._capital_base = capital_base self._capital_base = capital_base
self._emission_rate = emission_rate self._emission_rate = emission_rate
catalyst/gens/tradesimulation.py
+2 -1
View File
@@ -14,6 +14,7 @@
# limitations under the License. # limitations under the License.
from contextlib2 import ExitStack from contextlib2 import ExitStack
from logbook import Logger, Processor from logbook import Logger, Processor
from pandas.tslib import normalize_date
from catalyst.protocol import BarData from catalyst.protocol import BarData
from catalyst.utils.api_support import ZiplineAPI from catalyst.utils.api_support import ZiplineAPI
from six import viewkeys from six import viewkeys
@@ -228,7 +229,7 @@ class AlgorithmSimulator(object):
elif action == SESSION_END: elif action == SESSION_END:
# End of the session. # End of the session.
if emission_rate == 'daily': if emission_rate == 'daily':
handle_benchmark(dt).normalize() handle_benchmark(normalize_date(dt))
execute_order_cancellation_policy() execute_order_cancellation_policy()
yield self._get_daily_message(dt, algo, algo.perf_tracker) yield self._get_daily_message(dt, algo, algo.perf_tracker)
catalyst/marketplace/contract_enigma_abi.json
-302
View File
@@ -1,302 +0,0 @@
[
{
"constant": true,
"inputs": [],
"name": "name",
"outputs": [
{
"name": "",
"type": "string"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "approve",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "totalSupply",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_from",
"type": "address"
},
{
"name": "_to",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "transferFrom",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "INITIAL_SUPPLY",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "decimals",
"outputs": [
{
"name": "",
"type": "uint8"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_subtractedValue",
"type": "uint256"
}
],
"name": "decreaseApproval",
"outputs": [
{
"name": "success",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": false,
"inputs": [],
"name": "getAfterApproveTest",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [
{
"name": "_owner",
"type": "address"
}
],
"name": "balanceOf",
"outputs": [
{
"name": "balance",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": true,
"inputs": [],
"name": "symbol",
"outputs": [
{
"name": "",
"type": "string"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_to",
"type": "address"
},
{
"name": "_value",
"type": "uint256"
}
],
"name": "transfer",
"outputs": [
{
"name": "",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": false,
"inputs": [
{
"name": "_spender",
"type": "address"
},
{
"name": "_addedValue",
"type": "uint256"
}
],
"name": "increaseApproval",
"outputs": [
{
"name": "success",
"type": "bool"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "function"
},
{
"constant": true,
"inputs": [
{
"name": "_owner",
"type": "address"
},
{
"name": "_spender",
"type": "address"
}
],
"name": "allowance",
"outputs": [
{
"name": "",
"type": "uint256"
}
],
"payable": false,
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{
"name": "testValue",
"type": "address"
}
],
"payable": false,
"stateMutability": "nonpayable",
"type": "constructor"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"name": "owner",
"type": "address"
},
{
"indexed": true,
"name": "spender",
"type": "address"
},
{
"indexed": false,
"name": "value",
"type": "uint256"
}
],
"name": "Approval",
"type": "event"
},
{
"anonymous": false,
"inputs": [
{
"indexed": true,
"name": "from",
"type": "address"
},
{
"indexed": true,
"name": "to",
"type": "address"
},
{
"indexed": false,
"name": "value",
"type": "uint256"
}
],
"name": "Transfer",
"type": "event"
}
]
catalyst/marketplace/contract_enigma_address.txt
-1
View File
@@ -1 +0,0 @@
0xf0ee6b27b759c9893ce4f094b49ad28fd15a23e4
catalyst/marketplace/contract_marketplace_abi.json
-1
View File
File diff suppressed because one or more lines are too long
catalyst/marketplace/contract_marketplace_address.txt
-1
View File
@@ -1 +0,0 @@
0xa64927358a82254be92eb1f1cb01de68d1787004
catalyst/marketplace/marketplace.py
-814
View File
@@ -1,814 +0,0 @@
from __future__ import print_function
import glob
import json
import os
import re
import shutil
import sys
import time
import webbrowser
import bcolz
import logbook
import pandas as pd
import requests
from requests_toolbelt import MultipartDecoder
from requests_toolbelt.multipart.decoder import \
NonMultipartContentTypeException
from catalyst.constants import (
LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
from catalyst.exchange.utils.stats_utils import set_print_settings
from catalyst.marketplace.marketplace_errors import (
MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles, MarketplaceRequiresPython3)
from catalyst.marketplace.utils.auth_utils import get_key_secret, \
get_signed_headers
from catalyst.marketplace.utils.bundle_utils import merge_bundles
from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
to_grains
from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
get_data_source_folder, get_marketplace_folder, \
get_user_pubaddr, get_temp_bundles_folder, extract_bundle
from catalyst.utils.paths import ensure_directory
if sys.version_info.major < 3:
import urllib
else:
import urllib.request as urllib
log = logbook.Logger('Marketplace', level=LOG_LEVEL)
class Marketplace:
def __init__(self):
global Web3
try:
from web3 import Web3, HTTPProvider
except ImportError:
raise MarketplaceRequiresPython3()
self.addresses = get_user_pubaddr()
if self.addresses[0]['pubAddr'] == '':
raise MarketplacePubAddressEmpty(
filename=os.path.join(
get_marketplace_folder(), 'addresses.json')
)
self.default_account = self.addresses[0]['pubAddr']
self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE))
contract_url = urllib.urlopen(MARKETPLACE_CONTRACT)
self.mkt_contract_address = Web3.toChecksumAddress(
contract_url.readline().decode(
contract_url.info().get_content_charset()).strip())
abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
abi_url = abi_url.read().decode(
abi_url.info().get_content_charset())
abi = json.loads(abi_url)
self.mkt_contract = self.web3.eth.contract(
self.mkt_contract_address,
abi=abi,
)
contract_url = urllib.urlopen(ENIGMA_CONTRACT)
self.eng_contract_address = Web3.toChecksumAddress(
contract_url.readline().decode(
contract_url.info().get_content_charset()).strip())
abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
abi_url = abi_url.read().decode(
abi_url.info().get_content_charset())
abi = json.loads(abi_url)
self.eng_contract = self.web3.eth.contract(
self.eng_contract_address,
abi=abi,
)
# def get_data_sources_map(self):
# return [
# dict(
# name='Marketcap',
# desc='The marketcap value in USD.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily'],
# ),
# dict(
# name='GitHub',
# desc='The rate of development activity on GitHub.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily', 'hour'],
# ),
# dict(
# name='Influencers',
# desc='Tweets & related sentiments by selected influencers.',
# start_date=pd.to_datetime('2017-01-01'),
# end_date=pd.to_datetime('2018-01-15'),
# data_frequencies=['daily', 'hour', 'minute'],
# ),
# ]
def to_text(self, hex):
return Web3.toText(hex).rstrip('\0')
def choose_pubaddr(self):
if len(self.addresses) == 1:
address = self.addresses[0]['pubAddr']
address_i = 0
print('Using {} for this transaction.'.format(address))
else:
while True:
for i in range(0, len(self.addresses)):
print('{}\t{}\t{}\t{}'.format(
i,
self.addresses[i]['pubAddr'],
self.addresses[i]['wallet'].ljust(10),
self.addresses[i]['desc'])
)
address_i = int(input('Choose your address associated with '
'this transaction: [default: 0] ') or 0)
if not (0 <= address_i < len(self.addresses)):
print('Please choose a number between 0 and {}\n'.format(
len(self.addresses) - 1))
else:
address = Web3.toChecksumAddress(
self.addresses[address_i]['pubAddr'])
break
return address, address_i
def sign_transaction(self, tx):
url = 'https://www.mycrypto.com/#offline-transaction'
print('\nVisit {url} and enter the following parameters:\n\n'
'From Address:\t\t{_from}\n'
'\n\tClick the "Generate Information" button\n\n'
'To Address:\t\t{to}\n'
'Value / Amount to Send:\t{value}\n'
'Gas Limit:\t\t{gas}\n'
'Gas Price:\t\t[Accept the default value]\n'
'Nonce:\t\t\t{nonce}\n'
'Data:\t\t\t{data}\n'.format(
url=url,
_from=tx['from'],
to=tx['to'],
value=tx['value'],
gas=tx['gas'],
nonce=tx['nonce'],
data=tx['data'], )
)
webbrowser.open_new(url)
signed_tx = input('Copy and Paste the "Signed Transaction" '
'field here:\n')
if signed_tx.startswith('0x'):
signed_tx = signed_tx[2:]
return signed_tx
def check_transaction(self, tx_hash):
if 'ropsten' in ETH_REMOTE_NODE:
etherscan = 'https://ropsten.etherscan.io/tx/'
elif 'rinkeby' in ETH_REMOTE_NODE:
etherscan = 'https://rinkeby.etherscan.io/tx/'
else:
etherscan = 'https://etherscan.io/tx/'
etherscan = '{}{}'.format(etherscan, tx_hash)
print('\nYou can check the outcome of your transaction here:\n'
'{}\n\n'.format(etherscan))
def _list(self):
data_sources = self.mkt_contract.functions.getAllProviders().call()
data = []
for index, data_source in enumerate(data_sources):
if index > 0:
if 'test' not in Web3.toText(data_source).lower():
data.append(
dict(
dataset=self.to_text(data_source)
)
)
return pd.DataFrame(data)
def list(self):
df = self._list()
set_print_settings()
if df.empty:
print('There are no datasets available yet.')
else:
print(df)
def subscribe(self, dataset=None):
if dataset is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'subscribe to [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
dataset = df_sets.iloc[dataset_num]['dataset']
break
dataset = dataset.lower()
address = self.choose_pubaddr()[0]
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(dataset))
return
grains = provider_info[1]
price = from_grains(grains)
subscribed = self.mkt_contract.functions.checkAddressSubscription(
address, Web3.toHex(dataset)
).call()
if subscribed[5]:
print(
'\nYou are already subscribed to the "{}" dataset.\n'
'Your subscription started on {} UTC, and is valid until '
'{} UTC.'.format(
dataset,
pd.to_datetime(subscribed[3], unit='s', utc=True),
pd.to_datetime(subscribed[4], unit='s', utc=True)
)
)
return
print('\nThe price for a monthly subscription to this dataset is'
' {} ENG'.format(price))
print(
'Checking that the ENG balance in {} is greater than {} '
'ENG... '.format(address, price), end=''
)
wallet_address = address[2:]
balance = self.web3.eth.call({
'from': address,
'to': self.eng_contract_address,
'data': '0x70a08231000000000000000000000000{}'.format(
wallet_address
)
})
try:
balance = Web3.toInt(balance) # web3 >= 4.0.0b7
except TypeError:
balance = Web3.toInt(hexstr=balance) # web3 <= 4.0.0b6
if balance > grains:
print('OK.')
else:
print('FAIL.\n\nAddress {} balance is {} ENG,\nwhich is lower '
'than the price of the dataset that you are trying to\n'
'buy: {} ENG. Get enough ENG to cover the costs of the '
'monthly\nsubscription for what you are trying to buy, '
'and try again.'.format(
address, from_grains(balance), price))
return
while True:
agree_pay = input('Please confirm that you agree to pay {} ENG '
'for a monthly subscription to the dataset "{}" '
'starting today. [default: Y] '.format(
price, dataset)) or 'y'
if agree_pay.lower() not in ('y', 'n'):
print("Please answer Y or N.")
else:
if agree_pay.lower() == 'y':
break
else:
return
print('Ready to subscribe to dataset {}.\n'.format(dataset))
print('In order to execute the subscription, you will need to sign '
'two different transactions:\n'
'1. First transaction is to authorize the Marketplace contract '
'to spend {} ENG on your behalf.\n'
'2. Second transaction is the actual subscription for the '
'desired dataset'.format(price))
tx = self.eng_contract.functions.approve(
self.mkt_contract_address,
grains,
).buildTransaction(
{'from': address,
'nonce': self.web3.eth.getTransactionCount(address)}
)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
)
print(
'\nThis is the TxHash for this transaction: {}'.format(tx_hash)
)
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the first transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nFirst transaction successful!\n'
'Now processing second transaction.')
tx = self.mkt_contract.functions.subscribe(
Web3.toHex(dataset),
).buildTransaction({
'from': address,
'nonce': self.web3.eth.getTransactionCount(address)})
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(bin_hex(
self.web3.eth.sendRawTransaction(signed_tx)))
print('\nThis is the TxHash for this transaction: '
'{}'.format(tx_hash))
except Exception as e:
print('Unable to subscribe to data source: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the second transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nSecond transaction successful!\n'
'You have successfully subscribed to dataset {} with'
'address {}.\n'
'You can now ingest this dataset anytime during the '
'next month by running the following command:\n'
'catalyst marketplace ingest --dataset={}'.format(
dataset, address, dataset))
def process_temp_bundle(self, ds_name, path):
"""
Merge the temp bundle into the main bundle for the specified
data source.
Parameters
----------
ds_name
path
Returns
-------
"""
tmp_bundle = extract_bundle(path)
bundle_folder = get_data_source_folder(ds_name)
ensure_directory(bundle_folder)
if os.listdir(bundle_folder):
zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
merge_bundles(zsource, ztarget)
else:
shutil.rmtree(bundle_folder, ignore_errors=True)
os.rename(tmp_bundle, bundle_folder)
def ingest(self, ds_name=None, start=None, end=None, force_download=False):
if ds_name is None:
df_sets = self._list()
if df_sets.empty:
print('There are no datasets available yet.')
return
set_print_settings()
while True:
print(df_sets)
dataset_num = input('Choose the dataset you want to '
'ingest [0..{}]: '.format(
df_sets.size - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
if dataset_num not in range(0, df_sets.size):
print('Enter a number between 0 and {}'.format(
df_sets.size - 1))
else:
ds_name = df_sets.iloc[dataset_num]['dataset']
break
# ds_name = ds_name.lower()
# TODO: catch error conditions
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(ds_name)
).call()
if not provider_info[4]:
print('The requested "{}" dataset is not registered in '
'the Data Marketplace.'.format(ds_name))
return
address, address_i = self.choose_pubaddr()
fns = self.mkt_contract.functions
check_sub = fns.checkAddressSubscription(
address, Web3.toHex(ds_name)
).call()
if check_sub[0] != address or self.to_text(check_sub[1]) != ds_name:
print('You are not subscribed to dataset "{}" with address {}. '
'Plese subscribe first.'.format(ds_name, address))
return
if not check_sub[5]:
print('Your subscription to dataset "{}" expired on {} UTC.'
'Please renew your subscription by running:\n'
'catalyst marketplace subscribe --dataset={}'.format(
ds_name,
pd.to_datetime(check_sub[4], unit='s', utc=True),
ds_name)
)
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
secret = self.addresses[address_i]['secret']
else:
key, secret = get_key_secret(address,
self.addresses[address_i]['wallet'])
headers = get_signed_headers(ds_name, key, secret)
log.info('Starting download of dataset for ingestion...')
r = requests.post(
'{}/marketplace/ingest'.format(AUTH_SERVER),
headers=headers,
stream=True,
)
if r.status_code == 200:
log.info('Dataset downloaded successfully. Processing dataset...')
target_path = get_temp_bundles_folder()
try:
decoder = MultipartDecoder.from_response(r)
# with maybe_show_progress(
# iter(decoder.parts),
# True,
# label='Processing files') as part:
counter = 1
for part in decoder.parts:
log.info("Processing file {} of {}".format(
counter, len(decoder.parts)))
h = part.headers[b'Content-Disposition'].decode('utf-8')
# Extracting the filename from the header
name = re.search(r'filename="(.*)"', h).group(1)
filename = os.path.join(target_path, name)
with open(filename, 'wb') as f:
# for chunk in part.content.iter_content(
# chunk_size=1024):
# if chunk: # filter out keep-alive new chunks
# f.write(chunk)
f.write(part.content)
self.process_temp_bundle(ds_name, filename)
counter += 1
except NonMultipartContentTypeException:
response = r.json()
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=response,
)
else:
raise MarketplaceHTTPRequest(
request='ingest dataset',
error=r.status_code,
)
log.info('{} ingested successfully'.format(ds_name))
def get_dataset(self, ds_name, start=None, end=None):
ds_name = ds_name.lower()
# TODO: filter ctable by start and end date
bundle_folder = get_data_source_folder(ds_name)
z = bcolz.ctable(rootdir=bundle_folder, mode='r')
df = z.todataframe() # type: pd.DataFrame
df.set_index(['date', 'symbol'], drop=True, inplace=True)
# TODO: implement the filter more carefully
# if start and end is None:
# df = df.xs(start, level=0)
return df
def clean(self, ds_name=None, data_frequency=None):
if ds_name is None:
mktplace_root = get_marketplace_folder()
folders = [os.path.basename(f.rstrip('/'))
for f in glob.glob('{}/*/'.format(mktplace_root))
if 'temp_bundles' not in f]
while True:
for idx, f in enumerate(folders):
print('{}\t{}'.format(idx, f))
dataset_num = input('Choose the dataset you want to '
'clean [0..{}]: '.format(
len(folders) - 1))
try:
dataset_num = int(dataset_num)
except ValueError:
print('Enter a number between 0 and {}'.format(
len(folders) - 1))
else:
if dataset_num not in range(0, len(folders)):
print('Enter a number between 0 and {}'.format(
len(folders) - 1))
else:
ds_name = folders[dataset_num]
break
ds_name = ds_name.lower()
if data_frequency is None:
folder = get_data_source_folder(ds_name)
else:
folder = get_bundle_folder(ds_name, data_frequency)
shutil.rmtree(folder)
def create_metadata(self, key, secret, ds_name, data_frequency, desc,
has_history=True, has_live=True):
"""
Returns
-------
"""
headers = get_signed_headers(ds_name, key, secret)
r = requests.post(
'{}/marketplace/register'.format(AUTH_SERVER),
json=dict(
ds_name=ds_name,
desc=desc,
data_frequency=data_frequency,
has_history=has_history,
has_live=has_live,
),
headers=headers,
)
if r.status_code != 200:
raise MarketplaceHTTPRequest(
request='register', error=r.status_code
)
if 'error' in r.json():
raise MarketplaceHTTPRequest(
request='upload file', error=r.json()['error']
)
def register(self):
while True:
desc = input('Enter the name of the dataset to register: ')
dataset = desc.lower().strip()
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if provider_info[4]:
print('There is already a dataset registered under '
'the name "{}". Please choose a different '
'name.'.format(dataset))
else:
break
price = int(
input(
'Enter the price for a monthly subscription to '
'this dataset in ENG: '
)
)
while True:
freq = input('Enter the data frequency [daily, hourly, minute]: ')
if freq.lower() not in ('daily', 'hourly', 'minute'):
print('Not a valid frequency.')
else:
break
while True:
reg_pub = input(
'Does it include historical data? [default: Y]: '
) or 'y'
if reg_pub.lower() not in ('y', 'n'):
print('Please answer Y or N.')
else:
if reg_pub.lower() == 'y':
has_history = True
else:
has_history = False
break
while True:
reg_pub = input(
'Doest it include live data? [default: Y]: '
) or 'y'
if reg_pub.lower() not in ('y', 'n'):
print('Please answer Y or N.')
else:
if reg_pub.lower() == 'y':
has_live = True
else:
has_live = False
break
address, address_i = self.choose_pubaddr()
if 'key' in self.addresses[address_i]:
key = self.addresses[address_i]['key']
secret = self.addresses[address_i]['secret']
else:
key, secret = get_key_secret(address,
self.addresses[address_i]['wallet'])
grains = to_grains(price)
tx = self.mkt_contract.functions.register(
Web3.toHex(dataset),
grains,
address,
).buildTransaction(
{'from': address,
'nonce': self.web3.eth.getTransactionCount(address)}
)
signed_tx = self.sign_transaction(tx)
try:
tx_hash = '0x{}'.format(
bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
)
print(
'\nThis is the TxHash for this transaction: {}'.format(tx_hash)
)
except Exception as e:
print('Unable to register the requested dataset: {}'.format(e))
return
self.check_transaction(tx_hash)
print('Waiting for the transaction to succeed...')
while True:
try:
if self.web3.eth.getTransactionReceipt(tx_hash).status:
break
else:
print('\nTransaction failed. Aborting...')
return
except AttributeError:
pass
for i in range(0, 10):
print('.', end='', flush=True)
time.sleep(1)
print('\nWarming up the {} dataset'.format(dataset))
self.create_metadata(
key=key,
secret=secret,
ds_name=dataset,
data_frequency=freq,
desc=desc,
has_history=has_history,
has_live=has_live,
)
print('\n{} registered successfully'.format(dataset))
def publish(self, dataset, datadir, watch):
dataset = dataset.lower()
provider_info = self.mkt_contract.functions.getDataProviderInfo(
Web3.toHex(dataset)
).call()
if not provider_info[4]:
raise MarketplaceDatasetNotFound(dataset=dataset)
match = next(
(l for l in self.addresses if l['pubAddr'] == provider_info[0]),
None
)
if not match:
raise MarketplaceNoAddressMatch(
dataset=dataset,
address=provider_info[0])
print('Using address: {} to publish this dataset.'.format(
provider_info[0]))
if 'key' in match:
key = match['key']
secret = match['secret']
else:
key, secret = get_key_secret(provider_info[0], match['wallet'])
filenames = glob.glob(os.path.join(datadir, '*.csv'))
if not filenames:
raise MarketplaceNoCSVFiles(datadir=datadir)
files = []
for idx, file in enumerate(filenames):
log.info('Uploading file {} of {}: {}'.format(
idx+1, len(filenames), file))
files = []
files.append(('file', open(file, 'rb')))
headers = get_signed_headers(dataset, key, secret)
r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
files=files,
headers=headers)
if r.status_code != 200:
raise MarketplaceHTTPRequest(request='upload file',
error=r.status_code)
if 'error' in r.json():
raise MarketplaceHTTPRequest(request='upload file',
error=r.json()['error'])
log.info('File processed successfully.')
print('\nDataset {} uploaded and processed successfully.'.format(
dataset))
catalyst/marketplace/marketplace_errors.py
-97
View File
@@ -1,97 +0,0 @@
import sys
import traceback
from catalyst.errors import ZiplineError
def silent_except_hook(exctype, excvalue, exctraceback):
if exctype in [MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
MarketplaceNoAddressMatch, MarketplaceHTTPRequest,
MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch,
MarketplaceSubscriptionExpired, MarketplaceJSONError,
MarketplaceWalletNotSupported, MarketplaceEmptySignature,
MarketplaceRequiresPython3]:
fn = traceback.extract_tb(exctraceback)[-1][0]
ln = traceback.extract_tb(exctraceback)[-1][1]
print("Error traceback: {1} (line {2})\n"
"{0.__name__}: {3}".format(exctype, fn, ln, excvalue))
else:
sys.__excepthook__(exctype, excvalue, exctraceback)
sys.excepthook = silent_except_hook
class MarketplacePubAddressEmpty(ZiplineError):
msg = (
'Please enter your public address to use in the Data Marketplace '
'in the following file: {filename}'
).strip()
class MarketplaceDatasetNotFound(ZiplineError):
msg = (
'The dataset "{dataset}" is not registered in the Data Marketplace.'
).strip()
class MarketplaceNoAddressMatch(ZiplineError):
msg = (
'The address registered with the dataset {dataset}: {address} '
'does not match any of your addresses.'
).strip()
class MarketplaceHTTPRequest(ZiplineError):
msg = (
'Request to remote server to {request} failed: {error}'
).strip()
class MarketplaceNoCSVFiles(ZiplineError):
msg = (
'No CSV files found on {datadir} to upload.'
)
class MarketplaceContractDataNoMatch(ZiplineError):
msg = (
'The information found on the contract does not match the '
'requested data:\n{params}.'
)
class MarketplaceSubscriptionExpired(ZiplineError):
msg = (
'Your subscription to dataset "{dataset}" expired on {date} '
'and is no longer active. You have to subscribe again running the '
'following command:\n'
'catalyst marketplace subscribe --dataset={dataset}'
)
class MarketplaceWalletNotSupported(ZiplineError):
msg = (
'Wallet {wallet} is not supported.'
)
class MarketplaceEmptySignature(ZiplineError):
msg = (
'Signature cannot be empty.'
)
class MarketplaceJSONError(ZiplineError):
msg = (
'The configuration file {file} is malformed. Please correct '
'the following error:\n{error}'
)
class MarketplaceRequiresPython3(ZiplineError):
msg = (
'\nCatalyst requires Python3 to access the Enigma Data Marketplace.\n'
'If you want to use the Data Marketplace, you need to reinstall '
'Catalyst\nwith Python3. See the documentation website for additional '
'information.')
catalyst/marketplace/utils/auth_utils.py
-141
View File
@@ -1,141 +0,0 @@
import hashlib
import hmac
import webbrowser
import requests
import time
from catalyst.marketplace.marketplace_errors import (
MarketplaceHTTPRequest, MarketplaceWalletNotSupported,
MarketplaceEmptySignature)
from catalyst.marketplace.utils.path_utils import (
get_user_pubaddr, save_user_pubaddr)
from catalyst.constants import AUTH_SERVER, SUPPORTED_WALLETS
def get_key_secret(pubAddr, wallet):
"""
Obtain a new key/secret pair from authentication server
Parameters
----------
pubAddr: str
dataset: str
Returns
-------
key: str
secret: str
"""
session = requests.Session()
response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
headers={
'Authorization': 'Digest username="{0}"'.format(
pubAddr)})
if response.status_code != 401:
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error='Unexpected response code: '
'{}'.format(response.status_code))
header = response.headers.get('WWW-Authenticate')
auth_type, auth_info = header.split(None, 1)
d = requests.utils.parse_dict_header(auth_info)
nonce = 'Catalyst nonce: 0x{}'.format(d['nonce'])
if wallet in SUPPORTED_WALLETS:
url = 'https://www.mycrypto.com/signmsg.html'
print('\nObtaining a key/secret pair to streamline all future '
'requests with the authentication server.\n'
'Visit {url} and sign the '
'following message (copy the entire line, without the '
'line break at the end):\n\n{nonce}'.format(
url=url,
nonce=nonce))
webbrowser.open_new(url)
signature = input('\nCopy and Paste the "sig" field from '
'the signature here (without the double quotes, '
'only the HEX value):\n')
else:
raise MarketplaceWalletNotSupported(wallet=wallet)
if signature is None:
raise MarketplaceEmptySignature()
signature = signature[2:]
r = int(signature[0:64], base=16)
s = int(signature[64:128], base=16)
v = int(signature[128:130], base=16)
vrs = [v, r, s]
response = session.get('{}/marketplace/getkeysecret'.format(AUTH_SERVER),
headers={
'Authorization': 'Digest username="{0}",realm="{1}",'
'nonce="{2}",uri="/marketplace/getkeysecret",response="{3}",'
'opaque="{4}"'.format(pubAddr,
d['realm'],
d['nonce'],
','.join(str(e) for e in vrs+[wallet]),
d['opaque'])})
if response.status_code == 200:
if 'error' in response.json():
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error=str(response.json()['error']))
else:
addresses = get_user_pubaddr()
match = next((l for l in addresses if
l['pubAddr'].lower() == pubAddr.lower()), None)
match['key'] = response.json()['key']
match['secret'] = response.json()['secret']
addresses[addresses.index(match)] = match
save_user_pubaddr(addresses)
print('Key/secret pair retrieved successfully from server.')
return match['key'], match['secret']
else:
raise MarketplaceHTTPRequest(request=str('obtain key/secret'),
error=response.status_code)
def get_signed_headers(ds_name, key, secret):
"""
Return a new request header including the key / secret signature
Parameters
----------
ds_name
key
secret
Returns
-------
"""
nonce = str(int(time.time() * 1000))
signature = hmac.new(
secret.encode('utf-8'),
'{}{}'.format(ds_name, nonce).encode('utf-8'),
hashlib.sha512
).hexdigest()
headers = {
'Sign': signature,
'Key': key,
'Nonce': nonce,
'Dataset': ds_name,
}
return headers
catalyst/marketplace/utils/bundle_utils.py
-94
View File
@@ -1,94 +0,0 @@
import os
import random
import re
import shutil
import bcolz
import numpy as np
import pandas as pd
from six import string_types
def merge_bundles(zsource, ztarget):
"""
Merge
Parameters
----------
zsource
ztarget
Returns
-------
"""
# TODO: find a way to do this iteratively instead of in-memory
df_source = zsource.todataframe()
df_target = ztarget.todataframe()
df = pd.concat(
[df_source, df_target], ignore_index=True
) # type: pd.DataFrame
df.drop_duplicates(inplace=True)
df.set_index(['date', 'symbol'], drop=False, inplace=True)
sanitize_df(df)
dirname = os.path.basename(ztarget.rootdir)
bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
shutil.move(ztarget.rootdir, bak_dir)
z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
shutil.rmtree(bak_dir)
return z
def sanitize_df(df):
# Using a sampling method to identify dates for efficiency with
# large datasets
if len(df) > 100:
indexes = random.sample(range(0, len(df) - 1), 100)
elif len(df) > 1:
indexes = range(0, len(df) - 1)
else:
indexes = [0, ]
for column in df.columns:
is_date = False
for index in indexes:
value = df[column].iloc[index]
if not isinstance(value, string_types):
continue
# TODO: assuming that the date is at least daily
exp = re.compile(r'^\d{4}-\d{2}-\d{2}.*$')
matches = exp.findall(value)
if matches:
is_date = True
break
if is_date:
df[column] = pd.to_datetime(df[column])
else:
try:
ser = safely_reduce_dtype(df[column])
df[column] = ser
except Exception:
pass
return df
def safely_reduce_dtype(ser): # pandas.Series or numpy.array
orig_dtype = "".join(
[x for x in ser.dtype.name if x.isalpha()]) # float/int
mx = 1
for val in ser.values:
new_itemsize = np.min_scalar_type(val).itemsize
if mx < new_itemsize:
mx = new_itemsize
if orig_dtype == 'int':
mx = max(mx, 4)
new_dtype = orig_dtype + str(mx * 8)
return ser.astype(new_dtype)
catalyst/marketplace/utils/eth_utils.py
-82
View File
@@ -1,82 +0,0 @@
import binascii
# def bytes32(string):
# """
# Convert string to bytes32 data type for smart contract
# Parameters
# ----------
# string: str
# Returns
# -------
# list
# """
# return binascii.hexlify(string.encode('utf-8'))
# def b32_str(bytes32):
# """
# Convert bytes32 to string
# Parameters
# ----------
# input: bytes object
# Returns
# -------
# str
# """
# return binascii.unhexlify(
# bytes32.decode('utf-8').rstrip('\0')).decode('ascii')
def bin_hex(binary):
"""
Convert bytes32 to string
Parameters
----------
input: bytes object
Returns
-------
str
"""
return binascii.hexlify(binary).decode('utf-8')
def from_grains(amount):
"""
Convert from grains to cryptocurrency
Parameters
----------
input: amount
Returns
-------
int
"""
return amount // 10 ** 8
def to_grains(amount):
"""
Convert from cryptocurrency to grains
Parameters
----------
input: amount
Returns
-------
int
"""
return amount * 10 ** 8
catalyst/marketplace/utils/path_utils.py
-213
View File
@@ -1,213 +0,0 @@
import os
import json
import tarfile
from catalyst.constants import SUPPORTED_WALLETS
from catalyst.utils.deprecate import deprecated
from catalyst.utils.paths import data_root, ensure_directory
from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
def get_marketplace_folder(environ=None):
"""
The root path of the marketplace folder.
Parameters
----------
environ:
Returns
-------
str
"""
if not environ:
environ = os.environ
root = data_root(environ)
marketplace_folder = os.path.join(root, 'marketplace')
ensure_directory(marketplace_folder)
return marketplace_folder
def get_data_source_folder(data_source_name, environ=None):
"""
The root path of an data_source folder.
Parameters
----------
data_source_name: str
environ:
Returns
-------
str
"""
if not environ:
environ = os.environ
root = data_root(environ)
data_source_folder = os.path.join(root, 'marketplace', data_source_name)
ensure_directory(data_source_folder)
return data_source_folder
@deprecated
def get_bundle_folder(data_source_name, data_frequency, environ=None):
data_source_folder = get_data_source_folder(data_source_name, environ)
bundle_folder = os.path.join(data_source_folder, data_frequency)
ensure_directory(bundle_folder)
return bundle_folder
def get_temp_bundles_folder(environ=None):
"""
The temp folder for bundle downloads by algo name.
Parameters
----------
ds_name: str
environ:
Returns
-------
str
"""
root = data_root(environ)
folder = os.path.join(root, 'marketplace', 'temp_bundles')
ensure_directory(folder)
return folder
def extract_bundle(tar_filename):
"""
Extract a bcolz bundle.
Parameters
----------
ds_name
Returns
-------
str
"""
target_path = tar_filename.replace('.tar.gz', '')
with tarfile.open(tar_filename, 'r') as tar:
tar.extractall(target_path)
return target_path
def get_user_pubaddr(environ=None):
"""
The de-serialized contend of the user's addresses.json file.
Parameters
----------
environ:
Returns
-------
Object
"""
marketplace_folder = get_marketplace_folder(environ)
filename = os.path.join(marketplace_folder, 'addresses.json')
if os.path.isfile(filename):
with open(filename) as data_file:
try:
data = json.load(data_file)
except json.decoder.JSONDecodeError as e:
raise MarketplaceJSONError(file=filename, error=e)
try:
d = data[0]['pubAddr']
except Exception as e:
data = [data, ]
changed = False
for idx, d in enumerate(data):
try:
if d['wallet'] not in SUPPORTED_WALLETS:
data[idx]['wallet'] = _choose_wallet(
d['pubAddr'], False)
changed = True
except KeyError:
data[idx]['wallet'] = _choose_wallet(
d['pubAddr'], True)
changed = True
if changed:
save_user_pubaddr(data)
return data
else:
data = []
data.append(dict(pubAddr='', desc='', wallet=''))
with open(filename, 'w') as f:
json.dump(data, f, sort_keys=False, indent=2,
separators=(',', ':'))
return data
def _choose_wallet(pubAddr, missing):
while True:
if missing:
print('\nYou need to specify a wallet for address '
'{}.'.format(pubAddr))
else:
print('\nThe wallet specified for address {} is not '
'supported.'.format(pubAddr))
print('Please choose among the following options:')
for idx, wallet in enumerate(SUPPORTED_WALLETS):
print('{}\t{}'.format(idx, wallet))
lw = len(SUPPORTED_WALLETS)-1
w = input('Choose a number between 0 and {}: '.format(
lw))
try:
w = int(w)
except ValueError:
print('Enter a number between 0 and {}'.format(lw))
else:
if w not in range(0, lw+1):
print('Enter a number between 0 and '
'{}'.format(lw))
else:
return SUPPORTED_WALLETS[w]
def save_user_pubaddr(data, environ=None):
"""
Saves the user's public addresses and their related metadata in
the corresponding addresses.json file.
Parameters
----------
data: dict
Returns
-------
True
"""
marketplace_folder = get_marketplace_folder(environ)
filename = os.path.join(marketplace_folder, 'addresses.json')
with open(filename, 'w') as f:
json.dump(data, f, sort_keys=False, indent=2,
separators=(',', ':'))
return True
catalyst/patches/stats.py
-1112
View File
File diff suppressed because it is too large Load Diff
catalyst/pipeline/engine.py
+4 -8
View File
@@ -7,7 +7,6 @@ from abc import (
) )
from uuid import uuid4 from uuid import uuid4
import six
from six import ( from six import (
iteritems, iteritems,
with_metaclass, with_metaclass,
@@ -34,6 +33,7 @@ from catalyst.utils.sharedoc import copydoc
class PipelineEngine(with_metaclass(ABCMeta)): class PipelineEngine(with_metaclass(ABCMeta)):
@abstractmethod @abstractmethod
def run_pipeline(self, pipeline, start_date, end_date): def run_pipeline(self, pipeline, start_date, end_date):
""" """
@@ -118,7 +118,6 @@ class ExplodingPipelineEngine(PipelineEngine):
""" """
A PipelineEngine that doesn't do anything. A PipelineEngine that doesn't do anything.
""" """
def run_pipeline(self, pipeline, start_date, end_date): def run_pipeline(self, pipeline, start_date, end_date):
raise NoEngineRegistered( raise NoEngineRegistered(
"Attempted to run a pipeline but no pipeline " "Attempted to run a pipeline but no pipeline "
@@ -485,10 +484,8 @@ class SimplePipelineEngine(PipelineEngine):
) )
if isinstance(term, LoadableTerm): if isinstance(term, LoadableTerm):
term_key = loader_group_key(term)
# TODO: temp workaround
to_load = sorted( to_load = sorted(
six.next(six.itervalues(loader_groups)), loader_groups[loader_group_key(term)],
key=lambda t: t.dataset key=lambda t: t.dataset
) )
loader = get_loader(term) loader = get_loader(term)
@@ -568,10 +565,9 @@ class SimplePipelineEngine(PipelineEngine):
index=MultiIndex.from_arrays([empty_dates, empty_assets]), index=MultiIndex.from_arrays([empty_dates, empty_assets]),
) )
# TODO: not sure what's wrong with the resolved_assets resolved_assets = array(self._finder.retrieve_all(assets))
# resolved_assets = array(self._finder.retrieve_all(assets))
dates_kept = repeat_last_axis(dates.values, len(assets))[mask] dates_kept = repeat_last_axis(dates.values, len(assets))[mask]
assets_kept = repeat_first_axis(assets, len(dates))[mask] assets_kept = repeat_first_axis(resolved_assets, len(dates))[mask]
final_columns = {} final_columns = {}
for name in data: for name in data:
catalyst/pipeline/factors/equity/__init__.py
+3
View File
@@ -1,6 +1,9 @@
from .statistical import ( from .statistical import (
RollingPearson,
RollingLinearRegression,
RollingLinearRegressionOfReturns, RollingLinearRegressionOfReturns,
RollingPearsonOfReturns, RollingPearsonOfReturns,
RollingSpearman,
RollingSpearmanOfReturns, RollingSpearmanOfReturns,
) )
from .technical import ( from .technical import (
catalyst/pipeline/graph.py
+2 -2
View File
@@ -142,7 +142,7 @@ class TermGraph(object):
at the end of execution. at the end of execution.
""" """
refcounts = self.graph.out_degree() refcounts = self.graph.out_degree()
for t in list(self.outputs.values()): for t in self.outputs.values():
refcounts[t] += 1 refcounts[t] += 1
for t in initial_terms: for t in initial_terms:
@@ -238,7 +238,7 @@ class ExecutionPlan(TermGraph):
min_extra_rows=0): min_extra_rows=0):
super(ExecutionPlan, self).__init__(terms) super(ExecutionPlan, self).__init__(terms)
for term in list(terms.values()): for term in terms.values():
self.set_extra_rows( self.set_extra_rows(
term, term,
all_dates, all_dates,
catalyst/pipeline/loaders/equity_pricing_loader.py
+4 -8
View File
@@ -38,11 +38,9 @@ class USEquityPricingLoader(PipelineLoader):
def __init__(self, bundle, data_frequency, dataset): def __init__(self, bundle, data_frequency, dataset):
# TODO: This is currently broken, No Pipeline support for Catalyst if data_frequency == 'daily':
# if data_frequency == 'daily': reader = bundle.daily_bar_reader
# reader = bundle.daily_bar_reader elif daily_bar_reader == 'minute':
# elif daily_bar_reader == 'minute':
if data_frequency == 'minute':
reader = bundle.minute_bar_reader reader = bundle.minute_bar_reader
else: else:
raise ValueError( raise ValueError(
@@ -53,9 +51,7 @@ class USEquityPricingLoader(PipelineLoader):
if data_frequency == 'daily': if data_frequency == 'daily':
all_sessions = cal.all_sessions all_sessions = cal.all_sessions
# TODO: this cannot be right, but no pipeline support at the moment elif daily_bar_reader == 'minute':
# elif daily_bar_reader == 'minute':
elif data_frequency == 'minute':
reader = bundle.minute_bar_reader reader = bundle.minute_bar_reader
all_sessions = cal.all_minutes all_sessions = cal.all_minutes

Some files were not shown because too many files have changed in this diff Show More