From 3c4c6c3dfd6c4c02b1f667ee8e3164a7b239d681 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 8 Feb 2018 22:07:41 -0700 Subject: [PATCH 01/19] DOC: small edits, eliminating sphinx warnings --- README.rst | 4 +--- docs/source/videos.rst | 7 ++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index a39137fd..7bb7361c 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,7 @@ insights regarding a particular strategy's performance. Catalyst also supports live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, and Poloniex) with more being added over time. Catalyst empowers users to share and curate data and build profitable, data-driven investment strategies. Please -visit `enigma.co `_ to learn more about Catalyst, or -refer to the `whitepaper `_ for -further technical details. +visit `enigma.co `_ to learn more about Catalyst. Catalyst builds on top of the well-established `Zipline `_ project. We did our best to diff --git a/docs/source/videos.rst b/docs/source/videos.rst index 1db8ff28..0beb291b 100644 --- a/docs/source/videos.rst +++ b/docs/source/videos.rst @@ -11,6 +11,7 @@ Installation: MacOS | | + Installation: Windows --------------------- @@ -21,6 +22,7 @@ Where things go smoothly: | + Where things don't: .. raw:: html @@ -29,6 +31,7 @@ Where things don't: | | + Backtesting a Strategy ---------------------- @@ -44,6 +47,7 @@ sell. Hopefully, we’ll ride the waves. | | + Live Trading a Strategy ----------------------- @@ -54,5 +58,6 @@ in the previous video, we now take it to trade live against the Bittrex exchange .. raw:: html + +| | -| \ No newline at end of file From bc8bf6941deb27ad0f19d91ad3dc54b80901ee76 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Feb 2018 16:08:22 -0800 Subject: [PATCH 02/19] MAINT: contract+abi pointing to master, not develop --- catalyst/constants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index 912d3894..b29d6f62 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -27,20 +27,20 @@ AUTH_SERVER = 'https://data.enigma.co' # TODO: switch to mainnet ETH_REMOTE_NODE = 'https://ropsten.infura.io/' -# TODO: move to MASTER branch on github + MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/develop/catalyst/marketplace/' \ + 'catalyst/master/catalyst/marketplace/' \ 'contract_marketplace_address.txt' MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/develop/catalyst/marketplace/' \ + 'catalyst/master/catalyst/marketplace/' \ 'contract_marketplace_abi.json' # TODO: switch to mainnet ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'develop/catalyst/marketplace/' \ + 'master/catalyst/marketplace/' \ 'contract_enigma_address.txt' ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/develop/catalyst/marketplace/' \ + 'catalyst/master/catalyst/marketplace/' \ 'contract_enigma_abi.json' From 46f34d64a0bf368244632f754f01b071a817211c Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 13 Feb 2018 12:04:49 -0700 Subject: [PATCH 03/19] MAINT: conda env for Python3 --- etc/python2.7-environment.yml | 2 +- etc/python3.6-environment.yml | 95 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 etc/python3.6-environment.yml diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 4f06bbae..5147a1be 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -59,4 +59,4 @@ dependencies: - tables==3.4.2 - toolz==0.8.2 - urllib3==1.22 - - enigma-catalyst>=0.3 + - enigma-catalyst>=0.5 diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml new file mode 100644 index 00000000..446198e0 --- /dev/null +++ b/etc/python3.6-environment.yml @@ -0,0 +1,95 @@ +name: catalyst +channels: +- defaults +dependencies: +- ca-certificates=2017.08.26=ha1e5d58_0 +- certifi=2018.1.18=py36_0 +- intel-openmp=2018.0.0=h8158457_8 +- libcxx=4.0.1=h579ed51_0 +- libcxxabi=4.0.1=hebd6815_0 +- libedit=3.1=hb4e282d_0 +- libffi=3.2.1=h475c297_4 +- libgfortran=3.0.1=h93005f0_2 +- mkl=2018.0.1=hfbd8650_4 +- ncurses=6.0=hd04f020_2 +- numpy=1.14.0=py36h8a80b8c_1 +- openssl=1.0.2n=hdbc3d79_0 +- pip=9.0.1=py36h1555ced_4 +- python=3.6.4=hc167b69_1 +- readline=7.0=hc1231fa_4 +- scipy=1.0.0=py36h1de22e9_0 +- setuptools=38.4.0=py36_0 +- sqlite=3.22.0=h3efe00b_0 +- tk=8.6.7=h35a86e2_3 +- wheel=0.30.0=py36h5eb2c71_1 +- xz=5.2.3=h0278029_2 +- zlib=1.2.11=hf3cbc9b_2 +- pip: + - aiodns==1.1.1 + - aiohttp==3.0.1 + - alembic==0.9.7 + - async-timeout==2.0.0 + - attrdict==2.0.0 + - attrs==17.4.0 + - bcolz==0.12.1 + - boto3==1.5.27 + - botocore==1.8.41 + - bottleneck==1.2.1 + - cchardet==2.1.1 + - ccxt==1.10.1102 + - chardet==3.0.4 + - click==6.7 + - contextlib2==0.5.5 + - cyordereddict==1.0.0 + - cython==0.27.3 + - cytoolz==0.9.0 + - decorator==4.2.1 + - docutils==0.14 + - empyrical==0.2.1 + - enigma-catalyst>=0.5.3 + - eth-abi==1.0.0b0 + - eth-account==0.1.0a2 + - eth-keyfile==0.5.1 + - eth-keys==0.2.0b1 + - eth-rlp==0.1.0a2 + - eth-utils==1.0.0b1 + - hexbytes==0.1.0b0 + - idna==2.6 + - idna-ssl==1.0.0 + - intervaltree==2.1.0 + - jmespath==0.9.3 + - logbook==1.2.1 + - lru-dict==1.1.6 + - lxml==4.1.1 + - mako==1.0.7 + - markupsafe==1.0 + - multidict==4.1.0 + - multipledispatch==0.4.9 + - networkx==2.1 + - numexpr==2.6.4 + - pandas==0.19.2 + - pandas-datareader==0.6.0 + - patsy==0.5.0 + - pycares==2.3.0 + - pycryptodome==3.4.11 + - pysha3==1.0.2 + - python-dateutil==2.6.1 + - python-editor==1.0.3 + - pytz==2018.3 + - redo==1.6 + - requests==2.18.4 + - requests-file==1.4.3 + - requests-ftp==0.3.1 + - requests-toolbelt==0.8.0 + - rlp==0.6.0 + - s3transfer==0.1.12 + - six==1.11.0 + - sortedcontainers==1.5.9 + - sqlalchemy==1.2.2 + - statsmodels==0.8.0 + - tables==3.4.2 + - toolz==0.9.0 + - urllib3==1.22 + - web3==4.0.0b9 + - wrapt==1.10.11 + - yarl==1.1.0 From 9956b5462d3422743f3eccc6e14826597fce3b94 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 14 Feb 2018 09:27:53 -0700 Subject: [PATCH 04/19] Update python3.6-environment.yml --- etc/python3.6-environment.yml | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml index 446198e0..0e5eabbf 100644 --- a/etc/python3.6-environment.yml +++ b/etc/python3.6-environment.yml @@ -2,28 +2,28 @@ name: catalyst channels: - defaults dependencies: -- ca-certificates=2017.08.26=ha1e5d58_0 -- certifi=2018.1.18=py36_0 -- intel-openmp=2018.0.0=h8158457_8 -- libcxx=4.0.1=h579ed51_0 -- libcxxabi=4.0.1=hebd6815_0 -- libedit=3.1=hb4e282d_0 -- libffi=3.2.1=h475c297_4 -- libgfortran=3.0.1=h93005f0_2 -- mkl=2018.0.1=hfbd8650_4 -- ncurses=6.0=hd04f020_2 -- numpy=1.14.0=py36h8a80b8c_1 -- openssl=1.0.2n=hdbc3d79_0 -- pip=9.0.1=py36h1555ced_4 -- python=3.6.4=hc167b69_1 -- readline=7.0=hc1231fa_4 -- scipy=1.0.0=py36h1de22e9_0 +- ca-certificates=2017.08.26 +- certifi=2018.1.18 +- intel-openmp=2018.0.0 +- libcxx=4.0.1 +- libcxxabi=4.0.1 +- libedit=3.1 +- libffi=3.2.1 +- libgfortran=3.0.1 +- mkl=2018.0.1 +- ncurses=6.0 +- numpy=1.14.0 +- openssl=1.0.2n +- pip=9.0.1 +- python=3.6.4 +- readline=7.0 +- scipy=1.0.0 - setuptools=38.4.0=py36_0 -- sqlite=3.22.0=h3efe00b_0 -- tk=8.6.7=h35a86e2_3 -- wheel=0.30.0=py36h5eb2c71_1 -- xz=5.2.3=h0278029_2 -- zlib=1.2.11=hf3cbc9b_2 +- sqlite=3.22.0 +- tk=8.6.7 +- wheel=0.30.0 +- xz=5.2.3 +- zlib=1.2.11 - pip: - aiodns==1.1.1 - aiohttp==3.0.1 From ec5fdecf91ccdf856a1b605e775a9c5bf2eb408e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 16 Feb 2018 11:48:52 -0700 Subject: [PATCH 05/19] DOC: marketplace code examples --- .../examples/marketplace/github-research.py | 70 +++++++++++++++++++ .../mean_reversion_by_marketcap.py | 0 2 files changed, 70 insertions(+) create mode 100644 catalyst/examples/marketplace/github-research.py rename catalyst/examples/{ => marketplace}/mean_reversion_by_marketcap.py (100%) diff --git a/catalyst/examples/marketplace/github-research.py b/catalyst/examples/marketplace/github-research.py new file mode 100644 index 00000000..b2443597 --- /dev/null +++ b/catalyst/examples/marketplace/github-research.py @@ -0,0 +1,70 @@ +import pandas as pd +import matplotlib.pyplot as plt + +from catalyst import run_algorithm +from catalyst.api import symbol, get_dataset + +START = '2017-01-01' +END = '2017-12-31' + + +def initialize(context): + pass + + +def handle_data(context, data): + context.github = get_dataset('github') + context.github.sort_index(level=0, inplace=True) + + context.zec = data.history(symbol('zec_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + context.xmr = data.history(symbol('xmr_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + + +def analyze(context=None, results=None): + ax1 = plt.subplot(211) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'ZEC']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax1, color='blue') + ax1.legend(loc=2) + ax1.set_title('Zcash') + ax2 = ax1.twinx() + context.zec['price'].loc[START:END].plot(ax=ax2, color='green') + ax2.legend(loc=1) + + ax3 = plt.subplot(212) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'XMR']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax3, color='blue') + ax3.legend(loc=2) + ax3.set_title('Monero') + ax4 = ax3.twinx() + context.xmr['price'].loc[START:END].plot(ax=ax4, color='green') + ax4.legend(loc=1) + + plt.show() + + +if __name__ == '__main__': + run_algorithm( + capital_base=1000, + data_frequency='daily', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace='algo-github', + base_currency='usdt', + live=False, + start=pd.to_datetime(END, utc=True), + end=pd.to_datetime(END, utc=True), + ) diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/marketplace/mean_reversion_by_marketcap.py similarity index 100% rename from catalyst/examples/mean_reversion_by_marketcap.py rename to catalyst/examples/marketplace/mean_reversion_by_marketcap.py From 2577b53518a14eeba063f81e1fc7f93f881d12e6 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 22 Feb 2018 12:54:29 -0700 Subject: [PATCH 06/19] MAINT: conda environment updates --- etc/python2.7-environment.yml | 2 ++ etc/python3.6-environment.yml | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 5147a1be..ab530cb0 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -1,9 +1,11 @@ name: catalyst channels: - defaults +- conda-forge dependencies: - certifi=2016.2.28=py27_0 - mkl=2017.0.3 +- matplotlib=2.1.2=py36_0 - numpy=1.13.1=py27_0 - openssl=1.0.2l - pip=9.0.1=py27_1 diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml index 0e5eabbf..c93f3c82 100644 --- a/etc/python3.6-environment.yml +++ b/etc/python3.6-environment.yml @@ -1,22 +1,17 @@ name: catalyst channels: - defaults +- conda-forge dependencies: - ca-certificates=2017.08.26 - certifi=2018.1.18 - intel-openmp=2018.0.0 -- libcxx=4.0.1 -- libcxxabi=4.0.1 -- libedit=3.1 -- libffi=3.2.1 -- libgfortran=3.0.1 - mkl=2018.0.1 -- ncurses=6.0 - numpy=1.14.0 - openssl=1.0.2n +- matplotlib=2.1.2=py36_0 - pip=9.0.1 - python=3.6.4 -- readline=7.0 - scipy=1.0.0 - setuptools=38.4.0=py36_0 - sqlite=3.22.0 From 25e9f0f58f16651c86570c55ca704153d98dd35c Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 22 Feb 2018 22:09:51 -0700 Subject: [PATCH 07/19] BUG: reverts changed introduced in 00f232e2d7426b8593295ee3e4e6488a39a4866b --- catalyst/examples/dual_moving_average.py | 39 +++++++++--------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index 363edba1..2a1be0aa 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -20,8 +20,8 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages - short_window = 2 - long_window = 2 + short_window = 50 + long_window = 200 # Skip as many bars as long_window to properly compute the average context.i += 1 @@ -150,27 +150,16 @@ def analyze(context, perf): if __name__ == '__main__': + run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - simulate_orders=True, - live=True, - ) - # run_algorithm( - # capital_base=1000, - # data_frequency='minute', - # initialize=initialize, - # handle_data=handle_data, - # analyze=analyze, - # exchange_name='bitfinex', - # algo_namespace=NAMESPACE, - # base_currency='usd', - # start=pd.to_datetime('2017-9-22', utc=True), - # end=pd.to_datetime('2017-9-23', utc=True), - # ) + capital_base=1000, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=NAMESPACE, + base_currency='usd', + start=pd.to_datetime('2017-9-22', utc=True), + end=pd.to_datetime('2017-9-23', utc=True), + ) From 497212383abf3d3730fd45c586e0df87f8d64ab7 Mon Sep 17 00:00:00 2001 From: Matt Bornski Date: Mon, 26 Feb 2018 15:44:32 -0800 Subject: [PATCH 08/19] Python 3 returns bytes, the parsing functions are looking for strings --- catalyst/marketplace/marketplace.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a1ac263c..37ed2ec6 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -40,6 +40,13 @@ else: log = logbook.Logger('Marketplace', level=LOG_LEVEL) +def first_line_from_url(url): + request = urllib.urlopen(url) + first_line = request.readline() + if sys.version_info.major >= 3: + charset = request.info().get_content_charset() + first_line = first_line.decode(charset) + return first_line.strip() class Marketplace: def __init__(self): @@ -57,10 +64,8 @@ class Marketplace: self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) - contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) - self.mkt_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + first_line_from_url(MARKETPLACE_CONTRACT)) abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) abi = json.load(abi_url) @@ -70,10 +75,8 @@ class Marketplace: abi=abi, ) - contract_url = urllib.urlopen(ENIGMA_CONTRACT) - self.eng_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + first_line_from_url(ENIGMA_CONTRACT)) abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) abi = json.load(abi_url) From f990ecf14d58c79233eab7d122078b3182c480b4 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 2 Mar 2018 11:34:18 -0700 Subject: [PATCH 09/19] BUG: fix incompatibility with web3==4.0.0b11 --- catalyst/marketplace/marketplace.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 37ed2ec6..b188d6d6 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -40,13 +40,6 @@ else: log = logbook.Logger('Marketplace', level=LOG_LEVEL) -def first_line_from_url(url): - request = urllib.urlopen(url) - first_line = request.readline() - if sys.version_info.major >= 3: - charset = request.info().get_content_charset() - first_line = first_line.decode(charset) - return first_line.strip() class Marketplace: def __init__(self): @@ -65,7 +58,8 @@ class Marketplace: self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) self.mkt_contract_address = Web3.toChecksumAddress( - first_line_from_url(MARKETPLACE_CONTRACT)) + contract_url.readline().decode( + contract_url.info().get_content_charset()).strip()) abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) abi = json.load(abi_url) @@ -76,7 +70,8 @@ class Marketplace: ) self.eng_contract_address = Web3.toChecksumAddress( - first_line_from_url(ENIGMA_CONTRACT)) + contract_url.readline().decode( + contract_url.info().get_content_charset()).strip()) abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) abi = json.load(abi_url) From b4e5b699bd77eb78863208b76786f56290d72e74 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 2 Mar 2018 11:37:22 -0700 Subject: [PATCH 10/19] BUG: fix2 incompatibility with web3==4.0.0b11 --- catalyst/marketplace/marketplace.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index b188d6d6..731f116a 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -57,6 +57,8 @@ class Marketplace: self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) + contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) + self.mkt_contract_address = Web3.toChecksumAddress( contract_url.readline().decode( contract_url.info().get_content_charset()).strip()) @@ -69,6 +71,8 @@ class Marketplace: abi=abi, ) + contract_url = urllib.urlopen(ENIGMA_CONTRACT) + self.eng_contract_address = Web3.toChecksumAddress( contract_url.readline().decode( contract_url.info().get_content_charset()).strip()) From fa0e9332bfadcc33a7c5440885f75281ff6051a9 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 2 Mar 2018 11:43:53 -0700 Subject: [PATCH 11/19] MAINT: CLI info on marketplace cmds --- catalyst/__main__.py | 18 +++++++++++++++++- etc/requirements.txt | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 1b08f69d..03669486 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -767,12 +767,18 @@ def bundles(): @main.group() @click.pass_context def marketplace(ctx): + """Access the Enigma Data Marketplace to:\n + - Register and Publish new datasets (seller-side)\n + - Subscribe and Ingest premium datasets (buyer-side)\n + """ pass @marketplace.command() @click.pass_context def ls(ctx): + """List all available datasets. + """ click.echo('Listing of available data sources on the marketplace:', sys.stdout) marketplace = Marketplace() @@ -787,6 +793,8 @@ def ls(ctx): ) @click.pass_context def subscribe(ctx, dataset): + """Subscribe to an exisiting dataset. + """ if dataset is None: ctx.fail("must specify a dataset to subscribe to with '--dataset'\n" "List available dataset on the marketplace with " @@ -825,6 +833,8 @@ def subscribe(ctx, dataset): ) @click.pass_context def ingest(ctx, dataset, data_frequency, start, end): + """Ingest a dataset (requires subscription). + """ if dataset is None: ctx.fail("must specify a dataset to clean with '--dataset'\n" "List available dataset on the marketplace with " @@ -842,8 +852,10 @@ def ingest(ctx, dataset, data_frequency, start, end): ) @click.pass_context def clean(ctx, dataset): + """Clean/Remove local data for a given dataset. + """ if dataset is None: - ctx.fail("must specify a dataset to ingest with '--dataset'\n" + ctx.fail("must specify a dataset to clean up with '--dataset'\n" "List available dataset on the marketplace with " "'catalyst marketplace ls'") click.echo('Cleaning data source: {}'.format(dataset), sys.stdout) @@ -855,6 +867,8 @@ def clean(ctx, dataset): @marketplace.command() @click.pass_context def register(ctx): + """Register a new dataset. + """ marketplace = Marketplace() marketplace.register() @@ -878,6 +892,8 @@ def register(ctx): ) @click.pass_context def publish(ctx, dataset, datadir, watch): + """Publish data for a registered dataset. + """ marketplace = Marketplace() if dataset is None: ctx.fail("must specify a dataset to publish data for " diff --git a/etc/requirements.txt b/etc/requirements.txt index bac9666d..73573e45 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -84,5 +84,5 @@ tables==3.3.0 ccxt==1.10.1094 boto3==1.4.8 redo==1.6 -web3==4.0.0b7 +web3==4.0.0b11 requests-toolbelt==0.8.0 From d7b6cb8490ab45ee88dddaafa45ac3fbbf2a907d Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 2 Mar 2018 12:18:48 -0700 Subject: [PATCH 12/19] BUG: marketplace typo --- catalyst/marketplace/marketplace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 731f116a..68088c5c 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -623,7 +623,7 @@ class Marketplace: ) except Exception as e: - print('Unable to subscribe to data source: {}'.format(e)) + print('Unable to register the requested dataset: {}'.format(e)) return self.check_transaction(tx_hash) From 5197ab6cc2ed9df66508b9565f256aebda9258ce Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 2 Mar 2018 12:50:46 -0700 Subject: [PATCH 13/19] BUG: isolated Python3 depedency to the marketplace --- catalyst/marketplace/marketplace.py | 39 ++++++++++++---------- catalyst/marketplace/marketplace_errors.py | 11 +++++- etc/python2.7-environment.yml | 4 ++- etc/requirements.txt | 2 +- etc/requirements_marketplace.txt | 2 -- 5 files changed, 35 insertions(+), 23 deletions(-) delete mode 100644 etc/requirements_marketplace.txt diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 68088c5c..e972b750 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -23,7 +23,7 @@ from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, MarketplaceNoAddressMatch, MarketplaceHTTPRequest, - MarketplaceNoCSVFiles) + MarketplaceNoCSVFiles, MarketplaceRequiresPython3) from catalyst.marketplace.utils.auth_utils import get_key_secret, \ get_signed_headers from catalyst.marketplace.utils.bundle_utils import merge_bundles @@ -44,7 +44,10 @@ log = logbook.Logger('Marketplace', level=LOG_LEVEL) class Marketplace: def __init__(self): global Web3 - from web3 import Web3, HTTPProvider + try: + from web3 import Web3, HTTPProvider + except ImportError: + raise MarketplaceRequiresPython3() self.addresses = get_user_pubaddr() @@ -58,7 +61,7 @@ class Marketplace: self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) - + self.mkt_contract_address = Web3.toChecksumAddress( contract_url.readline().decode( contract_url.info().get_content_charset()).strip()) @@ -72,7 +75,7 @@ class Marketplace: ) contract_url = urllib.urlopen(ENIGMA_CONTRACT) - + self.eng_contract_address = Web3.toChecksumAddress( contract_url.readline().decode( contract_url.info().get_content_charset()).strip()) @@ -150,13 +153,13 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], ) - ) + _from=from_address, + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -261,14 +264,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -371,7 +374,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -428,10 +431,10 @@ class Marketplace: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' 'catalyst marketplace subscribe --dataset={}'.format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] diff --git a/catalyst/marketplace/marketplace_errors.py b/catalyst/marketplace/marketplace_errors.py index b6be1c3b..488c204f 100644 --- a/catalyst/marketplace/marketplace_errors.py +++ b/catalyst/marketplace/marketplace_errors.py @@ -9,7 +9,8 @@ def silent_except_hook(exctype, excvalue, exctraceback): MarketplaceNoAddressMatch, MarketplaceHTTPRequest, MarketplaceNoCSVFiles, MarketplaceContractDataNoMatch, MarketplaceSubscriptionExpired, MarketplaceJSONError, - MarketplaceWalletNotSupported, MarketplaceEmptySignature]: + MarketplaceWalletNotSupported, MarketplaceEmptySignature, + MarketplaceRequiresPython3]: fn = traceback.extract_tb(exctraceback)[-1][0] ln = traceback.extract_tb(exctraceback)[-1][1] print("Error traceback: {1} (line {2})\n" @@ -86,3 +87,11 @@ class MarketplaceJSONError(ZiplineError): 'The configuration file {file} is malformed. Please correct ' 'the following error:\n{error}' ) + + +class MarketplaceRequiresPython3(ZiplineError): + msg = ( + '\nCatalyst requires Python3 to access the Enigma Data Marketplace.\n' + 'If you want to use the Data Marketplace, you need to reinstall ' + 'Catalyst\nwith Python3. See the documentation website for additional ' + 'information.') diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index b2e2486c..3835d7d4 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -23,7 +23,9 @@ dependencies: - bottleneck==1.2.1 - chardet==3.0.4 - ccxt==1.10.1094 - - web3==4.0.0b7 +# The Enigma Data Marketplace requires Python3 because it depends on +# web3, which requires Python3, as building its dependencies breaks in Python2 +# - web3==4.0.0b7 - requests-toolbelt==0.8.0 - click==6.7 - contextlib2==0.5.5 diff --git a/etc/requirements.txt b/etc/requirements.txt index 73573e45..5be47e3d 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -84,5 +84,5 @@ tables==3.3.0 ccxt==1.10.1094 boto3==1.4.8 redo==1.6 -web3==4.0.0b11 +web3==4.0.0b11; python_version > '3.4' requests-toolbelt==0.8.0 diff --git a/etc/requirements_marketplace.txt b/etc/requirements_marketplace.txt deleted file mode 100644 index 2a56c200..00000000 --- a/etc/requirements_marketplace.txt +++ /dev/null @@ -1,2 +0,0 @@ -web3==4.0.0b7 -requests-toolbelt==0.8.0 From e59f46dfd653220a77fdf1ea1698e9137a14d2b9 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Sun, 4 Mar 2018 13:44:04 +0200 Subject: [PATCH 14/19] BLD: improve periods calculation --- catalyst/exchange/utils/exchange_utils.py | 18 +--- tests/exchange/test_exchange_utils.py | 109 ++++++++++++---------- 2 files changed, 65 insertions(+), 62 deletions(-) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 9a681068..28185091 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -16,7 +16,6 @@ from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ ExchangeJSONDecoder from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time -from catalyst.exchange.utils.datetime_utils import get_periods_range def get_sid(symbol): @@ -740,19 +739,10 @@ def get_candles_df(candles, field, freq, bar_count, end_dt=None): for asset in candles: asset_df = transform_candles_to_df(candles[asset]) - rounded_end_dt = end_dt.round(freq) - - periods = get_periods_range( - start_dt=None, end_dt=rounded_end_dt, - freq=freq, periods=bar_count - ) - - if rounded_end_dt > end_dt: - periods = periods[:-1] - elif rounded_end_dt <= end_dt: - periods = periods[1:] - - # periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq) + rounded_end_dt = end_dt.floor(freq) + periods = pd.date_range(end=rounded_end_dt, + periods=bar_count, + freq=freq) asset_df = forward_fill_df_if_needed(asset_df, periods) all_series[asset] = pd.Series(asset_df[field]) diff --git a/tests/exchange/test_exchange_utils.py b/tests/exchange/test_exchange_utils.py index ddc15cc9..2d3d1efe 100644 --- a/tests/exchange/test_exchange_utils.py +++ b/tests/exchange/test_exchange_utils.py @@ -2,6 +2,7 @@ from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, \ forward_fill_df_if_needed, get_candles_df from catalyst.testing.fixtures import WithLogger, ZiplineTestCase +from datetime import timedelta from pandas import Timestamp, DataFrame, concat import numpy as np @@ -15,9 +16,59 @@ class TestExchangeUtils(WithLogger, ZiplineTestCase): new_df.index.name = None return new_df + @classmethod + def verify_forward_fill_df_if_needed(cls, candles, periods, expected_df): + observed_df = forward_fill_df_if_needed( + transform_candles_to_df(candles), + periods) + assert (expected_df.equals(observed_df)) + + @classmethod + def verify_get_candles_df(cls, assets, candles, end_fixed_dt, + expected_df, check_next_candle=False): + # run on all the fields + for field in ['volume', 'open', 'close', 'high', 'low']: + + field_dt = cls.get_specific_field_from_df(expected_df, + field, + assets[0]) + # run on several timestamps + for delta in range(5): + end_dt = end_fixed_dt + timedelta(minutes=delta) + assert (field_dt.equals(get_candles_df({assets[0]: candles}, + field, '5T', 3, + end_dt=end_dt))) + + field_dt_a1 = cls.get_specific_field_from_df(expected_df, + field, + assets[0]) + field_dt_a2 = cls.get_specific_field_from_df(expected_df, + field, + assets[1]) + observed_df = get_candles_df({assets[0]: candles, + assets[1]: candles}, + field, '5T', 3, + end_dt=end_dt) + + assert (observed_df.equals(concat([field_dt_a1, field_dt_a2], + axis=1))) + + if check_next_candle: + # one candle forward + end_dt = end_fixed_dt + timedelta(minutes=6) + observed_df = get_candles_df({assets[0]: candles, + assets[1]: candles}, + field, '5T', 3, + end_dt=end_dt) + + assert (not observed_df.equals(concat([field_dt_a1, + field_dt_a2], + axis=1))) + assert (concat([field_dt_a1, field_dt_a2], + axis=1)[1:].equals(observed_df[:-1])) + def test_get_candles_df(self): - asset = 'btc_usdt' - asset2 = 'eth_usdt' + assets = ['btc_usdt', 'eth_usdt'] # test forward fill in the end candles = [{'high': 595, 'volume': 10, 'low': 594, @@ -51,20 +102,12 @@ class TestExchangeUtils(WithLogger, ZiplineTestCase): Timestamp('2018-03-01 09:50:00+0000', tz='UTC'), Timestamp('2018-03-01 09:55:00+0000', tz='UTC')] - observed_df = forward_fill_df_if_needed( - transform_candles_to_df(candles), - periods) expected_df = transform_candles_to_df(expected) - assert (expected_df.equals(observed_df)) - - for field in ['volume', 'open', 'close', 'high', 'low']: - field_dt = self.get_specific_field_from_df(expected_df, - field, - asset) - assert (field_dt.equals(get_candles_df({asset: candles}, - field, '5T', 3, - end_dt=periods[2]))) + self.verify_forward_fill_df_if_needed(candles, periods, + expected_df) + self.verify_get_candles_df(assets, candles, periods[2], + expected_df, True) # test forward fill in the middle candles = [{'high': 595, 'volume': 10, 'low': 594, @@ -94,28 +137,9 @@ class TestExchangeUtils(WithLogger, ZiplineTestCase): tz='UTC') }] - df = transform_candles_to_df(candles) - observed_df = forward_fill_df_if_needed(df, periods) expected_df = transform_candles_to_df(expected) - - assert (expected_df.equals(observed_df)) - - for field in ['volume', 'open', 'close', 'high', 'low']: - # test several assets as well - observed_df = get_candles_df({asset: candles, - asset2: candles}, - field, '5T', 3, - end_dt=periods[2]) - - field_dt_a1 = self.get_specific_field_from_df(expected_df, - field, - asset) - field_dt_a2 = self.get_specific_field_from_df(expected_df, - field, - asset2) - - assert(observed_df.equals(concat([field_dt_a1, field_dt_a2], - axis=1))) + self.verify_forward_fill_df_if_needed(candles, periods, expected_df) + self.verify_get_candles_df(assets, candles, periods[2], expected_df) # test "forward fill" at the beginning candles = [{'high': 595, 'volume': 10, 'low': 594, @@ -145,18 +169,7 @@ class TestExchangeUtils(WithLogger, ZiplineTestCase): tz='UTC') }] - df = transform_candles_to_df(candles) - observed_df = forward_fill_df_if_needed(df, periods) expected_df = transform_candles_to_df(expected) - - assert (expected_df.equals(observed_df)) + self.verify_forward_fill_df_if_needed(candles, periods, expected_df) # Not the same due to dropna - commenting out for now - """ - for field in ['volume', 'open', 'close', 'high', 'low']: - field_dt = self.get_specific_field_from_df(observed_df, - field, - asset) - assert(field_dt.equals(get_candles_df({asset:candles}, - field, '5T', 3, - end_dt=periods[2]))) - """ + # self.verify_get_candles_df(assets, candles, periods[2], expected_df) From aa520d5a8bc99b45704623088d76724cbdae6523 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 4 Mar 2018 17:23:12 +0200 Subject: [PATCH 15/19] STY: pep8 change in exchange_blotter --- catalyst/exchange/exchange_blotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index c82b2f00..3d85149e 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -68,7 +68,7 @@ class TradingPairFeeSchedule(CommissionModel): multiplier = maker \ if ((order.amount > 0 and order.limit < transaction.price) or (order.amount < 0 and order.limit > transaction.price)) \ - and order.limit_reached else taker + and order.limit_reached else taker fee = cost * multiplier return fee From ed406a30ffb8caed1f372a02236dca131c819320 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Sun, 4 Mar 2018 17:42:32 +0200 Subject: [PATCH 16/19] BLD: fix issue #260 - always request more data to avoid empty bars and always give the exact bar number --- catalyst/exchange/exchange.py | 73 ++++++++++++----------- catalyst/exchange/exchange_errors.py | 7 +++ catalyst/exchange/utils/exchange_utils.py | 2 +- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 310d12fc..e830b90a 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -1,5 +1,4 @@ import abc -import pytz from abc import ABCMeta, abstractmethod, abstractproperty from datetime import timedelta from time import sleep @@ -12,13 +11,14 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ SymbolNotFoundOnExchange, \ PricingDataNotLoadedError, \ - NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \ + NoDataAvailableOnExchange, NoValueForField, \ + NoCandlesReceivedFromExchange, \ TickerNotFoundError, NotEnoughCashError from catalyst.exchange.utils.datetime_utils import get_delta, \ get_periods_range, \ get_periods, get_start_dt, get_frequency from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ - resample_history_df, has_bundle + resample_history_df, has_bundle, get_candles_df from logbook import Logger log = Logger('Exchange', level=LOG_LEVEL) @@ -256,7 +256,8 @@ class Exchange: elif data_frequency is not None: applies = ( ( - data_frequency == 'minute' and a.end_minute is not None) + data_frequency == 'minute' and + a.end_minute is not None) or ( data_frequency == 'daily' and a.end_daily is not None) ) @@ -505,49 +506,50 @@ class Exchange: freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency, supported_freqs=['T', 'D', 'H'] ) + + # we want to avoid receiving empty candles + # so we request more than needed + requested_bar_count = bar_count + 30 # The get_history method supports multiple asset candles = self.get_candles( freq=freq, assets=assets, - bar_count=bar_count, + bar_count=requested_bar_count, end_dt=end_dt if not is_current else None, ) - series = dict() + # candles sanity check - verify no empty candles were received: for asset in candles: - if candles[asset]: - first_candle = candles[asset][0] - asset_series = self.get_series_from_candles( - candles=candles[asset], - start_dt=first_candle['last_traded'], + if not candles[asset]: + raise NoCandlesReceivedFromExchange( + bar_count=requested_bar_count, end_dt=end_dt, - data_frequency=frequency, - field=field, - ) + asset=asset, + exchange=self.name) - delta_candle_size = candle_size * 60 if unit == 'H' else candle_size - # Checking to make sure that the dates match - delta = get_delta(delta_candle_size, data_frequency) - adj_end_dt = end_dt - delta - last_traded = asset_series.index[-1] + series = get_candles_df(candles=candles, + field=field, + freq=frequency, + bar_count=requested_bar_count, + end_dt=end_dt) - if last_traded < adj_end_dt: - raise LastCandleTooEarlyError( - last_traded=last_traded, - end_dt=adj_end_dt, - exchange=self.name, - ) - else: # empty candle received - # because other assets are tz-aware, we need its tz to be set as well - asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc)) - - - series[asset] = asset_series + # TODO: consider how to approach this edge case + # delta_candle_size = candle_size * 60 if unit == 'H' else candle_size + # Checking to make sure that the dates match + # delta = get_delta(delta_candle_size, data_frequency) + # adj_end_dt = end_dt - delta + # last_traded = asset_series.index[-1] + # if last_traded < adj_end_dt: + # raise LastCandleTooEarlyError( + # last_traded=last_traded, + # end_dt=adj_end_dt, + # exchange=self.name, + # ) df = pd.DataFrame(series) - #df.dropna(inplace=True) # commented out due to issue 236 + df.dropna(inplace=True) - return df + return df.tail(bar_count) def get_history_window_with_bundle(self, assets, @@ -595,7 +597,8 @@ class Exchange: A dataframe containing the requested data. """ - # TODO: this function needs some work, we're currently using it just for benchmark data + # TODO: this function needs some work, + # we're currently using it just for benchmark data freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency ) @@ -621,7 +624,7 @@ class Exchange: start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ - if asset in series else start_dt + if asset in series else start_dt # The get_history method supports multiple asset # Use the original frequency to let each api optimize diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index d5af87c4..1d38cd18 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -322,3 +322,10 @@ class BalanceTooLowError(ZiplineError): 'add positions to hold a free amount greater than {amount}, or clean ' 'the state of this algo and restart.' ).strip() + + +class NoCandlesReceivedFromExchange(ZiplineError): + msg = ( + 'Although requesting {bar_count} candles until {end_dt} of asset {asset}, ' + 'an empty list of candles was received for {exchange}.' + ).strip() diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 28185091..50e5124a 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -734,7 +734,7 @@ def transform_candles_to_df(candles): return pd.DataFrame(candles).set_index('last_traded') -def get_candles_df(candles, field, freq, bar_count, end_dt=None): +def get_candles_df(candles, field, freq, bar_count, end_dt): all_series = dict() for asset in candles: From 09068a4c37800da69c14d423f5220251710a352a Mon Sep 17 00:00:00 2001 From: lenak25 Date: Sun, 4 Mar 2018 18:20:14 +0200 Subject: [PATCH 17/19] BLD: adjust the example to Python 3 --- catalyst/examples/portfolio_optimization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/examples/portfolio_optimization.py b/catalyst/examples/portfolio_optimization.py index 26c957b6..c6da1f1f 100644 --- a/catalyst/examples/portfolio_optimization.py +++ b/catalyst/examples/portfolio_optimization.py @@ -66,7 +66,7 @@ def handle_data(context, data): # Define portfolio optimization parameters n_portfolios = 50000 results_array = np.zeros((3 + context.nassets, n_portfolios)) - for p in xrange(n_portfolios): + for p in range(n_portfolios): weights = np.random.random(context.nassets) weights /= np.sum(weights) w = np.asmatrix(weights) From 60924711807f62dde30da9b3f6dc23f6128d779e Mon Sep 17 00:00:00 2001 From: lenak25 Date: Mon, 5 Mar 2018 20:42:14 +0200 Subject: [PATCH 18/19] DOC: add some commented TODOs --- catalyst/exchange/exchange.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index e830b90a..5af13e22 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -509,6 +509,8 @@ class Exchange: # we want to avoid receiving empty candles # so we request more than needed + # TODO: consider defining a const per asset + # and/or some retry mechanism (in each iteration request more data) requested_bar_count = bar_count + 30 # The get_history method supports multiple asset candles = self.get_candles( From cb4668f09350fcd74a020e227f9e2bf392a897d4 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Tue, 6 Mar 2018 00:05:29 +0200 Subject: [PATCH 19/19] BUG: #243 added a function which reduces open orders amount from calculated target/amount for target orders --- catalyst/exchange/exchange_algorithm.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index b9c319f1..abaaea14 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -163,6 +163,25 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): style) return amount, style + def _calculate_order_target_amount(self, asset, target): + """ + removes order amounts so we won't run into issues + when two orders are placed one after the other. + it then proceeds to removing positions amount at TradingAlgorithm + :param asset: + :param target: + :return: target + """ + if asset in self.blotter.open_orders: + for open_order in self.blotter.open_orders[asset]: + current_amount = open_order.amount + target -= current_amount + + target = super(ExchangeTradingAlgorithmBase, self). \ + _calculate_order_target_amount(asset, target) + + return target + def round_order(self, amount, asset): """ We need fractions with cryptocurrencies