From 3c4c6c3dfd6c4c02b1f667ee8e3164a7b239d681 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 8 Feb 2018 22:07:41 -0700 Subject: [PATCH 01/10] DOC: small edits, eliminating sphinx warnings --- README.rst | 4 +--- docs/source/videos.rst | 7 ++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index a39137fd..7bb7361c 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,7 @@ insights regarding a particular strategy's performance. Catalyst also supports live-trading of crypto-assets starting with three exchanges (Bitfinex, Bittrex, and Poloniex) with more being added over time. Catalyst empowers users to share and curate data and build profitable, data-driven investment strategies. Please -visit `enigma.co `_ to learn more about Catalyst, or -refer to the `whitepaper `_ for -further technical details. +visit `enigma.co `_ to learn more about Catalyst. Catalyst builds on top of the well-established `Zipline `_ project. We did our best to diff --git a/docs/source/videos.rst b/docs/source/videos.rst index 1db8ff28..0beb291b 100644 --- a/docs/source/videos.rst +++ b/docs/source/videos.rst @@ -11,6 +11,7 @@ Installation: MacOS | | + Installation: Windows --------------------- @@ -21,6 +22,7 @@ Where things go smoothly: | + Where things don't: .. raw:: html @@ -29,6 +31,7 @@ Where things don't: | | + Backtesting a Strategy ---------------------- @@ -44,6 +47,7 @@ sell. Hopefully, we’ll ride the waves. | | + Live Trading a Strategy ----------------------- @@ -54,5 +58,6 @@ in the previous video, we now take it to trade live against the Bittrex exchange .. raw:: html + +| | -| \ No newline at end of file From bc8bf6941deb27ad0f19d91ad3dc54b80901ee76 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Feb 2018 16:08:22 -0800 Subject: [PATCH 02/10] MAINT: contract+abi pointing to master, not develop --- catalyst/constants.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/catalyst/constants.py b/catalyst/constants.py index 912d3894..b29d6f62 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -27,20 +27,20 @@ AUTH_SERVER = 'https://data.enigma.co' # TODO: switch to mainnet ETH_REMOTE_NODE = 'https://ropsten.infura.io/' -# TODO: move to MASTER branch on github + MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/develop/catalyst/marketplace/' \ + 'catalyst/master/catalyst/marketplace/' \ 'contract_marketplace_address.txt' MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/develop/catalyst/marketplace/' \ + 'catalyst/master/catalyst/marketplace/' \ 'contract_marketplace_abi.json' # TODO: switch to mainnet ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'develop/catalyst/marketplace/' \ + 'master/catalyst/marketplace/' \ 'contract_enigma_address.txt' ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ - 'catalyst/develop/catalyst/marketplace/' \ + 'catalyst/master/catalyst/marketplace/' \ 'contract_enigma_abi.json' From 46f34d64a0bf368244632f754f01b071a817211c Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Tue, 13 Feb 2018 12:04:49 -0700 Subject: [PATCH 03/10] MAINT: conda env for Python3 --- etc/python2.7-environment.yml | 2 +- etc/python3.6-environment.yml | 95 +++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 etc/python3.6-environment.yml diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 4f06bbae..5147a1be 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -59,4 +59,4 @@ dependencies: - tables==3.4.2 - toolz==0.8.2 - urllib3==1.22 - - enigma-catalyst>=0.3 + - enigma-catalyst>=0.5 diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml new file mode 100644 index 00000000..446198e0 --- /dev/null +++ b/etc/python3.6-environment.yml @@ -0,0 +1,95 @@ +name: catalyst +channels: +- defaults +dependencies: +- ca-certificates=2017.08.26=ha1e5d58_0 +- certifi=2018.1.18=py36_0 +- intel-openmp=2018.0.0=h8158457_8 +- libcxx=4.0.1=h579ed51_0 +- libcxxabi=4.0.1=hebd6815_0 +- libedit=3.1=hb4e282d_0 +- libffi=3.2.1=h475c297_4 +- libgfortran=3.0.1=h93005f0_2 +- mkl=2018.0.1=hfbd8650_4 +- ncurses=6.0=hd04f020_2 +- numpy=1.14.0=py36h8a80b8c_1 +- openssl=1.0.2n=hdbc3d79_0 +- pip=9.0.1=py36h1555ced_4 +- python=3.6.4=hc167b69_1 +- readline=7.0=hc1231fa_4 +- scipy=1.0.0=py36h1de22e9_0 +- setuptools=38.4.0=py36_0 +- sqlite=3.22.0=h3efe00b_0 +- tk=8.6.7=h35a86e2_3 +- wheel=0.30.0=py36h5eb2c71_1 +- xz=5.2.3=h0278029_2 +- zlib=1.2.11=hf3cbc9b_2 +- pip: + - aiodns==1.1.1 + - aiohttp==3.0.1 + - alembic==0.9.7 + - async-timeout==2.0.0 + - attrdict==2.0.0 + - attrs==17.4.0 + - bcolz==0.12.1 + - boto3==1.5.27 + - botocore==1.8.41 + - bottleneck==1.2.1 + - cchardet==2.1.1 + - ccxt==1.10.1102 + - chardet==3.0.4 + - click==6.7 + - contextlib2==0.5.5 + - cyordereddict==1.0.0 + - cython==0.27.3 + - cytoolz==0.9.0 + - decorator==4.2.1 + - docutils==0.14 + - empyrical==0.2.1 + - enigma-catalyst>=0.5.3 + - eth-abi==1.0.0b0 + - eth-account==0.1.0a2 + - eth-keyfile==0.5.1 + - eth-keys==0.2.0b1 + - eth-rlp==0.1.0a2 + - eth-utils==1.0.0b1 + - hexbytes==0.1.0b0 + - idna==2.6 + - idna-ssl==1.0.0 + - intervaltree==2.1.0 + - jmespath==0.9.3 + - logbook==1.2.1 + - lru-dict==1.1.6 + - lxml==4.1.1 + - mako==1.0.7 + - markupsafe==1.0 + - multidict==4.1.0 + - multipledispatch==0.4.9 + - networkx==2.1 + - numexpr==2.6.4 + - pandas==0.19.2 + - pandas-datareader==0.6.0 + - patsy==0.5.0 + - pycares==2.3.0 + - pycryptodome==3.4.11 + - pysha3==1.0.2 + - python-dateutil==2.6.1 + - python-editor==1.0.3 + - pytz==2018.3 + - redo==1.6 + - requests==2.18.4 + - requests-file==1.4.3 + - requests-ftp==0.3.1 + - requests-toolbelt==0.8.0 + - rlp==0.6.0 + - s3transfer==0.1.12 + - six==1.11.0 + - sortedcontainers==1.5.9 + - sqlalchemy==1.2.2 + - statsmodels==0.8.0 + - tables==3.4.2 + - toolz==0.9.0 + - urllib3==1.22 + - web3==4.0.0b9 + - wrapt==1.10.11 + - yarl==1.1.0 From 9956b5462d3422743f3eccc6e14826597fce3b94 Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 14 Feb 2018 09:27:53 -0700 Subject: [PATCH 04/10] Update python3.6-environment.yml --- etc/python3.6-environment.yml | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml index 446198e0..0e5eabbf 100644 --- a/etc/python3.6-environment.yml +++ b/etc/python3.6-environment.yml @@ -2,28 +2,28 @@ name: catalyst channels: - defaults dependencies: -- ca-certificates=2017.08.26=ha1e5d58_0 -- certifi=2018.1.18=py36_0 -- intel-openmp=2018.0.0=h8158457_8 -- libcxx=4.0.1=h579ed51_0 -- libcxxabi=4.0.1=hebd6815_0 -- libedit=3.1=hb4e282d_0 -- libffi=3.2.1=h475c297_4 -- libgfortran=3.0.1=h93005f0_2 -- mkl=2018.0.1=hfbd8650_4 -- ncurses=6.0=hd04f020_2 -- numpy=1.14.0=py36h8a80b8c_1 -- openssl=1.0.2n=hdbc3d79_0 -- pip=9.0.1=py36h1555ced_4 -- python=3.6.4=hc167b69_1 -- readline=7.0=hc1231fa_4 -- scipy=1.0.0=py36h1de22e9_0 +- ca-certificates=2017.08.26 +- certifi=2018.1.18 +- intel-openmp=2018.0.0 +- libcxx=4.0.1 +- libcxxabi=4.0.1 +- libedit=3.1 +- libffi=3.2.1 +- libgfortran=3.0.1 +- mkl=2018.0.1 +- ncurses=6.0 +- numpy=1.14.0 +- openssl=1.0.2n +- pip=9.0.1 +- python=3.6.4 +- readline=7.0 +- scipy=1.0.0 - setuptools=38.4.0=py36_0 -- sqlite=3.22.0=h3efe00b_0 -- tk=8.6.7=h35a86e2_3 -- wheel=0.30.0=py36h5eb2c71_1 -- xz=5.2.3=h0278029_2 -- zlib=1.2.11=hf3cbc9b_2 +- sqlite=3.22.0 +- tk=8.6.7 +- wheel=0.30.0 +- xz=5.2.3 +- zlib=1.2.11 - pip: - aiodns==1.1.1 - aiohttp==3.0.1 From ec5fdecf91ccdf856a1b605e775a9c5bf2eb408e Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 16 Feb 2018 11:48:52 -0700 Subject: [PATCH 05/10] DOC: marketplace code examples --- .../examples/marketplace/github-research.py | 70 +++++++++++++++++++ .../mean_reversion_by_marketcap.py | 0 2 files changed, 70 insertions(+) create mode 100644 catalyst/examples/marketplace/github-research.py rename catalyst/examples/{ => marketplace}/mean_reversion_by_marketcap.py (100%) diff --git a/catalyst/examples/marketplace/github-research.py b/catalyst/examples/marketplace/github-research.py new file mode 100644 index 00000000..b2443597 --- /dev/null +++ b/catalyst/examples/marketplace/github-research.py @@ -0,0 +1,70 @@ +import pandas as pd +import matplotlib.pyplot as plt + +from catalyst import run_algorithm +from catalyst.api import symbol, get_dataset + +START = '2017-01-01' +END = '2017-12-31' + + +def initialize(context): + pass + + +def handle_data(context, data): + context.github = get_dataset('github') + context.github.sort_index(level=0, inplace=True) + + context.zec = data.history(symbol('zec_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + context.xmr = data.history(symbol('xmr_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + + +def analyze(context=None, results=None): + ax1 = plt.subplot(211) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'ZEC']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax1, color='blue') + ax1.legend(loc=2) + ax1.set_title('Zcash') + ax2 = ax1.twinx() + context.zec['price'].loc[START:END].plot(ax=ax2, color='green') + ax2.legend(loc=1) + + ax3 = plt.subplot(212) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'XMR']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax3, color='blue') + ax3.legend(loc=2) + ax3.set_title('Monero') + ax4 = ax3.twinx() + context.xmr['price'].loc[START:END].plot(ax=ax4, color='green') + ax4.legend(loc=1) + + plt.show() + + +if __name__ == '__main__': + run_algorithm( + capital_base=1000, + data_frequency='daily', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace='algo-github', + base_currency='usdt', + live=False, + start=pd.to_datetime(END, utc=True), + end=pd.to_datetime(END, utc=True), + ) diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/marketplace/mean_reversion_by_marketcap.py similarity index 100% rename from catalyst/examples/mean_reversion_by_marketcap.py rename to catalyst/examples/marketplace/mean_reversion_by_marketcap.py From 2577b53518a14eeba063f81e1fc7f93f881d12e6 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 22 Feb 2018 12:54:29 -0700 Subject: [PATCH 06/10] MAINT: conda environment updates --- etc/python2.7-environment.yml | 2 ++ etc/python3.6-environment.yml | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 5147a1be..ab530cb0 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -1,9 +1,11 @@ name: catalyst channels: - defaults +- conda-forge dependencies: - certifi=2016.2.28=py27_0 - mkl=2017.0.3 +- matplotlib=2.1.2=py36_0 - numpy=1.13.1=py27_0 - openssl=1.0.2l - pip=9.0.1=py27_1 diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml index 0e5eabbf..c93f3c82 100644 --- a/etc/python3.6-environment.yml +++ b/etc/python3.6-environment.yml @@ -1,22 +1,17 @@ name: catalyst channels: - defaults +- conda-forge dependencies: - ca-certificates=2017.08.26 - certifi=2018.1.18 - intel-openmp=2018.0.0 -- libcxx=4.0.1 -- libcxxabi=4.0.1 -- libedit=3.1 -- libffi=3.2.1 -- libgfortran=3.0.1 - mkl=2018.0.1 -- ncurses=6.0 - numpy=1.14.0 - openssl=1.0.2n +- matplotlib=2.1.2=py36_0 - pip=9.0.1 - python=3.6.4 -- readline=7.0 - scipy=1.0.0 - setuptools=38.4.0=py36_0 - sqlite=3.22.0 From 25e9f0f58f16651c86570c55ca704153d98dd35c Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 22 Feb 2018 22:09:51 -0700 Subject: [PATCH 07/10] BUG: reverts changed introduced in 00f232e2d7426b8593295ee3e4e6488a39a4866b --- catalyst/examples/dual_moving_average.py | 39 +++++++++--------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index 363edba1..2a1be0aa 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -20,8 +20,8 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages - short_window = 2 - long_window = 2 + short_window = 50 + long_window = 200 # Skip as many bars as long_window to properly compute the average context.i += 1 @@ -150,27 +150,16 @@ def analyze(context, perf): if __name__ == '__main__': + run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - simulate_orders=True, - live=True, - ) - # run_algorithm( - # capital_base=1000, - # data_frequency='minute', - # initialize=initialize, - # handle_data=handle_data, - # analyze=analyze, - # exchange_name='bitfinex', - # algo_namespace=NAMESPACE, - # base_currency='usd', - # start=pd.to_datetime('2017-9-22', utc=True), - # end=pd.to_datetime('2017-9-23', utc=True), - # ) + capital_base=1000, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=NAMESPACE, + base_currency='usd', + start=pd.to_datetime('2017-9-22', utc=True), + end=pd.to_datetime('2017-9-23', utc=True), + ) From 497212383abf3d3730fd45c586e0df87f8d64ab7 Mon Sep 17 00:00:00 2001 From: Matt Bornski Date: Mon, 26 Feb 2018 15:44:32 -0800 Subject: [PATCH 08/10] Python 3 returns bytes, the parsing functions are looking for strings --- catalyst/marketplace/marketplace.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index a1ac263c..37ed2ec6 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -40,6 +40,13 @@ else: log = logbook.Logger('Marketplace', level=LOG_LEVEL) +def first_line_from_url(url): + request = urllib.urlopen(url) + first_line = request.readline() + if sys.version_info.major >= 3: + charset = request.info().get_content_charset() + first_line = first_line.decode(charset) + return first_line.strip() class Marketplace: def __init__(self): @@ -57,10 +64,8 @@ class Marketplace: self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) - contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) - self.mkt_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + first_line_from_url(MARKETPLACE_CONTRACT)) abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) abi = json.load(abi_url) @@ -70,10 +75,8 @@ class Marketplace: abi=abi, ) - contract_url = urllib.urlopen(ENIGMA_CONTRACT) - self.eng_contract_address = Web3.toChecksumAddress( - contract_url.readline().strip()) + first_line_from_url(ENIGMA_CONTRACT)) abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) abi = json.load(abi_url) From f990ecf14d58c79233eab7d122078b3182c480b4 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 2 Mar 2018 11:34:18 -0700 Subject: [PATCH 09/10] BUG: fix incompatibility with web3==4.0.0b11 --- catalyst/marketplace/marketplace.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 37ed2ec6..b188d6d6 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -40,13 +40,6 @@ else: log = logbook.Logger('Marketplace', level=LOG_LEVEL) -def first_line_from_url(url): - request = urllib.urlopen(url) - first_line = request.readline() - if sys.version_info.major >= 3: - charset = request.info().get_content_charset() - first_line = first_line.decode(charset) - return first_line.strip() class Marketplace: def __init__(self): @@ -65,7 +58,8 @@ class Marketplace: self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) self.mkt_contract_address = Web3.toChecksumAddress( - first_line_from_url(MARKETPLACE_CONTRACT)) + contract_url.readline().decode( + contract_url.info().get_content_charset()).strip()) abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI) abi = json.load(abi_url) @@ -76,7 +70,8 @@ class Marketplace: ) self.eng_contract_address = Web3.toChecksumAddress( - first_line_from_url(ENIGMA_CONTRACT)) + contract_url.readline().decode( + contract_url.info().get_content_charset()).strip()) abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI) abi = json.load(abi_url) From b4e5b699bd77eb78863208b76786f56290d72e74 Mon Sep 17 00:00:00 2001 From: Victor Date: Fri, 2 Mar 2018 11:37:22 -0700 Subject: [PATCH 10/10] BUG: fix2 incompatibility with web3==4.0.0b11 --- catalyst/marketplace/marketplace.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index b188d6d6..731f116a 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -57,6 +57,8 @@ class Marketplace: self.web3 = Web3(HTTPProvider(ETH_REMOTE_NODE)) + contract_url = urllib.urlopen(MARKETPLACE_CONTRACT) + self.mkt_contract_address = Web3.toChecksumAddress( contract_url.readline().decode( contract_url.info().get_content_charset()).strip()) @@ -69,6 +71,8 @@ class Marketplace: abi=abi, ) + contract_url = urllib.urlopen(ENIGMA_CONTRACT) + self.eng_contract_address = Web3.toChecksumAddress( contract_url.readline().decode( contract_url.info().get_content_charset()).strip())