Merge branch 'develop'

README.rst

@@ -5,7 +5,6 @@
 
 |version tag|
 |version status|
-|forum|
 |discord|
 |twitter|
 
@@ -23,11 +22,9 @@ visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
 Catalyst builds on top of the well-established 
 `Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
 minimize structural changes to the general API to maximize compatibility with 
-existing trading algorithms, developer knowledge, and tutorials. Join us on the 
+existing trading algorithms, developer knowledge, and tutorials. Join us on 
-`Catalyst Forum <https://catalyst.enigma.co/>`_ for questions around Catalyst, 
+`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
-algorithmic trading and technical support. We also have a 
+for questions around Catalyst, algorithmic trading and technical support.
-`Discord <https://discord.gg/SJK32GY>`_ group with the *#catalyst_dev* and 
-*#catalyst_setup* dedicated channels.
 
 Overview
 ========
@@ -64,9 +61,6 @@ Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
 .. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
    :target: https://pypi.python.org/pypi/enigma-catalyst
 
-.. |forum| image:: https://img.shields.io/badge/forum-join-green.svg
-   :target: https://catalyst.enigma.co/
-
 .. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
    :target: https://discordapp.com/invite/SJK32GY
 

catalyst/assets/_assets.pyx

@@ -630,28 +630,23 @@ cdef class TradingPair(Asset):
         and whose second element is a tuple of all the attributes that should
         be serialized/deserialized during pickling.
         """
-        # added arguments for catalyst
+        #TODO: make sure that all fields set there
         return (self.__class__, (self.symbol,
                                  self.exchange,
                                  self.start_date,
                                  self.asset_name,
                                  self.sid,
                                  self.leverage,
-                                 self.end_daily,
-                                 self.end_minute,
                                  self.end_date,
-                                 self.exchange_symbol,
                                  self.first_traded,
                                  self.auto_close_date,
                                  self.exchange_full,
                                  self.min_trade_size,
                                  self.max_trade_size,
-                                 self.maker,
-                                 self.taker,
                                  self.lot,
                                  self.decimals,
-                                 self.trading_state,
+                                 self.taker,
-                                 self.data_source))
+                                 self.maker))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/exchange/exchange_data_portal.py

@@ -1,5 +1,4 @@
 import abc
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -301,34 +300,20 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         )
         adj_bar_count = candle_size * bar_count
 
-        if data_frequency == "minute":
+        if data_frequency == 'minute' and adj_data_frequency == 'daily':
-            # for minute frequency always request data until the
+            end_dt = end_dt.floor('1D')
-            # current minute (do not include the current minute)
-            last_dt_for_series = end_dt - datetime.timedelta(minutes=1)
-
-            # read the minute bundles for daily frequency to
-            # support last partial candle
-            # TODO: optimize this by applying this logic only for the last day
-            if adj_data_frequency == 'daily':
-                adj_data_frequency = 'minute'
-                adj_bar_count = adj_bar_count * 1440
-
-        else:  # data_frequency == "daily":
-            last_dt_for_series = end_dt
 
         series = bundle.get_history_window_series_and_load(
             assets=assets,
-            end_dt=last_dt_for_series,
+            end_dt=end_dt,
             bar_count=adj_bar_count,
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
         )
 
-        start_dt = get_start_dt(last_dt_for_series, adj_bar_count,
+        start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency)
-                                adj_data_frequency, False)
         df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
-
         return df
 
     def get_exchange_spot_value(self,

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +1 @@
-0xf0ee6b27b759c9893ce4f094b49ad28fd15a23e4
+0x39a54f480d922a58c963de8091a6c9afc69db2cf

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +1 @@
-0xa64927358a82254be92eb1f1cb01de68d1787004
+0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc

catalyst/marketplace/marketplace.py

@@ -20,6 +20,7 @@ from requests_toolbelt.multipart.decoder import \
 from catalyst.constants import (
     LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
     MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
+from catalyst.utils.cli import maybe_show_progress
 from catalyst.exchange.utils.stats_utils import set_print_settings
 from catalyst.marketplace.marketplace_errors import (
     MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
@@ -69,10 +70,7 @@ class Marketplace:
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
-        abi_url = abi_url.read().decode(
+        abi = json.load(abi_url)
-                abi_url.info().get_content_charset())
-
-        abi = json.loads(abi_url)
 
         self.mkt_contract = self.web3.eth.contract(
             self.mkt_contract_address,
@@ -86,10 +84,7 @@ class Marketplace:
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
-        abi_url = abi_url.read().decode(
+        abi = json.load(abi_url)
-                abi_url.info().get_content_charset())
-
-        abi = json.loads(abi_url)
 
         self.eng_contract = self.web3.eth.contract(
             self.eng_contract_address,
@@ -523,7 +518,7 @@ class Marketplace:
                 #     iter(decoder.parts),
                 #     True,
                 #     label='Processing files') as part:
-                counter = 1
+                counter = 0
                 for part in decoder.parts:
                     log.info("Processing file {} of {}".format(
                         counter, len(decoder.parts)))
@@ -643,7 +638,7 @@ class Marketplace:
     def register(self):
         while True:
             desc = input('Enter the name of the dataset to register: ')
-            dataset = desc.lower().strip()
+            dataset = desc.lower()
             provider_info = self.mkt_contract.functions.getDataProviderInfo(
                 Web3.toHex(dataset)
             ).call()
@@ -783,32 +778,26 @@ class Marketplace:
         else:
             key, secret = get_key_secret(provider_info[0], match['wallet'])
 
+        headers = get_signed_headers(dataset, key, secret)
         filenames = glob.glob(os.path.join(datadir, '*.csv'))
 
         if not filenames:
             raise MarketplaceNoCSVFiles(datadir=datadir)
 
         files = []
-        for idx, file in enumerate(filenames):
+        for file in filenames:
-            log.info('Uploading file {} of {}: {}'.format(
-                idx+1, len(filenames), file))
-            files = []
             files.append(('file', open(file, 'rb')))
 
-            headers = get_signed_headers(dataset, key, secret)
+        r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
-            r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
+                          files=files,
-                              files=files,
+                          headers=headers)
-                              headers=headers)
 
-            if r.status_code != 200:
+        if r.status_code != 200:
-                raise MarketplaceHTTPRequest(request='upload file',
+            raise MarketplaceHTTPRequest(request='upload file',
-                                             error=r.status_code)
+                                         error=r.status_code)
 
-            if 'error' in r.json():
+        if 'error' in r.json():
-                raise MarketplaceHTTPRequest(request='upload file',
+            raise MarketplaceHTTPRequest(request='upload file',
-                                             error=r.json()['error'])
+                                         error=r.json()['error'])
 
-            log.info('File processed successfully.')
+        print('Dataset {} uploaded successfully.'.format(dataset))
-
-        print('\nDataset {} uploaded and processed successfully.'.format(
-            dataset))

catalyst/marketplace/utils/auth_utils.py

@@ -123,7 +123,7 @@ def get_signed_headers(ds_name, key, secret):
     -------
 
     """
-    nonce = str(int(time.time() * 1000))
+    nonce = str(int(time.time()))
 
     signature = hmac.new(
         secret.encode('utf-8'),

docs/source/install.rst

@@ -562,10 +562,6 @@ If after following the instructions above, and going through the
 *Troubleshooting* sections, you still experience problems installing Catalyst,
 you can seek additional help through the following channels:
 
-- Join our `Catalyst Forum <https://catalyst.enigma.co/>`_, and browse a variety
-  of topics and conversations around common issues that others face when using
-  Catalyst, and how to resolve them. And join the conversation!
-
 - Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over 
   the #catalyst_dev channel where many other users (as well as the project 
   developers) hang out, and can assist you with your particular issue. The 

docs/source/releases.rst

@@ -2,29 +2,6 @@
 Release Notes
 =============
 
-Version 0.5.8
-^^^^^^^^^^^^^
-**Release Date**: 2018-03-29
-
-Bug Fixes
-~~~~~~~~~
-- Fix Data Marketplace release on mainnet
-
-Version 0.5.7
-^^^^^^^^^^^^^
-**Release Date**: 2018-03-29
-
-Build
-~~~~~
-- Data Marketplace deployed on mainnet.
-- Added progress indicators for publishing data, and made the data publishing
-  synchronous to provide feedback to the publisher.
-
-Bug Fixes
-~~~~~~~~~
-- fixes in storing and loading the state :issue:`214`, 
-  :issue:`287`
-
 Version 0.5.6
 ^^^^^^^^^^^^^
 **Release Date**: 2018-03-22
@@ -32,9 +9,9 @@ Version 0.5.6
 Build
 ~~~~~
 - Data Marketplace: ensures compatibility across wallets, now fully supporting 
-  ``ledger``, ``trezor``, ``keystore``, ``private key``. Partial support for 
+`ledger`, `trezor`, `keystore`, `private key`. Partial support for `metamask` 
-  ``metamask`` (includes sign_msg, but not sign_tx). Current support for 
+(includes sign_msg, but not sign_tx). Current support for `Digital Bitbox` is 
-  ``Digital Bitbox`` is unknown, but believed to be supported.
+unknown.
 - Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto.
 - Data Marketplace: Added progress indicator for data ingestion.
 

etc/python2.7-environment.yml

@@ -5,7 +5,6 @@ channels:
 dependencies:
 - certifi=2016.2.28=py27_0
 - mkl=2017.0.3
-- matplotlib=2.1.2=py36_0
 - numpy=1.13.1=py27_0
 - openssl=1.0.2l
 - pip=9.0.1=py27_1
@@ -40,7 +39,7 @@ dependencies:
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.1.0
+  - matplotlib==2.1.2
   - multipledispatch==0.4.9
   - networkx==2.0
   - numexpr==2.6.4