Merge branch 'develop'

README.rst

@@ -5,7 +5,6 @@
 
 |version tag|
 |version status|
-|forum|
 |discord|
 |twitter|
 
@@ -23,11 +22,9 @@ visit `enigma.co <https://www.enigma.co>`_ to learn more about Catalyst.
 Catalyst builds on top of the well-established 
 `Zipline <https://github.com/quantopian/zipline>`_ project. We did our best to 
 minimize structural changes to the general API to maximize compatibility with 
-existing trading algorithms, developer knowledge, and tutorials. Join us on the 
+existing trading algorithms, developer knowledge, and tutorials. Join us on 
-`Catalyst Forum <https://catalyst.enigma.co/>`_ for questions around Catalyst, 
+`Discord <https://discord.gg/SJK32GY>`_ where we have a *#catalyst_dev* channel 
-algorithmic trading and technical support. We also have a 
+for questions around Catalyst, algorithmic trading and technical support.
-`Discord <https://discord.gg/SJK32GY>`_ group with the *#catalyst_dev* and 
-*#catalyst_setup* dedicated channels.
 
 Overview
 ========
@@ -64,9 +61,6 @@ Go to our `Documentation Website <https://enigmampc.github.io/catalyst/>`_.
 .. |version status| image:: https://img.shields.io/pypi/pyversions/enigma-catalyst.svg
    :target: https://pypi.python.org/pypi/enigma-catalyst
 
-.. |forum| image:: https://img.shields.io/badge/forum-join-green.svg
-   :target: https://catalyst.enigma.co/
-
 .. |discord| image:: https://img.shields.io/badge/discord-join%20chat-green.svg
    :target: https://discordapp.com/invite/SJK32GY
 

catalyst/__main__.py

@@ -580,7 +580,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name),
+    click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
                sys.stdout)
     exchange_bundle.ingest(
         data_frequency=data_frequency,

catalyst/assets/_assets.pyx

@@ -630,28 +630,23 @@ cdef class TradingPair(Asset):
         and whose second element is a tuple of all the attributes that should
         be serialized/deserialized during pickling.
         """
-        # added arguments for catalyst
+        #TODO: make sure that all fields set there
         return (self.__class__, (self.symbol,
                                  self.exchange,
                                  self.start_date,
                                  self.asset_name,
                                  self.sid,
                                  self.leverage,
-                                 self.end_daily,
-                                 self.end_minute,
                                  self.end_date,
-                                 self.exchange_symbol,
                                  self.first_traded,
                                  self.auto_close_date,
                                  self.exchange_full,
                                  self.min_trade_size,
                                  self.max_trade_size,
-                                 self.maker,
-                                 self.taker,
                                  self.lot,
                                  self.decimals,
-                                 self.trading_state,
+                                 self.taker,
-                                 self.data_source))
+                                 self.maker))
 
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)

catalyst/constants.py

@@ -43,6 +43,3 @@ ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
 ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                       'catalyst/master/catalyst/marketplace/' \
                       'contract_enigma_abi.json'
-
-SUPPORTED_WALLETS = ['metamask', 'ledger', 'trezor', 'bitbox', 'keystore',
-                     'key']

catalyst/exchange/exchange.py

@@ -199,8 +199,12 @@ class Exchange:
                 )
                 assets.append(asset)
 
-            except SymbolNotFoundOnExchange as e:
+            except SymbolNotFoundOnExchange:
-                log.warn(e)
+                log.debug(
+                    'skipping non-existent market {} {}'.format(
+                        self.name, symbol
+                    )
+                )
         return assets
 
     def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,

catalyst/exchange/exchange_bundle.py

@@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
 from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_df_from_arrays, get_assets
-from catalyst.exchange.utils.datetime_utils import get_start_dt, \
+from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
     get_period_label, get_month_start_end, get_year_start_end
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
     save_exchange_symbols, mixin_market_params, get_catalyst_symbol
@@ -598,7 +598,6 @@ class ExchangeBundle:
         # we want to give an end_date far in time
         writer = self.get_writer(start_dt, end_dt, data_frequency)
         if show_breakdown:
-            if chunks:
             for asset in chunks:
                 with maybe_show_progress(
                     chunks[asset],
@@ -620,8 +619,8 @@ class ExchangeBundle:
                         )
         else:
             all_chunks = list(chain.from_iterable(itervalues(chunks)))
+
             # We sort the chunks by end date to ingest most recent data first
-            if all_chunks:
             all_chunks.sort(
                 key=lambda chunk: pd.to_datetime(chunk['period'])
             )

catalyst/exchange/exchange_data_portal.py

@@ -1,5 +1,4 @@
 import abc
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -301,34 +300,20 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         )
         adj_bar_count = candle_size * bar_count
 
-        if data_frequency == "minute":
+        if data_frequency == 'minute' and adj_data_frequency == 'daily':
-            # for minute frequency always request data until the
+            end_dt = end_dt.floor('1D')
-            # current minute (do not include the current minute)
-            last_dt_for_series = end_dt - datetime.timedelta(minutes=1)
-
-            # read the minute bundles for daily frequency to
-            # support last partial candle
-            # TODO: optimize this by applying this logic only for the last day
-            if adj_data_frequency == 'daily':
-                adj_data_frequency = 'minute'
-                adj_bar_count = adj_bar_count * 1440
-
-        else:  # data_frequency == "daily":
-            last_dt_for_series = end_dt
 
         series = bundle.get_history_window_series_and_load(
             assets=assets,
-            end_dt=last_dt_for_series,
+            end_dt=end_dt,
             bar_count=adj_bar_count,
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
         )
 
-        start_dt = get_start_dt(last_dt_for_series, adj_bar_count,
+        start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency)
-                                adj_data_frequency, False)
         df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
-
         return df
 
     def get_exchange_spot_value(self,

catalyst/finance/trading.py

@@ -95,24 +95,11 @@ class TradingEnvironment(object):
         if not trading_calendar:
             trading_calendar = get_calendar("NYSE")
 
-        # todo: uncomment and add a well defined benchmark
+        self.benchmark_returns, self.treasury_curves = load(
-        # self.benchmark_returns, self.treasury_curves = load(
+            trading_calendar.day,
-        #     trading_calendar.day,
+            trading_calendar.schedule.index,
-        #     trading_calendar.schedule.index,
+            self.bm_symbol,
-        #     self.bm_symbol,
+        )
-        #     exchange=exchange,
-        # )
-
-        start_data = get_calendar('OPEN').first_trading_session
-        end_data = pd.Timestamp.utcnow()
-        treasure_cols = ['1month', '3month', '6month', '1year', '2year',
-                         '3year', '5year', '7year', '10year', '20year', '30year']
-        self.benchmark_returns = pd.DataFrame(data=0.001,
-                                              index=pd.date_range(start_data, end_data),
-                                              columns=['close'])
-        self.treasury_curves = pd.DataFrame(data=0.001,
-                                            index=pd.date_range(start_data, end_data),
-                                            columns=treasure_cols)
 
         self.exchange_tz = exchange_tz
 

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +1 @@
-0xf0ee6b27b759c9893ce4f094b49ad28fd15a23e4
+0x39a54f480d922a58c963de8091a6c9afc69db2cf

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +1 @@
-0xa64927358a82254be92eb1f1cb01de68d1787004
+0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc

catalyst/marketplace/marketplace.py

@@ -69,10 +69,7 @@ class Marketplace:
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(MARKETPLACE_CONTRACT_ABI)
-        abi_url = abi_url.read().decode(
+        abi = json.load(abi_url)
-                abi_url.info().get_content_charset())
-
-        abi = json.loads(abi_url)
 
         self.mkt_contract = self.web3.eth.contract(
             self.mkt_contract_address,
@@ -86,10 +83,7 @@ class Marketplace:
                 contract_url.info().get_content_charset()).strip())
 
         abi_url = urllib.urlopen(ENIGMA_CONTRACT_ABI)
-        abi_url = abi_url.read().decode(
+        abi = json.load(abi_url)
-                abi_url.info().get_content_charset())
-
-        abi = json.loads(abi_url)
 
         self.eng_contract = self.web3.eth.contract(
             self.eng_contract_address,
@@ -132,10 +126,9 @@ class Marketplace:
         else:
             while True:
                 for i in range(0, len(self.addresses)):
-                    print('{}\t{}\t{}\t{}'.format(
+                    print('{}\t{}\t{}'.format(
                         i,
                         self.addresses[i]['pubAddr'],
-                        self.addresses[i]['wallet'].ljust(10),
                         self.addresses[i]['desc'])
                     )
                 address_i = int(input('Choose your address associated with '
@@ -152,7 +145,7 @@ class Marketplace:
 
     def sign_transaction(self, tx):
 
-        url = 'https://www.mycrypto.com/#offline-transaction'
+        url = 'https://www.myetherwallet.com/#offline-transaction'
         print('\nVisit {url} and enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
@@ -437,9 +430,10 @@ class Marketplace:
             merge_bundles(zsource, ztarget)
 
         else:
-            shutil.rmtree(bundle_folder, ignore_errors=True)
             os.rename(tmp_bundle, bundle_folder)
 
+        pass
+
     def ingest(self, ds_name=None, start=None, end=None, force_download=False):
 
         if ds_name is None:
@@ -504,29 +498,20 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address,
+            key, secret = get_key_secret(address)
-                                         self.addresses[address_i]['wallet'])
 
         headers = get_signed_headers(ds_name, key, secret)
-        log.info('Starting download of dataset for ingestion...')
+        log.debug('Starting download of dataset for ingestion...')
         r = requests.post(
             '{}/marketplace/ingest'.format(AUTH_SERVER),
             headers=headers,
             stream=True,
         )
         if r.status_code == 200:
-            log.info('Dataset downloaded successfully. Processing dataset...')
             target_path = get_temp_bundles_folder()
             try:
                 decoder = MultipartDecoder.from_response(r)
-                # with maybe_show_progress(
-                #     iter(decoder.parts),
-                #     True,
-                #     label='Processing files') as part:
-                counter = 1
                 for part in decoder.parts:
-                    log.info("Processing file {} of {}".format(
-                        counter, len(decoder.parts)))
                     h = part.headers[b'Content-Disposition'].decode('utf-8')
                     # Extracting the filename from the header
                     name = re.search(r'filename="(.*)"', h).group(1)
@@ -540,7 +525,6 @@ class Marketplace:
                         f.write(part.content)
 
                     self.process_temp_bundle(ds_name, filename)
-                    counter += 1
 
             except NonMultipartContentTypeException:
                 response = r.json()
@@ -608,6 +592,7 @@ class Marketplace:
             folder = get_bundle_folder(ds_name, data_frequency)
 
         shutil.rmtree(folder)
+        pass
 
     def create_metadata(self, key, secret, ds_name, data_frequency, desc,
                         has_history=True, has_live=True):
@@ -643,7 +628,7 @@ class Marketplace:
     def register(self):
         while True:
             desc = input('Enter the name of the dataset to register: ')
-            dataset = desc.lower().strip()
+            dataset = desc.lower()
             provider_info = self.mkt_contract.functions.getDataProviderInfo(
                 Web3.toHex(dataset)
             ).call()
@@ -699,8 +684,7 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address,
+            key, secret = get_key_secret(address)
-                                         self.addresses[address_i]['wallet'])
 
         grains = to_grains(price)
 
@@ -781,21 +765,18 @@ class Marketplace:
             key = match['key']
             secret = match['secret']
         else:
-            key, secret = get_key_secret(provider_info[0], match['wallet'])
+            key, secret = get_key_secret(provider_info[0])
 
+        headers = get_signed_headers(dataset, key, secret)
         filenames = glob.glob(os.path.join(datadir, '*.csv'))
 
         if not filenames:
             raise MarketplaceNoCSVFiles(datadir=datadir)
 
         files = []
-        for idx, file in enumerate(filenames):
+        for file in filenames:
-            log.info('Uploading file {} of {}: {}'.format(
-                idx+1, len(filenames), file))
-            files = []
             files.append(('file', open(file, 'rb')))
 
-            headers = get_signed_headers(dataset, key, secret)
         r = requests.post('{}/marketplace/publish'.format(AUTH_SERVER),
                           files=files,
                           headers=headers)
@@ -808,7 +789,4 @@ class Marketplace:
             raise MarketplaceHTTPRequest(request='upload file',
                                          error=r.json()['error'])
 
-            log.info('File processed successfully.')
+        print('Dataset {} uploaded successfully.'.format(dataset))
-
-        print('\nDataset {} uploaded and processed successfully.'.format(
-            dataset))

catalyst/marketplace/utils/auth_utils.py

@@ -10,10 +10,10 @@ from catalyst.marketplace.marketplace_errors import (
     MarketplaceEmptySignature)
 from catalyst.marketplace.utils.path_utils import (
     get_user_pubaddr, save_user_pubaddr)
-from catalyst.constants import AUTH_SERVER, SUPPORTED_WALLETS
+from catalyst.constants import AUTH_SERVER
 
 
-def get_key_secret(pubAddr, wallet):
+def get_key_secret(pubAddr, wallet='mew'):
     """
     Obtain a new key/secret pair from authentication server
 
@@ -43,22 +43,21 @@ def get_key_secret(pubAddr, wallet):
     auth_type, auth_info = header.split(None, 1)
     d = requests.utils.parse_dict_header(auth_info)
 
-    nonce = 'Catalyst nonce: 0x{}'.format(d['nonce'])
+    nonce = '0x{}'.format(d['nonce'])
 
-    if wallet in SUPPORTED_WALLETS:
+    if wallet == 'mew':
-        url = 'https://www.mycrypto.com/signmsg.html'
+        url = 'https://www.myetherwallet.com/signmsg.html'
 
         print('\nObtaining a key/secret pair to streamline all future '
               'requests with the authentication server.\n'
               'Visit {url} and sign the '
-              'following message (copy the entire line, without the '
+              'following message:\n{nonce}'.format(
-              'line break at the end):\n\n{nonce}'.format(
                 url=url,
                 nonce=nonce))
 
         webbrowser.open_new(url)
 
-        signature = input('\nCopy and Paste the "sig" field from '
+        signature = input('Copy and Paste the "sig" field from '
                           'the signature here (without the double quotes, '
                           'only the HEX value):\n')
     else:
@@ -92,8 +91,7 @@ def get_key_secret(pubAddr, wallet):
             addresses = get_user_pubaddr()
 
             match = next((l for l in addresses if
-                          l['pubAddr'].lower() == pubAddr.lower()), None)
+                          l['pubAddr'] == pubAddr), None)
-
             match['key'] = response.json()['key']
             match['secret'] = response.json()['secret']
 
@@ -123,7 +121,7 @@ def get_signed_headers(ds_name, key, secret):
     -------
 
     """
-    nonce = str(int(time.time() * 1000))
+    nonce = str(int(time.time()))
 
     signature = hmac.new(
         secret.encode('utf-8'),

catalyst/marketplace/utils/path_utils.py

@@ -2,7 +2,6 @@ import os
 import json
 import tarfile
 
-from catalyst.constants import SUPPORTED_WALLETS
 from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_root, ensure_directory
 from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
@@ -132,63 +131,17 @@ def get_user_pubaddr(environ=None):
             try:
                 d = data[0]['pubAddr']
             except Exception as e:
-                data = [data, ]
+                return [data, ]
-
-            changed = False
-
-            for idx, d in enumerate(data):
-                try:
-                    if d['wallet'] not in SUPPORTED_WALLETS:
-                        data[idx]['wallet'] = _choose_wallet(
-                            d['pubAddr'], False)
-                        changed = True
-                except KeyError:
-                    data[idx]['wallet'] = _choose_wallet(
-                        d['pubAddr'], True)
-                    changed = True
-
-            if changed:
-                save_user_pubaddr(data)
-
             return data
-
     else:
         data = []
-        data.append(dict(pubAddr='', desc='', wallet=''))
+        data.append(dict(pubAddr='', desc=''))
         with open(filename, 'w') as f:
             json.dump(data, f, sort_keys=False, indent=2,
                       separators=(',', ':'))
             return data
 
 
-def _choose_wallet(pubAddr, missing):
-    while True:
-        if missing:
-            print('\nYou need to specify a wallet for address '
-                  '{}.'.format(pubAddr))
-        else:
-            print('\nThe wallet specified for address {} is not '
-                  'supported.'.format(pubAddr))
-
-        print('Please choose among the following options:')
-        for idx, wallet in enumerate(SUPPORTED_WALLETS):
-            print('{}\t{}'.format(idx, wallet))
-
-        lw = len(SUPPORTED_WALLETS)-1
-        w = input('Choose a number between 0 and {}: '.format(
-                    lw))
-        try:
-            w = int(w)
-        except ValueError:
-            print('Enter a number between 0 and {}'.format(lw))
-        else:
-            if w not in range(0, lw+1):
-                print('Enter a number between 0 and '
-                      '{}'.format(lw))
-            else:
-                return SUPPORTED_WALLETS[w]
-
-
 def save_user_pubaddr(data, environ=None):
     """
     Saves the user's public addresses and their related metadata in

docs/source/install.rst

@@ -562,10 +562,6 @@ If after following the instructions above, and going through the
 *Troubleshooting* sections, you still experience problems installing Catalyst,
 you can seek additional help through the following channels:
 
-- Join our `Catalyst Forum <https://catalyst.enigma.co/>`_, and browse a variety
-  of topics and conversations around common issues that others face when using
-  Catalyst, and how to resolve them. And join the conversation!
-
 - Join our `Discord community <https://discord.gg/SJK32GY>`_, and head over 
   the #catalyst_dev channel where many other users (as well as the project 
   developers) hang out, and can assist you with your particular issue. The 

docs/source/releases.rst

@@ -2,47 +2,6 @@
 Release Notes
 =============
 
-Version 0.5.8
-^^^^^^^^^^^^^
-**Release Date**: 2018-03-29
-
-Bug Fixes
-~~~~~~~~~
-- Fix Data Marketplace release on mainnet
-
-Version 0.5.7
-^^^^^^^^^^^^^
-**Release Date**: 2018-03-29
-
-Build
-~~~~~
-- Data Marketplace deployed on mainnet.
-- Added progress indicators for publishing data, and made the data publishing
-  synchronous to provide feedback to the publisher.
-
-Bug Fixes
-~~~~~~~~~
-- fixes in storing and loading the state :issue:`214`, 
-  :issue:`287`
-
-Version 0.5.6
-^^^^^^^^^^^^^
-**Release Date**: 2018-03-22
-
-Build
-~~~~~
-- Data Marketplace: ensures compatibility across wallets, now fully supporting 
-  ``ledger``, ``trezor``, ``keystore``, ``private key``. Partial support for 
-  ``metamask`` (includes sign_msg, but not sign_tx). Current support for 
-  ``Digital Bitbox`` is unknown, but believed to be supported.
-- Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto.
-- Data Marketplace: Added progress indicator for data ingestion.
-
-Bug Fixes
-~~~~~~~~~
-- Changed benchmark to be constant, so it doesn't ingest data at all. Temporary
-  fix for :issue:`271`, :issue:`285`
-
 Version 0.5.5
 ^^^^^^^^^^^^^
 **Release Date**: 2018-03-19

etc/python2.7-environment.yml

@@ -5,7 +5,6 @@ channels:
 dependencies:
 - certifi=2016.2.28=py27_0
 - mkl=2017.0.3
-- matplotlib=2.1.2=py36_0
 - numpy=1.13.1=py27_0
 - openssl=1.0.2l
 - pip=9.0.1=py27_1
@@ -40,7 +39,7 @@ dependencies:
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.1.0
+  - matplotlib==2.1.2
   - multipledispatch==0.4.9
   - networkx==2.0
   - numexpr==2.6.4