Merge branch 'develop'

catalyst/__main__.py

@@ -580,7 +580,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
+    click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name),
                sys.stdout)
     exchange_bundle.ingest(
         data_frequency=data_frequency,

catalyst/constants.py

@@ -43,3 +43,6 @@ ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
 ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                       'catalyst/master/catalyst/marketplace/' \
                       'contract_enigma_abi.json'
+
+SUPPORTED_WALLETS = ['metamask', 'ledger', 'trezor', 'bitbox', 'keystore',
+                     'key']

catalyst/exchange/exchange.py

@@ -199,12 +199,8 @@ class Exchange:
                 )
                 assets.append(asset)
 
-            except SymbolNotFoundOnExchange:
-                log.debug(
-                    'skipping non-existent market {} {}'.format(
-                        self.name, symbol
-                    )
-                )
+            except SymbolNotFoundOnExchange as e:
+                log.warn(e)
         return assets
 
     def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,

catalyst/exchange/exchange_bundle.py

@@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
 from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_df_from_arrays, get_assets
-from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
+from catalyst.exchange.utils.datetime_utils import get_start_dt, \
     get_period_label, get_month_start_end, get_year_start_end
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
     save_exchange_symbols, mixin_market_params, get_catalyst_symbol
@@ -232,12 +232,12 @@ class ExchangeBundle:
 
                 problem = '{name} ({start_dt} to {end_dt}) has empty ' \
                           'periods: {dates}'.format(
-                    name=asset.symbol,
-                    start_dt=asset.start_date.strftime(
-                        DATE_TIME_FORMAT),
-                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                    dates=[date.strftime(
-                        DATE_TIME_FORMAT) for date in dates])
+                                name=asset.symbol,
+                                start_dt=asset.start_date.strftime(
+                                    DATE_TIME_FORMAT),
+                                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                                dates=[date.strftime(
+                                    DATE_TIME_FORMAT) for date in dates])
 
                 if empty_rows_behavior == 'warn':
                     log.warn(problem)
@@ -286,12 +286,12 @@ class ExchangeBundle:
 
             problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
                       'identical close values on: {dates}'.format(
-                name=asset.symbol,
-                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
-                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                threshold=threshold,
-                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
-                       for date in dates])
+                        name=asset.symbol,
+                        start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                        end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                        threshold=threshold,
+                        dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                               for date in dates])
 
             problems.append(problem)
 
@@ -458,7 +458,7 @@ class ExchangeBundle:
                 last_entry = None
 
         if start is None or \
-            (earliest_trade is not None and earliest_trade > start):
+                (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
         if last_entry is not None and (end is None or end > last_entry):
@@ -598,16 +598,41 @@ class ExchangeBundle:
         # we want to give an end_date far in time
         writer = self.get_writer(start_dt, end_dt, data_frequency)
         if show_breakdown:
-            for asset in chunks:
+            if chunks:
+                for asset in chunks:
+                    with maybe_show_progress(
+                        chunks[asset],
+                        show_progress,
+                        label='Ingesting {frequency} price data for '
+                              '{symbol} on {exchange}'.format(
+                            exchange=self.exchange_name,
+                            frequency=data_frequency,
+                            symbol=asset.symbol
+                            )) as it:
+                        for chunk in it:
+                            problems += self.ingest_ctable(
+                                asset=chunk['asset'],
+                                data_frequency=data_frequency,
+                                period=chunk['period'],
+                                writer=writer,
+                                empty_rows_behavior='strip',
+                                cleanup=True
+                            )
+        else:
+            all_chunks = list(chain.from_iterable(itervalues(chunks)))
+            # We sort the chunks by end date to ingest most recent data first
+            if all_chunks:
+                all_chunks.sort(
+                    key=lambda chunk: pd.to_datetime(chunk['period'])
+                )
                 with maybe_show_progress(
-                    chunks[asset],
+                    all_chunks,
                     show_progress,
-                    label='Ingesting {frequency} price data for '
-                          '{symbol} on {exchange}'.format(
+                    label='Ingesting {frequency} price data on '
+                          '{exchange}'.format(
                         exchange=self.exchange_name,
                         frequency=data_frequency,
-                        symbol=asset.symbol
-                    )) as it:
+                        )) as it:
                     for chunk in it:
                         problems += self.ingest_ctable(
                             asset=chunk['asset'],
@@ -617,30 +642,6 @@ class ExchangeBundle:
                             empty_rows_behavior='strip',
                             cleanup=True
                         )
-        else:
-            all_chunks = list(chain.from_iterable(itervalues(chunks)))
-
-            # We sort the chunks by end date to ingest most recent data first
-            all_chunks.sort(
-                key=lambda chunk: pd.to_datetime(chunk['period'])
-            )
-            with maybe_show_progress(
-                all_chunks,
-                show_progress,
-                label='Ingesting {frequency} price data on '
-                      '{exchange}'.format(
-                    exchange=self.exchange_name,
-                    frequency=data_frequency,
-                )) as it:
-                for chunk in it:
-                    problems += self.ingest_ctable(
-                        asset=chunk['asset'],
-                        data_frequency=data_frequency,
-                        period=chunk['period'],
-                        writer=writer,
-                        empty_rows_behavior='strip',
-                        cleanup=True
-                    )
 
         if show_report and len(problems) > 0:
             log.info('problems during ingestion:{}\n'.format(

catalyst/finance/trading.py

@@ -95,11 +95,24 @@ class TradingEnvironment(object):
         if not trading_calendar:
             trading_calendar = get_calendar("NYSE")
 
-        self.benchmark_returns, self.treasury_curves = load(
-            trading_calendar.day,
-            trading_calendar.schedule.index,
-            self.bm_symbol,
-        )
+        # todo: uncomment and add a well defined benchmark
+        # self.benchmark_returns, self.treasury_curves = load(
+        #     trading_calendar.day,
+        #     trading_calendar.schedule.index,
+        #     self.bm_symbol,
+        #     exchange=exchange,
+        # )
+
+        start_data = get_calendar('OPEN').first_trading_session
+        end_data = pd.Timestamp.utcnow()
+        treasure_cols = ['1month', '3month', '6month', '1year', '2year',
+                         '3year', '5year', '7year', '10year', '20year', '30year']
+        self.benchmark_returns = pd.DataFrame(data=0.001,
+                                              index=pd.date_range(start_data, end_data),
+                                              columns=['close'])
+        self.treasury_curves = pd.DataFrame(data=0.001,
+                                            index=pd.date_range(start_data, end_data),
+                                            columns=treasure_cols)
 
         self.exchange_tz = exchange_tz
 

catalyst/marketplace/marketplace.py

@@ -20,6 +20,7 @@ from requests_toolbelt.multipart.decoder import \
 from catalyst.constants import (
     LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
     MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
+from catalyst.utils.cli import maybe_show_progress
 from catalyst.exchange.utils.stats_utils import set_print_settings
 from catalyst.marketplace.marketplace_errors import (
     MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
@@ -126,9 +127,10 @@ class Marketplace:
         else:
             while True:
                 for i in range(0, len(self.addresses)):
-                    print('{}\t{}\t{}'.format(
+                    print('{}\t{}\t{}\t{}'.format(
                         i,
                         self.addresses[i]['pubAddr'],
+                        self.addresses[i]['wallet'].ljust(10),
                         self.addresses[i]['desc'])
                     )
                 address_i = int(input('Choose your address associated with '
@@ -145,7 +147,7 @@ class Marketplace:
 
     def sign_transaction(self, tx):
 
-        url = 'https://www.myetherwallet.com/#offline-transaction'
+        url = 'https://www.mycrypto.com/#offline-transaction'
         print('\nVisit {url} and enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
@@ -430,10 +432,9 @@ class Marketplace:
             merge_bundles(zsource, ztarget)
 
         else:
+            shutil.rmtree(bundle_folder, ignore_errors=True)
             os.rename(tmp_bundle, bundle_folder)
 
-        pass
-
     def ingest(self, ds_name=None, start=None, end=None, force_download=False):
 
         if ds_name is None:
@@ -498,20 +499,29 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address)
+            key, secret = get_key_secret(address,
+                                         self.addresses[address_i]['wallet'])
 
         headers = get_signed_headers(ds_name, key, secret)
-        log.debug('Starting download of dataset for ingestion...')
+        log.info('Starting download of dataset for ingestion...')
         r = requests.post(
             '{}/marketplace/ingest'.format(AUTH_SERVER),
             headers=headers,
             stream=True,
         )
         if r.status_code == 200:
+            log.info('Dataset downloaded successfully. Processing dataset...')
             target_path = get_temp_bundles_folder()
             try:
                 decoder = MultipartDecoder.from_response(r)
+                # with maybe_show_progress(
+                #     iter(decoder.parts),
+                #     True,
+                #     label='Processing files') as part:
+                counter = 0
                 for part in decoder.parts:
+                    log.info("Processing file {} of {}".format(
+                        counter, len(decoder.parts)))
                     h = part.headers[b'Content-Disposition'].decode('utf-8')
                     # Extracting the filename from the header
                     name = re.search(r'filename="(.*)"', h).group(1)
@@ -525,6 +535,7 @@ class Marketplace:
                         f.write(part.content)
 
                     self.process_temp_bundle(ds_name, filename)
+                    counter += 1
 
             except NonMultipartContentTypeException:
                 response = r.json()
@@ -592,7 +603,6 @@ class Marketplace:
             folder = get_bundle_folder(ds_name, data_frequency)
 
         shutil.rmtree(folder)
-        pass
 
     def create_metadata(self, key, secret, ds_name, data_frequency, desc,
                         has_history=True, has_live=True):
@@ -684,7 +694,8 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address)
+            key, secret = get_key_secret(address,
+                                         self.addresses[address_i]['wallet'])
 
         grains = to_grains(price)
 
@@ -765,7 +776,7 @@ class Marketplace:
             key = match['key']
             secret = match['secret']
         else:
-            key, secret = get_key_secret(provider_info[0])
+            key, secret = get_key_secret(provider_info[0], match['wallet'])
 
         headers = get_signed_headers(dataset, key, secret)
         filenames = glob.glob(os.path.join(datadir, '*.csv'))

catalyst/marketplace/utils/auth_utils.py

@@ -10,10 +10,10 @@ from catalyst.marketplace.marketplace_errors import (
     MarketplaceEmptySignature)
 from catalyst.marketplace.utils.path_utils import (
     get_user_pubaddr, save_user_pubaddr)
-from catalyst.constants import AUTH_SERVER
+from catalyst.constants import AUTH_SERVER, SUPPORTED_WALLETS
 
 
-def get_key_secret(pubAddr, wallet='mew'):
+def get_key_secret(pubAddr, wallet):
     """
     Obtain a new key/secret pair from authentication server
 
@@ -43,21 +43,22 @@ def get_key_secret(pubAddr, wallet='mew'):
     auth_type, auth_info = header.split(None, 1)
     d = requests.utils.parse_dict_header(auth_info)
 
-    nonce = '0x{}'.format(d['nonce'])
+    nonce = 'Catalyst nonce: 0x{}'.format(d['nonce'])
 
-    if wallet == 'mew':
-        url = 'https://www.myetherwallet.com/signmsg.html'
+    if wallet in SUPPORTED_WALLETS:
+        url = 'https://www.mycrypto.com/signmsg.html'
 
         print('\nObtaining a key/secret pair to streamline all future '
               'requests with the authentication server.\n'
               'Visit {url} and sign the '
-              'following message:\n{nonce}'.format(
+              'following message (copy the entire line, without the '
+              'line break at the end):\n\n{nonce}'.format(
                 url=url,
                 nonce=nonce))
 
         webbrowser.open_new(url)
 
-        signature = input('Copy and Paste the "sig" field from '
+        signature = input('\nCopy and Paste the "sig" field from '
                           'the signature here (without the double quotes, '
                           'only the HEX value):\n')
     else:
@@ -91,7 +92,8 @@ def get_key_secret(pubAddr, wallet='mew'):
             addresses = get_user_pubaddr()
 
             match = next((l for l in addresses if
-                          l['pubAddr'] == pubAddr), None)
+                          l['pubAddr'].lower() == pubAddr.lower()), None)
+
             match['key'] = response.json()['key']
             match['secret'] = response.json()['secret']
 

catalyst/marketplace/utils/path_utils.py

@@ -2,6 +2,7 @@ import os
 import json
 import tarfile
 
+from catalyst.constants import SUPPORTED_WALLETS
 from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_root, ensure_directory
 from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
@@ -131,17 +132,63 @@ def get_user_pubaddr(environ=None):
             try:
                 d = data[0]['pubAddr']
             except Exception as e:
-                return [data, ]
+                data = [data, ]
+
+            changed = False
+
+            for idx, d in enumerate(data):
+                try:
+                    if d['wallet'] not in SUPPORTED_WALLETS:
+                        data[idx]['wallet'] = _choose_wallet(
+                            d['pubAddr'], False)
+                        changed = True
+                except KeyError:
+                    data[idx]['wallet'] = _choose_wallet(
+                        d['pubAddr'], True)
+                    changed = True
+
+            if changed:
+                save_user_pubaddr(data)
+
             return data
+
     else:
         data = []
-        data.append(dict(pubAddr='', desc=''))
+        data.append(dict(pubAddr='', desc='', wallet=''))
         with open(filename, 'w') as f:
             json.dump(data, f, sort_keys=False, indent=2,
                       separators=(',', ':'))
             return data
 
 
+def _choose_wallet(pubAddr, missing):
+    while True:
+        if missing:
+            print('\nYou need to specify a wallet for address '
+                  '{}.'.format(pubAddr))
+        else:
+            print('\nThe wallet specified for address {} is not '
+                  'supported.'.format(pubAddr))
+
+        print('Please choose among the following options:')
+        for idx, wallet in enumerate(SUPPORTED_WALLETS):
+            print('{}\t{}'.format(idx, wallet))
+
+        lw = len(SUPPORTED_WALLETS)-1
+        w = input('Choose a number between 0 and {}: '.format(
+                    lw))
+        try:
+            w = int(w)
+        except ValueError:
+            print('Enter a number between 0 and {}'.format(lw))
+        else:
+            if w not in range(0, lw+1):
+                print('Enter a number between 0 and '
+                      '{}'.format(lw))
+            else:
+                return SUPPORTED_WALLETS[w]
+
+
 def save_user_pubaddr(data, environ=None):
     """
     Saves the user's public addresses and their related metadata in

docs/source/releases.rst

@@ -2,6 +2,24 @@
 Release Notes
 =============
 
+Version 0.5.6
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-22
+
+Build
+~~~~~
+- Data Marketplace: ensures compatibility across wallets, now fully supporting 
+`ledger`, `trezor`, `keystore`, `private key`. Partial support for `metamask` 
+(includes sign_msg, but not sign_tx). Current support for `Digital Bitbox` is 
+unknown.
+- Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto.
+- Data Marketplace: Added progress indicator for data ingestion.
+
+Bug Fixes
+~~~~~~~~~
+- Changed benchmark to be constant, so it doesn't ingest data at all. Temporary
+  fix for :issue:`271`, :issue:`285`
+
 Version 0.5.5
 ^^^^^^^^^^^^^
 **Release Date**: 2018-03-19