diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 39b5e277..1b02a6d0 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -580,7 +580,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end, exchange_bundle = ExchangeBundle(exchange_name) - click.echo('Ingesting exchange bundle {}...'.format(exchange_name), + click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name), sys.stdout) exchange_bundle.ingest( data_frequency=data_frequency, diff --git a/catalyst/constants.py b/catalyst/constants.py index 3369e412..e40680c7 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -43,3 +43,6 @@ ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \ ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \ 'catalyst/master/catalyst/marketplace/' \ 'contract_enigma_abi.json' + +SUPPORTED_WALLETS = ['metamask', 'ledger', 'trezor', 'bitbox', 'keystore', + 'key'] diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index e5bea0fe..40238b90 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -199,12 +199,8 @@ class Exchange: ) assets.append(asset) - except SymbolNotFoundOnExchange: - log.debug( - 'skipping non-existent market {} {}'.format( - self.name, symbol - ) - ) + except SymbolNotFoundOnExchange as e: + log.warn(e) return assets def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False, diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 569fa6e5..a9138b23 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \ PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ get_bcolz_chunk, get_df_from_arrays, get_assets -from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \ +from catalyst.exchange.utils.datetime_utils import get_start_dt, \ get_period_label, get_month_start_end, get_year_start_end from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ save_exchange_symbols, mixin_market_params, get_catalyst_symbol @@ -232,12 +232,12 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has empty ' \ 'periods: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime( - DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - dates=[date.strftime( - DATE_TIME_FORMAT) for date in dates]) + name=asset.symbol, + start_dt=asset.start_date.strftime( + DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + dates=[date.strftime( + DATE_TIME_FORMAT) for date in dates]) if empty_rows_behavior == 'warn': log.warn(problem) @@ -286,12 +286,12 @@ class ExchangeBundle: problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \ 'identical close values on: {dates}'.format( - name=asset.symbol, - start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), - end_dt=end_dt.strftime(DATE_TIME_FORMAT), - threshold=threshold, - dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) - for date in dates]) + name=asset.symbol, + start_dt=asset.start_date.strftime(DATE_TIME_FORMAT), + end_dt=end_dt.strftime(DATE_TIME_FORMAT), + threshold=threshold, + dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT) + for date in dates]) problems.append(problem) @@ -458,7 +458,7 @@ class ExchangeBundle: last_entry = None if start is None or \ - (earliest_trade is not None and earliest_trade > start): + (earliest_trade is not None and earliest_trade > start): start = earliest_trade if last_entry is not None and (end is None or end > last_entry): @@ -598,16 +598,41 @@ class ExchangeBundle: # we want to give an end_date far in time writer = self.get_writer(start_dt, end_dt, data_frequency) if show_breakdown: - for asset in chunks: + if chunks: + for asset in chunks: + with maybe_show_progress( + chunks[asset], + show_progress, + label='Ingesting {frequency} price data for ' + '{symbol} on {exchange}'.format( + exchange=self.exchange_name, + frequency=data_frequency, + symbol=asset.symbol + )) as it: + for chunk in it: + problems += self.ingest_ctable( + asset=chunk['asset'], + data_frequency=data_frequency, + period=chunk['period'], + writer=writer, + empty_rows_behavior='strip', + cleanup=True + ) + else: + all_chunks = list(chain.from_iterable(itervalues(chunks))) + # We sort the chunks by end date to ingest most recent data first + if all_chunks: + all_chunks.sort( + key=lambda chunk: pd.to_datetime(chunk['period']) + ) with maybe_show_progress( - chunks[asset], + all_chunks, show_progress, - label='Ingesting {frequency} price data for ' - '{symbol} on {exchange}'.format( + label='Ingesting {frequency} price data on ' + '{exchange}'.format( exchange=self.exchange_name, frequency=data_frequency, - symbol=asset.symbol - )) as it: + )) as it: for chunk in it: problems += self.ingest_ctable( asset=chunk['asset'], @@ -617,30 +642,6 @@ class ExchangeBundle: empty_rows_behavior='strip', cleanup=True ) - else: - all_chunks = list(chain.from_iterable(itervalues(chunks))) - - # We sort the chunks by end date to ingest most recent data first - all_chunks.sort( - key=lambda chunk: pd.to_datetime(chunk['period']) - ) - with maybe_show_progress( - all_chunks, - show_progress, - label='Ingesting {frequency} price data on ' - '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, - )) as it: - for chunk in it: - problems += self.ingest_ctable( - asset=chunk['asset'], - data_frequency=data_frequency, - period=chunk['period'], - writer=writer, - empty_rows_behavior='strip', - cleanup=True - ) if show_report and len(problems) > 0: log.info('problems during ingestion:{}\n'.format( diff --git a/catalyst/finance/trading.py b/catalyst/finance/trading.py index 9d097cf7..bab89aac 100644 --- a/catalyst/finance/trading.py +++ b/catalyst/finance/trading.py @@ -95,11 +95,24 @@ class TradingEnvironment(object): if not trading_calendar: trading_calendar = get_calendar("NYSE") - self.benchmark_returns, self.treasury_curves = load( - trading_calendar.day, - trading_calendar.schedule.index, - self.bm_symbol, - ) + # todo: uncomment and add a well defined benchmark + # self.benchmark_returns, self.treasury_curves = load( + # trading_calendar.day, + # trading_calendar.schedule.index, + # self.bm_symbol, + # exchange=exchange, + # ) + + start_data = get_calendar('OPEN').first_trading_session + end_data = pd.Timestamp.utcnow() + treasure_cols = ['1month', '3month', '6month', '1year', '2year', + '3year', '5year', '7year', '10year', '20year', '30year'] + self.benchmark_returns = pd.DataFrame(data=0.001, + index=pd.date_range(start_data, end_data), + columns=['close']) + self.treasury_curves = pd.DataFrame(data=0.001, + index=pd.date_range(start_data, end_data), + columns=treasure_cols) self.exchange_tz = exchange_tz diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index f72bd661..a703bb70 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -20,6 +20,7 @@ from requests_toolbelt.multipart.decoder import \ from catalyst.constants import ( LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT, MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI) +from catalyst.utils.cli import maybe_show_progress from catalyst.exchange.utils.stats_utils import set_print_settings from catalyst.marketplace.marketplace_errors import ( MarketplacePubAddressEmpty, MarketplaceDatasetNotFound, @@ -126,9 +127,10 @@ class Marketplace: else: while True: for i in range(0, len(self.addresses)): - print('{}\t{}\t{}'.format( + print('{}\t{}\t{}\t{}'.format( i, self.addresses[i]['pubAddr'], + self.addresses[i]['wallet'].ljust(10), self.addresses[i]['desc']) ) address_i = int(input('Choose your address associated with ' @@ -145,7 +147,7 @@ class Marketplace: def sign_transaction(self, tx): - url = 'https://www.myetherwallet.com/#offline-transaction' + url = 'https://www.mycrypto.com/#offline-transaction' print('\nVisit {url} and enter the following parameters:\n\n' 'From Address:\t\t{_from}\n' '\n\tClick the "Generate Information" button\n\n' @@ -430,10 +432,9 @@ class Marketplace: merge_bundles(zsource, ztarget) else: + shutil.rmtree(bundle_folder, ignore_errors=True) os.rename(tmp_bundle, bundle_folder) - pass - def ingest(self, ds_name=None, start=None, end=None, force_download=False): if ds_name is None: @@ -498,20 +499,29 @@ class Marketplace: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: - key, secret = get_key_secret(address) + key, secret = get_key_secret(address, + self.addresses[address_i]['wallet']) headers = get_signed_headers(ds_name, key, secret) - log.debug('Starting download of dataset for ingestion...') + log.info('Starting download of dataset for ingestion...') r = requests.post( '{}/marketplace/ingest'.format(AUTH_SERVER), headers=headers, stream=True, ) if r.status_code == 200: + log.info('Dataset downloaded successfully. Processing dataset...') target_path = get_temp_bundles_folder() try: decoder = MultipartDecoder.from_response(r) + # with maybe_show_progress( + # iter(decoder.parts), + # True, + # label='Processing files') as part: + counter = 0 for part in decoder.parts: + log.info("Processing file {} of {}".format( + counter, len(decoder.parts))) h = part.headers[b'Content-Disposition'].decode('utf-8') # Extracting the filename from the header name = re.search(r'filename="(.*)"', h).group(1) @@ -525,6 +535,7 @@ class Marketplace: f.write(part.content) self.process_temp_bundle(ds_name, filename) + counter += 1 except NonMultipartContentTypeException: response = r.json() @@ -592,7 +603,6 @@ class Marketplace: folder = get_bundle_folder(ds_name, data_frequency) shutil.rmtree(folder) - pass def create_metadata(self, key, secret, ds_name, data_frequency, desc, has_history=True, has_live=True): @@ -684,7 +694,8 @@ class Marketplace: key = self.addresses[address_i]['key'] secret = self.addresses[address_i]['secret'] else: - key, secret = get_key_secret(address) + key, secret = get_key_secret(address, + self.addresses[address_i]['wallet']) grains = to_grains(price) @@ -765,7 +776,7 @@ class Marketplace: key = match['key'] secret = match['secret'] else: - key, secret = get_key_secret(provider_info[0]) + key, secret = get_key_secret(provider_info[0], match['wallet']) headers = get_signed_headers(dataset, key, secret) filenames = glob.glob(os.path.join(datadir, '*.csv')) diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index f4d893a8..5eb0b66a 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -10,10 +10,10 @@ from catalyst.marketplace.marketplace_errors import ( MarketplaceEmptySignature) from catalyst.marketplace.utils.path_utils import ( get_user_pubaddr, save_user_pubaddr) -from catalyst.constants import AUTH_SERVER +from catalyst.constants import AUTH_SERVER, SUPPORTED_WALLETS -def get_key_secret(pubAddr, wallet='mew'): +def get_key_secret(pubAddr, wallet): """ Obtain a new key/secret pair from authentication server @@ -43,21 +43,22 @@ def get_key_secret(pubAddr, wallet='mew'): auth_type, auth_info = header.split(None, 1) d = requests.utils.parse_dict_header(auth_info) - nonce = '0x{}'.format(d['nonce']) + nonce = 'Catalyst nonce: 0x{}'.format(d['nonce']) - if wallet == 'mew': - url = 'https://www.myetherwallet.com/signmsg.html' + if wallet in SUPPORTED_WALLETS: + url = 'https://www.mycrypto.com/signmsg.html' print('\nObtaining a key/secret pair to streamline all future ' 'requests with the authentication server.\n' 'Visit {url} and sign the ' - 'following message:\n{nonce}'.format( + 'following message (copy the entire line, without the ' + 'line break at the end):\n\n{nonce}'.format( url=url, nonce=nonce)) webbrowser.open_new(url) - signature = input('Copy and Paste the "sig" field from ' + signature = input('\nCopy and Paste the "sig" field from ' 'the signature here (without the double quotes, ' 'only the HEX value):\n') else: @@ -91,7 +92,8 @@ def get_key_secret(pubAddr, wallet='mew'): addresses = get_user_pubaddr() match = next((l for l in addresses if - l['pubAddr'] == pubAddr), None) + l['pubAddr'].lower() == pubAddr.lower()), None) + match['key'] = response.json()['key'] match['secret'] = response.json()['secret'] diff --git a/catalyst/marketplace/utils/path_utils.py b/catalyst/marketplace/utils/path_utils.py index fd4ee663..4d230271 100644 --- a/catalyst/marketplace/utils/path_utils.py +++ b/catalyst/marketplace/utils/path_utils.py @@ -2,6 +2,7 @@ import os import json import tarfile +from catalyst.constants import SUPPORTED_WALLETS from catalyst.utils.deprecate import deprecated from catalyst.utils.paths import data_root, ensure_directory from catalyst.marketplace.marketplace_errors import MarketplaceJSONError @@ -131,17 +132,63 @@ def get_user_pubaddr(environ=None): try: d = data[0]['pubAddr'] except Exception as e: - return [data, ] + data = [data, ] + + changed = False + + for idx, d in enumerate(data): + try: + if d['wallet'] not in SUPPORTED_WALLETS: + data[idx]['wallet'] = _choose_wallet( + d['pubAddr'], False) + changed = True + except KeyError: + data[idx]['wallet'] = _choose_wallet( + d['pubAddr'], True) + changed = True + + if changed: + save_user_pubaddr(data) + return data + else: data = [] - data.append(dict(pubAddr='', desc='')) + data.append(dict(pubAddr='', desc='', wallet='')) with open(filename, 'w') as f: json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':')) return data +def _choose_wallet(pubAddr, missing): + while True: + if missing: + print('\nYou need to specify a wallet for address ' + '{}.'.format(pubAddr)) + else: + print('\nThe wallet specified for address {} is not ' + 'supported.'.format(pubAddr)) + + print('Please choose among the following options:') + for idx, wallet in enumerate(SUPPORTED_WALLETS): + print('{}\t{}'.format(idx, wallet)) + + lw = len(SUPPORTED_WALLETS)-1 + w = input('Choose a number between 0 and {}: '.format( + lw)) + try: + w = int(w) + except ValueError: + print('Enter a number between 0 and {}'.format(lw)) + else: + if w not in range(0, lw+1): + print('Enter a number between 0 and ' + '{}'.format(lw)) + else: + return SUPPORTED_WALLETS[w] + + def save_user_pubaddr(data, environ=None): """ Saves the user's public addresses and their related metadata in diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 479b4558..73e9e5ff 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,6 +2,24 @@ Release Notes ============= +Version 0.5.6 +^^^^^^^^^^^^^ +**Release Date**: 2018-03-22 + +Build +~~~~~ +- Data Marketplace: ensures compatibility across wallets, now fully supporting +`ledger`, `trezor`, `keystore`, `private key`. Partial support for `metamask` +(includes sign_msg, but not sign_tx). Current support for `Digital Bitbox` is +unknown. +- Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto. +- Data Marketplace: Added progress indicator for data ingestion. + +Bug Fixes +~~~~~~~~~ +- Changed benchmark to be constant, so it doesn't ingest data at all. Temporary + fix for :issue:`271`, :issue:`285` + Version 0.5.5 ^^^^^^^^^^^^^ **Release Date**: 2018-03-19