diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 03669486..74b65819 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -795,10 +795,6 @@ def ls(ctx): def subscribe(ctx, dataset): """Subscribe to an exisiting dataset. """ - if dataset is None: - ctx.fail("must specify a dataset to subscribe to with '--dataset'\n" - "List available dataset on the marketplace with " - "'catalyst marketplace ls'") marketplace = Marketplace() marketplace.subscribe(dataset) @@ -835,11 +831,6 @@ def subscribe(ctx, dataset): def ingest(ctx, dataset, data_frequency, start, end): """Ingest a dataset (requires subscription). """ - if dataset is None: - ctx.fail("must specify a dataset to clean with '--dataset'\n" - "List available dataset on the marketplace with " - "'catalyst marketplace ls'") - click.echo('Ingesting data: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.ingest(dataset, data_frequency, start, end) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index e972b750..225b2459 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -7,6 +7,7 @@ import re import shutil import sys import time +import webbrowser import bcolz import logbook @@ -141,10 +142,10 @@ class Marketplace: return address, address_i - def sign_transaction(self, from_address, tx): + def sign_transaction(self, tx): - print('\nVisit https://www.myetherwallet.com/#offline-transaction and ' - 'enter the following parameters:\n\n' + url = 'https://www.myetherwallet.com/#offline-transaction' + print('\nVisit {url} and enter the following parameters:\n\n' 'From Address:\t\t{_from}\n' '\n\tClick the "Generate Information" button\n\n' 'To Address:\t\t{to}\n' @@ -153,7 +154,8 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - _from=from_address, + url=url, + _from=tx['from'], to=tx['to'], value=tx['value'], gas=tx['gas'], @@ -161,6 +163,8 @@ class Marketplace: data=tx['data'], ) ) + webbrowser.open_new(url) + signed_tx = input('Copy and Paste the "Signed Transaction" ' 'field here:\n') @@ -180,8 +184,7 @@ class Marketplace: print('\nYou can check the outcome of your transaction here:\n' '{}\n\n'.format(etherscan)) - def list(self): - + def _list(self): data_sources = self.mkt_contract.functions.getAllProviders().call() data = [] @@ -193,15 +196,44 @@ class Marketplace: dataset=self.to_text(data_source) ) ) + return pd.DataFrame(data) + + def list(self): + df = self._list() - df = pd.DataFrame(data) set_print_settings() if df.empty: print('There are no datasets available yet.') else: print(df) - def subscribe(self, dataset): + def subscribe(self, dataset=None): + + if dataset is None: + + df_sets = self._list() + if df_sets.empty: + print('There are no datasets available yet.') + return + + set_print_settings() + while True: + print(df_sets) + dataset_num = input('Choose the dataset you want to ' + 'subscribe to [0..{}]: '.format( + df_sets.size-1)) + try: + dataset_num = int(dataset_num) + except ValueError: + print('Enter a number between 0 and {}'.format( + df_sets.size-1)) + else: + if dataset_num not in range(0, df_sets.size): + print('Enter a number between 0 and {}'.format( + df_sets.size-1)) + else: + dataset = df_sets.iloc[dataset_num]['dataset'] + break dataset = dataset.lower() @@ -292,13 +324,14 @@ class Marketplace: self.mkt_contract_address, grains, ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)} + {'from': address, + 'nonce': self.web3.eth.getTransactionCount(address)} ) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) - signed_tx = self.sign_transaction(address, tx) + signed_tx = self.sign_transaction(tx) try: tx_hash = '0x{}'.format( bin_hex(self.web3.eth.sendRawTransaction(signed_tx)) @@ -332,14 +365,15 @@ class Marketplace: 'Now processing second transaction.') tx = self.mkt_contract.functions.subscribe( - Web3.toHex(dataset), - ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)}) + Web3.toHex(dataset), + ).buildTransaction({ + 'from': address, + 'nonce': self.web3.eth.getTransactionCount(address)}) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) - signed_tx = self.sign_transaction(address, tx) + signed_tx = self.sign_transaction(tx) try: tx_hash = '0x{}'.format(bin_hex( @@ -402,7 +436,33 @@ class Marketplace: pass - def ingest(self, ds_name, start=None, end=None, force_download=False): + def ingest(self, ds_name=None, start=None, end=None, force_download=False): + + if ds_name is None: + + df_sets = self._list() + if df_sets.empty: + print('There are no datasets available yet.') + return + + set_print_settings() + while True: + print(df_sets) + dataset_num = input('Choose the dataset you want to ' + 'ingest [0..{}]: '.format( + df_sets.size-1)) + try: + dataset_num = int(dataset_num) + except ValueError: + print('Enter a number between 0 and {}'.format( + df_sets.size-1)) + else: + if dataset_num not in range(0, df_sets.size): + print('Enter a number between 0 and {}'.format( + df_sets.size-1)) + else: + ds_name = df_sets.iloc[dataset_num]['dataset'] + break # ds_name = ds_name.lower() @@ -609,13 +669,14 @@ class Marketplace: grains, address, ).buildTransaction( - {'nonce': self.web3.eth.getTransactionCount(address)} + {'from': address, + 'nonce': self.web3.eth.getTransactionCount(address)} ) if 'ropsten' in ETH_REMOTE_NODE: tx['gas'] = min(int(tx['gas'] * 1.5), 4700000) - signed_tx = self.sign_transaction(address, tx) + signed_tx = self.sign_transaction(tx) try: tx_hash = '0x{}'.format( diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 014887a6..086e6b2c 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -29,7 +29,7 @@ def merge_bundles(zsource, ztarget): dirname = os.path.basename(ztarget.rootdir) bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname)) - os.rename(ztarget.rootdir, bak_dir) + shutil.move(ztarget.rootdir, bak_dir) z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir) shutil.rmtree(bak_dir) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 2e9981fa..fcf25f15 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -10,6 +10,7 @@ import click import pandas as pd from six import string_types +import catalyst from catalyst.data.bundles import load from catalyst.data.data_portal import DataPortal from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \ @@ -23,7 +24,7 @@ try: from pygments.formatters import TerminalFormatter PYGMENTS = True -except: +except ImportError: PYGMENTS = False from toolz import valfilter, concatv from functools import partial @@ -151,6 +152,7 @@ def _run(handle_data, 'We encourage you to report any issue on GitHub: ' 'https://github.com/enigmampc/catalyst/issues' ) + log.info('Catalyst version {}'.format(catalyst.__version__)) sleep(3) if live: @@ -261,6 +263,15 @@ def _run(handle_data, # We still need to support bundles for other misc data, but we # can handle this later. + if start != pd.tslib.normalize_date(start) or \ + end != pd.tslib.normalize_date(end): + # todo: add to Sim_Params the option to start & end at specific times + log.warn( + "Catalyst currently starts and ends on the start and " + "end of the dates specified, respectively. We hope to " + "Modify this and support specific times in a future release." + ) + data = DataPortalExchangeBacktest( exchange_names=[exchange_name for exchange_name in exchanges], asset_finder=None,