From f0606b5ea4f8ae32804e058b6a235eac834e269b Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Mar 2018 09:50:20 -0700 Subject: [PATCH] BLD: [mktplace] clean --dataset param optional --- catalyst/__main__.py | 6 --- catalyst/marketplace/marketplace.py | 69 ++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 74b65819..f6505054 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -845,14 +845,8 @@ def ingest(ctx, dataset, data_frequency, start, end): def clean(ctx, dataset): """Clean/Remove local data for a given dataset. """ - if dataset is None: - ctx.fail("must specify a dataset to clean up with '--dataset'\n" - "List available dataset on the marketplace with " - "'catalyst marketplace ls'") - click.echo('Cleaning data source: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.clean(dataset) - click.echo('Done', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 22344c1b..ec2627f4 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -155,14 +155,14 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - url=url, - _from=tx['from'], - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], ) - ) + url=url, + _from=tx['from'], + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) webbrowser.open_new(url) @@ -222,7 +222,7 @@ class Marketplace: print(df_sets) dataset_num = input('Choose the dataset you want to ' 'subscribe to [0..{}]: '.format( - df_sets.size - 1)) + df_sets.size - 1)) try: dataset_num = int(dataset_num) except ValueError: @@ -297,14 +297,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -409,7 +409,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -429,6 +429,7 @@ class Marketplace: bundle_folder = os.path.join( get_data_source_folder(ds_name), 'bundle' ) + ensure_directory(bundle_folder) if os.listdir(bundle_folder): zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r') @@ -454,7 +455,7 @@ class Marketplace: print(df_sets) dataset_num = input('Choose the dataset you want to ' 'ingest [0..{}]: '.format( - df_sets.size - 1)) + df_sets.size - 1)) try: dataset_num = int(dataset_num) except ValueError: @@ -495,10 +496,10 @@ class Marketplace: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' 'catalyst marketplace subscribe --dataset={}'.format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -562,14 +563,40 @@ class Marketplace: return df - def clean(self, data_source_name, data_frequency=None): - data_source_name = data_source_name.lower() + def clean(self, ds_name=None, data_frequency=None): + + if ds_name is None: + mktplace_root = get_marketplace_folder() + folders = [os.path.basename(f.rstrip('/')) + for f in glob.glob('{}/*/'.format(mktplace_root)) + if 'temp_bundles' not in f] + + while True: + for idx, f in enumerate(folders): + print('{}\t{}'.format(idx, f)) + dataset_num = input('Choose the dataset you want to ' + 'clean [0..{}]: '.format( + len(folders) - 1)) + try: + dataset_num = int(dataset_num) + except ValueError: + print('Enter a number between 0 and {}'.format( + len(folders) - 1)) + else: + if dataset_num not in range(0, len(folders)): + print('Enter a number between 0 and {}'.format( + len(folders) - 1)) + else: + ds_name = folders[dataset_num] + break + + ds_name = ds_name.lower() if data_frequency is None: - folder = get_data_source_folder(data_source_name) + folder = get_data_source_folder(ds_name) else: - folder = get_bundle_folder(data_source_name, data_frequency) + folder = get_bundle_folder(ds_name, data_frequency) shutil.rmtree(folder) pass