From b0dce13672e23d7acd8f6e9dc7c6c2ed6bbfc0c5 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Fri, 9 Mar 2018 11:01:19 +0200 Subject: [PATCH 1/5] BUG: removed get_open_orders from buy_low_sell_high --- catalyst/examples/buy_low_sell_high.py | 1 - 1 file changed, 1 deletion(-) diff --git a/catalyst/examples/buy_low_sell_high.py b/catalyst/examples/buy_low_sell_high.py index 075e2f71..ed5e211c 100644 --- a/catalyst/examples/buy_low_sell_high.py +++ b/catalyst/examples/buy_low_sell_high.py @@ -7,7 +7,6 @@ from catalyst.api import ( order_target_percent, symbol, record, - get_open_orders, ) from catalyst.exchange.utils.stats_utils import get_pretty_stats from catalyst.utils.run_algo import run_algorithm From f0606b5ea4f8ae32804e058b6a235eac834e269b Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Mar 2018 09:50:20 -0700 Subject: [PATCH 2/5] BLD: [mktplace] clean --dataset param optional --- catalyst/__main__.py | 6 --- catalyst/marketplace/marketplace.py | 69 ++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 74b65819..f6505054 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -845,14 +845,8 @@ def ingest(ctx, dataset, data_frequency, start, end): def clean(ctx, dataset): """Clean/Remove local data for a given dataset. """ - if dataset is None: - ctx.fail("must specify a dataset to clean up with '--dataset'\n" - "List available dataset on the marketplace with " - "'catalyst marketplace ls'") - click.echo('Cleaning data source: {}'.format(dataset), sys.stdout) marketplace = Marketplace() marketplace.clean(dataset) - click.echo('Done', sys.stdout) @marketplace.command() diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index 22344c1b..ec2627f4 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -155,14 +155,14 @@ class Marketplace: 'Gas Price:\t\t[Accept the default value]\n' 'Nonce:\t\t\t{nonce}\n' 'Data:\t\t\t{data}\n'.format( - url=url, - _from=tx['from'], - to=tx['to'], - value=tx['value'], - gas=tx['gas'], - nonce=tx['nonce'], - data=tx['data'], ) - ) + url=url, + _from=tx['from'], + to=tx['to'], + value=tx['value'], + gas=tx['gas'], + nonce=tx['nonce'], + data=tx['data'], ) + ) webbrowser.open_new(url) @@ -222,7 +222,7 @@ class Marketplace: print(df_sets) dataset_num = input('Choose the dataset you want to ' 'subscribe to [0..{}]: '.format( - df_sets.size - 1)) + df_sets.size - 1)) try: dataset_num = int(dataset_num) except ValueError: @@ -297,14 +297,14 @@ class Marketplace: 'buy: {} ENG. Get enough ENG to cover the costs of the ' 'monthly\nsubscription for what you are trying to buy, ' 'and try again.'.format( - address, from_grains(balance), price)) + address, from_grains(balance), price)) return while True: agree_pay = input('Please confirm that you agree to pay {} ENG ' 'for a monthly subscription to the dataset "{}" ' 'starting today. [default: Y] '.format( - price, dataset)) or 'y' + price, dataset)) or 'y' if agree_pay.lower() not in ('y', 'n'): print("Please answer Y or N.") else: @@ -409,7 +409,7 @@ class Marketplace: 'You can now ingest this dataset anytime during the ' 'next month by running the following command:\n' 'catalyst marketplace ingest --dataset={}'.format( - dataset, address, dataset)) + dataset, address, dataset)) def process_temp_bundle(self, ds_name, path): """ @@ -429,6 +429,7 @@ class Marketplace: bundle_folder = os.path.join( get_data_source_folder(ds_name), 'bundle' ) + ensure_directory(bundle_folder) if os.listdir(bundle_folder): zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r') @@ -454,7 +455,7 @@ class Marketplace: print(df_sets) dataset_num = input('Choose the dataset you want to ' 'ingest [0..{}]: '.format( - df_sets.size - 1)) + df_sets.size - 1)) try: dataset_num = int(dataset_num) except ValueError: @@ -495,10 +496,10 @@ class Marketplace: print('Your subscription to dataset "{}" expired on {} UTC.' 'Please renew your subscription by running:\n' 'catalyst marketplace subscribe --dataset={}'.format( - ds_name, - pd.to_datetime(check_sub[4], unit='s', utc=True), - ds_name) - ) + ds_name, + pd.to_datetime(check_sub[4], unit='s', utc=True), + ds_name) + ) if 'key' in self.addresses[address_i]: key = self.addresses[address_i]['key'] @@ -562,14 +563,40 @@ class Marketplace: return df - def clean(self, data_source_name, data_frequency=None): - data_source_name = data_source_name.lower() + def clean(self, ds_name=None, data_frequency=None): + + if ds_name is None: + mktplace_root = get_marketplace_folder() + folders = [os.path.basename(f.rstrip('/')) + for f in glob.glob('{}/*/'.format(mktplace_root)) + if 'temp_bundles' not in f] + + while True: + for idx, f in enumerate(folders): + print('{}\t{}'.format(idx, f)) + dataset_num = input('Choose the dataset you want to ' + 'clean [0..{}]: '.format( + len(folders) - 1)) + try: + dataset_num = int(dataset_num) + except ValueError: + print('Enter a number between 0 and {}'.format( + len(folders) - 1)) + else: + if dataset_num not in range(0, len(folders)): + print('Enter a number between 0 and {}'.format( + len(folders) - 1)) + else: + ds_name = folders[dataset_num] + break + + ds_name = ds_name.lower() if data_frequency is None: - folder = get_data_source_folder(data_source_name) + folder = get_data_source_folder(ds_name) else: - folder = get_bundle_folder(data_source_name, data_frequency) + folder = get_bundle_folder(ds_name, data_frequency) shutil.rmtree(folder) pass From 4694372496a6e1d008dd1bad540162e596729ecf Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Mar 2018 09:56:09 -0700 Subject: [PATCH 3/5] BUG: [mktplace] ingest mismatch dest. folder --- catalyst/marketplace/marketplace.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/catalyst/marketplace/marketplace.py b/catalyst/marketplace/marketplace.py index ec2627f4..fbaaed3a 100644 --- a/catalyst/marketplace/marketplace.py +++ b/catalyst/marketplace/marketplace.py @@ -426,10 +426,7 @@ class Marketplace: """ tmp_bundle = extract_bundle(path) - bundle_folder = os.path.join( - get_data_source_folder(ds_name), 'bundle' - ) - + bundle_folder = get_data_source_folder(ds_name) ensure_directory(bundle_folder) if os.listdir(bundle_folder): zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r') From 93ca4e990d73b1ef0bfdc7746e1aa185631c0b28 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 9 Mar 2018 13:20:50 -0500 Subject: [PATCH 4/5] BUG: fixed a dependency issue --- tests/exchange/test_bundle.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index c66fcfb4..d88723b2 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -11,7 +11,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle, \ BUNDLE_NAME_TEMPLATE from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \ get_df_from_arrays -from exchange.utils.datetime_utils import get_start_dt +from catalyst.exchange.utils.datetime_utils import get_start_dt from catalyst.exchange.utils.exchange_utils import get_exchange_folder from catalyst.exchange.utils.factory import get_exchange from catalyst.exchange.utils.stats_utils import df_to_string @@ -42,7 +42,7 @@ class TestExchangeBundle: def test_ingest_minute(self): data_frequency = 'minute' - exchange_name = 'poloniex' + exchange_name = 'binance' exchange = get_exchange(exchange_name) exchange_bundle = ExchangeBundle(exchange) @@ -50,8 +50,8 @@ class TestExchangeBundle: exchange.get_asset('eth_btc') ] - start = pd.to_datetime('2016-03-01', utc=True) - end = pd.to_datetime('2017-11-1', utc=True) + start = pd.to_datetime('2018-03-01', utc=True) + end = pd.to_datetime('2018-03-8', utc=True) log.info('ingesting exchange bundle {}'.format(exchange_name)) exchange_bundle.ingest( @@ -101,7 +101,7 @@ class TestExchangeBundle: # data_frequency = 'daily' # include_symbols = 'neo_btc,bch_btc,eth_btc' - exchange_name = 'bitfinex' + exchange_name = 'binance' data_frequency = 'minute' exchange = get_exchange(exchange_name) From 8f7d67817056e6ec9fba2a292f2c43ecb215c49c Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 9 Mar 2018 12:44:01 -0700 Subject: [PATCH 5/5] BUG: [mktplace]: fix sanitize_df to handle 1-row DFs --- catalyst/marketplace/utils/bundle_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/catalyst/marketplace/utils/bundle_utils.py b/catalyst/marketplace/utils/bundle_utils.py index 3fc9a45a..e489c0b5 100644 --- a/catalyst/marketplace/utils/bundle_utils.py +++ b/catalyst/marketplace/utils/bundle_utils.py @@ -47,8 +47,10 @@ def sanitize_df(df): # large datasets if len(df) > 100: indexes = random.sample(range(0, len(df) - 1), 100) - else: + elif len(df) > 1: indexes = range(0, len(df) - 1) + else: + indexes = [0, ] for column in df.columns: is_date = False