Merge remote-tracking branch 'remotes/origin/develop' into new_exchange_config

# Conflicts:
#	tests/exchange/test_bundle.py

catalyst/__main__.py

@@ -845,14 +845,8 @@ def ingest(ctx, dataset, data_frequency, start, end):
 def clean(ctx, dataset):
     """Clean/Remove local data for a given dataset.
     """
-    if dataset is None:
-        ctx.fail("must specify a dataset to clean up with '--dataset'\n"
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
-    click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
     marketplace = Marketplace()
     marketplace.clean(dataset)
-    click.echo('Done', sys.stdout)
 
 
 @marketplace.command()

catalyst/examples/buy_low_sell_high.py

@@ -7,7 +7,6 @@ from catalyst.api import (
     order_target_percent,
     symbol,
     record,
-    get_open_orders,
 )
 from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm

catalyst/marketplace/marketplace.py

@@ -155,14 +155,14 @@ class Marketplace:
               'Gas Price:\t\t[Accept the default value]\n'
               'Nonce:\t\t\t{nonce}\n'
               'Data:\t\t\t{data}\n'.format(
-            url=url,
-            _from=tx['from'],
-            to=tx['to'],
-            value=tx['value'],
-            gas=tx['gas'],
-            nonce=tx['nonce'],
-            data=tx['data'], )
-        )
+                url=url,
+                _from=tx['from'],
+                to=tx['to'],
+                value=tx['value'],
+                gas=tx['gas'],
+                nonce=tx['nonce'],
+                data=tx['data'], )
+              )
 
         webbrowser.open_new(url)
 
@@ -222,7 +222,7 @@ class Marketplace:
                 print(df_sets)
                 dataset_num = input('Choose the dataset you want to '
                                     'subscribe to [0..{}]: '.format(
-                    df_sets.size - 1))
+                                        df_sets.size - 1))
                 try:
                     dataset_num = int(dataset_num)
                 except ValueError:
@@ -297,14 +297,14 @@ class Marketplace:
                   'buy: {} ENG. Get enough ENG to cover the costs of the '
                   'monthly\nsubscription for what you are trying to buy, '
                   'and try again.'.format(
-                address, from_grains(balance), price))
+                    address, from_grains(balance), price))
             return
 
         while True:
             agree_pay = input('Please confirm that you agree to pay {} ENG '
                               'for a monthly subscription to the dataset "{}" '
                               'starting today. [default: Y] '.format(
-                price, dataset)) or 'y'
+                                price, dataset)) or 'y'
             if agree_pay.lower() not in ('y', 'n'):
                 print("Please answer Y or N.")
             else:
@@ -409,7 +409,7 @@ class Marketplace:
               'You can now ingest this dataset anytime during the '
               'next month by running the following command:\n'
               'catalyst marketplace ingest --dataset={}'.format(
-            dataset, address, dataset))
+                dataset, address, dataset))
 
     def process_temp_bundle(self, ds_name, path):
         """
@@ -426,9 +426,7 @@ class Marketplace:
 
         """
         tmp_bundle = extract_bundle(path)
-        bundle_folder = os.path.join(
-            get_data_source_folder(ds_name), 'bundle'
-        )
+        bundle_folder = get_data_source_folder(ds_name)
         ensure_directory(bundle_folder)
         if os.listdir(bundle_folder):
             zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
@@ -454,7 +452,7 @@ class Marketplace:
                 print(df_sets)
                 dataset_num = input('Choose the dataset you want to '
                                     'ingest [0..{}]: '.format(
-                    df_sets.size - 1))
+                                        df_sets.size - 1))
                 try:
                     dataset_num = int(dataset_num)
                 except ValueError:
@@ -495,10 +493,10 @@ class Marketplace:
             print('Your subscription to dataset "{}" expired on {} UTC.'
                   'Please renew your subscription by running:\n'
                   'catalyst marketplace subscribe --dataset={}'.format(
-                ds_name,
-                pd.to_datetime(check_sub[4], unit='s', utc=True),
-                ds_name)
-            )
+                    ds_name,
+                    pd.to_datetime(check_sub[4], unit='s', utc=True),
+                    ds_name)
+                  )
 
         if 'key' in self.addresses[address_i]:
             key = self.addresses[address_i]['key']
@@ -562,14 +560,40 @@ class Marketplace:
 
         return df
 
-    def clean(self, data_source_name, data_frequency=None):
-        data_source_name = data_source_name.lower()
+    def clean(self, ds_name=None, data_frequency=None):
+
+        if ds_name is None:
+            mktplace_root = get_marketplace_folder()
+            folders = [os.path.basename(f.rstrip('/'))
+                       for f in glob.glob('{}/*/'.format(mktplace_root))
+                       if 'temp_bundles' not in f]
+
+            while True:
+                for idx, f in enumerate(folders):
+                    print('{}\t{}'.format(idx, f))
+                dataset_num = input('Choose the dataset you want to '
+                                    'clean [0..{}]: '.format(
+                                        len(folders) - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        len(folders) - 1))
+                else:
+                    if dataset_num not in range(0, len(folders)):
+                        print('Enter a number between 0 and {}'.format(
+                            len(folders) - 1))
+                    else:
+                        ds_name = folders[dataset_num]
+                        break
+
+        ds_name = ds_name.lower()
 
         if data_frequency is None:
-            folder = get_data_source_folder(data_source_name)
+            folder = get_data_source_folder(ds_name)
 
         else:
-            folder = get_bundle_folder(data_source_name, data_frequency)
+            folder = get_bundle_folder(ds_name, data_frequency)
 
         shutil.rmtree(folder)
         pass

catalyst/marketplace/utils/bundle_utils.py

@@ -47,8 +47,10 @@ def sanitize_df(df):
     # large datasets
     if len(df) > 100:
         indexes = random.sample(range(0, len(df) - 1), 100)
-    else:
+    elif len(df) > 1:
         indexes = range(0, len(df) - 1)
+    else:
+        indexes = [0, ]
 
     for column in df.columns:
         is_date = False

tests/exchange/test_bundle.py

@@ -43,7 +43,7 @@ class TestExchangeBundle:
 
     def test_ingest_minute(self):
         data_frequency = 'minute'
-        exchange_name = 'poloniex'
+        exchange_name = 'binance'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange_name)
@@ -51,10 +51,8 @@ class TestExchangeBundle:
             exchange.get_asset('bch_eth')
         ]
 
-        start = pd.to_datetime('2018-01-01', utc=True)
-        end = pd.to_datetime('2018-03-01', utc=True)
-        # start = None
-        # end = None
+        start = pd.to_datetime('2018-03-01', utc=True)
+        end = pd.to_datetime('2018-03-8', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -111,7 +109,7 @@ class TestExchangeBundle:
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'bitfinex'
+        exchange_name = 'binance'
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)