Merge remote-tracking branch 'remotes/origin/develop' into new_exchange_config

catalyst/__main__.py

@@ -795,10 +795,6 @@ def ls(ctx):
 def subscribe(ctx, dataset):
     """Subscribe to an exisiting dataset.
     """
-    if dataset is None:
-        ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
     marketplace = Marketplace()
     marketplace.subscribe(dataset)
 
@@ -835,11 +831,6 @@ def subscribe(ctx, dataset):
 def ingest(ctx, dataset, data_frequency, start, end):
     """Ingest a dataset (requires subscription).
     """
-    if dataset is None:
-        ctx.fail("must specify a dataset to clean with '--dataset'\n"
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
-    click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
     marketplace = Marketplace()
     marketplace.ingest(dataset, data_frequency, start, end)
 

catalyst/marketplace/marketplace.py

@@ -7,6 +7,7 @@ import re
 import shutil
 import sys
 import time
+import webbrowser
 
 import bcolz
 import logbook
@@ -141,10 +142,10 @@ class Marketplace:
 
         return address, address_i
 
-    def sign_transaction(self, from_address, tx):
+    def sign_transaction(self, tx):
 
-        print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
-              'enter the following parameters:\n\n'
+        url = 'https://www.myetherwallet.com/#offline-transaction'
+        print('\nVisit {url} and enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
               'To Address:\t\t{to}\n'
@@ -153,7 +154,8 @@ class Marketplace:
               'Gas Price:\t\t[Accept the default value]\n'
               'Nonce:\t\t\t{nonce}\n'
               'Data:\t\t\t{data}\n'.format(
-                _from=from_address,
+                url=url,
+                _from=tx['from'],
                 to=tx['to'],
                 value=tx['value'],
                 gas=tx['gas'],
@@ -161,6 +163,8 @@ class Marketplace:
                 data=tx['data'], )
               )
 
+        webbrowser.open_new(url)
+
         signed_tx = input('Copy and Paste the "Signed Transaction" '
                           'field here:\n')
 
@@ -180,8 +184,7 @@ class Marketplace:
         print('\nYou can check the outcome of your transaction here:\n'
               '{}\n\n'.format(etherscan))
 
-    def list(self):
-
+    def _list(self):
         data_sources = self.mkt_contract.functions.getAllProviders().call()
 
         data = []
@@ -193,15 +196,44 @@ class Marketplace:
                             dataset=self.to_text(data_source)
                         )
                     )
+        return pd.DataFrame(data)
+
+    def list(self):
+        df = self._list()
 
-        df = pd.DataFrame(data)
         set_print_settings()
         if df.empty:
             print('There are no datasets available yet.')
         else:
             print(df)
 
-    def subscribe(self, dataset):
+    def subscribe(self, dataset=None):
+
+        if dataset is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'subscribe to [0..{}]: '.format(
+                                        df_sets.size-1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size-1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size-1))
+                    else:
+                        dataset = df_sets.iloc[dataset_num]['dataset']
+                        break
 
         dataset = dataset.lower()
 
@@ -292,13 +324,14 @@ class Marketplace:
             self.mkt_contract_address,
             grains,
         ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
         if 'ropsten' in ETH_REMOTE_NODE:
             tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
 
-        signed_tx = self.sign_transaction(address, tx)
+        signed_tx = self.sign_transaction(tx)
         try:
             tx_hash = '0x{}'.format(
                 bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
@@ -332,14 +365,15 @@ class Marketplace:
               'Now processing second transaction.')
 
         tx = self.mkt_contract.functions.subscribe(
-            Web3.toHex(dataset),
-        ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)})
+                Web3.toHex(dataset),
+                ).buildTransaction({
+                    'from': address,
+                    'nonce': self.web3.eth.getTransactionCount(address)})
 
         if 'ropsten' in ETH_REMOTE_NODE:
             tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
 
-        signed_tx = self.sign_transaction(address, tx)
+        signed_tx = self.sign_transaction(tx)
 
         try:
             tx_hash = '0x{}'.format(bin_hex(
@@ -402,7 +436,33 @@ class Marketplace:
 
         pass
 
-    def ingest(self, ds_name, start=None, end=None, force_download=False):
+    def ingest(self, ds_name=None, start=None, end=None, force_download=False):
+
+        if ds_name is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'ingest [0..{}]: '.format(
+                                        df_sets.size-1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size-1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size-1))
+                    else:
+                        ds_name = df_sets.iloc[dataset_num]['dataset']
+                        break
 
         # ds_name = ds_name.lower()
 
@@ -609,13 +669,14 @@ class Marketplace:
             grains,
             address,
         ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
         if 'ropsten' in ETH_REMOTE_NODE:
             tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
 
-        signed_tx = self.sign_transaction(address, tx)
+        signed_tx = self.sign_transaction(tx)
 
         try:
             tx_hash = '0x{}'.format(

catalyst/marketplace/utils/bundle_utils.py

@@ -29,7 +29,7 @@ def merge_bundles(zsource, ztarget):
 
     dirname = os.path.basename(ztarget.rootdir)
     bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
-    os.rename(ztarget.rootdir, bak_dir)
+    shutil.move(ztarget.rootdir, bak_dir)
 
     z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
     shutil.rmtree(bak_dir)

catalyst/utils/run_algo.py

@@ -10,6 +10,7 @@ import click
 import pandas as pd
 from six import string_types
 
+import catalyst
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
@@ -23,7 +24,7 @@ try:
     from pygments.formatters import TerminalFormatter
 
     PYGMENTS = True
-except:
+except ImportError:
     PYGMENTS = False
 from toolz import valfilter, concatv
 from functools import partial
@@ -151,6 +152,7 @@ def _run(handle_data,
         'We encourage you to report any issue on GitHub: '
         'https://github.com/enigmampc/catalyst/issues'
     )
+    log.info('Catalyst version {}'.format(catalyst.__version__))
     sleep(3)
 
     if live:
@@ -261,6 +263,15 @@ def _run(handle_data,
         # We still need to support bundles for other misc data, but we
         # can handle this later.
 
+        if start != pd.tslib.normalize_date(start) or \
+                        end != pd.tslib.normalize_date(end):
+            # todo: add to Sim_Params the option to start & end at specific times 
+            log.warn(
+                "Catalyst currently starts and ends on the start and "
+                "end of the dates specified, respectively. We hope to "
+                "Modify this and support specific times in a future release."
+            )
+
         data = DataPortalExchangeBacktest(
             exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,