Merge branch 'develop'

BUG: Error when ingesting marketcap on windows

catalyst/__main__.py

@@ -14,7 +14,6 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
-from catalyst.utils.run_server import run_server
 
 try:
     __IPYTHON__
@@ -506,370 +505,6 @@ def live(ctx,
     return perf
 
 
-@main.command(name='serve')
-@click.option(
-    '-f',
-    '--algofile',
-    default=None,
-    type=click.File('r'),
-    help='The file that contains the algorithm to run.',
-)
-@click.option(
-    '-t',
-    '--algotext',
-    help='The algorithm script to run.',
-)
-@click.option(
-    '-D',
-    '--define',
-    multiple=True,
-    help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be"
-         " any python expression. These are evaluated in order so they"
-         " may refer to previously defined names.",
-)
-@click.option(
-    '--data-frequency',
-    type=click.Choice({'daily', 'minute'}),
-    default='daily',
-    show_default=True,
-    help='The data frequency of the simulation.',
-)
-@click.option(
-    '--capital-base',
-    type=float,
-    show_default=True,
-    help='The starting capital for the simulation.',
-)
-@click.option(
-    '-b',
-    '--bundle',
-    default='poloniex',
-    metavar='BUNDLE-NAME',
-    show_default=True,
-    help='The data bundle to use for the simulation.',
-)
-@click.option(
-    '--bundle-timestamp',
-    type=Timestamp(),
-    default=pd.Timestamp.utcnow(),
-    show_default=False,
-    help='The date to lookup data on or before.\n'
-         '[default: <current-time>]'
-)
-@click.option(
-    '-s',
-    '--start',
-    type=Date(tz='utc', as_timestamp=True),
-    help='The start date of the simulation.',
-)
-@click.option(
-    '-e',
-    '--end',
-    type=Date(tz='utc', as_timestamp=True),
-    help='The end date of the simulation.',
-)
-@click.option(
-    '-o',
-    '--output',
-    default='-',
-    metavar='FILENAME',
-    show_default=True,
-    help="The location to write the perf data. If this is '-' the perf"
-         " will be written to stdout.",
-)
-@click.option(
-    '--print-algo/--no-print-algo',
-    is_flag=True,
-    default=False,
-    help='Print the algorithm to stdout.',
-)
-@ipython_only(click.option(
-    '--local-namespace/--no-local-namespace',
-    is_flag=True,
-    default=None,
-    help='Should the algorithm methods be resolved in the local namespace.'
-))
-@click.option(
-    '-x',
-    '--exchange-name',
-    help='The name of the targeted exchange.',
-)
-@click.option(
-    '-n',
-    '--algo-namespace',
-    help='A label assigned to the algorithm for data storage purposes.'
-)
-@click.option(
-    '-c',
-    '--base-currency',
-    help='The base currency used to calculate statistics '
-         '(e.g. usd, btc, eth).',
-)
-@click.pass_context
-def run(ctx,
-        algofile,
-        algotext,
-        define,
-        data_frequency,
-        capital_base,
-        bundle,
-        bundle_timestamp,
-        start,
-        end,
-        output,
-        print_algo,
-        local_namespace,
-        exchange_name,
-        algo_namespace,
-        base_currency):
-    """Run a backtest for the given algorithm on the server.
-    """
-
-    if (algotext is not None) == (algofile is not None):
-        ctx.fail(
-            "must specify exactly one of '-f' / '--algofile' or"
-            " '-t' / '--algotext'",
-        )
-
-    # check that the start and end dates are passed correctly
-    if start is None and end is None:
-        # check both at the same time to avoid the case where a user
-        # does not pass either of these and then passes the first only
-        # to be told they need to pass the second argument also
-        ctx.fail(
-            "must specify dates with '-s' / '--start' and '-e' / '--end'"
-            " in backtest mode",
-        )
-    if start is None:
-        ctx.fail("must specify a start date with '-s' / '--start'"
-                 " in backtest mode")
-    if end is None:
-        ctx.fail("must specify an end date with '-e' / '--end'"
-                 " in backtest mode")
-
-    if exchange_name is None:
-        ctx.fail("must specify an exchange name '-x'")
-
-    if base_currency is None:
-        ctx.fail("must specify a base currency with '-c' in backtest mode")
-
-    if capital_base is None:
-        ctx.fail("must specify a capital base with '--capital-base'")
-
-    click.echo('Running in backtesting mode.', sys.stdout)
-
-    perf = run_server(
-        initialize=None,
-        handle_data=None,
-        before_trading_start=None,
-        analyze=None,
-        algofile=algofile,
-        algotext=algotext,
-        defines=define,
-        data_frequency=data_frequency,
-        capital_base=capital_base,
-        data=None,
-        bundle=bundle,
-        bundle_timestamp=bundle_timestamp,
-        start=start,
-        end=end,
-        output=output,
-        print_algo=print_algo,
-        local_namespace=local_namespace,
-        environ=os.environ,
-        live=False,
-        exchange=exchange_name,
-        algo_namespace=algo_namespace,
-        base_currency=base_currency,
-        analyze_live=None,
-        live_graph=False,
-        simulate_orders=True,
-        auth_aliases=None,
-        stats_output=None,
-    )
-
-    if output == '-':
-        click.echo(str(perf), sys.stdout)
-    elif output != os.devnull:  # make the catalyst magic not write any data
-        perf.to_pickle(output)
-
-    return perf
-
-
-@main.command(name='serve-live')
-@click.option(
-    '-f',
-    '--algofile',
-    default=None,
-    type=click.File('r'),
-    help='The file that contains the algorithm to run.',
-)
-@click.option(
-    '--capital-base',
-    type=float,
-    show_default=True,
-    help='The amount of capital (in base_currency) allocated to trading.',
-)
-@click.option(
-    '-t',
-    '--algotext',
-    help='The algorithm script to run.',
-)
-@click.option(
-    '-D',
-    '--define',
-    multiple=True,
-    help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be"
-         " any python expression. These are evaluated in order so they"
-         " may refer to previously defined names.",
-)
-@click.option(
-    '-o',
-    '--output',
-    default='-',
-    metavar='FILENAME',
-    show_default=True,
-    help="The location to write the perf data. If this is '-' the perf will"
-         " be written to stdout.",
-)
-@click.option(
-    '--print-algo/--no-print-algo',
-    is_flag=True,
-    default=False,
-    help='Print the algorithm to stdout.',
-)
-@ipython_only(click.option(
-    '--local-namespace/--no-local-namespace',
-    is_flag=True,
-    default=None,
-    help='Should the algorithm methods be resolved in the local namespace.'
-))
-@click.option(
-    '-x',
-    '--exchange-name',
-    help='The name of the targeted exchange.',
-)
-@click.option(
-    '-n',
-    '--algo-namespace',
-    help='A label assigned to the algorithm for data storage purposes.'
-)
-@click.option(
-    '-c',
-    '--base-currency',
-    help='The base currency used to calculate statistics '
-         '(e.g. usd, btc, eth).',
-)
-@click.option(
-    '-e',
-    '--end',
-    type=Date(tz='utc', as_timestamp=True),
-    help='An optional end date at which to stop the execution.',
-)
-@click.option(
-    '--live-graph/--no-live-graph',
-    is_flag=True,
-    default=False,
-    help='Display live graph.',
-)
-@click.option(
-    '--simulate-orders/--no-simulate-orders',
-    is_flag=True,
-    default=True,
-    help='Simulating orders enable the paper trading mode. No orders will be '
-         'sent to the exchange unless set to false.',
-)
-@click.option(
-    '--auth-aliases',
-    default=None,
-    help='Authentication file aliases for the specified exchanges. By default,'
-         'each exchange uses the "auth.json" file in the exchange folder. '
-         'Specifying an "auth2" alias would use "auth2.json". It should be '
-         'specified like this: "[exchange_name],[alias],..." For example, '
-         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
-)
-@click.pass_context
-def serve_live(ctx,
-               algofile,
-               capital_base,
-               algotext,
-               define,
-               output,
-               print_algo,
-               local_namespace,
-               exchange_name,
-               algo_namespace,
-               base_currency,
-               end,
-               live_graph,
-               auth_aliases,
-               simulate_orders):
-    """Trade live with the given algorithm on the server.
-    """
-    if (algotext is not None) == (algofile is not None):
-        ctx.fail(
-            "must specify exactly one of '-f' / '--algofile' or"
-            " '-t' / '--algotext'",
-        )
-
-    if exchange_name is None:
-        ctx.fail("must specify an exchange name '-x'")
-
-    if algo_namespace is None:
-        ctx.fail("must specify an algorithm name '-n' in live execution mode")
-
-    if base_currency is None:
-        ctx.fail("must specify a base currency '-c' in live execution mode")
-
-    if capital_base is None:
-        ctx.fail("must specify a capital base with '--capital-base'")
-
-    if simulate_orders:
-        click.echo('Running in paper trading mode.', sys.stdout)
-
-    else:
-        click.echo('Running in live trading mode.', sys.stdout)
-
-    perf = run_server(
-        initialize=None,
-        handle_data=None,
-        before_trading_start=None,
-        analyze=None,
-            algofile=algofile,
-        algotext=algotext,
-        defines=define,
-        data_frequency=None,
-        capital_base=capital_base,
-        data=None,
-        bundle=None,
-        bundle_timestamp=None,
-        start=None,
-        end=end,
-        output=output,
-        print_algo=print_algo,
-        local_namespace=local_namespace,
-        environ=os.environ,
-        live=True,
-        exchange=exchange_name,
-        algo_namespace=algo_namespace,
-        base_currency=base_currency,
-        live_graph=live_graph,
-        analyze_live=None,
-        simulate_orders=simulate_orders,
-        auth_aliases=auth_aliases,
-        stats_output=None,
-    )
-
-    if output == '-':
-        click.echo(str(perf), sys.stdout)
-    elif output != os.devnull:  # make the catalyst magic not write any data
-        perf.to_pickle(output)
-
-    return perf
-
-
 @main.command(name='ingest-exchange')
 @click.option(
     '-x',
@@ -945,7 +580,7 @@ def ingest_exchange(ctx, exchange_name, data_frequency, start, end,
 
     exchange_bundle = ExchangeBundle(exchange_name)
 
-    click.echo('Ingesting exchange bundle {}...'.format(exchange_name),
+    click.echo('Trying to ingest exchange bundle {}...'.format(exchange_name),
                sys.stdout)
     exchange_bundle.ingest(
         data_frequency=data_frequency,
@@ -1132,12 +767,18 @@ def bundles():
 @main.group()
 @click.pass_context
 def marketplace(ctx):
+    """Access the Enigma Data Marketplace to:\n
+    - Register and Publish new datasets (seller-side)\n
+    - Subscribe and Ingest premium datasets (buyer-side)\n
+    """
     pass
 
 
 @marketplace.command()
 @click.pass_context
 def ls(ctx):
+    """List all available datasets.
+    """
     click.echo('Listing of available data sources on the marketplace:',
                sys.stdout)
     marketplace = Marketplace()
@@ -1152,10 +793,8 @@ def ls(ctx):
 )
 @click.pass_context
 def subscribe(ctx, dataset):
-    if dataset is None:
+    """Subscribe to an existing dataset.
-        ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
+    """
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
     marketplace = Marketplace()
     marketplace.subscribe(dataset)
 
@@ -1190,11 +829,8 @@ def subscribe(ctx, dataset):
 )
 @click.pass_context
 def ingest(ctx, dataset, data_frequency, start, end):
-    if dataset is None:
+    """Ingest a dataset (requires subscription).
-        ctx.fail("must specify a dataset to clean with '--dataset'\n"
+    """
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
-    click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
     marketplace = Marketplace()
     marketplace.ingest(dataset, data_frequency, start, end)
 
@@ -1207,19 +843,17 @@ def ingest(ctx, dataset, data_frequency, start, end):
 )
 @click.pass_context
 def clean(ctx, dataset):
-    if dataset is None:
+    """Clean/Remove local data for a given dataset.
-        ctx.fail("must specify a dataset to ingest with '--dataset'\n"
+    """
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
-    click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
     marketplace = Marketplace()
     marketplace.clean(dataset)
-    click.echo('Done', sys.stdout)
 
 
 @marketplace.command()
 @click.pass_context
 def register(ctx):
+    """Register a new dataset.
+    """
     marketplace = Marketplace()
     marketplace.register()
 
@@ -1243,6 +877,8 @@ def register(ctx):
 )
 @click.pass_context
 def publish(ctx, dataset, datadir, watch):
+    """Publish data for a registered dataset.
+    """
     marketplace = Marketplace()
     if dataset is None:
         ctx.fail("must specify a dataset to publish data for "

catalyst/constants.py

@@ -25,8 +25,7 @@ AUTO_INGEST = False
 AUTH_SERVER = 'https://data.enigma.co'
 
 # TODO: switch to mainnet
-ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
+ETH_REMOTE_NODE = 'https://rinkeby.infura.io/'
-
 
 MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
                        'catalyst/master/catalyst/marketplace/' \
@@ -37,10 +36,13 @@ MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                            'contract_marketplace_abi.json'
 
 # TODO: switch to mainnet
-ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
+ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
-                  'master/catalyst/marketplace/' \
+                  'catalyst/master/catalyst/marketplace/' \
                   'contract_enigma_address.txt'
 
 ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                       'catalyst/master/catalyst/marketplace/' \
                       'contract_enigma_abi.json'
+
+SUPPORTED_WALLETS = ['metamask', 'ledger', 'trezor', 'bitbox', 'keystore',
+                     'key']

catalyst/examples/buy_low_sell_high.py

@@ -7,7 +7,6 @@ from catalyst.api import (
     order_target_percent,
     symbol,
     record,
-    get_open_orders,
 )
 from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm

catalyst/examples/portfolio_optimization.py

@@ -66,7 +66,7 @@ def handle_data(context, data):
         # Define portfolio optimization parameters
         n_portfolios = 50000
         results_array = np.zeros((3 + context.nassets, n_portfolios))
-        for p in xrange(n_portfolios):
+        for p in range(n_portfolios):
             weights = np.random.random(context.nassets)
             weights /= np.sum(weights)
             w = np.asmatrix(weights)

catalyst/examples/simple_loop.py

@@ -26,7 +26,7 @@ def handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='2H'
+        frequency='30T'
     )
     last_traded = prices.index[-1]
     log.info('last candle date: {}'.format(last_traded))

catalyst/exchange/ccxt/ccxt_exchange.py

@@ -190,6 +190,9 @@ class CCXT(Exchange):
                 if data_frequency == 'minute' and not freq.endswith('T'):
                     continue
 
+                elif data_frequency == 'hourly' and not freq.endswith('D'):
+                    continue
+
                 elif data_frequency == 'daily' and not freq.endswith('D'):
                     continue
 

catalyst/exchange/exchange.py

@@ -1,5 +1,4 @@
 import abc
-import pytz
 from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 from time import sleep
@@ -12,13 +11,15 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
+    NoDataAvailableOnExchange, NoValueForField, \
+    NoCandlesReceivedFromExchange, \
     TickerNotFoundError, NotEnoughCashError
 from catalyst.exchange.utils.datetime_utils import get_delta, \
     get_periods_range, \
-    get_periods, get_start_dt, get_frequency
+    get_periods, get_start_dt, get_frequency, \
+    get_candles_number_from_minutes
 from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
-    resample_history_df, has_bundle
+    resample_history_df, has_bundle, get_candles_df
 from logbook import Logger
 
 log = Logger('Exchange', level=LOG_LEVEL)
@@ -198,12 +199,8 @@ class Exchange:
                 )
                 assets.append(asset)
 
-            except SymbolNotFoundOnExchange:
+            except SymbolNotFoundOnExchange as e:
-                log.debug(
+                log.warn(e)
-                    'skipping non-existent market {} {}'.format(
-                        self.name, symbol
-                    )
-                )
         return assets
 
     def get_asset(self, symbol, data_frequency=None, is_exchange_symbol=False,
@@ -256,7 +253,8 @@ class Exchange:
             elif data_frequency is not None:
                 applies = (
                     (
-                        data_frequency == 'minute' and a.end_minute is not None)
+                        data_frequency == 'minute' and
+                        a.end_minute is not None)
                     or (
                         data_frequency == 'daily' and a.end_daily is not None)
                 )
@@ -505,49 +503,60 @@ class Exchange:
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency, supported_freqs=['T', 'D', 'H']
         )
+
+        # we want to avoid receiving empty candles
+        # so we request more than needed
+        # TODO: consider defining a const per asset
+        # and/or some retry mechanism (in each iteration request more data)
+        kExtra_minutes_candles = 150
+        requested_bar_count = bar_count + \
+            get_candles_number_from_minutes(unit,
+                                            candle_size,
+                                            kExtra_minutes_candles)
+
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
             assets=assets,
-            bar_count=bar_count,
+            bar_count=requested_bar_count,
             end_dt=end_dt if not is_current else None,
         )
 
-        series = dict()
+        # candles sanity check - verify no empty candles were received:
         for asset in candles:
-            if candles[asset]:
+            if not candles[asset]:
-                first_candle = candles[asset][0]
+                raise NoCandlesReceivedFromExchange(
-                asset_series = self.get_series_from_candles(
+                    bar_count=requested_bar_count,
-                    candles=candles[asset],
-                    start_dt=first_candle['last_traded'],
                     end_dt=end_dt,
-                    data_frequency=frequency,
+                    asset=asset,
-                    field=field,
+                    exchange=self.name)
-                )
 
-                delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
+        # for avoiding unnecessary forward fill end_dt is taken back one second
-                # Checking to make sure that the dates match
+        forward_fill_till_dt = end_dt - timedelta(seconds=1)
-                delta = get_delta(delta_candle_size, data_frequency)
-                adj_end_dt = end_dt - delta
-                last_traded = asset_series.index[-1]
 
-                if last_traded < adj_end_dt:
+        series = get_candles_df(candles=candles,
-                    raise LastCandleTooEarlyError(
+                                field=field,
-                        last_traded=last_traded,
+                                freq=frequency,
-                        end_dt=adj_end_dt,
+                                bar_count=requested_bar_count,
-                        exchange=self.name,
+                                end_dt=forward_fill_till_dt)
-                    )
-            else: # empty candle received
-                # because other assets are tz-aware, we need its tz to be set as well
-                asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc))
 
-
+        # TODO: consider how to approach this edge case
-            series[asset] = asset_series
+        # delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
+        # Checking to make sure that the dates match
+        # delta = get_delta(delta_candle_size, data_frequency)
+        # adj_end_dt = end_dt - delta
+        # last_traded = asset_series.index[-1]
+        # if last_traded < adj_end_dt:
+        #    raise LastCandleTooEarlyError(
+        #        last_traded=last_traded,
+        #        end_dt=adj_end_dt,
+        #        exchange=self.name,
+        #    )
 
         df = pd.DataFrame(series)
-        #df.dropna(inplace=True) # commented out due to issue 236
+        df.dropna(inplace=True)
 
-        return df
+        return df.tail(bar_count)
 
     def get_history_window_with_bundle(self,
                                        assets,
@@ -595,9 +604,10 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
-        # TODO: this function needs some work, we're currently using it just for benchmark data
+        # TODO: this function needs some work,
+        # we're currently using it just for benchmark data
         freq, candle_size, unit, data_frequency = get_frequency(
-            frequency, data_frequency
+            frequency, data_frequency, supported_freqs=['T', 'D']
         )
         adj_bar_count = candle_size * bar_count
         try:
@@ -621,7 +631,7 @@ class Exchange:
                 start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
                 trailing_dt = \
                     series[asset].index[-1] + get_delta(1, data_frequency) \
-                        if asset in series else start_dt
+                    if asset in series else start_dt
 
                 # The get_history method supports multiple asset
                 # Use the original frequency to let each api optimize

catalyst/exchange/exchange_algorithm.py

@@ -163,6 +163,25 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
                                                         style)
         return amount, style
 
+    def _calculate_order_target_amount(self, asset, target):
+        """
+        removes order amounts so we won't run into issues
+        when two orders are placed one after the other.
+        it then proceeds to removing positions amount at TradingAlgorithm
+        :param asset:
+        :param target:
+        :return: target
+        """
+        if asset in self.blotter.open_orders:
+            for open_order in self.blotter.open_orders[asset]:
+                current_amount = open_order.amount
+                target -= current_amount
+
+        target = super(ExchangeTradingAlgorithmBase, self). \
+            _calculate_order_target_amount(asset, target)
+
+        return target
+
     def round_order(self, amount, asset):
         """
         We need fractions with cryptocurrencies

catalyst/exchange/exchange_bundle.py

@@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
     PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError
 from catalyst.exchange.utils.bundle_utils import range_in_bundle, \
     get_bcolz_chunk, get_df_from_arrays, get_assets
-from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \
+from catalyst.exchange.utils.datetime_utils import get_start_dt, \
     get_period_label, get_month_start_end, get_year_start_end
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \
     save_exchange_symbols, mixin_market_params, get_catalyst_symbol
@@ -232,12 +232,12 @@ class ExchangeBundle:
 
                 problem = '{name} ({start_dt} to {end_dt}) has empty ' \
                           'periods: {dates}'.format(
-                    name=asset.symbol,
+                                name=asset.symbol,
-                    start_dt=asset.start_date.strftime(
+                                start_dt=asset.start_date.strftime(
-                        DATE_TIME_FORMAT),
+                                    DATE_TIME_FORMAT),
-                    end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                    dates=[date.strftime(
+                                dates=[date.strftime(
-                        DATE_TIME_FORMAT) for date in dates])
+                                    DATE_TIME_FORMAT) for date in dates])
 
                 if empty_rows_behavior == 'warn':
                     log.warn(problem)
@@ -286,12 +286,12 @@ class ExchangeBundle:
 
             problem = '{name} ({start_dt} to {end_dt}) has {threshold} ' \
                       'identical close values on: {dates}'.format(
-                name=asset.symbol,
+                        name=asset.symbol,
-                start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
+                        start_dt=asset.start_date.strftime(DATE_TIME_FORMAT),
-                end_dt=end_dt.strftime(DATE_TIME_FORMAT),
+                        end_dt=end_dt.strftime(DATE_TIME_FORMAT),
-                threshold=threshold,
+                        threshold=threshold,
-                dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
+                        dates=[pd.to_datetime(date).strftime(DATE_TIME_FORMAT)
-                       for date in dates])
+                               for date in dates])
 
             problems.append(problem)
 
@@ -458,7 +458,7 @@ class ExchangeBundle:
                 last_entry = None
 
         if start is None or \
-            (earliest_trade is not None and earliest_trade > start):
+                (earliest_trade is not None and earliest_trade > start):
             start = earliest_trade
 
         if last_entry is not None and (end is None or end > last_entry):
@@ -598,16 +598,41 @@ class ExchangeBundle:
         # we want to give an end_date far in time
         writer = self.get_writer(start_dt, end_dt, data_frequency)
         if show_breakdown:
-            for asset in chunks:
+            if chunks:
+                for asset in chunks:
+                    with maybe_show_progress(
+                        chunks[asset],
+                        show_progress,
+                        label='Ingesting {frequency} price data for '
+                              '{symbol} on {exchange}'.format(
+                            exchange=self.exchange_name,
+                            frequency=data_frequency,
+                            symbol=asset.symbol
+                            )) as it:
+                        for chunk in it:
+                            problems += self.ingest_ctable(
+                                asset=chunk['asset'],
+                                data_frequency=data_frequency,
+                                period=chunk['period'],
+                                writer=writer,
+                                empty_rows_behavior='strip',
+                                cleanup=True
+                            )
+        else:
+            all_chunks = list(chain.from_iterable(itervalues(chunks)))
+            # We sort the chunks by end date to ingest most recent data first
+            if all_chunks:
+                all_chunks.sort(
+                    key=lambda chunk: pd.to_datetime(chunk['period'])
+                )
                 with maybe_show_progress(
-                    chunks[asset],
+                    all_chunks,
                     show_progress,
-                    label='Ingesting {frequency} price data for '
+                    label='Ingesting {frequency} price data on '
-                          '{symbol} on {exchange}'.format(
+                          '{exchange}'.format(
                         exchange=self.exchange_name,
                         frequency=data_frequency,
-                        symbol=asset.symbol
+                        )) as it:
-                    )) as it:
                     for chunk in it:
                         problems += self.ingest_ctable(
                             asset=chunk['asset'],
@@ -617,30 +642,6 @@ class ExchangeBundle:
                             empty_rows_behavior='strip',
                             cleanup=True
                         )
-        else:
-            all_chunks = list(chain.from_iterable(itervalues(chunks)))
-
-            # We sort the chunks by end date to ingest most recent data first
-            all_chunks.sort(
-                key=lambda chunk: pd.to_datetime(chunk['period'])
-            )
-            with maybe_show_progress(
-                all_chunks,
-                show_progress,
-                label='Ingesting {frequency} price data on '
-                      '{exchange}'.format(
-                    exchange=self.exchange_name,
-                    frequency=data_frequency,
-                )) as it:
-                for chunk in it:
-                    problems += self.ingest_ctable(
-                        asset=chunk['asset'],
-                        data_frequency=data_frequency,
-                        period=chunk['period'],
-                        writer=writer,
-                        empty_rows_behavior='strip',
-                        cleanup=True
-                    )
 
         if show_report and len(problems) > 0:
             log.info('problems during ingestion:{}\n'.format(
@@ -843,7 +844,6 @@ class ExchangeBundle:
         field: str
         data_frequency: str
         algo_end_dt: pd.Timestamp
-        force_auto_ingest: 
 
         Returns
         -------

catalyst/exchange/exchange_data_portal.py

@@ -296,7 +296,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         bundle = self.exchange_bundles[exchange_name]  # type: ExchangeBundle
 
         freq, candle_size, unit, adj_data_frequency = get_frequency(
-            frequency, data_frequency
+            frequency, data_frequency, supported_freqs=['T', 'D']
         )
         adj_bar_count = candle_size * bar_count
 
@@ -312,7 +312,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
             algo_end_dt=self._last_available_session,
         )
 
-        start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
+        start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency)
         df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
         return df
 

catalyst/exchange/exchange_errors.py

@@ -322,3 +322,10 @@ class BalanceTooLowError(ZiplineError):
         'add positions to hold a free amount greater than {amount}, or clean '
         'the state of this algo and restart.'
     ).strip()
+
+
+class NoCandlesReceivedFromExchange(ZiplineError):
+    msg = (
+        'Although requesting {bar_count} candles until {end_dt} of asset {asset}, '
+        'an empty list of candles was received for {exchange}.'
+    ).strip()

catalyst/exchange/utils/datetime_utils.py

@@ -1,4 +1,5 @@
 import calendar
+import math
 import re
 from datetime import datetime, timedelta, date
 
@@ -248,9 +249,12 @@ def get_year_start_end(dt, first_day=None, last_day=None):
     return year_start, year_end
 
 
-def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
+def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']):
     """
-    Get the frequency parameters.
+    Takes an arbitrary candle size (e.g. 15T) and converts to the lowest
+    common denominator supported by the data bundles (e.g. 1T). The data
+    bundles only support 1T and 1D frequencies. If another frequency
+    is requested, Catalyst must request the underlying data and resample.
 
     Notes
     -----
@@ -305,14 +309,14 @@ def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'T']):
         data_frequency = 'minute'
 
     elif unit.lower() == 'h':
+        data_frequency = 'minute'
+
         if 'H' in supported_freqs:
             unit = 'H'
             alias = '{}H'.format(candle_size)
-
         else:
             candle_size = candle_size * 60
             alias = '{}T'.format(candle_size)
-            data_frequency = 'minute'
 
     else:
         raise InvalidHistoryFrequencyAlias(freq=freq)
@@ -326,3 +330,33 @@ def from_ms_timestamp(ms):
 
 def get_epoch():
     return pd.to_datetime('1970-1-1', utc=True)
+
+
+def get_candles_number_from_minutes(unit, candle_size, minutes):
+    """
+    Get the number of bars needed for the given time interval
+    in minutes.
+
+    Notes
+    -----
+    Supports only "T", "D" and "H" units
+
+    Parameters
+    ----------
+    unit: str
+    candle_size : int
+    minutes: int
+
+    Returns
+    -------
+    int
+
+    """
+    if unit == "T":
+        res = (float(minutes) / candle_size)
+    elif unit == "H":
+        res = (minutes / 60.0) / candle_size
+    else:  # unit == "D"
+        res = (minutes / 1440.0) / candle_size
+
+    return int(math.ceil(res))

catalyst/exchange/utils/exchange_utils.py

@@ -734,7 +734,7 @@ def transform_candles_to_df(candles):
     return pd.DataFrame(candles).set_index('last_traded')
 
 
-def get_candles_df(candles, field, freq, bar_count, end_dt=None):
+def get_candles_df(candles, field, freq, bar_count, end_dt):
     all_series = dict()
 
     for asset in candles:

catalyst/finance/trading.py

@@ -95,11 +95,24 @@ class TradingEnvironment(object):
         if not trading_calendar:
             trading_calendar = get_calendar("NYSE")
 
-        self.benchmark_returns, self.treasury_curves = load(
+        # todo: uncomment and add a well defined benchmark
-            trading_calendar.day,
+        # self.benchmark_returns, self.treasury_curves = load(
-            trading_calendar.schedule.index,
+        #     trading_calendar.day,
-            self.bm_symbol,
+        #     trading_calendar.schedule.index,
-        )
+        #     self.bm_symbol,
+        #     exchange=exchange,
+        # )
+
+        start_data = get_calendar('OPEN').first_trading_session
+        end_data = pd.Timestamp.utcnow()
+        treasure_cols = ['1month', '3month', '6month', '1year', '2year',
+                         '3year', '5year', '7year', '10year', '20year', '30year']
+        self.benchmark_returns = pd.DataFrame(data=0.001,
+                                              index=pd.date_range(start_data, end_data),
+                                              columns=['close'])
+        self.treasury_curves = pd.DataFrame(data=0.001,
+                                            index=pd.date_range(start_data, end_data),
+                                            columns=treasure_cols)
 
         self.exchange_tz = exchange_tz
 

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +1 @@
-0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10
+0x39a54f480d922a58c963de8091a6c9afc69db2cf

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +1 @@
-0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4
+0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc

catalyst/marketplace/marketplace.py

@@ -7,6 +7,7 @@ import re
 import shutil
 import sys
 import time
+import webbrowser
 
 import bcolz
 import logbook
@@ -19,6 +20,7 @@ from requests_toolbelt.multipart.decoder import \
 from catalyst.constants import (
     LOG_LEVEL, AUTH_SERVER, ETH_REMOTE_NODE, MARKETPLACE_CONTRACT,
     MARKETPLACE_CONTRACT_ABI, ENIGMA_CONTRACT, ENIGMA_CONTRACT_ABI)
+from catalyst.utils.cli import maybe_show_progress
 from catalyst.exchange.utils.stats_utils import set_print_settings
 from catalyst.marketplace.marketplace_errors import (
     MarketplacePubAddressEmpty, MarketplaceDatasetNotFound,
@@ -32,6 +34,7 @@ from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
 from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
     get_data_source_folder, get_marketplace_folder, \
     get_user_pubaddr, get_temp_bundles_folder, extract_bundle
+from catalyst.utils.paths import ensure_directory
 
 if sys.version_info.major < 3:
     import urllib
@@ -124,9 +127,10 @@ class Marketplace:
         else:
             while True:
                 for i in range(0, len(self.addresses)):
-                    print('{}\t{}\t{}'.format(
+                    print('{}\t{}\t{}\t{}'.format(
                         i,
                         self.addresses[i]['pubAddr'],
+                        self.addresses[i]['wallet'].ljust(10),
                         self.addresses[i]['desc'])
                     )
                 address_i = int(input('Choose your address associated with '
@@ -141,10 +145,10 @@ class Marketplace:
 
         return address, address_i
 
-    def sign_transaction(self, from_address, tx):
+    def sign_transaction(self, tx):
 
-        print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
+        url = 'https://www.mycrypto.com/#offline-transaction'
-              'enter the following parameters:\n\n'
+        print('\nVisit {url} and enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
               'To Address:\t\t{to}\n'
@@ -153,7 +157,8 @@ class Marketplace:
               'Gas Price:\t\t[Accept the default value]\n'
               'Nonce:\t\t\t{nonce}\n'
               'Data:\t\t\t{data}\n'.format(
-                _from=from_address,
+                url=url,
+                _from=tx['from'],
                 to=tx['to'],
                 value=tx['value'],
                 gas=tx['gas'],
@@ -161,6 +166,8 @@ class Marketplace:
                 data=tx['data'], )
               )
 
+        webbrowser.open_new(url)
+
         signed_tx = input('Copy and Paste the "Signed Transaction" '
                           'field here:\n')
 
@@ -172,16 +179,17 @@ class Marketplace:
     def check_transaction(self, tx_hash):
 
         if 'ropsten' in ETH_REMOTE_NODE:
-            etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
+            etherscan = 'https://ropsten.etherscan.io/tx/'
-                tx_hash)
+        elif 'rinkeby' in ETH_REMOTE_NODE:
+            etherscan = 'https://rinkeby.etherscan.io/tx/'
         else:
-            etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
+            etherscan = 'https://etherscan.io/tx/'
+        etherscan = '{}{}'.format(etherscan, tx_hash)
 
         print('\nYou can check the outcome of your transaction here:\n'
               '{}\n\n'.format(etherscan))
 
-    def list(self):
+    def _list(self):
-
         data_sources = self.mkt_contract.functions.getAllProviders().call()
 
         data = []
@@ -193,15 +201,44 @@ class Marketplace:
                             dataset=self.to_text(data_source)
                         )
                     )
+        return pd.DataFrame(data)
+
+    def list(self):
+        df = self._list()
 
-        df = pd.DataFrame(data)
         set_print_settings()
         if df.empty:
             print('There are no datasets available yet.')
         else:
             print(df)
 
-    def subscribe(self, dataset):
+    def subscribe(self, dataset=None):
+
+        if dataset is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'subscribe to [0..{}]: '.format(
+                                        df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        dataset = df_sets.iloc[dataset_num]['dataset']
+                        break
 
         dataset = dataset.lower()
 
@@ -292,13 +329,11 @@ class Marketplace:
             self.mkt_contract_address,
             grains,
         ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
-        if 'ropsten' in ETH_REMOTE_NODE:
+        signed_tx = self.sign_transaction(tx)
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
         try:
             tx_hash = '0x{}'.format(
                 bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
@@ -333,13 +368,11 @@ class Marketplace:
 
         tx = self.mkt_contract.functions.subscribe(
             Web3.toHex(dataset),
-        ).buildTransaction(
+        ).buildTransaction({
-            {'nonce': self.web3.eth.getTransactionCount(address)})
+            'from': address,
+            'nonce': self.web3.eth.getTransactionCount(address)})
 
-        if 'ropsten' in ETH_REMOTE_NODE:
+        signed_tx = self.sign_transaction(tx)
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
 
         try:
             tx_hash = '0x{}'.format(bin_hex(
@@ -392,17 +425,43 @@ class Marketplace:
         """
         tmp_bundle = extract_bundle(path)
         bundle_folder = get_data_source_folder(ds_name)
+        ensure_directory(bundle_folder)
         if os.listdir(bundle_folder):
             zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
             ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
             merge_bundles(zsource, ztarget)
 
         else:
+            shutil.rmtree(bundle_folder, ignore_errors=True)
             os.rename(tmp_bundle, bundle_folder)
 
-        pass
+    def ingest(self, ds_name=None, start=None, end=None, force_download=False):
 
-    def ingest(self, ds_name, start=None, end=None, force_download=False):
+        if ds_name is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'ingest [0..{}]: '.format(
+                                        df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        ds_name = df_sets.iloc[dataset_num]['dataset']
+                        break
 
         # ds_name = ds_name.lower()
 
@@ -440,20 +499,29 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address)
+            key, secret = get_key_secret(address,
+                                         self.addresses[address_i]['wallet'])
 
         headers = get_signed_headers(ds_name, key, secret)
-        log.debug('Starting download of dataset for ingestion...')
+        log.info('Starting download of dataset for ingestion...')
         r = requests.post(
             '{}/marketplace/ingest'.format(AUTH_SERVER),
             headers=headers,
             stream=True,
         )
         if r.status_code == 200:
+            log.info('Dataset downloaded successfully. Processing dataset...')
             target_path = get_temp_bundles_folder()
             try:
                 decoder = MultipartDecoder.from_response(r)
+                # with maybe_show_progress(
+                #     iter(decoder.parts),
+                #     True,
+                #     label='Processing files') as part:
+                counter = 0
                 for part in decoder.parts:
+                    log.info("Processing file {} of {}".format(
+                        counter, len(decoder.parts)))
                     h = part.headers[b'Content-Disposition'].decode('utf-8')
                     # Extracting the filename from the header
                     name = re.search(r'filename="(.*)"', h).group(1)
@@ -467,6 +535,7 @@ class Marketplace:
                         f.write(part.content)
 
                     self.process_temp_bundle(ds_name, filename)
+                    counter += 1
 
             except NonMultipartContentTypeException:
                 response = r.json()
@@ -498,17 +567,42 @@ class Marketplace:
 
         return df
 
-    def clean(self, data_source_name, data_frequency=None):
+    def clean(self, ds_name=None, data_frequency=None):
-        data_source_name = data_source_name.lower()
+
+        if ds_name is None:
+            mktplace_root = get_marketplace_folder()
+            folders = [os.path.basename(f.rstrip('/'))
+                       for f in glob.glob('{}/*/'.format(mktplace_root))
+                       if 'temp_bundles' not in f]
+
+            while True:
+                for idx, f in enumerate(folders):
+                    print('{}\t{}'.format(idx, f))
+                dataset_num = input('Choose the dataset you want to '
+                                    'clean [0..{}]: '.format(
+                                        len(folders) - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        len(folders) - 1))
+                else:
+                    if dataset_num not in range(0, len(folders)):
+                        print('Enter a number between 0 and {}'.format(
+                            len(folders) - 1))
+                    else:
+                        ds_name = folders[dataset_num]
+                        break
+
+        ds_name = ds_name.lower()
 
         if data_frequency is None:
-            folder = get_data_source_folder(data_source_name)
+            folder = get_data_source_folder(ds_name)
 
         else:
-            folder = get_bundle_folder(data_source_name, data_frequency)
+            folder = get_bundle_folder(ds_name, data_frequency)
 
         shutil.rmtree(folder)
-        pass
 
     def create_metadata(self, key, secret, ds_name, data_frequency, desc,
                         has_history=True, has_live=True):
@@ -600,7 +694,8 @@ class Marketplace:
             key = self.addresses[address_i]['key']
             secret = self.addresses[address_i]['secret']
         else:
-            key, secret = get_key_secret(address)
+            key, secret = get_key_secret(address,
+                                         self.addresses[address_i]['wallet'])
 
         grains = to_grains(price)
 
@@ -609,13 +704,11 @@ class Marketplace:
             grains,
             address,
         ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
-        if 'ropsten' in ETH_REMOTE_NODE:
+        signed_tx = self.sign_transaction(tx)
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
 
         try:
             tx_hash = '0x{}'.format(
@@ -683,7 +776,7 @@ class Marketplace:
             key = match['key']
             secret = match['secret']
         else:
-            key, secret = get_key_secret(provider_info[0])
+            key, secret = get_key_secret(provider_info[0], match['wallet'])
 
         headers = get_signed_headers(dataset, key, secret)
         filenames = glob.glob(os.path.join(datadir, '*.csv'))

catalyst/marketplace/utils/auth_utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import hmac
+import webbrowser
 
 import requests
 import time
@@ -9,10 +10,10 @@ from catalyst.marketplace.marketplace_errors import (
     MarketplaceEmptySignature)
 from catalyst.marketplace.utils.path_utils import (
     get_user_pubaddr, save_user_pubaddr)
-from catalyst.constants import AUTH_SERVER
+from catalyst.constants import AUTH_SERVER, SUPPORTED_WALLETS
 
 
-def get_key_secret(pubAddr, wallet='mew'):
+def get_key_secret(pubAddr, wallet):
     """
     Obtain a new key/secret pair from authentication server
 
@@ -42,14 +43,22 @@ def get_key_secret(pubAddr, wallet='mew'):
     auth_type, auth_info = header.split(None, 1)
     d = requests.utils.parse_dict_header(auth_info)
 
-    nonce = '0x{}'.format(d['nonce'])
+    nonce = 'Catalyst nonce: 0x{}'.format(d['nonce'])
+
+    if wallet in SUPPORTED_WALLETS:
+        url = 'https://www.mycrypto.com/signmsg.html'
 
-    if wallet == 'mew':
         print('\nObtaining a key/secret pair to streamline all future '
               'requests with the authentication server.\n'
-              'Visit https://www.myetherwallet.com/signmsg.html and sign the '
+              'Visit {url} and sign the '
-              'following message:\n{}'.format(nonce))
+              'following message (copy the entire line, without the '
-        signature = input('Copy and Paste the "sig" field from '
+              'line break at the end):\n\n{nonce}'.format(
+                url=url,
+                nonce=nonce))
+
+        webbrowser.open_new(url)
+
+        signature = input('\nCopy and Paste the "sig" field from '
                           'the signature here (without the double quotes, '
                           'only the HEX value):\n')
     else:
@@ -83,7 +92,8 @@ def get_key_secret(pubAddr, wallet='mew'):
             addresses = get_user_pubaddr()
 
             match = next((l for l in addresses if
-                          l['pubAddr'] == pubAddr), None)
+                          l['pubAddr'].lower() == pubAddr.lower()), None)
+
             match['key'] = response.json()['key']
             match['secret'] = response.json()['secret']
 

catalyst/marketplace/utils/bundle_utils.py

@@ -1,8 +1,12 @@
 import os
+import random
+import re
 import shutil
 
 import bcolz
+import numpy as np
 import pandas as pd
+from six import string_types
 
 
 def merge_bundles(zsource, ztarget):
@@ -27,10 +31,64 @@ def merge_bundles(zsource, ztarget):
     df.drop_duplicates(inplace=True)
     df.set_index(['date', 'symbol'], drop=False, inplace=True)
 
+    sanitize_df(df)
+
     dirname = os.path.basename(ztarget.rootdir)
     bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
-    os.rename(ztarget.rootdir, bak_dir)
+    shutil.move(ztarget.rootdir, bak_dir)
 
     z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
     shutil.rmtree(bak_dir)
     return z
+
+
+def sanitize_df(df):
+    # Using a sampling method to identify dates for efficiency with
+    # large datasets
+    if len(df) > 100:
+        indexes = random.sample(range(0, len(df) - 1), 100)
+    elif len(df) > 1:
+        indexes = range(0, len(df) - 1)
+    else:
+        indexes = [0, ]
+
+    for column in df.columns:
+        is_date = False
+        for index in indexes:
+            value = df[column].iloc[index]
+            if not isinstance(value, string_types):
+                continue
+
+            # TODO: assuming that the date is at least daily
+            exp = re.compile(r'^\d{4}-\d{2}-\d{2}.*$')
+            matches = exp.findall(value)
+
+            if matches:
+                is_date = True
+                break
+
+        if is_date:
+            df[column] = pd.to_datetime(df[column])
+
+        else:
+            try:
+                ser = safely_reduce_dtype(df[column])
+                df[column] = ser
+            except Exception:
+                pass
+
+    return df
+
+
+def safely_reduce_dtype(ser):  # pandas.Series or numpy.array
+    orig_dtype = "".join(
+        [x for x in ser.dtype.name if x.isalpha()])  # float/int
+    mx = 1
+    for val in ser.values:
+        new_itemsize = np.min_scalar_type(val).itemsize
+        if mx < new_itemsize:
+            mx = new_itemsize
+        if orig_dtype == 'int':
+            mx = max(mx, 4)
+    new_dtype = orig_dtype + str(mx * 8)
+    return ser.astype(new_dtype)

catalyst/marketplace/utils/path_utils.py

@@ -2,6 +2,7 @@ import os
 import json
 import tarfile
 
+from catalyst.constants import SUPPORTED_WALLETS
 from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import data_root, ensure_directory
 from catalyst.marketplace.marketplace_errors import MarketplaceJSONError
@@ -131,17 +132,63 @@ def get_user_pubaddr(environ=None):
             try:
                 d = data[0]['pubAddr']
             except Exception as e:
-                return [data, ]
+                data = [data, ]
+
+            changed = False
+
+            for idx, d in enumerate(data):
+                try:
+                    if d['wallet'] not in SUPPORTED_WALLETS:
+                        data[idx]['wallet'] = _choose_wallet(
+                            d['pubAddr'], False)
+                        changed = True
+                except KeyError:
+                    data[idx]['wallet'] = _choose_wallet(
+                        d['pubAddr'], True)
+                    changed = True
+
+            if changed:
+                save_user_pubaddr(data)
+
             return data
+
     else:
         data = []
-        data.append(dict(pubAddr='', desc=''))
+        data.append(dict(pubAddr='', desc='', wallet=''))
         with open(filename, 'w') as f:
             json.dump(data, f, sort_keys=False, indent=2,
                       separators=(',', ':'))
             return data
 
 
+def _choose_wallet(pubAddr, missing):
+    while True:
+        if missing:
+            print('\nYou need to specify a wallet for address '
+                  '{}.'.format(pubAddr))
+        else:
+            print('\nThe wallet specified for address {} is not '
+                  'supported.'.format(pubAddr))
+
+        print('Please choose among the following options:')
+        for idx, wallet in enumerate(SUPPORTED_WALLETS):
+            print('{}\t{}'.format(idx, wallet))
+
+        lw = len(SUPPORTED_WALLETS)-1
+        w = input('Choose a number between 0 and {}: '.format(
+                    lw))
+        try:
+            w = int(w)
+        except ValueError:
+            print('Enter a number between 0 and {}'.format(lw))
+        else:
+            if w not in range(0, lw+1):
+                print('Enter a number between 0 and '
+                      '{}'.format(lw))
+            else:
+                return SUPPORTED_WALLETS[w]
+
+
 def save_user_pubaddr(data, environ=None):
     """
     Saves the user's public addresses and their related metadata in

catalyst/support/issue_227.py

@@ -0,0 +1,49 @@
+import pytz
+from datetime import datetime
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+coin = 'btc'
+base_currency = 'usd'
+n_candles = 5
+
+
+def initialize(context):
+    context.symbol = symbol('%s_%s' % (coin, base_currency))
+
+
+def handle_data_polo_partial_candles(context, data):
+    history = data.history(symbol('btc_usdt'), ['volume'],
+                           bar_count=10,
+                           frequency='4H')
+    print('\nnow: %s\n%s' % (data.current_dt, history))
+    if not hasattr(context, 'i'):
+        context.i = 0
+    context.i += 1
+    if context.i > 5:
+        raise Exception('stop')
+
+
+live = False
+
+if live:
+    run_algorithm(initialize=lambda ctx: True,
+                  handle_data=handle_data_polo_partial_candles,
+                  exchange_name='poloniex',
+                  base_currency='usdt',
+                  algo_namespace='ns',
+                  live=True,
+                  data_frequency='minute',
+                  capital_base=3000)
+else:
+    run_algorithm(initialize=lambda ctx: True,
+                  handle_data=handle_data_polo_partial_candles,
+                  exchange_name='poloniex',
+                  base_currency='usdt',
+                  algo_namespace='ns',
+                  live=False,
+                  data_frequency='minute',
+                  capital_base=3000,
+                  start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc),
+                  end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc)
+                  )

catalyst/support/issue_274.py

@@ -0,0 +1,35 @@
+import pytz
+from datetime import datetime
+from catalyst.api import symbol
+from catalyst.utils.run_algo import run_algorithm
+
+coin = 'btc'
+base_currency = 'usd'
+
+
+def initialize(context):
+    context.symbol = symbol('%s_%s' % (coin, base_currency))
+
+
+def handle_data_polo_partial_candles(context, data):
+    history = data.history(symbol('btc_usdt'), ['volume'],
+                           bar_count=10,
+                           frequency='1D')
+    print('\nnow: %s\n%s' % (data.current_dt, history))
+    if not hasattr(context, 'i'):
+        context.i = 0
+    context.i += 1
+    if context.i > 5:
+        raise Exception('stop')
+
+
+run_algorithm(initialize=lambda ctx: True,
+              handle_data=handle_data_polo_partial_candles,
+              exchange_name='poloniex',
+              base_currency='usdt',
+              algo_namespace='ns',
+              live=False,
+              data_frequency='minute',
+              capital_base=3000,
+              start=datetime(2018, 2, 2, 0, 0, 0, 0, pytz.utc),
+              end=datetime(2018, 2, 20, 0, 0, 0, 0, pytz.utc))

catalyst/utils/run_algo.py

@@ -10,6 +10,7 @@ import click
 import pandas as pd
 from six import string_types
 
+import catalyst
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
@@ -23,7 +24,7 @@ try:
     from pygments.formatters import TerminalFormatter
 
     PYGMENTS = True
-except:
+except ImportError:
     PYGMENTS = False
 from toolz import valfilter, concatv
 from functools import partial
@@ -151,6 +152,7 @@ def _run(handle_data,
         'We encourage you to report any issue on GitHub: '
         'https://github.com/enigmampc/catalyst/issues'
     )
+    log.info('Catalyst version {}'.format(catalyst.__version__))
     sleep(3)
 
     if live:
@@ -261,6 +263,15 @@ def _run(handle_data,
         # We still need to support bundles for other misc data, but we
         # can handle this later.
 
+        if start != pd.tslib.normalize_date(start) or \
+                        end != pd.tslib.normalize_date(end):
+            # todo: add to Sim_Params the option to start & end at specific times 
+            log.warn(
+                "Catalyst currently starts and ends on the start and "
+                "end of the dates specified, respectively. We hope to "
+                "Modify this and support specific times in a future release."
+            )
+
         data = DataPortalExchangeBacktest(
             exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,

catalyst/utils/run_server.py

@@ -1,103 +0,0 @@
-#!flask/bin/python
-import base64
-
-import requests
-import pandas as pd
-import json
-
-
-def convert_date(date):
-    """
-    when transferring dates by json,
-    converts it to str
-    :param date:
-    :return: str(date)
-    """
-    if isinstance(date, pd.Timestamp):
-        return date.__str__()
-
-
-def run_server(
-        initialize,
-        handle_data,
-        before_trading_start,
-        analyze,
-        algofile,
-        algotext,
-        defines,
-        data_frequency,
-        capital_base,
-        data,
-        bundle,
-        bundle_timestamp,
-        start,
-        end,
-        output,
-        print_algo,
-        local_namespace,
-        environ,
-        live,
-        exchange,
-        algo_namespace,
-        base_currency,
-        live_graph,
-        analyze_live,
-        simulate_orders,
-        auth_aliases,
-        stats_output,
-        ):
-
-    # address to send
-    url = 'http://sandbox.enigma.co/api/catalyst/serve'
-    # url = 'http://127.0.0.1:5000/api/catalyst/serve'
-
-    # argument preparation - encode the file for transfer
-    if algotext:
-        algotext = base64.b64encode(algotext)
-    else:
-        algotext = base64.b64encode(bytes(algofile.read(), 'utf-8')).decode('utf-8')
-        algofile = None
-
-    json_file = {'arguments': {
-        'initialize': initialize,
-        'handle_data': handle_data,
-        'before_trading_start': before_trading_start,
-        'analyze': analyze,
-        'algotext': algotext,
-        'defines': defines,
-        'data_frequency': data_frequency,
-        'capital_base': capital_base,
-        'data': data,
-        'bundle': bundle,
-        'bundle_timestamp': bundle_timestamp,
-        'start': start,
-        'end': end,
-        'local_namespace': local_namespace,
-        'environ': None,
-        'analyze_live': analyze_live,
-        'stats_output': stats_output,
-        'algofile': algofile,
-        'output': output,
-        'print_algo': print_algo,
-        'live': live,
-        'exchange': exchange,
-        'algo_namespace': algo_namespace,
-        'base_currency': base_currency,
-        'live_graph': live_graph,
-        'simulate_orders': simulate_orders,
-        'auth_aliases': auth_aliases,
-    }}
-
-    response = requests.post(url,
-                             json=json.dumps(
-                                    json_file,
-                                    default=convert_date
-                                )
-                             )
-
-    if response.status_code == 500:
-        raise Exception("issues with cloud connections, "
-                        "unable to run catalyst on the cloud")
-    received_data = response.json()
-    cloud_log_tail = base64.b64decode(received_data["log"])
-    print(cloud_log_tail)

docs/source/install.rst

@@ -143,7 +143,7 @@ with the following steps:
 
    .. code-block:: bash
 
-      conda create --name catalyst python=2.7 scipy zlib
+      conda create --name catalyst python=3.6 scipy zlib
 
 3. Activate the environment:
 
@@ -314,6 +314,16 @@ Troubleshooting ``pip`` Install
 
       $ sudo apt-get install python-dev
 
+----
+
+**Issue**:
+   Missing TA_Lib
+
+**Solution**:
+   Follow `these instructions
+   <https://mrjbq7.github.io/ta-lib/install.html>`_ to install the TA_Lib Python wrapper
+   (and if needed, its underlying C library as well).
+
 .. _pipenv:
 
 Installing with ``pipenv``

docs/source/releases.rst

@@ -2,6 +2,70 @@
 Release Notes
 =============
 
+Version 0.5.6
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-22
+
+Build
+~~~~~
+- Data Marketplace: ensures compatibility across wallets, now fully supporting 
+`ledger`, `trezor`, `keystore`, `private key`. Partial support for `metamask` 
+(includes sign_msg, but not sign_tx). Current support for `Digital Bitbox` is 
+unknown.
+- Data Marketplace: Switched online provider from MyEtherWallet to MyCrypto.
+- Data Marketplace: Added progress indicator for data ingestion.
+
+Bug Fixes
+~~~~~~~~~
+- Changed benchmark to be constant, so it doesn't ingest data at all. Temporary
+  fix for :issue:`271`, :issue:`285`
+
+Version 0.5.5
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-19
+
+Bug Fixes
+~~~~~~~~~
+- Fixed an issue with the data history in daily frequency :issue:`274`
+- Fix hourly frequency issues :issue:`227` and :issue:`114`
+
+Version 0.5.4
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-14
+
+Build
+~~~~~
+- Switched Data Marketplace from Ropstein testnet to Rinkeby testnet after 
+  incorporating changes resulting from the marketplace contract audit
+- Several usability improvements of the Data Marketplace that make the 
+  `--dataset` parameter optional. If it is not included in the command line, 
+  will list available datasets, and let you choose interactively.
+
+Bug Fixes
+~~~~~~~~~
+- Fix Binance requirement of symbol to be included in the cancelled order 
+  :issue:`204`
+- Fix `notenoughcasherror` when an open order is filled minutes later 
+  :issue:`237`
+- Properly handle of empty candles received from exchanges :issue:`236`
+- Added a function to reduce open orders amount from calculated target/amount 
+  for target orders :issue:`243`
+- Fix missing file in live trading mode on date change :issue:`252`, 
+  :issue:`253`
+- Upgraded Data Marketplace to Web3==4.0.0b11, which was breaking some 
+  functionality from prior version 4.0.0b7 :issue:`257`
+- Always request more data to avoid empty bars and always give the exact bar
+  number :issue:`260`
+
+Documentation
+~~~~~~~~~~~~~
+- PyCharm documentation :issue:`195`
+- Added TA-Lib troubleshooting instructions
+- Added instructions on how to create a Conda environment for Python 3.6, and
+  updated Visual C++ instructions for Windows and Python 3
+- Linking example algorithms in the documentation to their sources
+
+
 Version 0.5.3
 ^^^^^^^^^^^^^
 **Release Date**: 2018-02-09

etc/python2.7-environment.yml

@@ -5,7 +5,6 @@ channels:
 dependencies:
 - certifi=2016.2.28=py27_0
 - mkl=2017.0.3
-- matplotlib=2.1.2=py36_0
 - numpy=1.13.1=py27_0
 - openssl=1.0.2l
 - pip=9.0.1=py27_1
@@ -40,7 +39,7 @@ dependencies:
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.1.0
+  - matplotlib==2.1.2
   - multipledispatch==0.4.9
   - networkx==2.0
   - numexpr==2.6.4

tests/exchange/test_bundle.py

@@ -11,7 +11,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
 from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
     get_df_from_arrays
-from exchange.utils.datetime_utils import get_start_dt
+from catalyst.exchange.utils.datetime_utils import get_start_dt
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder
 from catalyst.exchange.utils.factory import get_exchange
 from catalyst.exchange.utils.stats_utils import df_to_string
@@ -42,7 +42,7 @@ class TestExchangeBundle:
 
     def test_ingest_minute(self):
         data_frequency = 'minute'
-        exchange_name = 'poloniex'
+        exchange_name = 'binance'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
@@ -50,8 +50,8 @@ class TestExchangeBundle:
             exchange.get_asset('eth_btc')
         ]
 
-        start = pd.to_datetime('2016-03-01', utc=True)
+        start = pd.to_datetime('2018-03-01', utc=True)
-        end = pd.to_datetime('2017-11-1', utc=True)
+        end = pd.to_datetime('2018-03-8', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -101,7 +101,7 @@ class TestExchangeBundle:
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'bitfinex'
+        exchange_name = 'binance'
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)

tests/marketplace/test_marketplace.py

@@ -1,6 +1,5 @@
 from catalyst.marketplace.marketplace import Marketplace
 from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
-import pandas as pd
 
 
 class TestMarketplace(WithLogger, ZiplineTestCase):
@@ -16,12 +15,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
 
     def test_subscribe(self):
         marketplace = Marketplace()
-        marketplace.subscribe('marketcap2222')
+        marketplace.subscribe('marketcap')
         pass
 
     def test_ingest(self):
         marketplace = Marketplace()
-        ds_def = marketplace.ingest('github')
+        ds_def = marketplace.ingest('marketcap')
         pass
 
     def test_publish(self):