Merge branch 'develop'

catalyst/__main__.py

@@ -14,7 +14,6 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
-from catalyst.utils.run_server import run_server
 
 try:
     __IPYTHON__
@@ -506,370 +505,6 @@ def live(ctx,
     return perf
 
 
-@main.command(name='serve')
-@click.option(
-    '-f',
-    '--algofile',
-    default=None,
-    type=click.File('r'),
-    help='The file that contains the algorithm to run.',
-)
-@click.option(
-    '-t',
-    '--algotext',
-    help='The algorithm script to run.',
-)
-@click.option(
-    '-D',
-    '--define',
-    multiple=True,
-    help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be"
-         " any python expression. These are evaluated in order so they"
-         " may refer to previously defined names.",
-)
-@click.option(
-    '--data-frequency',
-    type=click.Choice({'daily', 'minute'}),
-    default='daily',
-    show_default=True,
-    help='The data frequency of the simulation.',
-)
-@click.option(
-    '--capital-base',
-    type=float,
-    show_default=True,
-    help='The starting capital for the simulation.',
-)
-@click.option(
-    '-b',
-    '--bundle',
-    default='poloniex',
-    metavar='BUNDLE-NAME',
-    show_default=True,
-    help='The data bundle to use for the simulation.',
-)
-@click.option(
-    '--bundle-timestamp',
-    type=Timestamp(),
-    default=pd.Timestamp.utcnow(),
-    show_default=False,
-    help='The date to lookup data on or before.\n'
-         '[default: <current-time>]'
-)
-@click.option(
-    '-s',
-    '--start',
-    type=Date(tz='utc', as_timestamp=True),
-    help='The start date of the simulation.',
-)
-@click.option(
-    '-e',
-    '--end',
-    type=Date(tz='utc', as_timestamp=True),
-    help='The end date of the simulation.',
-)
-@click.option(
-    '-o',
-    '--output',
-    default='-',
-    metavar='FILENAME',
-    show_default=True,
-    help="The location to write the perf data. If this is '-' the perf"
-         " will be written to stdout.",
-)
-@click.option(
-    '--print-algo/--no-print-algo',
-    is_flag=True,
-    default=False,
-    help='Print the algorithm to stdout.',
-)
-@ipython_only(click.option(
-    '--local-namespace/--no-local-namespace',
-    is_flag=True,
-    default=None,
-    help='Should the algorithm methods be resolved in the local namespace.'
-))
-@click.option(
-    '-x',
-    '--exchange-name',
-    help='The name of the targeted exchange.',
-)
-@click.option(
-    '-n',
-    '--algo-namespace',
-    help='A label assigned to the algorithm for data storage purposes.'
-)
-@click.option(
-    '-c',
-    '--base-currency',
-    help='The base currency used to calculate statistics '
-         '(e.g. usd, btc, eth).',
-)
-@click.pass_context
-def run(ctx,
-        algofile,
-        algotext,
-        define,
-        data_frequency,
-        capital_base,
-        bundle,
-        bundle_timestamp,
-        start,
-        end,
-        output,
-        print_algo,
-        local_namespace,
-        exchange_name,
-        algo_namespace,
-        base_currency):
-    """Run a backtest for the given algorithm on the server.
-    """
-
-    if (algotext is not None) == (algofile is not None):
-        ctx.fail(
-            "must specify exactly one of '-f' / '--algofile' or"
-            " '-t' / '--algotext'",
-        )
-
-    # check that the start and end dates are passed correctly
-    if start is None and end is None:
-        # check both at the same time to avoid the case where a user
-        # does not pass either of these and then passes the first only
-        # to be told they need to pass the second argument also
-        ctx.fail(
-            "must specify dates with '-s' / '--start' and '-e' / '--end'"
-            " in backtest mode",
-        )
-    if start is None:
-        ctx.fail("must specify a start date with '-s' / '--start'"
-                 " in backtest mode")
-    if end is None:
-        ctx.fail("must specify an end date with '-e' / '--end'"
-                 " in backtest mode")
-
-    if exchange_name is None:
-        ctx.fail("must specify an exchange name '-x'")
-
-    if base_currency is None:
-        ctx.fail("must specify a base currency with '-c' in backtest mode")
-
-    if capital_base is None:
-        ctx.fail("must specify a capital base with '--capital-base'")
-
-    click.echo('Running in backtesting mode.', sys.stdout)
-
-    perf = run_server(
-        initialize=None,
-        handle_data=None,
-        before_trading_start=None,
-        analyze=None,
-        algofile=algofile,
-        algotext=algotext,
-        defines=define,
-        data_frequency=data_frequency,
-        capital_base=capital_base,
-        data=None,
-        bundle=bundle,
-        bundle_timestamp=bundle_timestamp,
-        start=start,
-        end=end,
-        output=output,
-        print_algo=print_algo,
-        local_namespace=local_namespace,
-        environ=os.environ,
-        live=False,
-        exchange=exchange_name,
-        algo_namespace=algo_namespace,
-        base_currency=base_currency,
-        analyze_live=None,
-        live_graph=False,
-        simulate_orders=True,
-        auth_aliases=None,
-        stats_output=None,
-    )
-
-    if output == '-':
-        click.echo(str(perf), sys.stdout)
-    elif output != os.devnull:  # make the catalyst magic not write any data
-        perf.to_pickle(output)
-
-    return perf
-
-
-@main.command(name='serve-live')
-@click.option(
-    '-f',
-    '--algofile',
-    default=None,
-    type=click.File('r'),
-    help='The file that contains the algorithm to run.',
-)
-@click.option(
-    '--capital-base',
-    type=float,
-    show_default=True,
-    help='The amount of capital (in base_currency) allocated to trading.',
-)
-@click.option(
-    '-t',
-    '--algotext',
-    help='The algorithm script to run.',
-)
-@click.option(
-    '-D',
-    '--define',
-    multiple=True,
-    help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be"
-         " any python expression. These are evaluated in order so they"
-         " may refer to previously defined names.",
-)
-@click.option(
-    '-o',
-    '--output',
-    default='-',
-    metavar='FILENAME',
-    show_default=True,
-    help="The location to write the perf data. If this is '-' the perf will"
-         " be written to stdout.",
-)
-@click.option(
-    '--print-algo/--no-print-algo',
-    is_flag=True,
-    default=False,
-    help='Print the algorithm to stdout.',
-)
-@ipython_only(click.option(
-    '--local-namespace/--no-local-namespace',
-    is_flag=True,
-    default=None,
-    help='Should the algorithm methods be resolved in the local namespace.'
-))
-@click.option(
-    '-x',
-    '--exchange-name',
-    help='The name of the targeted exchange.',
-)
-@click.option(
-    '-n',
-    '--algo-namespace',
-    help='A label assigned to the algorithm for data storage purposes.'
-)
-@click.option(
-    '-c',
-    '--base-currency',
-    help='The base currency used to calculate statistics '
-         '(e.g. usd, btc, eth).',
-)
-@click.option(
-    '-e',
-    '--end',
-    type=Date(tz='utc', as_timestamp=True),
-    help='An optional end date at which to stop the execution.',
-)
-@click.option(
-    '--live-graph/--no-live-graph',
-    is_flag=True,
-    default=False,
-    help='Display live graph.',
-)
-@click.option(
-    '--simulate-orders/--no-simulate-orders',
-    is_flag=True,
-    default=True,
-    help='Simulating orders enable the paper trading mode. No orders will be '
-         'sent to the exchange unless set to false.',
-)
-@click.option(
-    '--auth-aliases',
-    default=None,
-    help='Authentication file aliases for the specified exchanges. By default,'
-         'each exchange uses the "auth.json" file in the exchange folder. '
-         'Specifying an "auth2" alias would use "auth2.json". It should be '
-         'specified like this: "[exchange_name],[alias],..." For example, '
-         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
-)
-@click.pass_context
-def serve_live(ctx,
-               algofile,
-               capital_base,
-               algotext,
-               define,
-               output,
-               print_algo,
-               local_namespace,
-               exchange_name,
-               algo_namespace,
-               base_currency,
-               end,
-               live_graph,
-               auth_aliases,
-               simulate_orders):
-    """Trade live with the given algorithm on the server.
-    """
-    if (algotext is not None) == (algofile is not None):
-        ctx.fail(
-            "must specify exactly one of '-f' / '--algofile' or"
-            " '-t' / '--algotext'",
-        )
-
-    if exchange_name is None:
-        ctx.fail("must specify an exchange name '-x'")
-
-    if algo_namespace is None:
-        ctx.fail("must specify an algorithm name '-n' in live execution mode")
-
-    if base_currency is None:
-        ctx.fail("must specify a base currency '-c' in live execution mode")
-
-    if capital_base is None:
-        ctx.fail("must specify a capital base with '--capital-base'")
-
-    if simulate_orders:
-        click.echo('Running in paper trading mode.', sys.stdout)
-
-    else:
-        click.echo('Running in live trading mode.', sys.stdout)
-
-    perf = run_server(
-        initialize=None,
-        handle_data=None,
-        before_trading_start=None,
-        analyze=None,
-            algofile=algofile,
-        algotext=algotext,
-        defines=define,
-        data_frequency=None,
-        capital_base=capital_base,
-        data=None,
-        bundle=None,
-        bundle_timestamp=None,
-        start=None,
-        end=end,
-        output=output,
-        print_algo=print_algo,
-        local_namespace=local_namespace,
-        environ=os.environ,
-        live=True,
-        exchange=exchange_name,
-        algo_namespace=algo_namespace,
-        base_currency=base_currency,
-        live_graph=live_graph,
-        analyze_live=None,
-        simulate_orders=simulate_orders,
-        auth_aliases=auth_aliases,
-        stats_output=None,
-    )
-
-    if output == '-':
-        click.echo(str(perf), sys.stdout)
-    elif output != os.devnull:  # make the catalyst magic not write any data
-        perf.to_pickle(output)
-
-    return perf
-
-
 @main.command(name='ingest-exchange')
 @click.option(
     '-x',
@@ -1132,12 +767,18 @@ def bundles():
 @main.group()
 @click.pass_context
 def marketplace(ctx):
+    """Access the Enigma Data Marketplace to:\n
+    - Register and Publish new datasets (seller-side)\n
+    - Subscribe and Ingest premium datasets (buyer-side)\n
+    """
     pass
 
 
 @marketplace.command()
 @click.pass_context
 def ls(ctx):
+    """List all available datasets.
+    """
     click.echo('Listing of available data sources on the marketplace:',
                sys.stdout)
     marketplace = Marketplace()
@@ -1152,10 +793,8 @@ def ls(ctx):
 )
 @click.pass_context
 def subscribe(ctx, dataset):
-    if dataset is None:
-        ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
+    """Subscribe to an existing dataset.
+    """
     marketplace = Marketplace()
     marketplace.subscribe(dataset)
 
@@ -1190,11 +829,8 @@ def subscribe(ctx, dataset):
 )
 @click.pass_context
 def ingest(ctx, dataset, data_frequency, start, end):
-    if dataset is None:
-        ctx.fail("must specify a dataset to clean with '--dataset'\n"
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
-    click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
+    """Ingest a dataset (requires subscription).
+    """
     marketplace = Marketplace()
     marketplace.ingest(dataset, data_frequency, start, end)
 
@@ -1207,19 +843,17 @@ def ingest(ctx, dataset, data_frequency, start, end):
 )
 @click.pass_context
 def clean(ctx, dataset):
-    if dataset is None:
-        ctx.fail("must specify a dataset to ingest with '--dataset'\n"
-                 "List available dataset on the marketplace with "
-                 "'catalyst marketplace ls'")
-    click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
+    """Clean/Remove local data for a given dataset.
+    """
     marketplace = Marketplace()
     marketplace.clean(dataset)
-    click.echo('Done', sys.stdout)
 
 
 @marketplace.command()
 @click.pass_context
 def register(ctx):
+    """Register a new dataset.
+    """
     marketplace = Marketplace()
     marketplace.register()
 
@@ -1243,6 +877,8 @@ def register(ctx):
 )
 @click.pass_context
 def publish(ctx, dataset, datadir, watch):
+    """Publish data for a registered dataset.
+    """
     marketplace = Marketplace()
     if dataset is None:
         ctx.fail("must specify a dataset to publish data for "

catalyst/constants.py

@@ -25,8 +25,7 @@ AUTO_INGEST = False
 AUTH_SERVER = 'https://data.enigma.co'
 
 # TODO: switch to mainnet
-ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
-
+ETH_REMOTE_NODE = 'https://rinkeby.infura.io/'
 
 MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
                        'catalyst/master/catalyst/marketplace/' \
@@ -37,8 +36,8 @@ MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                            'contract_marketplace_abi.json'
 
 # TODO: switch to mainnet
-ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
-                  'master/catalyst/marketplace/' \
+ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
+                  'catalyst/master/catalyst/marketplace/' \
                   'contract_enigma_address.txt'
 
 ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \

catalyst/examples/buy_low_sell_high.py

@@ -7,7 +7,6 @@ from catalyst.api import (
     order_target_percent,
     symbol,
     record,
-    get_open_orders,
 )
 from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm

catalyst/examples/portfolio_optimization.py

@@ -66,7 +66,7 @@ def handle_data(context, data):
         # Define portfolio optimization parameters
         n_portfolios = 50000
         results_array = np.zeros((3 + context.nassets, n_portfolios))
-        for p in xrange(n_portfolios):
+        for p in range(n_portfolios):
             weights = np.random.random(context.nassets)
             weights /= np.sum(weights)
             w = np.asmatrix(weights)

catalyst/examples/simple_loop.py

@@ -26,7 +26,7 @@ def handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='2H'
+        frequency='30T'
     )
     last_traded = prices.index[-1]
     log.info('last candle date: {}'.format(last_traded))

catalyst/exchange/exchange.py

@@ -1,5 +1,4 @@
 import abc
-import pytz
 from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 from time import sleep
@@ -12,13 +11,16 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
+    NoDataAvailableOnExchange, NoValueForField, \
+    NoCandlesReceivedFromExchange, \
+    InvalidHistoryFrequencyAlias,  \
     TickerNotFoundError, NotEnoughCashError
 from catalyst.exchange.utils.datetime_utils import get_delta, \
     get_periods_range, \
-    get_periods, get_start_dt, get_frequency
+    get_periods, get_start_dt, get_frequency, \
+    get_candles_number_from_minutes
 from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
-    resample_history_df, has_bundle
+    resample_history_df, has_bundle, get_candles_df
 from logbook import Logger
 
 log = Logger('Exchange', level=LOG_LEVEL)
@@ -256,7 +258,8 @@ class Exchange:
             elif data_frequency is not None:
                 applies = (
                     (
-                        data_frequency == 'minute' and a.end_minute is not None)
+                        data_frequency == 'minute' and
+                        a.end_minute is not None)
                     or (
                         data_frequency == 'daily' and a.end_daily is not None)
                 )
@@ -505,49 +508,64 @@ class Exchange:
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency, supported_freqs=['T', 'D', 'H']
         )
+
+        if unit == 'H':
+            raise InvalidHistoryFrequencyAlias(
+                freq=frequency)
+
+        # we want to avoid receiving empty candles
+        # so we request more than needed
+        # TODO: consider defining a const per asset
+        # and/or some retry mechanism (in each iteration request more data)
+        kExtra_minutes_candles = 150
+        requested_bar_count = bar_count + \
+            get_candles_number_from_minutes(unit,
+                                            candle_size,
+                                            kExtra_minutes_candles)
+
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
             assets=assets,
-            bar_count=bar_count,
+            bar_count=requested_bar_count,
             end_dt=end_dt if not is_current else None,
         )
 
-        series = dict()
+        # candles sanity check - verify no empty candles were received:
         for asset in candles:
-            if candles[asset]:
-                first_candle = candles[asset][0]
-                asset_series = self.get_series_from_candles(
-                    candles=candles[asset],
-                    start_dt=first_candle['last_traded'],
+            if not candles[asset]:
+                raise NoCandlesReceivedFromExchange(
+                    bar_count=requested_bar_count,
                     end_dt=end_dt,
-                    data_frequency=frequency,
-                    field=field,
-                )
+                    asset=asset,
+                    exchange=self.name)
 
-                delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
-                # Checking to make sure that the dates match
-                delta = get_delta(delta_candle_size, data_frequency)
-                adj_end_dt = end_dt - delta
-                last_traded = asset_series.index[-1]
+        # for avoiding unnecessary forward fill end_dt is taken back one second
+        forward_fill_till_dt = end_dt - timedelta(seconds=1)
 
-                if last_traded < adj_end_dt:
-                    raise LastCandleTooEarlyError(
-                        last_traded=last_traded,
-                        end_dt=adj_end_dt,
-                        exchange=self.name,
-                    )
-            else: # empty candle received
-                # because other assets are tz-aware, we need its tz to be set as well
-                asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc))
+        series = get_candles_df(candles=candles,
+                                field=field,
+                                freq=frequency,
+                                bar_count=requested_bar_count,
+                                end_dt=forward_fill_till_dt)
 
-
-            series[asset] = asset_series
+        # TODO: consider how to approach this edge case
+        # delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
+        # Checking to make sure that the dates match
+        # delta = get_delta(delta_candle_size, data_frequency)
+        # adj_end_dt = end_dt - delta
+        # last_traded = asset_series.index[-1]
+        # if last_traded < adj_end_dt:
+        #    raise LastCandleTooEarlyError(
+        #        last_traded=last_traded,
+        #        end_dt=adj_end_dt,
+        #        exchange=self.name,
+        #    )
 
         df = pd.DataFrame(series)
-        #df.dropna(inplace=True) # commented out due to issue 236
+        df.dropna(inplace=True)
 
-        return df
+        return df.tail(bar_count)
 
     def get_history_window_with_bundle(self,
                                        assets,
@@ -595,7 +613,8 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
-        # TODO: this function needs some work, we're currently using it just for benchmark data
+        # TODO: this function needs some work,
+        # we're currently using it just for benchmark data
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )
@@ -621,7 +640,7 @@ class Exchange:
                 start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
                 trailing_dt = \
                     series[asset].index[-1] + get_delta(1, data_frequency) \
-                        if asset in series else start_dt
+                    if asset in series else start_dt
 
                 # The get_history method supports multiple asset
                 # Use the original frequency to let each api optimize

catalyst/exchange/exchange_algorithm.py

@@ -163,6 +163,25 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
                                                         style)
         return amount, style
 
+    def _calculate_order_target_amount(self, asset, target):
+        """
+        removes order amounts so we won't run into issues
+        when two orders are placed one after the other.
+        it then proceeds to removing positions amount at TradingAlgorithm
+        :param asset:
+        :param target:
+        :return: target
+        """
+        if asset in self.blotter.open_orders:
+            for open_order in self.blotter.open_orders[asset]:
+                current_amount = open_order.amount
+                target -= current_amount
+
+        target = super(ExchangeTradingAlgorithmBase, self). \
+            _calculate_order_target_amount(asset, target)
+
+        return target
+
     def round_order(self, amount, asset):
         """
         We need fractions with cryptocurrencies

catalyst/exchange/exchange_bundle.py

@@ -843,7 +843,6 @@ class ExchangeBundle:
         field: str
         data_frequency: str
         algo_end_dt: pd.Timestamp
-        force_auto_ingest: 
 
         Returns
         -------

catalyst/exchange/exchange_errors.py

@@ -322,3 +322,10 @@ class BalanceTooLowError(ZiplineError):
         'add positions to hold a free amount greater than {amount}, or clean '
         'the state of this algo and restart.'
     ).strip()
+
+
+class NoCandlesReceivedFromExchange(ZiplineError):
+    msg = (
+        'Although requesting {bar_count} candles until {end_dt} of asset {asset}, '
+        'an empty list of candles was received for {exchange}.'
+    ).strip()

catalyst/exchange/utils/datetime_utils.py

@@ -1,4 +1,5 @@
 import calendar
+import math
 import re
 from datetime import datetime, timedelta, date
 
@@ -326,3 +327,33 @@ def from_ms_timestamp(ms):
 
 def get_epoch():
     return pd.to_datetime('1970-1-1', utc=True)
+
+
+def get_candles_number_from_minutes(unit, candle_size, minutes):
+    """
+    Get the number of bars needed for the given time interval
+    in minutes.
+
+    Notes
+    -----
+    Supports only "T", "D" and "H" units
+
+    Parameters
+    ----------
+    unit: str
+    candle_size : int
+    minutes: int
+
+    Returns
+    -------
+    int
+
+    """
+    if unit == "T":
+        res = (float(minutes) / candle_size)
+    elif unit == "H":
+        res = (minutes / 60.0) / candle_size
+    else:  # unit == "D"
+        res = (minutes / 1440.0) / candle_size
+
+    return int(math.ceil(res))

catalyst/exchange/utils/exchange_utils.py

@@ -734,7 +734,7 @@ def transform_candles_to_df(candles):
     return pd.DataFrame(candles).set_index('last_traded')
 
 
-def get_candles_df(candles, field, freq, bar_count, end_dt=None):
+def get_candles_df(candles, field, freq, bar_count, end_dt):
     all_series = dict()
 
     for asset in candles:

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +1 @@
-0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10
+0x39a54f480d922a58c963de8091a6c9afc69db2cf

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +1 @@
-0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4
+0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc

catalyst/marketplace/marketplace.py

@@ -7,6 +7,7 @@ import re
 import shutil
 import sys
 import time
+import webbrowser
 
 import bcolz
 import logbook
@@ -32,6 +33,7 @@ from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
 from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
     get_data_source_folder, get_marketplace_folder, \
     get_user_pubaddr, get_temp_bundles_folder, extract_bundle
+from catalyst.utils.paths import ensure_directory
 
 if sys.version_info.major < 3:
     import urllib
@@ -141,10 +143,10 @@ class Marketplace:
 
         return address, address_i
 
-    def sign_transaction(self, from_address, tx):
+    def sign_transaction(self, tx):
 
-        print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
-              'enter the following parameters:\n\n'
+        url = 'https://www.myetherwallet.com/#offline-transaction'
+        print('\nVisit {url} and enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
               'To Address:\t\t{to}\n'
@@ -153,7 +155,8 @@ class Marketplace:
               'Gas Price:\t\t[Accept the default value]\n'
               'Nonce:\t\t\t{nonce}\n'
               'Data:\t\t\t{data}\n'.format(
-                _from=from_address,
+                url=url,
+                _from=tx['from'],
                 to=tx['to'],
                 value=tx['value'],
                 gas=tx['gas'],
@@ -161,6 +164,8 @@ class Marketplace:
                 data=tx['data'], )
               )
 
+        webbrowser.open_new(url)
+
         signed_tx = input('Copy and Paste the "Signed Transaction" '
                           'field here:\n')
 
@@ -172,16 +177,17 @@ class Marketplace:
     def check_transaction(self, tx_hash):
 
         if 'ropsten' in ETH_REMOTE_NODE:
-            etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
-                tx_hash)
+            etherscan = 'https://ropsten.etherscan.io/tx/'
+        elif 'rinkeby' in ETH_REMOTE_NODE:
+            etherscan = 'https://rinkeby.etherscan.io/tx/'
         else:
-            etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
+            etherscan = 'https://etherscan.io/tx/'
+        etherscan = '{}{}'.format(etherscan, tx_hash)
 
         print('\nYou can check the outcome of your transaction here:\n'
               '{}\n\n'.format(etherscan))
 
-    def list(self):
-
+    def _list(self):
         data_sources = self.mkt_contract.functions.getAllProviders().call()
 
         data = []
@@ -193,15 +199,44 @@ class Marketplace:
                             dataset=self.to_text(data_source)
                         )
                     )
+        return pd.DataFrame(data)
+
+    def list(self):
+        df = self._list()
 
-        df = pd.DataFrame(data)
         set_print_settings()
         if df.empty:
             print('There are no datasets available yet.')
         else:
             print(df)
 
-    def subscribe(self, dataset):
+    def subscribe(self, dataset=None):
+
+        if dataset is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'subscribe to [0..{}]: '.format(
+                                        df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        dataset = df_sets.iloc[dataset_num]['dataset']
+                        break
 
         dataset = dataset.lower()
 
@@ -292,13 +327,11 @@ class Marketplace:
             self.mkt_contract_address,
             grains,
         ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
-        if 'ropsten' in ETH_REMOTE_NODE:
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
+        signed_tx = self.sign_transaction(tx)
         try:
             tx_hash = '0x{}'.format(
                 bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
@@ -333,13 +366,11 @@ class Marketplace:
 
         tx = self.mkt_contract.functions.subscribe(
             Web3.toHex(dataset),
-        ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)})
+        ).buildTransaction({
+            'from': address,
+            'nonce': self.web3.eth.getTransactionCount(address)})
 
-        if 'ropsten' in ETH_REMOTE_NODE:
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
+        signed_tx = self.sign_transaction(tx)
 
         try:
             tx_hash = '0x{}'.format(bin_hex(
@@ -392,6 +423,7 @@ class Marketplace:
         """
         tmp_bundle = extract_bundle(path)
         bundle_folder = get_data_source_folder(ds_name)
+        ensure_directory(bundle_folder)
         if os.listdir(bundle_folder):
             zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
             ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
@@ -402,7 +434,33 @@ class Marketplace:
 
         pass
 
-    def ingest(self, ds_name, start=None, end=None, force_download=False):
+    def ingest(self, ds_name=None, start=None, end=None, force_download=False):
+
+        if ds_name is None:
+
+            df_sets = self._list()
+            if df_sets.empty:
+                print('There are no datasets available yet.')
+                return
+
+            set_print_settings()
+            while True:
+                print(df_sets)
+                dataset_num = input('Choose the dataset you want to '
+                                    'ingest [0..{}]: '.format(
+                                        df_sets.size - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        df_sets.size - 1))
+                else:
+                    if dataset_num not in range(0, df_sets.size):
+                        print('Enter a number between 0 and {}'.format(
+                            df_sets.size - 1))
+                    else:
+                        ds_name = df_sets.iloc[dataset_num]['dataset']
+                        break
 
         # ds_name = ds_name.lower()
 
@@ -498,14 +556,40 @@ class Marketplace:
 
         return df
 
-    def clean(self, data_source_name, data_frequency=None):
-        data_source_name = data_source_name.lower()
+    def clean(self, ds_name=None, data_frequency=None):
+
+        if ds_name is None:
+            mktplace_root = get_marketplace_folder()
+            folders = [os.path.basename(f.rstrip('/'))
+                       for f in glob.glob('{}/*/'.format(mktplace_root))
+                       if 'temp_bundles' not in f]
+
+            while True:
+                for idx, f in enumerate(folders):
+                    print('{}\t{}'.format(idx, f))
+                dataset_num = input('Choose the dataset you want to '
+                                    'clean [0..{}]: '.format(
+                                        len(folders) - 1))
+                try:
+                    dataset_num = int(dataset_num)
+                except ValueError:
+                    print('Enter a number between 0 and {}'.format(
+                        len(folders) - 1))
+                else:
+                    if dataset_num not in range(0, len(folders)):
+                        print('Enter a number between 0 and {}'.format(
+                            len(folders) - 1))
+                    else:
+                        ds_name = folders[dataset_num]
+                        break
+
+        ds_name = ds_name.lower()
 
         if data_frequency is None:
-            folder = get_data_source_folder(data_source_name)
+            folder = get_data_source_folder(ds_name)
 
         else:
-            folder = get_bundle_folder(data_source_name, data_frequency)
+            folder = get_bundle_folder(ds_name, data_frequency)
 
         shutil.rmtree(folder)
         pass
@@ -609,13 +693,11 @@ class Marketplace:
             grains,
             address,
         ).buildTransaction(
-            {'nonce': self.web3.eth.getTransactionCount(address)}
+            {'from': address,
+             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
-        if 'ropsten' in ETH_REMOTE_NODE:
-            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
-
-        signed_tx = self.sign_transaction(address, tx)
+        signed_tx = self.sign_transaction(tx)
 
         try:
             tx_hash = '0x{}'.format(

catalyst/marketplace/utils/bundle_utils.py

@@ -1,8 +1,12 @@
 import os
+import random
+import re
 import shutil
 
 import bcolz
+import numpy as np
 import pandas as pd
+from six import string_types
 
 
 def merge_bundles(zsource, ztarget):
@@ -27,10 +31,64 @@ def merge_bundles(zsource, ztarget):
     df.drop_duplicates(inplace=True)
     df.set_index(['date', 'symbol'], drop=False, inplace=True)
 
+    sanitize_df(df)
+
     dirname = os.path.basename(ztarget.rootdir)
     bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
-    os.rename(ztarget.rootdir, bak_dir)
+    shutil.move(ztarget.rootdir, bak_dir)
 
     z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
     shutil.rmtree(bak_dir)
     return z
+
+
+def sanitize_df(df):
+    # Using a sampling method to identify dates for efficiency with
+    # large datasets
+    if len(df) > 100:
+        indexes = random.sample(range(0, len(df) - 1), 100)
+    elif len(df) > 1:
+        indexes = range(0, len(df) - 1)
+    else:
+        indexes = [0, ]
+
+    for column in df.columns:
+        is_date = False
+        for index in indexes:
+            value = df[column].iloc[index]
+            if not isinstance(value, string_types):
+                continue
+
+            # TODO: assuming that the date is at least daily
+            exp = re.compile(r'^\d{4}-\d{2}-\d{2}.*$')
+            matches = exp.findall(value)
+
+            if matches:
+                is_date = True
+                break
+
+        if is_date:
+            df[column] = pd.to_datetime(df[column])
+
+        else:
+            try:
+                ser = safely_reduce_dtype(df[column])
+                df[column] = ser
+            except Exception:
+                pass
+
+    return df
+
+
+def safely_reduce_dtype(ser):  # pandas.Series or numpy.array
+    orig_dtype = "".join(
+        [x for x in ser.dtype.name if x.isalpha()])  # float/int
+    mx = 1
+    for val in ser.values:
+        new_itemsize = np.min_scalar_type(val).itemsize
+        if mx < new_itemsize:
+            mx = new_itemsize
+        if orig_dtype == 'int':
+            mx = max(mx, 4)
+    new_dtype = orig_dtype + str(mx * 8)
+    return ser.astype(new_dtype)

catalyst/utils/run_algo.py

@@ -10,6 +10,7 @@ import click
 import pandas as pd
 from six import string_types
 
+import catalyst
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
@@ -23,7 +24,7 @@ try:
     from pygments.formatters import TerminalFormatter
 
     PYGMENTS = True
-except:
+except ImportError:
     PYGMENTS = False
 from toolz import valfilter, concatv
 from functools import partial
@@ -151,6 +152,7 @@ def _run(handle_data,
         'We encourage you to report any issue on GitHub: '
         'https://github.com/enigmampc/catalyst/issues'
     )
+    log.info('Catalyst version {}'.format(catalyst.__version__))
     sleep(3)
 
     if live:
@@ -261,6 +263,15 @@ def _run(handle_data,
         # We still need to support bundles for other misc data, but we
         # can handle this later.
 
+        if start != pd.tslib.normalize_date(start) or \
+                        end != pd.tslib.normalize_date(end):
+            # todo: add to Sim_Params the option to start & end at specific times 
+            log.warn(
+                "Catalyst currently starts and ends on the start and "
+                "end of the dates specified, respectively. We hope to "
+                "Modify this and support specific times in a future release."
+            )
+
         data = DataPortalExchangeBacktest(
             exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,

catalyst/utils/run_server.py

@@ -1,103 +0,0 @@
-#!flask/bin/python
-import base64
-
-import requests
-import pandas as pd
-import json
-
-
-def convert_date(date):
-    """
-    when transferring dates by json,
-    converts it to str
-    :param date:
-    :return: str(date)
-    """
-    if isinstance(date, pd.Timestamp):
-        return date.__str__()
-
-
-def run_server(
-        initialize,
-        handle_data,
-        before_trading_start,
-        analyze,
-        algofile,
-        algotext,
-        defines,
-        data_frequency,
-        capital_base,
-        data,
-        bundle,
-        bundle_timestamp,
-        start,
-        end,
-        output,
-        print_algo,
-        local_namespace,
-        environ,
-        live,
-        exchange,
-        algo_namespace,
-        base_currency,
-        live_graph,
-        analyze_live,
-        simulate_orders,
-        auth_aliases,
-        stats_output,
-        ):
-
-    # address to send
-    url = 'http://sandbox.enigma.co/api/catalyst/serve'
-    # url = 'http://127.0.0.1:5000/api/catalyst/serve'
-
-    # argument preparation - encode the file for transfer
-    if algotext:
-        algotext = base64.b64encode(algotext)
-    else:
-        algotext = base64.b64encode(bytes(algofile.read(), 'utf-8')).decode('utf-8')
-        algofile = None
-
-    json_file = {'arguments': {
-        'initialize': initialize,
-        'handle_data': handle_data,
-        'before_trading_start': before_trading_start,
-        'analyze': analyze,
-        'algotext': algotext,
-        'defines': defines,
-        'data_frequency': data_frequency,
-        'capital_base': capital_base,
-        'data': data,
-        'bundle': bundle,
-        'bundle_timestamp': bundle_timestamp,
-        'start': start,
-        'end': end,
-        'local_namespace': local_namespace,
-        'environ': None,
-        'analyze_live': analyze_live,
-        'stats_output': stats_output,
-        'algofile': algofile,
-        'output': output,
-        'print_algo': print_algo,
-        'live': live,
-        'exchange': exchange,
-        'algo_namespace': algo_namespace,
-        'base_currency': base_currency,
-        'live_graph': live_graph,
-        'simulate_orders': simulate_orders,
-        'auth_aliases': auth_aliases,
-    }}
-
-    response = requests.post(url,
-                             json=json.dumps(
-                                    json_file,
-                                    default=convert_date
-                                )
-                             )
-
-    if response.status_code == 500:
-        raise Exception("issues with cloud connections, "
-                        "unable to run catalyst on the cloud")
-    received_data = response.json()
-    cloud_log_tail = base64.b64decode(received_data["log"])
-    print(cloud_log_tail)

docs/source/install.rst

@@ -314,6 +314,16 @@ Troubleshooting ``pip`` Install
 
       $ sudo apt-get install python-dev
 
+----
+
+**Issue**:
+   Missing TA_Lib
+
+**Solution**:
+   Follow `these instructions
+   <https://mrjbq7.github.io/ta-lib/install.html>`_ to install the TA_Lib Python wrapper
+   (and if needed, its underlying C library as well).
+
 .. _pipenv:
 
 Installing with ``pipenv``

docs/source/releases.rst

@@ -2,6 +2,43 @@
 Release Notes
 =============
 
+Version 0.5.4
+^^^^^^^^^^^^^
+**Release Date**: 2018-03-14
+
+Build
+~~~~~
+- Switched Data Marketplace from Ropstein testnet to Rinkeby testnet after 
+  incorporating changes resulting from the marketplace contract audit
+- Several usability improvements of the Data Marketplace that make the 
+  `--dataset` parameter optional. If it is not included in the command line, 
+  will list available datasets, and let you choose interactively.
+
+Bug Fixes
+~~~~~~~~~
+- Fix Binance requirement of symbol to be included in the cancelled order 
+  :issue:`204`
+- Fix `notenoughcasherror` when an open order is filled minutes later 
+  :issue:`237`
+- Properly handle of empty candles received from exchanges :issue:`236`
+- Added a function to reduce open orders amount from calculated target/amount 
+  for target orders :issue:`243`
+- Fix missing file in live trading mode on date change :issue:`252`, 
+  :issue:`253`
+- Upgraded Data Marketplace to Web3==4.0.0b11, which was breaking some 
+  functionality from prior version 4.0.0b7 :issue:`257`
+- Always request more data to avoid empty bars and always give the exact bar  
+  number :issue:`260`
+
+Documentation
+~~~~~~~~~~~~~
+- PyCharm documentation :issue:`195`
+- Added TA-Lib troubleshooting instructions
+- Added instructions on how to create a Conda environment for Python 3.6, and
+  updated Visual C++ instructions for Windows and Python 3
+- Linking example algorithms in the documentation to their sources
+
+
 Version 0.5.3
 ^^^^^^^^^^^^^
 **Release Date**: 2018-02-09

tests/exchange/test_bundle.py

@@ -11,7 +11,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
 from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
     get_df_from_arrays
-from exchange.utils.datetime_utils import get_start_dt
+from catalyst.exchange.utils.datetime_utils import get_start_dt
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder
 from catalyst.exchange.utils.factory import get_exchange
 from catalyst.exchange.utils.stats_utils import df_to_string
@@ -42,7 +42,7 @@ class TestExchangeBundle:
 
     def test_ingest_minute(self):
         data_frequency = 'minute'
-        exchange_name = 'poloniex'
+        exchange_name = 'binance'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
@@ -50,8 +50,8 @@ class TestExchangeBundle:
             exchange.get_asset('eth_btc')
         ]
 
-        start = pd.to_datetime('2016-03-01', utc=True)
-        end = pd.to_datetime('2017-11-1', utc=True)
+        start = pd.to_datetime('2018-03-01', utc=True)
+        end = pd.to_datetime('2018-03-8', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -101,7 +101,7 @@ class TestExchangeBundle:
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'bitfinex'
+        exchange_name = 'binance'
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)

tests/marketplace/test_marketplace.py

@@ -1,6 +1,5 @@
 from catalyst.marketplace.marketplace import Marketplace
 from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
-import pandas as pd
 
 
 class TestMarketplace(WithLogger, ZiplineTestCase):
@@ -16,12 +15,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
 
     def test_subscribe(self):
         marketplace = Marketplace()
-        marketplace.subscribe('marketcap2222')
+        marketplace.subscribe('marketcap')
         pass
 
     def test_ingest(self):
         marketplace = Marketplace()
-        ds_def = marketplace.ingest('github')
+        ds_def = marketplace.ingest('marketcap')
         pass
 
     def test_publish(self):