DEV: encoding compatible to unix as well

catalyst/__main__.py

@@ -14,6 +14,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.utils.exchange_utils import delete_algo_folder
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
+from catalyst.utils.run_server import run_server
 
 try:
     __IPYTHON__
@@ -505,6 +506,370 @@ def live(ctx,
     return perf
 
 
+@main.command(name='serve')
+@click.option(
+    '-f',
+    '--algofile',
+    default=None,
+    type=click.File('r'),
+    help='The file that contains the algorithm to run.',
+)
+@click.option(
+    '-t',
+    '--algotext',
+    help='The algorithm script to run.',
+)
+@click.option(
+    '-D',
+    '--define',
+    multiple=True,
+    help="Define a name to be bound in the namespace before executing"
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
+)
+@click.option(
+    '--data-frequency',
+    type=click.Choice({'daily', 'minute'}),
+    default='daily',
+    show_default=True,
+    help='The data frequency of the simulation.',
+)
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The starting capital for the simulation.',
+)
+@click.option(
+    '-b',
+    '--bundle',
+    default='poloniex',
+    metavar='BUNDLE-NAME',
+    show_default=True,
+    help='The data bundle to use for the simulation.',
+)
+@click.option(
+    '--bundle-timestamp',
+    type=Timestamp(),
+    default=pd.Timestamp.utcnow(),
+    show_default=False,
+    help='The date to lookup data on or before.\n'
+         '[default: <current-time>]'
+)
+@click.option(
+    '-s',
+    '--start',
+    type=Date(tz='utc', as_timestamp=True),
+    help='The start date of the simulation.',
+)
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='The end date of the simulation.',
+)
+@click.option(
+    '-o',
+    '--output',
+    default='-',
+    metavar='FILENAME',
+    show_default=True,
+    help="The location to write the perf data. If this is '-' the perf"
+         " will be written to stdout.",
+)
+@click.option(
+    '--print-algo/--no-print-algo',
+    is_flag=True,
+    default=False,
+    help='Print the algorithm to stdout.',
+)
+@ipython_only(click.option(
+    '--local-namespace/--no-local-namespace',
+    is_flag=True,
+    default=None,
+    help='Should the algorithm methods be resolved in the local namespace.'
+))
+@click.option(
+    '-x',
+    '--exchange-name',
+    help='The name of the targeted exchange.',
+)
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='A label assigned to the algorithm for data storage purposes.'
+)
+@click.option(
+    '-c',
+    '--base-currency',
+    help='The base currency used to calculate statistics '
+         '(e.g. usd, btc, eth).',
+)
+@click.pass_context
+def run(ctx,
+        algofile,
+        algotext,
+        define,
+        data_frequency,
+        capital_base,
+        bundle,
+        bundle_timestamp,
+        start,
+        end,
+        output,
+        print_algo,
+        local_namespace,
+        exchange_name,
+        algo_namespace,
+        base_currency):
+    """Run a backtest for the given algorithm on the server.
+    """
+
+    if (algotext is not None) == (algofile is not None):
+        ctx.fail(
+            "must specify exactly one of '-f' / '--algofile' or"
+            " '-t' / '--algotext'",
+        )
+
+    # check that the start and end dates are passed correctly
+    if start is None and end is None:
+        # check both at the same time to avoid the case where a user
+        # does not pass either of these and then passes the first only
+        # to be told they need to pass the second argument also
+        ctx.fail(
+            "must specify dates with '-s' / '--start' and '-e' / '--end'"
+            " in backtest mode",
+        )
+    if start is None:
+        ctx.fail("must specify a start date with '-s' / '--start'"
+                 " in backtest mode")
+    if end is None:
+        ctx.fail("must specify an end date with '-e' / '--end'"
+                 " in backtest mode")
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    if base_currency is None:
+        ctx.fail("must specify a base currency with '-c' in backtest mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    click.echo('Running in backtesting mode.', sys.stdout)
+
+    perf = run_server(
+        initialize=None,
+        handle_data=None,
+        before_trading_start=None,
+        analyze=None,
+        algofile=algofile,
+        algotext=algotext,
+        defines=define,
+        data_frequency=data_frequency,
+        capital_base=capital_base,
+        data=None,
+        bundle=bundle,
+        bundle_timestamp=bundle_timestamp,
+        start=start,
+        end=end,
+        output=output,
+        print_algo=print_algo,
+        local_namespace=local_namespace,
+        environ=os.environ,
+        live=False,
+        exchange=exchange_name,
+        algo_namespace=algo_namespace,
+        base_currency=base_currency,
+        analyze_live=None,
+        live_graph=False,
+        simulate_orders=True,
+        auth_aliases=None,
+        stats_output=None,
+    )
+
+    if output == '-':
+        click.echo(str(perf), sys.stdout)
+    elif output != os.devnull:  # make the catalyst magic not write any data
+        perf.to_pickle(output)
+
+    return perf
+
+
+@main.command(name='serve-live')
+@click.option(
+    '-f',
+    '--algofile',
+    default=None,
+    type=click.File('r'),
+    help='The file that contains the algorithm to run.',
+)
+@click.option(
+    '--capital-base',
+    type=float,
+    show_default=True,
+    help='The amount of capital (in base_currency) allocated to trading.',
+)
+@click.option(
+    '-t',
+    '--algotext',
+    help='The algorithm script to run.',
+)
+@click.option(
+    '-D',
+    '--define',
+    multiple=True,
+    help="Define a name to be bound in the namespace before executing"
+         " the algotext. For example '-Dname=value'. The value may be"
+         " any python expression. These are evaluated in order so they"
+         " may refer to previously defined names.",
+)
+@click.option(
+    '-o',
+    '--output',
+    default='-',
+    metavar='FILENAME',
+    show_default=True,
+    help="The location to write the perf data. If this is '-' the perf will"
+         " be written to stdout.",
+)
+@click.option(
+    '--print-algo/--no-print-algo',
+    is_flag=True,
+    default=False,
+    help='Print the algorithm to stdout.',
+)
+@ipython_only(click.option(
+    '--local-namespace/--no-local-namespace',
+    is_flag=True,
+    default=None,
+    help='Should the algorithm methods be resolved in the local namespace.'
+))
+@click.option(
+    '-x',
+    '--exchange-name',
+    help='The name of the targeted exchange.',
+)
+@click.option(
+    '-n',
+    '--algo-namespace',
+    help='A label assigned to the algorithm for data storage purposes.'
+)
+@click.option(
+    '-c',
+    '--base-currency',
+    help='The base currency used to calculate statistics '
+         '(e.g. usd, btc, eth).',
+)
+@click.option(
+    '-e',
+    '--end',
+    type=Date(tz='utc', as_timestamp=True),
+    help='An optional end date at which to stop the execution.',
+)
+@click.option(
+    '--live-graph/--no-live-graph',
+    is_flag=True,
+    default=False,
+    help='Display live graph.',
+)
+@click.option(
+    '--simulate-orders/--no-simulate-orders',
+    is_flag=True,
+    default=True,
+    help='Simulating orders enable the paper trading mode. No orders will be '
+         'sent to the exchange unless set to false.',
+)
+@click.option(
+    '--auth-aliases',
+    default=None,
+    help='Authentication file aliases for the specified exchanges. By default,'
+         'each exchange uses the "auth.json" file in the exchange folder. '
+         'Specifying an "auth2" alias would use "auth2.json". It should be '
+         'specified like this: "[exchange_name],[alias],..." For example, '
+         '"binance,auth2" or "binance,auth2,bittrex,auth2".',
+)
+@click.pass_context
+def serve_live(ctx,
+               algofile,
+               capital_base,
+               algotext,
+               define,
+               output,
+               print_algo,
+               local_namespace,
+               exchange_name,
+               algo_namespace,
+               base_currency,
+               end,
+               live_graph,
+               auth_aliases,
+               simulate_orders):
+    """Trade live with the given algorithm on the server.
+    """
+    if (algotext is not None) == (algofile is not None):
+        ctx.fail(
+            "must specify exactly one of '-f' / '--algofile' or"
+            " '-t' / '--algotext'",
+        )
+
+    if exchange_name is None:
+        ctx.fail("must specify an exchange name '-x'")
+
+    if algo_namespace is None:
+        ctx.fail("must specify an algorithm name '-n' in live execution mode")
+
+    if base_currency is None:
+        ctx.fail("must specify a base currency '-c' in live execution mode")
+
+    if capital_base is None:
+        ctx.fail("must specify a capital base with '--capital-base'")
+
+    if simulate_orders:
+        click.echo('Running in paper trading mode.', sys.stdout)
+
+    else:
+        click.echo('Running in live trading mode.', sys.stdout)
+
+    perf = run_server(
+        initialize=None,
+        handle_data=None,
+        before_trading_start=None,
+        analyze=None,
+            algofile=algofile,
+        algotext=algotext,
+        defines=define,
+        data_frequency=None,
+        capital_base=capital_base,
+        data=None,
+        bundle=None,
+        bundle_timestamp=None,
+        start=None,
+        end=end,
+        output=output,
+        print_algo=print_algo,
+        local_namespace=local_namespace,
+        environ=os.environ,
+        live=True,
+        exchange=exchange_name,
+        algo_namespace=algo_namespace,
+        base_currency=base_currency,
+        live_graph=live_graph,
+        analyze_live=None,
+        simulate_orders=simulate_orders,
+        auth_aliases=auth_aliases,
+        stats_output=None,
+    )
+
+    if output == '-':
+        click.echo(str(perf), sys.stdout)
+    elif output != os.devnull:  # make the catalyst magic not write any data
+        perf.to_pickle(output)
+
+    return perf
+
+
 @main.command(name='ingest-exchange')
 @click.option(
     '-x',
@@ -767,18 +1132,12 @@ def bundles():
 @main.group()
 @click.pass_context
 def marketplace(ctx):
-    """Access the Enigma Data Marketplace to:\n
-    - Register and Publish new datasets (seller-side)\n
-    - Subscribe and Ingest premium datasets (buyer-side)\n
-    """
     pass
 
 
 @marketplace.command()
 @click.pass_context
 def ls(ctx):
-    """List all available datasets.
-    """
     click.echo('Listing of available data sources on the marketplace:',
                sys.stdout)
     marketplace = Marketplace()
@@ -793,8 +1152,10 @@ def ls(ctx):
 )
 @click.pass_context
 def subscribe(ctx, dataset):
-    """Subscribe to an existing dataset.
+    if dataset is None:
-    """
+        ctx.fail("must specify a dataset to subscribe to with '--dataset'\n"
+                 "List available dataset on the marketplace with "
+                 "'catalyst marketplace ls'")
     marketplace = Marketplace()
     marketplace.subscribe(dataset)
 
@@ -829,8 +1190,11 @@ def subscribe(ctx, dataset):
 )
 @click.pass_context
 def ingest(ctx, dataset, data_frequency, start, end):
-    """Ingest a dataset (requires subscription).
+    if dataset is None:
-    """
+        ctx.fail("must specify a dataset to clean with '--dataset'\n"
+                 "List available dataset on the marketplace with "
+                 "'catalyst marketplace ls'")
+    click.echo('Ingesting data: {}'.format(dataset), sys.stdout)
     marketplace = Marketplace()
     marketplace.ingest(dataset, data_frequency, start, end)
 
@@ -843,17 +1207,19 @@ def ingest(ctx, dataset, data_frequency, start, end):
 )
 @click.pass_context
 def clean(ctx, dataset):
-    """Clean/Remove local data for a given dataset.
+    if dataset is None:
-    """
+        ctx.fail("must specify a dataset to ingest with '--dataset'\n"
+                 "List available dataset on the marketplace with "
+                 "'catalyst marketplace ls'")
+    click.echo('Cleaning data source: {}'.format(dataset), sys.stdout)
     marketplace = Marketplace()
     marketplace.clean(dataset)
+    click.echo('Done', sys.stdout)
 
 
 @marketplace.command()
 @click.pass_context
 def register(ctx):
-    """Register a new dataset.
-    """
     marketplace = Marketplace()
     marketplace.register()
 
@@ -877,8 +1243,6 @@ def register(ctx):
 )
 @click.pass_context
 def publish(ctx, dataset, datadir, watch):
-    """Publish data for a registered dataset.
-    """
     marketplace = Marketplace()
     if dataset is None:
         ctx.fail("must specify a dataset to publish data for "

catalyst/constants.py

@@ -25,7 +25,8 @@ AUTO_INGEST = False
 AUTH_SERVER = 'https://data.enigma.co'
 
 # TODO: switch to mainnet
-ETH_REMOTE_NODE = 'https://rinkeby.infura.io/'
+ETH_REMOTE_NODE = 'https://ropsten.infura.io/'
+
 
 MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
                        'catalyst/master/catalyst/marketplace/' \
@@ -36,8 +37,8 @@ MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                            'contract_marketplace_abi.json'
 
 # TODO: switch to mainnet
-ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
+ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
-                  'catalyst/master/catalyst/marketplace/' \
+                  'master/catalyst/marketplace/' \
                   'contract_enigma_address.txt'
 
 ENIGMA_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \

catalyst/examples/buy_low_sell_high.py

@@ -7,6 +7,7 @@ from catalyst.api import (
     order_target_percent,
     symbol,
     record,
+    get_open_orders,
 )
 from catalyst.exchange.utils.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm

catalyst/examples/portfolio_optimization.py

@@ -66,7 +66,7 @@ def handle_data(context, data):
         # Define portfolio optimization parameters
         n_portfolios = 50000
         results_array = np.zeros((3 + context.nassets, n_portfolios))
-        for p in range(n_portfolios):
+        for p in xrange(n_portfolios):
             weights = np.random.random(context.nassets)
             weights /= np.sum(weights)
             w = np.asmatrix(weights)

catalyst/examples/simple_loop.py

@@ -26,7 +26,7 @@ def handle_data(context, data):
         context.asset,
         fields='price',
         bar_count=20,
-        frequency='30T'
+        frequency='2H'
     )
     last_traded = prices.index[-1]
     log.info('last candle date: {}'.format(last_traded))

catalyst/exchange/exchange.py

@@ -1,4 +1,5 @@
 import abc
+import pytz
 from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 from time import sleep
@@ -11,16 +12,13 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     SymbolNotFoundOnExchange, \
     PricingDataNotLoadedError, \
-    NoDataAvailableOnExchange, NoValueForField, \
+    NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \
-    NoCandlesReceivedFromExchange, \
-    InvalidHistoryFrequencyAlias,  \
     TickerNotFoundError, NotEnoughCashError
 from catalyst.exchange.utils.datetime_utils import get_delta, \
     get_periods_range, \
-    get_periods, get_start_dt, get_frequency, \
+    get_periods, get_start_dt, get_frequency
-    get_candles_number_from_minutes
 from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \
-    resample_history_df, has_bundle, get_candles_df
+    resample_history_df, has_bundle
 from logbook import Logger
 
 log = Logger('Exchange', level=LOG_LEVEL)
@@ -258,8 +256,7 @@ class Exchange:
             elif data_frequency is not None:
                 applies = (
                     (
-                        data_frequency == 'minute' and
+                        data_frequency == 'minute' and a.end_minute is not None)
-                        a.end_minute is not None)
                     or (
                         data_frequency == 'daily' and a.end_daily is not None)
                 )
@@ -508,64 +505,49 @@ class Exchange:
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency, supported_freqs=['T', 'D', 'H']
         )
-
-        if unit == 'H':
-            raise InvalidHistoryFrequencyAlias(
-                freq=frequency)
-
-        # we want to avoid receiving empty candles
-        # so we request more than needed
-        # TODO: consider defining a const per asset
-        # and/or some retry mechanism (in each iteration request more data)
-        kExtra_minutes_candles = 150
-        requested_bar_count = bar_count + \
-            get_candles_number_from_minutes(unit,
-                                            candle_size,
-                                            kExtra_minutes_candles)
-
         # The get_history method supports multiple asset
         candles = self.get_candles(
             freq=freq,
             assets=assets,
-            bar_count=requested_bar_count,
+            bar_count=bar_count,
             end_dt=end_dt if not is_current else None,
         )
 
-        # candles sanity check - verify no empty candles were received:
+        series = dict()
         for asset in candles:
-            if not candles[asset]:
+            if candles[asset]:
-                raise NoCandlesReceivedFromExchange(
+                first_candle = candles[asset][0]
-                    bar_count=requested_bar_count,
+                asset_series = self.get_series_from_candles(
+                    candles=candles[asset],
+                    start_dt=first_candle['last_traded'],
                     end_dt=end_dt,
-                    asset=asset,
+                    data_frequency=frequency,
-                    exchange=self.name)
-
-        # for avoiding unnecessary forward fill end_dt is taken back one second
-        forward_fill_till_dt = end_dt - timedelta(seconds=1)
-
-        series = get_candles_df(candles=candles,
                     field=field,
-                                freq=frequency,
+                )
-                                bar_count=requested_bar_count,
-                                end_dt=forward_fill_till_dt)
 
-        # TODO: consider how to approach this edge case
+                delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
-        # delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
                 # Checking to make sure that the dates match
-        # delta = get_delta(delta_candle_size, data_frequency)
+                delta = get_delta(delta_candle_size, data_frequency)
-        # adj_end_dt = end_dt - delta
+                adj_end_dt = end_dt - delta
-        # last_traded = asset_series.index[-1]
+                last_traded = asset_series.index[-1]
-        # if last_traded < adj_end_dt:
+
-        #    raise LastCandleTooEarlyError(
+                if last_traded < adj_end_dt:
-        #        last_traded=last_traded,
+                    raise LastCandleTooEarlyError(
-        #        end_dt=adj_end_dt,
+                        last_traded=last_traded,
-        #        exchange=self.name,
+                        end_dt=adj_end_dt,
-        #    )
+                        exchange=self.name,
+                    )
+            else: # empty candle received
+                # because other assets are tz-aware, we need its tz to be set as well
+                asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc))
+
+
+            series[asset] = asset_series
 
         df = pd.DataFrame(series)
-        df.dropna(inplace=True)
+        #df.dropna(inplace=True) # commented out due to issue 236
 
-        return df.tail(bar_count)
+        return df
 
     def get_history_window_with_bundle(self,
                                        assets,
@@ -613,8 +595,7 @@ class Exchange:
             A dataframe containing the requested data.
 
         """
-        # TODO: this function needs some work,
+        # TODO: this function needs some work, we're currently using it just for benchmark data
-        # we're currently using it just for benchmark data
         freq, candle_size, unit, data_frequency = get_frequency(
             frequency, data_frequency
         )

catalyst/exchange/exchange_algorithm.py

@@ -163,25 +163,6 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm):
                                                         style)
         return amount, style
 
-    def _calculate_order_target_amount(self, asset, target):
-        """
-        removes order amounts so we won't run into issues
-        when two orders are placed one after the other.
-        it then proceeds to removing positions amount at TradingAlgorithm
-        :param asset:
-        :param target:
-        :return: target
-        """
-        if asset in self.blotter.open_orders:
-            for open_order in self.blotter.open_orders[asset]:
-                current_amount = open_order.amount
-                target -= current_amount
-
-        target = super(ExchangeTradingAlgorithmBase, self). \
-            _calculate_order_target_amount(asset, target)
-
-        return target
-
     def round_order(self, amount, asset):
         """
         We need fractions with cryptocurrencies

catalyst/exchange/exchange_bundle.py

@@ -843,6 +843,7 @@ class ExchangeBundle:
         field: str
         data_frequency: str
         algo_end_dt: pd.Timestamp
+        force_auto_ingest: 
 
         Returns
         -------

catalyst/exchange/exchange_errors.py

@@ -322,10 +322,3 @@ class BalanceTooLowError(ZiplineError):
         'add positions to hold a free amount greater than {amount}, or clean '
         'the state of this algo and restart.'
     ).strip()
-
-
-class NoCandlesReceivedFromExchange(ZiplineError):
-    msg = (
-        'Although requesting {bar_count} candles until {end_dt} of asset {asset}, '
-        'an empty list of candles was received for {exchange}.'
-    ).strip()

catalyst/exchange/utils/datetime_utils.py

@@ -1,5 +1,4 @@
 import calendar
-import math
 import re
 from datetime import datetime, timedelta, date
 
@@ -327,33 +326,3 @@ def from_ms_timestamp(ms):
 
 def get_epoch():
     return pd.to_datetime('1970-1-1', utc=True)
-
-
-def get_candles_number_from_minutes(unit, candle_size, minutes):
-    """
-    Get the number of bars needed for the given time interval
-    in minutes.
-
-    Notes
-    -----
-    Supports only "T", "D" and "H" units
-
-    Parameters
-    ----------
-    unit: str
-    candle_size : int
-    minutes: int
-
-    Returns
-    -------
-    int
-
-    """
-    if unit == "T":
-        res = (float(minutes) / candle_size)
-    elif unit == "H":
-        res = (minutes / 60.0) / candle_size
-    else:  # unit == "D"
-        res = (minutes / 1440.0) / candle_size
-
-    return int(math.ceil(res))

catalyst/exchange/utils/exchange_utils.py

@@ -734,7 +734,7 @@ def transform_candles_to_df(candles):
     return pd.DataFrame(candles).set_index('last_traded')
 
 
-def get_candles_df(candles, field, freq, bar_count, end_dt):
+def get_candles_df(candles, field, freq, bar_count, end_dt=None):
     all_series = dict()
 
     for asset in candles:

catalyst/marketplace/contract_enigma_address.txt

@@ -1 +1 @@
-0x39a54f480d922a58c963de8091a6c9afc69db2cf
+0x7fAec9aaE31BE428DeAAE1be8195dF609079Fd10

catalyst/marketplace/contract_marketplace_address.txt

@@ -1 +1 @@
-0xa2b37c6cd52f60fd4eb46ca59fafcf22d081aebc
+0x3985f5de8fddf2e8f7705cd360b498bf35ebfbc4

catalyst/marketplace/marketplace.py

@@ -7,7 +7,6 @@ import re
 import shutil
 import sys
 import time
-import webbrowser
 
 import bcolz
 import logbook
@@ -33,7 +32,6 @@ from catalyst.marketplace.utils.eth_utils import bin_hex, from_grains, \
 from catalyst.marketplace.utils.path_utils import get_bundle_folder, \
     get_data_source_folder, get_marketplace_folder, \
     get_user_pubaddr, get_temp_bundles_folder, extract_bundle
-from catalyst.utils.paths import ensure_directory
 
 if sys.version_info.major < 3:
     import urllib
@@ -143,10 +141,10 @@ class Marketplace:
 
         return address, address_i
 
-    def sign_transaction(self, tx):
+    def sign_transaction(self, from_address, tx):
 
-        url = 'https://www.myetherwallet.com/#offline-transaction'
+        print('\nVisit https://www.myetherwallet.com/#offline-transaction and '
-        print('\nVisit {url} and enter the following parameters:\n\n'
+              'enter the following parameters:\n\n'
               'From Address:\t\t{_from}\n'
               '\n\tClick the "Generate Information" button\n\n'
               'To Address:\t\t{to}\n'
@@ -155,8 +153,7 @@ class Marketplace:
               'Gas Price:\t\t[Accept the default value]\n'
               'Nonce:\t\t\t{nonce}\n'
               'Data:\t\t\t{data}\n'.format(
-                url=url,
+                _from=from_address,
-                _from=tx['from'],
                 to=tx['to'],
                 value=tx['value'],
                 gas=tx['gas'],
@@ -164,8 +161,6 @@ class Marketplace:
                 data=tx['data'], )
               )
 
-        webbrowser.open_new(url)
-
         signed_tx = input('Copy and Paste the "Signed Transaction" '
                           'field here:\n')
 
@@ -177,17 +172,16 @@ class Marketplace:
     def check_transaction(self, tx_hash):
 
         if 'ropsten' in ETH_REMOTE_NODE:
-            etherscan = 'https://ropsten.etherscan.io/tx/'
+            etherscan = 'https://ropsten.etherscan.io/tx/{}'.format(
-        elif 'rinkeby' in ETH_REMOTE_NODE:
+                tx_hash)
-            etherscan = 'https://rinkeby.etherscan.io/tx/'
         else:
-            etherscan = 'https://etherscan.io/tx/'
+            etherscan = 'https://etherscan.io/tx/{}'.format(tx_hash)
-        etherscan = '{}{}'.format(etherscan, tx_hash)
 
         print('\nYou can check the outcome of your transaction here:\n'
               '{}\n\n'.format(etherscan))
 
-    def _list(self):
+    def list(self):
+
         data_sources = self.mkt_contract.functions.getAllProviders().call()
 
         data = []
@@ -199,44 +193,15 @@ class Marketplace:
                             dataset=self.to_text(data_source)
                         )
                     )
-        return pd.DataFrame(data)
-
-    def list(self):
-        df = self._list()
 
+        df = pd.DataFrame(data)
         set_print_settings()
         if df.empty:
             print('There are no datasets available yet.')
         else:
             print(df)
 
-    def subscribe(self, dataset=None):
+    def subscribe(self, dataset):
-
-        if dataset is None:
-
-            df_sets = self._list()
-            if df_sets.empty:
-                print('There are no datasets available yet.')
-                return
-
-            set_print_settings()
-            while True:
-                print(df_sets)
-                dataset_num = input('Choose the dataset you want to '
-                                    'subscribe to [0..{}]: '.format(
-                                        df_sets.size - 1))
-                try:
-                    dataset_num = int(dataset_num)
-                except ValueError:
-                    print('Enter a number between 0 and {}'.format(
-                        df_sets.size - 1))
-                else:
-                    if dataset_num not in range(0, df_sets.size):
-                        print('Enter a number between 0 and {}'.format(
-                            df_sets.size - 1))
-                    else:
-                        dataset = df_sets.iloc[dataset_num]['dataset']
-                        break
 
         dataset = dataset.lower()
 
@@ -327,11 +292,13 @@ class Marketplace:
             self.mkt_contract_address,
             grains,
         ).buildTransaction(
-            {'from': address,
+            {'nonce': self.web3.eth.getTransactionCount(address)}
-             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
-        signed_tx = self.sign_transaction(tx)
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(address, tx)
         try:
             tx_hash = '0x{}'.format(
                 bin_hex(self.web3.eth.sendRawTransaction(signed_tx))
@@ -366,11 +333,13 @@ class Marketplace:
 
         tx = self.mkt_contract.functions.subscribe(
             Web3.toHex(dataset),
-        ).buildTransaction({
+        ).buildTransaction(
-            'from': address,
+            {'nonce': self.web3.eth.getTransactionCount(address)})
-            'nonce': self.web3.eth.getTransactionCount(address)})
 
-        signed_tx = self.sign_transaction(tx)
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(address, tx)
 
         try:
             tx_hash = '0x{}'.format(bin_hex(
@@ -423,7 +392,6 @@ class Marketplace:
         """
         tmp_bundle = extract_bundle(path)
         bundle_folder = get_data_source_folder(ds_name)
-        ensure_directory(bundle_folder)
         if os.listdir(bundle_folder):
             zsource = bcolz.ctable(rootdir=tmp_bundle, mode='r')
             ztarget = bcolz.ctable(rootdir=bundle_folder, mode='r')
@@ -434,33 +402,7 @@ class Marketplace:
 
         pass
 
-    def ingest(self, ds_name=None, start=None, end=None, force_download=False):
+    def ingest(self, ds_name, start=None, end=None, force_download=False):
-
-        if ds_name is None:
-
-            df_sets = self._list()
-            if df_sets.empty:
-                print('There are no datasets available yet.')
-                return
-
-            set_print_settings()
-            while True:
-                print(df_sets)
-                dataset_num = input('Choose the dataset you want to '
-                                    'ingest [0..{}]: '.format(
-                                        df_sets.size - 1))
-                try:
-                    dataset_num = int(dataset_num)
-                except ValueError:
-                    print('Enter a number between 0 and {}'.format(
-                        df_sets.size - 1))
-                else:
-                    if dataset_num not in range(0, df_sets.size):
-                        print('Enter a number between 0 and {}'.format(
-                            df_sets.size - 1))
-                    else:
-                        ds_name = df_sets.iloc[dataset_num]['dataset']
-                        break
 
         # ds_name = ds_name.lower()
 
@@ -556,40 +498,14 @@ class Marketplace:
 
         return df
 
-    def clean(self, ds_name=None, data_frequency=None):
+    def clean(self, data_source_name, data_frequency=None):
-
+        data_source_name = data_source_name.lower()
-        if ds_name is None:
-            mktplace_root = get_marketplace_folder()
-            folders = [os.path.basename(f.rstrip('/'))
-                       for f in glob.glob('{}/*/'.format(mktplace_root))
-                       if 'temp_bundles' not in f]
-
-            while True:
-                for idx, f in enumerate(folders):
-                    print('{}\t{}'.format(idx, f))
-                dataset_num = input('Choose the dataset you want to '
-                                    'clean [0..{}]: '.format(
-                                        len(folders) - 1))
-                try:
-                    dataset_num = int(dataset_num)
-                except ValueError:
-                    print('Enter a number between 0 and {}'.format(
-                        len(folders) - 1))
-                else:
-                    if dataset_num not in range(0, len(folders)):
-                        print('Enter a number between 0 and {}'.format(
-                            len(folders) - 1))
-                    else:
-                        ds_name = folders[dataset_num]
-                        break
-
-        ds_name = ds_name.lower()
 
         if data_frequency is None:
-            folder = get_data_source_folder(ds_name)
+            folder = get_data_source_folder(data_source_name)
 
         else:
-            folder = get_bundle_folder(ds_name, data_frequency)
+            folder = get_bundle_folder(data_source_name, data_frequency)
 
         shutil.rmtree(folder)
         pass
@@ -693,11 +609,13 @@ class Marketplace:
             grains,
             address,
         ).buildTransaction(
-            {'from': address,
+            {'nonce': self.web3.eth.getTransactionCount(address)}
-             'nonce': self.web3.eth.getTransactionCount(address)}
         )
 
-        signed_tx = self.sign_transaction(tx)
+        if 'ropsten' in ETH_REMOTE_NODE:
+            tx['gas'] = min(int(tx['gas'] * 1.5), 4700000)
+
+        signed_tx = self.sign_transaction(address, tx)
 
         try:
             tx_hash = '0x{}'.format(

catalyst/marketplace/utils/bundle_utils.py

@@ -1,12 +1,8 @@
 import os
-import random
-import re
 import shutil
 
 import bcolz
-import numpy as np
 import pandas as pd
-from six import string_types
 
 
 def merge_bundles(zsource, ztarget):
@@ -31,64 +27,10 @@ def merge_bundles(zsource, ztarget):
     df.drop_duplicates(inplace=True)
     df.set_index(['date', 'symbol'], drop=False, inplace=True)
 
-    sanitize_df(df)
-
     dirname = os.path.basename(ztarget.rootdir)
     bak_dir = ztarget.rootdir.replace(dirname, '.{}'.format(dirname))
-    shutil.move(ztarget.rootdir, bak_dir)
+    os.rename(ztarget.rootdir, bak_dir)
 
     z = bcolz.ctable.fromdataframe(df=df, rootdir=ztarget.rootdir)
     shutil.rmtree(bak_dir)
     return z
-
-
-def sanitize_df(df):
-    # Using a sampling method to identify dates for efficiency with
-    # large datasets
-    if len(df) > 100:
-        indexes = random.sample(range(0, len(df) - 1), 100)
-    elif len(df) > 1:
-        indexes = range(0, len(df) - 1)
-    else:
-        indexes = [0, ]
-
-    for column in df.columns:
-        is_date = False
-        for index in indexes:
-            value = df[column].iloc[index]
-            if not isinstance(value, string_types):
-                continue
-
-            # TODO: assuming that the date is at least daily
-            exp = re.compile(r'^\d{4}-\d{2}-\d{2}.*$')
-            matches = exp.findall(value)
-
-            if matches:
-                is_date = True
-                break
-
-        if is_date:
-            df[column] = pd.to_datetime(df[column])
-
-        else:
-            try:
-                ser = safely_reduce_dtype(df[column])
-                df[column] = ser
-            except Exception:
-                pass
-
-    return df
-
-
-def safely_reduce_dtype(ser):  # pandas.Series or numpy.array
-    orig_dtype = "".join(
-        [x for x in ser.dtype.name if x.isalpha()])  # float/int
-    mx = 1
-    for val in ser.values:
-        new_itemsize = np.min_scalar_type(val).itemsize
-        if mx < new_itemsize:
-            mx = new_itemsize
-        if orig_dtype == 'int':
-            mx = max(mx, 4)
-    new_dtype = orig_dtype + str(mx * 8)
-    return ser.astype(new_dtype)

catalyst/utils/run_algo.py

@@ -10,7 +10,6 @@ import click
 import pandas as pd
 from six import string_types
 
-import catalyst
 from catalyst.data.bundles import load
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \
@@ -24,7 +23,7 @@ try:
     from pygments.formatters import TerminalFormatter
 
     PYGMENTS = True
-except ImportError:
+except:
     PYGMENTS = False
 from toolz import valfilter, concatv
 from functools import partial
@@ -152,7 +151,6 @@ def _run(handle_data,
         'We encourage you to report any issue on GitHub: '
         'https://github.com/enigmampc/catalyst/issues'
     )
-    log.info('Catalyst version {}'.format(catalyst.__version__))
     sleep(3)
 
     if live:
@@ -263,15 +261,6 @@ def _run(handle_data,
         # We still need to support bundles for other misc data, but we
         # can handle this later.
 
-        if start != pd.tslib.normalize_date(start) or \
-                        end != pd.tslib.normalize_date(end):
-            # todo: add to Sim_Params the option to start & end at specific times 
-            log.warn(
-                "Catalyst currently starts and ends on the start and "
-                "end of the dates specified, respectively. We hope to "
-                "Modify this and support specific times in a future release."
-            )
-
         data = DataPortalExchangeBacktest(
             exchange_names=[exchange_name for exchange_name in exchanges],
             asset_finder=None,

catalyst/utils/run_server.py

@@ -0,0 +1,103 @@
+#!flask/bin/python
+import base64
+
+import requests
+import pandas as pd
+import json
+
+
+def convert_date(date):
+    """
+    when transferring dates by json,
+    converts it to str
+    :param date:
+    :return: str(date)
+    """
+    if isinstance(date, pd.Timestamp):
+        return date.__str__()
+
+
+def run_server(
+        initialize,
+        handle_data,
+        before_trading_start,
+        analyze,
+        algofile,
+        algotext,
+        defines,
+        data_frequency,
+        capital_base,
+        data,
+        bundle,
+        bundle_timestamp,
+        start,
+        end,
+        output,
+        print_algo,
+        local_namespace,
+        environ,
+        live,
+        exchange,
+        algo_namespace,
+        base_currency,
+        live_graph,
+        analyze_live,
+        simulate_orders,
+        auth_aliases,
+        stats_output,
+        ):
+
+    # address to send
+    url = 'http://sandbox.enigma.co/api/catalyst/serve'
+    # url = 'http://127.0.0.1:5000/api/catalyst/serve'
+
+    # argument preparation - encode the file for transfer
+    if algotext:
+        algotext = base64.b64encode(algotext)
+    else:
+        algotext = base64.b64encode(bytes(algofile.read(), 'utf-8')).decode('utf-8')
+        algofile = None
+
+    json_file = {'arguments': {
+        'initialize': initialize,
+        'handle_data': handle_data,
+        'before_trading_start': before_trading_start,
+        'analyze': analyze,
+        'algotext': algotext,
+        'defines': defines,
+        'data_frequency': data_frequency,
+        'capital_base': capital_base,
+        'data': data,
+        'bundle': bundle,
+        'bundle_timestamp': bundle_timestamp,
+        'start': start,
+        'end': end,
+        'local_namespace': local_namespace,
+        'environ': None,
+        'analyze_live': analyze_live,
+        'stats_output': stats_output,
+        'algofile': algofile,
+        'output': output,
+        'print_algo': print_algo,
+        'live': live,
+        'exchange': exchange,
+        'algo_namespace': algo_namespace,
+        'base_currency': base_currency,
+        'live_graph': live_graph,
+        'simulate_orders': simulate_orders,
+        'auth_aliases': auth_aliases,
+    }}
+
+    response = requests.post(url,
+                             json=json.dumps(
+                                    json_file,
+                                    default=convert_date
+                                )
+                             )
+
+    if response.status_code == 500:
+        raise Exception("issues with cloud connections, "
+                        "unable to run catalyst on the cloud")
+    received_data = response.json()
+    cloud_log_tail = base64.b64decode(received_data["log"])
+    print(cloud_log_tail)

docs/source/install.rst

@@ -314,16 +314,6 @@ Troubleshooting ``pip`` Install
 
       $ sudo apt-get install python-dev
 
-----
-
-**Issue**:
-   Missing TA_Lib
-
-**Solution**:
-   Follow `these instructions
-   <https://mrjbq7.github.io/ta-lib/install.html>`_ to install the TA_Lib Python wrapper
-   (and if needed, its underlying C library as well).
-
 .. _pipenv:
 
 Installing with ``pipenv``

docs/source/releases.rst

@@ -2,43 +2,6 @@
 Release Notes
 =============
 
-Version 0.5.4
-^^^^^^^^^^^^^
-**Release Date**: 2018-03-14
-
-Build
-~~~~~
-- Switched Data Marketplace from Ropstein testnet to Rinkeby testnet after 
-  incorporating changes resulting from the marketplace contract audit
-- Several usability improvements of the Data Marketplace that make the 
-  `--dataset` parameter optional. If it is not included in the command line, 
-  will list available datasets, and let you choose interactively.
-
-Bug Fixes
-~~~~~~~~~
-- Fix Binance requirement of symbol to be included in the cancelled order 
-  :issue:`204`
-- Fix `notenoughcasherror` when an open order is filled minutes later 
-  :issue:`237`
-- Properly handle of empty candles received from exchanges :issue:`236`
-- Added a function to reduce open orders amount from calculated target/amount 
-  for target orders :issue:`243`
-- Fix missing file in live trading mode on date change :issue:`252`, 
-  :issue:`253`
-- Upgraded Data Marketplace to Web3==4.0.0b11, which was breaking some 
-  functionality from prior version 4.0.0b7 :issue:`257`
-- Always request more data to avoid empty bars and always give the exact bar  
-  number :issue:`260`
-
-Documentation
-~~~~~~~~~~~~~
-- PyCharm documentation :issue:`195`
-- Added TA-Lib troubleshooting instructions
-- Added instructions on how to create a Conda environment for Python 3.6, and
-  updated Visual C++ instructions for Windows and Python 3
-- Linking example algorithms in the documentation to their sources
-
-
 Version 0.5.3
 ^^^^^^^^^^^^^
 **Release Date**: 2018-02-09

tests/exchange/test_bundle.py

@@ -11,7 +11,7 @@ from catalyst.exchange.exchange_bundle import ExchangeBundle, \
     BUNDLE_NAME_TEMPLATE
 from catalyst.exchange.utils.bundle_utils import get_bcolz_chunk, \
     get_df_from_arrays
-from catalyst.exchange.utils.datetime_utils import get_start_dt
+from exchange.utils.datetime_utils import get_start_dt
 from catalyst.exchange.utils.exchange_utils import get_exchange_folder
 from catalyst.exchange.utils.factory import get_exchange
 from catalyst.exchange.utils.stats_utils import df_to_string
@@ -42,7 +42,7 @@ class TestExchangeBundle:
 
     def test_ingest_minute(self):
         data_frequency = 'minute'
-        exchange_name = 'binance'
+        exchange_name = 'poloniex'
 
         exchange = get_exchange(exchange_name)
         exchange_bundle = ExchangeBundle(exchange)
@@ -50,8 +50,8 @@ class TestExchangeBundle:
             exchange.get_asset('eth_btc')
         ]
 
-        start = pd.to_datetime('2018-03-01', utc=True)
+        start = pd.to_datetime('2016-03-01', utc=True)
-        end = pd.to_datetime('2018-03-8', utc=True)
+        end = pd.to_datetime('2017-11-1', utc=True)
 
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
@@ -101,7 +101,7 @@ class TestExchangeBundle:
         # data_frequency = 'daily'
         # include_symbols = 'neo_btc,bch_btc,eth_btc'
 
-        exchange_name = 'binance'
+        exchange_name = 'bitfinex'
         data_frequency = 'minute'
 
         exchange = get_exchange(exchange_name)

tests/marketplace/test_marketplace.py

@@ -1,5 +1,6 @@
 from catalyst.marketplace.marketplace import Marketplace
 from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+import pandas as pd
 
 
 class TestMarketplace(WithLogger, ZiplineTestCase):
@@ -15,12 +16,12 @@ class TestMarketplace(WithLogger, ZiplineTestCase):
 
     def test_subscribe(self):
         marketplace = Marketplace()
-        marketplace.subscribe('marketcap')
+        marketplace.subscribe('marketcap2222')
         pass
 
     def test_ingest(self):
         marketplace = Marketplace()
-        ds_def = marketplace.ingest('marketcap')
+        ds_def = marketplace.ingest('github')
         pass
 
     def test_publish(self):