Adjusted common files

catalyst/__main__.py

@@ -28,9 +28,9 @@ except NameError:
     '--strict-extensions/--non-strict-extensions',
     is_flag=True,
     help='If --strict-extensions is passed then catalyst will not run if it'
-    ' cannot load all of the specified extensions. If this is not passed or'
-    ' --non-strict-extensions is passed then the failure will be logged but'
-    ' execution will continue.',
+         ' cannot load all of the specified extensions. If this is not passed or'
+         ' --non-strict-extensions is passed then the failure will be logged but'
+         ' execution will continue.',
 )
 @click.option(
     '--default-extension/--no-default-extension',
@@ -64,6 +64,7 @@ def extract_option_object(option):
     option_object : click.Option
         The option object that this decorator will create.
     """
+
     @option
     def opt():
         pass
@@ -95,7 +96,9 @@ def ipython_only(option):
         def _(*args, **kwargs):
             kwargs[argname] = None
             return f(*args, **kwargs)
+
         return _
+
     return d
 
 
@@ -117,9 +120,9 @@ def ipython_only(option):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-    " the algotext. For example '-Dname=value'. The value may be any python"
-    " expression. These are evaluated in order so they may refer to previously"
-    " defined names.",
+         " the algotext. For example '-Dname=value'. The value may be any python"
+         " expression. These are evaluated in order so they may refer to previously"
+         " defined names.",
 )
 @click.option(
     '--data-frequency',
@@ -149,7 +152,7 @@ def ipython_only(option):
     default=pd.Timestamp.utcnow(),
     show_default=False,
     help='The date to lookup data on or before.\n'
-    '[default: <current-time>]'
+         '[default: <current-time>]'
 )
 @click.option(
     '-s',
@@ -170,7 +173,7 @@ def ipython_only(option):
     metavar='FILENAME',
     show_default=True,
     help="The location to write the perf data. If this is '-' the perf will"
-    " be written to stdout.",
+         " be written to stdout.",
 )
 @click.option(
     '--print-algo/--no-print-algo',
@@ -184,6 +187,29 @@ def ipython_only(option):
     default=None,
     help='Should the algorithm methods be resolved in the local namespace.'
 ))
+@click.option(
+    '--live/--no-live',
+    is_flag=True,
+    default=False,
+    help='Enable live trading.',
+)
+@click.option(
+    '-x',
+    '--exchange-name',
+    type=click.Choice({'bitfinex'}),
+    help='The name of the exchange (supported: bitfinex).',
+)
+@click.option(
+    '-n',
+    '--algo-name',
+    help='A label assigned to the algorithm for tracking purposes.',
+)
+@click.option(
+    '-c',
+    '--reference-currency',
+    help='The reference currency used to calculate statistics '
+         '(e.g. usd, btc, eth).',
+)
 @click.pass_context
 def run(ctx,
         algofile,
@@ -197,21 +223,37 @@ def run(ctx,
         end,
         output,
         print_algo,
-        local_namespace):
+        local_namespace,
+        live,
+        exchange_name,
+        algo_namespace,
+        base_currency):
     """Run a backtest for the given algorithm.
     """
-    # check that the start and end dates are passed correctly
-    if start is None and end is None:
-        # check both at the same time to avoid the case where a user
-        # does not pass either of these and then passes the first only
-        # to be told they need to pass the second argument also
-        ctx.fail(
-            "must specify dates with '-s' / '--start' and '-e' / '--end'",
-        )
-    if start is None:
-        ctx.fail("must specify a start date with '-s' / '--start'")
-    if end is None:
-        ctx.fail("must specify an end date with '-e' / '--end'")
+
+    if live:
+        if exchange_name is None:
+            ctx.fail("must specify an exchange name '-x' in live execution "
+                     "mode '--live'")
+        if algo_namespace is None:
+            ctx.fail("must specify an algorithm name '-n' in live execution "
+                     "mode '--live'")
+        if base_currency is None:
+            ctx.fail("must specify a reference currency '-c' in live "
+                     "execution mode '--live'")
+    else:
+        # check that the start and end dates are passed correctly
+        if start is None and end is None:
+            # check both at the same time to avoid the case where a user
+            # does not pass either of these and then passes the first only
+            # to be told they need to pass the second argument also
+            ctx.fail(
+                "must specify dates with '-s' / '--start' and '-e' / '--end'",
+            )
+        if start is None:
+            ctx.fail("must specify a start date with '-s' / '--start'")
+        if end is None:
+            ctx.fail("must specify an end date with '-e' / '--end'")
 
     if (algotext is not None) == (algofile is not None):
         ctx.fail(
@@ -238,6 +280,10 @@ def run(ctx,
         print_algo=print_algo,
         local_namespace=local_namespace,
         environ=os.environ,
+        live=live,
+        exchange=exchange_name,
+        algo_namespace=algo_namespace,
+        base_currency=base_currency
     )
 
     if output == '-':
@@ -265,11 +311,11 @@ def catalyst_magic(line, cell=None):
                 '--algotext', cell,
                 '--output', os.devnull,  # don't write the results by default
             ] + ([
-                # these options are set when running in line magic mode
-                # set a non None algo text to use the ipython user_ns
-                '--algotext', '',
-                '--local-namespace',
-            ] if cell is None else []) + line.split(),
+                     # these options are set when running in line magic mode
+                     # set a non None algo text to use the ipython user_ns
+                     '--algotext', '',
+                     '--local-namespace',
+                 ] if cell is None else []) + line.split(),
             '%s%%catalyst' % ((cell or '') and '%'),
             # don't use system exit and propogate errors to the caller
             standalone_mode=False,

catalyst/exchange/exchange_utils.py

@@ -9,7 +9,7 @@ from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
 from catalyst.utils.paths import data_root, ensure_directory
 
 SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
-              'live-trading/catalyst/exchange/symbols/{exchange}.json'
+              'exchange-trading/catalyst/exchange/{exchange}/symbols.json'
 
 
 def get_exchange_folder(exchange_name, environ=None):

catalyst/utils/run_algo.py

@@ -3,12 +3,18 @@ import re
 from runpy import run_path
 import sys
 import warnings
+from time import sleep
+from datetime import timedelta
+
+import pandas as pd
 
 import click
+
 try:
     from pygments import highlight
     from pygments.lexers import PythonLexer
     from pygments.formatters import TerminalFormatter
+
     PYGMENTS = True
 except:
     PYGMENTS = False
@@ -29,6 +35,21 @@ from catalyst.utils.calendars import get_calendar
 from catalyst.utils.factory import create_simulation_parameters
 import catalyst.utils.paths as pth
 
+from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm
+from catalyst.exchange.data_portal_exchange import DataPortalExchange
+from catalyst.exchange.bitfinex.bitfinex import Bitfinex
+from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
+from catalyst.exchange.exchange_portfolio import ExchangePortfolio
+from catalyst.exchange.exchange_errors import (
+    ExchangeRequestError,
+    ExchangeRequestErrorTooManyAttempts
+)
+from catalyst.exchange.exchange_utils import get_exchange_auth, \
+    get_algo_object
+from logbook import Logger
+
+log = Logger('run_algo')
+
 
 class _RunAlgoError(click.ClickException, ValueError):
     """Signal an error that should have a different message if invoked from
@@ -68,7 +89,11 @@ def _run(handle_data,
          output,
          print_algo,
          local_namespace,
-         environ):
+         environ,
+         live,
+         exchange,
+         algo_namespace,
+         base_currency):
     """Run a backtest for the given algorithm.
 
     This is shared between the cli and :func:`catalyst.run_algo`.
@@ -117,6 +142,18 @@ def _run(handle_data,
         else:
             click.echo(algotext)
 
+    if exchange is not None:
+        start = pd.Timestamp.utcnow()
+        end = start + timedelta(minutes=1439)
+
+    open_calendar = get_calendar('OPEN')
+    sim_params = create_simulation_parameters(
+        start=start,
+        end=end,
+        capital_base=capital_base,
+        data_frequency=data_frequency,
+        emission_rate=data_frequency,
+    )
     if bundle is not None:
         bundles = bundle.split(',')
 
@@ -146,8 +183,6 @@ def _run(handle_data,
                     str(bundle_data.asset_finder.engine.url),
                 )
 
-            open_calendar = get_calendar('OPEN')
-
             env = TradingEnvironment(
                 load=partial(load_crypto_market_data, environ=environ),
                 bm_symbol='USDT_BTC',
@@ -179,16 +214,16 @@ def _run(handle_data,
 
             if b == 'poloniex':
                 return CryptoPricingLoader(
-                           bundle_data,
-                           data_frequency,
-                           CryptoPricing,
-                       )
+                    bundle_data,
+                    data_frequency,
+                    CryptoPricing,
+                )
             elif b == 'quandl':
                 return USEquityPricingLoader(
-                           bundle_data,
-                           data_frequency,
-                           USEquityPricing,
-                       )
+                    bundle_data,
+                    data_frequency,
+                    USEquityPricing,
+                )
             raise ValueError(
                 "No PipelineLoader registered for bundle %s." % b
             )
@@ -205,20 +240,65 @@ def _run(handle_data,
             )
 
     else:
-        env = TradingEnvironment(environ=environ)
-        choose_loader = None
+        if live and exchange is not None:
+            env = TradingEnvironment(
+                environ=environ,
+                exchange_tz="UTC",
+                asset_db_path=None
+            )
+            env.asset_finder = AssetFinderExchange(exchange)
 
-    perf = TradingAlgorithm(
+            data = DataPortalExchange(
+                exchange=exchange,
+                asset_finder=env.asset_finder,
+                trading_calendar=open_calendar,
+                first_trading_day=pd.to_datetime('today', utc=True)
+            )
+            choose_loader = None
+
+            def update_portfolio(attempt_index=0):
+                """
+                Fetch the portfolio for the exchange
+                We can't continue on error because it is required to bootstrap
+                the algorithm.
+                :param attempt_index:
+                :return:
+                """
+                try:
+                    exchange.update_portfolio()
+                    return exchange.portfolio
+                except ExchangeRequestError as e:
+                    if attempt_index < 20:
+                        sleep(5)
+                        return update_portfolio(attempt_index + 1)
+                    else:
+                        raise ExchangeRequestErrorTooManyAttempts(
+                            attempts=attempt_index,
+                            error=e
+                        )
+
+            portfolio = update_portfolio()
+            sim_params = create_simulation_parameters(
+                start=start,
+                end=end,
+                capital_base=portfolio.starting_cash,
+                emission_rate='minute',
+                data_frequency='minute'
+            )
+        else:
+            env = TradingEnvironment(environ=environ)
+            choose_loader = None
+
+    TradingAlgorithmClass = (
+        partial(ExchangeTradingAlgorithm, exchange=exchange,
+                algo_namespace=algo_namespace)
+        if live and exchange else TradingAlgorithm)
+
+    perf = TradingAlgorithmClass(
         namespace=namespace,
         env=env,
         get_pipeline_loader=choose_loader,
-        sim_params=create_simulation_parameters(
-            start=start,
-            end=end,
-            capital_base=capital_base,
-            data_frequency=data_frequency,
-            emission_rate=data_frequency,
-        ),
+        sim_params=sim_params,
         **{
             'initialize': initialize,
             'handle_data': handle_data,
@@ -294,10 +374,10 @@ def load_extensions(default, extensions, strict, environ, reload=False):
             _loaded_extensions.add(ext)
 
 
-def run_algorithm(start,
-                  end,
-                  initialize,
-                  capital_base,
+def run_algorithm(initialize,
+                  capital_base=None,
+                  start=None,
+                  end=None,
                   handle_data=None,
                   before_trading_start=None,
                   analyze=None,
@@ -308,7 +388,11 @@ def run_algorithm(start,
                   default_extension=True,
                   extensions=(),
                   strict_extensions=True,
-                  environ=os.environ):
+                  environ=os.environ,
+                  live=False,
+                  exchange_name=None,
+                  base_currency=None,
+                  algo_namespace=None):
     """Run a trading algorithm.
 
     Parameters
@@ -362,6 +446,12 @@ def run_algorithm(start,
     environ : mapping[str -> str], optional
         The os environment to use. Many extensions use this to get parameters.
         This defaults to ``os.environ``.
+    live: execute live trading
+    exchange_conn: The exchange connection parameters
+
+    Supported Exchanges
+    -------------------
+    bitfinex
 
     Returns
     -------
@@ -372,26 +462,53 @@ def run_algorithm(start,
     --------
     catalyst.data.bundles.bundles : The available data bundles.
     """
+    mode = 'live' if live else 'backtest'
+    log.info('running algo in {mode} mode'.format(mode=mode))
     load_extensions(default_extension, extensions, strict_extensions, environ)
 
-    non_none_data = valfilter(bool, {
-        'data': data is not None,
-        'bundle': bundle is not None,
-    })
-    if not non_none_data:
-        # if neither data nor bundle are passed use 'quantopian-quandl'
-        bundle = 'quantopian-quandl'
+    exchange = None
+    if mode == 'backtest':
+        non_none_data = valfilter(bool, {
+            'data': data is not None,
+            'bundle': bundle is not None,
+        })
+        if not non_none_data:
+            # if neither data nor bundle are passed use 'quantopian-quandl'
+            bundle = 'quantopian-quandl'
 
-    elif len(non_none_data) != 1:
-        raise ValueError(
-            'must specify one of `data`, `data_portal`, or `bundle`,'
-            ' got: %r' % non_none_data,
-        )
+        elif len(non_none_data) != 1:
+            raise ValueError(
+                'must specify one of `data`, `data_portal`, or `bundle`,'
+                ' got: %r' % non_none_data,
+            )
 
-    elif 'bundle' not in non_none_data and bundle_timestamp is not None:
-        raise ValueError(
-            'cannot specify `bundle_timestamp` without passing `bundle`',
-        )
+        elif 'bundle' not in non_none_data and bundle_timestamp is not None:
+            raise ValueError(
+                'cannot specify `bundle_timestamp` without passing `bundle`',
+            )
+    else:
+        if exchange_name is not None:
+            portfolio = get_algo_object(
+                algo_name=algo_namespace,
+                key='portfolio_{}'.format(exchange_name),
+                environ=environ
+            )
+            if portfolio is None:
+                portfolio = ExchangePortfolio(
+                    start_date=pd.Timestamp.utcnow()
+                )
+
+            exchange_auth = get_exchange_auth(exchange_name)
+            if exchange_name == 'bitfinex':
+                exchange = Bitfinex(
+                    key=exchange_auth['key'],
+                    secret=exchange_auth['secret'].encode('UTF-8'),
+                    base_currency=base_currency,
+                    portfolio=portfolio
+                )
+            else:
+                raise NotImplementedError(
+                    'exchange not supported: %s' % exchange_name)
 
     return _run(
         handle_data=handle_data,
@@ -412,4 +529,8 @@ def run_algorithm(start,
         print_algo=False,
         local_namespace=False,
         environ=environ,
+        live=live,
+        exchange=exchange,
+        algo_namespace=algo_namespace,
+        base_currency=base_currency
     )