Merge branch 'develop'

.gitignore

@@ -78,7 +78,3 @@ zipline.iml
 ./data
 
 TAGS
-
-python2
-python3
-scratch

catalyst/__init__.py

@@ -79,7 +79,6 @@ __all__ = [
     'gens',
     'run_algorithm',
     'utils',
-    'exchange',
 ]
 
 from ._version import get_versions

catalyst/__main__.py

@@ -28,9 +28,9 @@ except NameError:
     '--strict-extensions/--non-strict-extensions',
     is_flag=True,
     help='If --strict-extensions is passed then catalyst will not run if it'
-         ' cannot load all of the specified extensions. If this is not passed or'
+    ' cannot load all of the specified extensions. If this is not passed or'
-         ' --non-strict-extensions is passed then the failure will be logged but'
+    ' --non-strict-extensions is passed then the failure will be logged but'
-         ' execution will continue.',
+    ' execution will continue.',
 )
 @click.option(
     '--default-extension/--no-default-extension',
@@ -64,7 +64,6 @@ def extract_option_object(option):
     option_object : click.Option
         The option object that this decorator will create.
     """
-
     @option
     def opt():
         pass
@@ -96,9 +95,7 @@ def ipython_only(option):
         def _(*args, **kwargs):
             kwargs[argname] = None
             return f(*args, **kwargs)
-
         return _
-
     return d
 
 
@@ -120,9 +117,9 @@ def ipython_only(option):
     '--define',
     multiple=True,
     help="Define a name to be bound in the namespace before executing"
-         " the algotext. For example '-Dname=value'. The value may be any python"
+    " the algotext. For example '-Dname=value'. The value may be any python"
-         " expression. These are evaluated in order so they may refer to previously"
+    " expression. These are evaluated in order so they may refer to previously"
-         " defined names.",
+    " defined names.",
 )
 @click.option(
     '--data-frequency',
@@ -152,7 +149,7 @@ def ipython_only(option):
     default=pd.Timestamp.utcnow(),
     show_default=False,
     help='The date to lookup data on or before.\n'
-         '[default: <current-time>]'
+    '[default: <current-time>]'
 )
 @click.option(
     '-s',
@@ -173,7 +170,7 @@ def ipython_only(option):
     metavar='FILENAME',
     show_default=True,
     help="The location to write the perf data. If this is '-' the perf will"
-         " be written to stdout.",
+    " be written to stdout.",
 )
 @click.option(
     '--print-algo/--no-print-algo',
@@ -181,41 +178,12 @@ def ipython_only(option):
     default=False,
     help='Print the algorithm to stdout.',
 )
-@click.option(
-    '-s',
-    '--start',
-    type=Date(tz='utc', as_timestamp=True),
-    help='The start date of the simulation.',
-)
 @ipython_only(click.option(
     '--local-namespace/--no-local-namespace',
     is_flag=True,
     default=None,
     help='Should the algorithm methods be resolved in the local namespace.'
 ))
-@click.option(
-    '--live/--no-live',
-    is_flag=True,
-    default=False,
-    help='Enable live trading.',
-)
-@click.option(
-    '-x',
-    '--exchange-name',
-    type=click.Choice({'bitfinex'}),
-    help='The name of the exchange (supported: bitfinex).',
-)
-@click.option(
-    '-n',
-    '--algo-name',
-    help='A label assigned to the algorithm for tracking purposes.',
-)
-@click.option(
-    '-c',
-    '--reference-currency',
-    help='The reference currency used to calculate statistics '
-         '(e.g. usd, btc, eth).',
-)
 @click.pass_context
 def run(ctx,
         algofile,
@@ -229,37 +197,21 @@ def run(ctx,
         end,
         output,
         print_algo,
-        local_namespace,
+        local_namespace):
-        live,
-        exchange_name,
-        algo_namespace,
-        base_currency):
     """Run a backtest for the given algorithm.
     """
-
+    # check that the start and end dates are passed correctly
-    if live:
+    if start is None and end is None:
-        if exchange_name is None:
+        # check both at the same time to avoid the case where a user
-            ctx.fail("must specify an exchange name '-x' in live execution "
+        # does not pass either of these and then passes the first only
-                     "mode '--live'")
+        # to be told they need to pass the second argument also
-        if algo_namespace is None:
+        ctx.fail(
-            ctx.fail("must specify an algorithm name '-n' in live execution "
+            "must specify dates with '-s' / '--start' and '-e' / '--end'",
-                     "mode '--live'")
+        )
-        if base_currency is None:
+    if start is None:
-            ctx.fail("must specify a reference currency '-c' in live "
+        ctx.fail("must specify a start date with '-s' / '--start'")
-                     "execution mode '--live'")
+    if end is None:
-    else:
+        ctx.fail("must specify an end date with '-e' / '--end'")
-        # check that the start and end dates are passed correctly
-        if start is None and end is None:
-            # check both at the same time to avoid the case where a user
-            # does not pass either of these and then passes the first only
-            # to be told they need to pass the second argument also
-            ctx.fail(
-                "must specify dates with '-s' / '--start' and '-e' / '--end'",
-            )
-        if start is None:
-            ctx.fail("must specify a start date with '-s' / '--start'")
-        if end is None:
-            ctx.fail("must specify an end date with '-e' / '--end'")
 
     if (algotext is not None) == (algofile is not None):
         ctx.fail(
@@ -286,10 +238,6 @@ def run(ctx,
         print_algo=print_algo,
         local_namespace=local_namespace,
         environ=os.environ,
-        live=live,
-        exchange=exchange_name,
-        algo_namespace=algo_namespace,
-        base_currency=base_currency
     )
 
     if output == '-':
@@ -317,11 +265,11 @@ def catalyst_magic(line, cell=None):
                 '--algotext', cell,
                 '--output', os.devnull,  # don't write the results by default
             ] + ([
-                     # these options are set when running in line magic mode
+                # these options are set when running in line magic mode
-                     # set a non None algo text to use the ipython user_ns
+                # set a non None algo text to use the ipython user_ns
-                     '--algotext', '',
+                '--algotext', '',
-                     '--local-namespace',
+                '--local-namespace',
-                 ] if cell is None else []) + line.split(),
+            ] if cell is None else []) + line.split(),
             '%s%%catalyst' % ((cell or '') and '%'),
             # don't use system exit and propogate errors to the caller
             standalone_mode=False,

catalyst/algorithm.py

@@ -1138,8 +1138,6 @@ class TradingAlgorithm(object):
         
         freq = self.sim_params.data_frequency
 
-        freq = self.sim_params.data_frequency
-
         date_rule = date_rule or date_rules.every_day()
         if freq is 'daily':
             # ignore time rule in daily mode

catalyst/data/bundles/poloniex.py

@@ -42,8 +42,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def tar_url(self):
         return (
-            'https://www.dropbox.com/s/9naqffawnq8o4r2/'
+            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
-            'poloniex-bundle.tar?dl=1'
         )
 
     @lazyval
@@ -103,7 +102,10 @@ class PoloniexBundle(BaseCryptoPricingBundle):
         )
         raw.set_index('date', inplace=True)
 
-        scale = 1
+        # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
+        # on disk, which we compensate here to get the right pricing amounts
+        # ref: data/us_equity_pricing.py
+        scale = 1000
         raw.loc[:, 'open'] /= scale
         raw.loc[:, 'high'] /= scale
         raw.loc[:, 'low'] /= scale
@@ -164,4 +166,4 @@ register_bundle(PoloniexBundle, ['USDT_BTC',])
 For a production environment make sure to use (to bundle all pairs):
 register_bundle(PoloniexBundle)
 '''
-register_bundle(PoloniexBundle)
+register_bundle(PoloniexBundle, create_writers=False)

catalyst/data/loader.py

@@ -96,16 +96,15 @@ def has_data_for_dates(series_or_df, first_date, last_date):
     first, last = dts[[0, -1]].tz_localize(None)
     return (first <= first_date.tz_localize(None)) and (last >= last_date.tz_localize(None))
 
-def load_crypto_market_data(trading_day=None,
+def load_crypto_market_data(trading_day=None, trading_days=None, bm_symbol='USDT_BTC',
-                            trading_days=None,
+                            bundle=None, bundle_data=None, environ=None):
-                            bm_symbol='USDT_BTC',
+
-                            environ=None):
     if trading_day is None:
         trading_day = get_calendar('OPEN').trading_day
     if trading_days is None:
         trading_days = get_calendar('OPEN').all_sessions
 
-    first_date = trading_days[0]
+    first_date = trading_days[1]
     now = pd.Timestamp.utcnow()
 
     # We expect to have benchmark and treasury data that's current up until
@@ -122,7 +121,14 @@ def load_crypto_market_data(trading_day=None,
 
     # We'll attempt to download new data if the latest entry in our cache is
     # before this date.
-    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
+    if(bundle_data):
+        # If we are using the bundle to retrieve the cryptobenchmark, find the last
+        # date for which there is trading data in the bundle
+        asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
+        ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
+        last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
+    else:
+        last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
     
     br = ensure_crypto_benchmark_data(
         bm_symbol,
@@ -132,11 +138,13 @@ def load_crypto_market_data(trading_day=None,
         # We need the trading_day to figure out the close prior to the first
         # date so that we can compute returns for the first date.
         trading_day,
+        bundle,
+        bundle_data,
         environ,
     )
     # Override first_date for treasury data since we have it for many more years
     # and is independent of crypto data
-    first_date_treasury = pd.Timestamp('1990-01-01', tz='UTC') 
+    first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC') 
     tc = ensure_treasury_data(
         bm_symbol,
         first_date_treasury,
@@ -240,6 +248,8 @@ def ensure_crypto_benchmark_data(symbol,
                                  last_date,
                                  now,
                                  trading_day,
+                                 bundle,
+                                 bundle_data,
                                  environ=None):
 
     filename = get_benchmark_filename(symbol)
@@ -248,7 +258,7 @@ def ensure_crypto_benchmark_data(symbol,
         ('Loading benchmark data for {symbol!r} '
             'from {first_date} to {last_date}'),
         symbol=symbol,
-        first_date=first_date - trading_day,
+        first_date=first_date,
         last_date=last_date
     )
 
@@ -261,34 +271,57 @@ def ensure_crypto_benchmark_data(symbol,
         environ,
     )
 
-
     if data is not None:
         return data
 
     # If no cached data was found or it was missing any dates then download the
     # necessary data.
-    logger.info(
-        ('Downloading benchmark data for {symbol!r} '
-            'from {first_date} to {last_date}'),
-        symbol=symbol,
-        first_date=first_date - trading_day,
-        last_date=last_date
-    )
 
-    # Load benchmark symbol from Poloniex API
+    if(bundle == 'poloniex'):
-    try:
+        '''
-        bundle = PoloniexBundle()
+        If we're using the Poloniex bundle, we'll get the benchmark from the bundle
-        bench_raw = bundle._fetch_symbol_frame(
+        instead of downloading it from Poloniex every time we need it.
-            None,
+        Poloniex has a captcha for API queries originating from outside the US that 
-            symbol,
+        prevents users abroad from getting Catalyst to work
-            get_calendar(bundle.calendar_name),
+        '''
-            first_date - trading_day,
+        logger.info(
-            last_date,
+            ('Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
-            'daily',
+            symbol=symbol, first_date=first_date, last_date=last_date)
-        )
+
-    except (OSError, IOError, HTTPError):
+        asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,as_of_date=None)
-        logger.exception('Failed to fetch new crypto benchmark returns')
+        fields = ['day', 'close']
-        raise
+        raw = bundle_data.daily_bar_reader.load_raw_arrays(
+            columns=fields,
+            start_date=first_date - trading_day,
+            end_date=last_date,
+            assets=[asset,])
+        bench_raw = pd.concat([pd.DataFrame(raw[0], columns=['date']),pd.DataFrame(raw[1], columns=['close'])], axis=1)
+        bench_raw['date'] = pd.to_datetime(bench_raw['date'],unit='s')
+        bench_raw.set_index('date', inplace=True)
+        bench_raw.sort_index(inplace=True)
+        bench_raw = bench_raw[pd.to_datetime(first_date - trading_day):pd.to_datetime(last_date)]
+
+    else:
+        # This is how it used to be: downloading the benchmark everytime. 
+        # Leaving this code here to be repurposed in the future for other bundles.
+        logger.info(
+            ('Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
+            symbol=symbol, first_date=first_date, last_date=last_date)
+
+        # Load benchmark symbol from Poloniex API
+        try:
+            bundle = PoloniexBundle()
+            bench_raw = bundle._fetch_symbol_frame(
+                None,
+                symbol,
+                get_calendar(bundle.calendar_name),
+                first_date - trading_day,
+                last_date,
+                'daily',
+            )
+        except (OSError, IOError, HTTPError):
+            logger.exception('Failed to fetch new crypto benchmark returns')
+            raise
 
     # select close column and compute percent change between days
     daily_close = bench_raw[['close']]
@@ -518,7 +551,7 @@ def _load_cached_data(filename, first_date, last_date, now, resource_name,
             )
 
     logger.info(
-        "Cache at {path} does not have data from {start} to {end}.\n",
+        "Cache at {path} does not have data from {start} to {end}.",
         start=first_date,
         end=last_date,
         path=path,

catalyst/examples/buy_and_hodl.py

@@ -23,7 +23,6 @@ from catalyst.api import (
     get_open_orders,
 )
 
-
 def initialize(context):
     context.ASSET_NAME = 'USDT_BTC'
     context.TARGET_HODL_RATIO = 0.8
@@ -42,8 +41,6 @@ def initialize(context):
 def handle_data(context, data):
     context.i += 1
 
-    print 'i:', context.i
-
     starting_cash = context.portfolio.starting_cash
     target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
     reserve_value = context.RESERVE_RATIO * starting_cash
@@ -73,6 +70,7 @@ def handle_data(context, data):
 
     record(
         price=price,
+        volume=data[context.asset].volume,
         cash=cash,
         starting_cash=context.portfolio.starting_cash,
         leverage=context.account.leverage,
@@ -80,12 +78,13 @@ def handle_data(context, data):
 
 def analyze(context=None, results=None):
     import matplotlib.pyplot as plt
+
     # Plot the portfolio and asset data.
-    ax1 = plt.subplot(511)
+    ax1 = plt.subplot(611)
     results[['portfolio_value']].plot(ax=ax1)
     ax1.set_ylabel('Portfolio Value (USD)')
 
-    ax2 = plt.subplot(512, sharex=ax1)
+    ax2 = plt.subplot(612, sharex=ax1)
     ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
     (context.TICK_SIZE * results[['price']]).plot(ax=ax2)
 
@@ -101,11 +100,11 @@ def analyze(context=None, results=None):
         color='g',
     )
 
-    ax3 = plt.subplot(513, sharex=ax1)
+    ax3 = plt.subplot(613, sharex=ax1)
     results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
     ax3.set_ylabel('Leverage ')
 
-    ax4 = plt.subplot(514, sharex=ax1)
+    ax4 = plt.subplot(614, sharex=ax1)
     results[['starting_cash', 'cash']].plot(ax=ax4)
     ax4.set_ylabel('Cash (USD)')
 
@@ -119,7 +118,7 @@ def analyze(context=None, results=None):
         'benchmark_period_return',
     ]]
 
-    ax5 = plt.subplot(515, sharex=ax1)
+    ax5 = plt.subplot(615, sharex=ax1)
     results[[
         'treasury',
         'algorithm',
@@ -127,6 +126,10 @@ def analyze(context=None, results=None):
     ]].plot(ax=ax5)
     ax5.set_ylabel('Percent Change')
 
+    ax6 = plt.subplot(616, sharex=ax1)
+    results[['volume']].plot(ax=ax6)
+    ax6.set_ylabel('Volume (mCoins/5min)')
+
     plt.legend(loc=3)
 
     # Show the plot.

catalyst/examples/buy_and_hold_test.py

@@ -1,79 +0,0 @@
-from catalyst.utils.run_algo import run_algorithm
-from datetime import datetime
-import pytz
-
-from catalyst.api import (
-    order_target_value,
-    symbol,
-    record,
-    cancel_order,
-    get_open_orders,
-)
-
-
-def initialize(context):
-    context.ASSET_NAME = 'USDT_BTC'
-    context.TARGET_HODL_RATIO = 0.8
-    context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
-
-    # For all trading pairs in the poloniex bundle, the default denomination
-    # currently supported by Catalyst is 1/1000th of a full coin. Use this
-    # constant to scale the price of up to that of a full coin if desired.
-    context.TICK_SIZE = 1000.0
-
-    context.is_buying = True
-    context.asset = symbol(context.ASSET_NAME)
-
-    context.i = 0
-
-
-def handle_data(context, data):
-    context.i += 1
-
-    print 'i:', context.i
-
-    starting_cash = context.portfolio.starting_cash
-    target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
-    reserve_value = context.RESERVE_RATIO * starting_cash
-
-    # Cancel any outstanding orders
-    orders = get_open_orders(context.asset) or []
-    for order in orders:
-        cancel_order(order)
-
-    # Stop buying after passing the reserve threshold
-    cash = context.portfolio.cash
-    if cash <= reserve_value:
-        context.is_buying = False
-
-    # Retrieve current asset price from pricing data
-    price = data[context.asset].price
-
-    # Check if still buying and could (approximately) afford another purchase
-    if context.is_buying and cash > price:
-        # Place order to make position in asset equal to target_hodl_value
-        order_target_value(
-            context.asset,
-            target_hodl_value,
-            limit_price=price * 1.1,
-            stop_price=price * 0.9,
-        )
-
-    record(
-        price=price,
-        cash=cash,
-        starting_cash=context.portfolio.starting_cash,
-        leverage=context.account.leverage,
-    )
-
-
-start = datetime(2015, 3, 1, 0, 0, 0, 0, pytz.utc)
-end = datetime(2017, 6, 28, 0, 0, 0, 0, pytz.utc)
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    start=start,
-    end=end,
-    capital_base=100000,
-    bundle='poloniex'
-)

catalyst/examples/buy_the_dip_live.py

@@ -1,155 +0,0 @@
-import talib
-from logbook import Logger
-
-from catalyst.api import (
-    order,
-    order_target_percent,
-    symbol,
-    record,
-    get_open_orders,
-)
-from catalyst.utils.run_algo import run_algorithm
-
-algo_namespace = 'buy_the_dip_live'
-log = Logger(algo_namespace)
-
-
-def initialize(context):
-    log.info('initializing algo')
-    context.ASSET_NAME = 'XRP_USD'
-    context.asset = symbol(context.ASSET_NAME)
-
-    context.TARGET_POSITIONS = 5000
-    context.PROFIT_TARGET = 0.1
-    context.SLIPPAGE_ALLOWED = 0.02
-
-    context.retry_check_open_orders = 10
-    context.retry_update_portfolio = 10
-    context.retry_order = 5
-
-    context.errors = []
-    pass
-
-
-def _handle_data(context, data):
-    prices = data.history(
-        context.asset,
-        fields='price',
-        bar_count=20,
-        frequency='15m'
-    )
-    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
-    log.info('got rsi: {}'.format(rsi))
-
-    # Buying more when RSI is low, this should lower our cost basis
-    if rsi <= 30:
-        buy_increment = 50
-    elif rsi <= 40:
-        buy_increment = 20
-    elif rsi <= 70:
-        buy_increment = 5
-    else:
-        buy_increment = None
-
-    cash = context.portfolio.cash
-    log.info('base currency available: {cash}'.format(cash=cash))
-
-    price = data.current(context.asset, 'price')
-    log.info('got price {price}'.format(price=price))
-
-    record(
-        price=price,
-        rsi=rsi,
-    )
-
-    orders = get_open_orders(context.asset)
-    if orders:
-        log.info('skipping bar until all open orders execute')
-        return
-
-    is_buy = False
-    cost_basis = None
-    if context.asset in context.portfolio.positions:
-        position = context.portfolio.positions[context.asset]
-
-        cost_basis = position.cost_basis
-        log.info(
-            'found {amount} positions with cost basis {cost_basis}'.format(
-                amount=position.amount,
-                cost_basis=cost_basis
-            )
-        )
-
-        if position.amount >= context.TARGET_POSITIONS:
-            log.info('reached positions target: {}'.format(position.amount))
-            return
-
-        if price < cost_basis:
-            is_buy = True
-        elif position.amount > 0 and \
-                        price > cost_basis * (1 + context.PROFIT_TARGET):
-            profit = (price * position.amount) - (cost_basis * position.amount)
-            log.info('closing position, taking profit: {}'.format(profit))
-            order_target_percent(
-                asset=context.asset,
-                target=0,
-                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
-            )
-        else:
-            log.info('no buy or sell opportunity found')
-    else:
-        is_buy = True
-
-    if is_buy:
-        if buy_increment is None:
-            log.info('the rsi is too high to consider buying {}'.format(rsi))
-            return
-
-        if price * buy_increment > cash:
-            log.info('not enough base currency to consider buying')
-            return
-
-        log.info(
-            'buying position cheaper than cost basis {} < {}'.format(
-                price,
-                cost_basis
-            )
-        )
-        order(
-            asset=context.asset,
-            amount=buy_increment,
-            limit_price=price * (1 + context.SLIPPAGE_ALLOWED)
-        )
-
-
-def handle_data(context, data):
-    log.info('handling bar {}'.format(data.current_dt))
-    # try:
-    _handle_data(context, data)
-    # except Exception as e:
-    #     log.warn('aborting the bar on error {}'.format(e))
-    #     context.errors.append(e)
-
-    log.info('completed bar {}, total execution errors {}'.format(
-        data.current_dt,
-        len(context.errors)
-    ))
-
-    if len(context.errors) > 0:
-        log.info('the errors:\n{}'.format(context.errors))
-
-
-def analyze(context, stats):
-    log.info('the full stats:\n{}'.format(stats.head()))
-    pass
-
-
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace=algo_namespace,
-    base_currency='usd'
-)

catalyst/exchange/algorithm_exchange.py

@@ -1,437 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-import signal
-import sys
-import pickle
-from datetime import timedelta
-from time import sleep
-from os import listdir
-from os.path import isfile, join
-
-import logbook
-import pandas as pd
-
-import catalyst.protocol as zp
-from catalyst.algorithm import TradingAlgorithm
-from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
-    BcolzMinuteBarReader
-from catalyst.errors import OrderInBeforeTradingStart
-from catalyst.exchange.exchange_clock import ExchangeClock
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    ExchangePortfolioDataError,
-    ExchangeTransactionError
-)
-from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \
-    save_algo_object, get_algo_object, get_algo_folder
-from catalyst.finance.performance.period import calc_period_stats
-from catalyst.gens.tradesimulation import AlgorithmSimulator
-from catalyst.utils.api_support import (
-    api_method,
-    disallowed_in_before_trading_start)
-from catalyst.utils.input_validation import error_keywords
-
-log = logbook.Logger("ExchangeTradingAlgorithm")
-
-
-class ExchangeAlgorithmExecutor(AlgorithmSimulator):
-    def __init__(self, *args, **kwargs):
-        super(self.__class__, self).__init__(*args, **kwargs)
-
-
-class ExchangeTradingAlgorithm(TradingAlgorithm):
-    def __init__(self, *args, **kwargs):
-        self.exchange = kwargs.pop('exchange', None)
-        self.algo_namespace = kwargs.pop('algo_namespace', None)
-        self.orders = {}
-        self.is_running = True
-
-        self.retry_check_open_orders = 5
-        self.retry_update_portfolio = 5
-        self.retry_get_open_orders = 5
-        self.retry_order = 2
-        self.retry_delay = 5
-
-        super(self.__class__, self).__init__(*args, **kwargs)
-        self._create_minute_writer()
-
-        signal.signal(signal.SIGINT, self.signal_handler)
-
-        log.info('exchange trading algorithm successfully initialized')
-
-    def _create_minute_writer(self):
-        root = get_exchange_minute_writer_root(self.exchange.name)
-        filename = os.path.join(root, 'metadata.json')
-
-        if os.path.isfile(filename):
-            writer = BcolzMinuteBarWriter.open(
-                root, self.sim_params.end_session)
-        else:
-            writer = BcolzMinuteBarWriter(
-                rootdir=root,
-                calendar=self.trading_calendar,
-                minutes_per_day=1440,
-                start_session=self.sim_params.start_session,
-                end_session=self.sim_params.end_session,
-                write_metadata=True
-            )
-
-        self.exchange.minute_writer = writer
-        self.exchange.minute_reader = BcolzMinuteBarReader(root)
-
-    def signal_handler(self, signal, frame):
-        self.is_running = False
-
-        log.info('You pressed Ctrl+C!')
-
-        stats = None
-        try:
-            algo_folder = get_algo_folder(self.algo_namespace)
-            folder = join(algo_folder, 'daily_perf')
-            files = [f for f in listdir(folder) if isfile(join(folder, f))]
-
-            daily_perf_list = []
-            for item in files:
-                filename = join(folder, item)
-                with open(filename, 'rb') as handle:
-                    daily_perf_list.append(pickle.load(handle))
-
-            stats = pd.DataFrame(daily_perf_list)
-            stats.set_index('period_close', drop=True, inplace=True)
-
-        except Exception as e:
-            log.warn('Unable to compute daily stats: {}'.format(e))
-
-        self.analyze(stats)
-        sys.exit(0)
-
-    def _create_clock(self):
-
-        # The calendar's execution times are the minutes over which we actually
-        # want to run the clock. Typically the execution times simply adhere to
-        # the market open and close times. In the case of the futures calendar,
-        # for example, we only want to simulate over a subset of the full 24
-        # hour calendar, so the execution times dictate a market open time of
-        # 6:31am US/Eastern and a close of 5:00pm US/Eastern.
-
-        # In our case, we are trading around the clock, so the market close
-        # corresponds to the last minute of the day.
-
-        # This method is taken from TradingAlgorithm.
-        # The clock has been replaced to use RealtimeClock
-        # TODO: should we apply a time skew? not sure to understand the utility.
-        return ExchangeClock(
-            self.sim_params.sessions,
-            time_skew=self.exchange.time_skew
-        )
-
-    def _create_generator(self, sim_params):
-        if self.perf_tracker is None:
-            self.perf_tracker = get_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker'
-            )
-
-        # Call the simulation trading algorithm for side-effects:
-        # it creates the perf tracker
-        TradingAlgorithm._create_generator(self, sim_params)
-        self.trading_client = ExchangeAlgorithmExecutor(
-            self,
-            sim_params,
-            self.data_portal,
-            self._create_clock(),
-            self._create_benchmark_source(),
-            self.restrictions,
-            universe_func=self._calculate_universe
-        )
-
-        return self.trading_client.transform()
-
-    def updated_portfolio(self):
-        """
-        We skip the entire performance tracker business and update the
-        portfolio directly.
-        :return:
-        """
-        return self.exchange.portfolio
-
-    def updated_account(self):
-        return self.exchange.account
-
-    def _update_portfolio(self, attempt_index=0):
-        try:
-            self.exchange.update_portfolio()
-
-            # Applying the updated last_sales_price to the positions
-            # in the performance tracker. This seems a bit redundant
-            # but it will make sense when we have multiple exchange portfolios
-            # feeding into the same performance tracker.
-            tracker = self.perf_tracker.todays_performance.position_tracker
-            for asset in self.exchange.portfolio.positions:
-                position = self.exchange.portfolio.positions[asset]
-                tracker.update_position(
-                    asset=asset,
-                    last_sale_date=position.last_sale_date,
-                    last_sale_price=position.last_sale_price
-                )
-        except ExchangeRequestError as e:
-            log.warn(
-                'update portfolio attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_update_portfolio:
-                sleep(self.retry_delay)
-                self._update_portfolio(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='update-portfolio',
-                    attempts=attempt_index,
-                    error=e
-                )
-
-    def _check_open_orders(self, attempt_index=0):
-        try:
-            return self.exchange.check_open_orders()
-        except ExchangeRequestError as e:
-            log.warn(
-                'check open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_check_open_orders:
-                sleep(self.retry_delay)
-                return self._check_open_orders(attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='order-status',
-                    attempts=attempt_index,
-                    error=e
-                )
-
-    def prepare_period_stats(self, start_dt, end_dt):
-        """
-        Creates a dictionary representing the state of the tracker.
-
-
-        I rewrote this in an attempt to better control the stats.
-        I don't want things to happen magically through complex logic
-        pertaining to backtesting.
-
-        """
-        tracker = self.perf_tracker
-        period = tracker.todays_performance
-
-        pos_stats = period.position_tracker.stats()
-        period_stats = calc_period_stats(pos_stats, period.ending_cash)
-
-        stats = dict(
-            period_start=tracker.period_start,
-            period_end=tracker.period_end,
-            capital_base=tracker.capital_base,
-            progress=tracker.progress,
-            ending_value=period.ending_value,
-            ending_exposure=period.ending_exposure,
-            capital_used=period.cash_flow,
-            starting_value=period.starting_value,
-            starting_exposure=period.starting_exposure,
-            starting_cash=period.starting_cash,
-            ending_cash=period.ending_cash,
-            portfolio_value=period.ending_cash + period.ending_value,
-            pnl=period.pnl,
-            returns=period.returns,
-            period_open=period.period_open,
-            period_close=period.period_close,
-            gross_leverage=period_stats.gross_leverage,
-            net_leverage=period_stats.net_leverage,
-            short_exposure=pos_stats.short_exposure,
-            long_exposure=pos_stats.long_exposure,
-            short_value=pos_stats.short_value,
-            long_value=pos_stats.long_value,
-            longs_count=pos_stats.longs_count,
-            shorts_count=pos_stats.shorts_count,
-        )
-
-        # Merging cumulative risk
-        stats.update(tracker.cumulative_risk_metrics.to_dict())
-
-        # Merging latest recorded variables
-        stats.update(self.recorded_vars)
-
-        stats['positions'] = period.position_tracker.get_positions_list()
-
-        # we want the key to be absent, not just empty
-        # Only include transactions for given dt
-        stats['transactions'] = dict()
-        for date in period.processed_transactions:
-            if start_dt <= date < end_dt:
-                stats['transactions'][date] = \
-                    period.processed_transactions[date]
-
-        stats['orders'] = dict()
-        for date in period.orders_by_modified:
-            if start_dt <= date < end_dt:
-                stats['orders'][date] = \
-                    period.orders_by_modified[date]
-
-        return stats
-
-    def handle_data(self, data):
-        if not self.is_running:
-            return
-
-        self._update_portfolio()
-
-        transactions = self._check_open_orders()
-        for transaction in transactions:
-            self.perf_tracker.process_transaction(transaction)
-
-        if self._handle_data:
-            self._handle_data(self, data)
-
-        # Unlike trading controls which remain constant unless placing an
-        # order, account controls can change each bar. Thus, must check
-        # every bar no matter if the algorithm places an order or not.
-        self.validate_account_controls()
-
-        try:
-            # Since the clock runs 24/7, I trying to disable the daily
-            # Performance tracker and keep only minute and cumulative
-            self.perf_tracker.update_performance()
-
-            # TODO: save for future use?
-            minute_stats = self.prepare_period_stats(
-                data.current_dt, data.current_dt + timedelta(minutes=1))
-            log.debug('the minute performance:\n{}'.format(minute_stats))
-
-            today = pd.to_datetime('today', utc=True)
-            daily_stats = self.prepare_period_stats(
-                start_dt=today,
-                end_dt=pd.Timestamp.utcnow()
-            )
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key=today.strftime('%Y-%m-%d'),
-                obj=daily_stats,
-                rel_path='daily_perf'
-            )
-
-        except Exception as e:
-            log.warn('unable to calculate performance: {}'.format(e))
-
-        try:
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key='perf_tracker',
-                obj=self.perf_tracker
-            )
-        except Exception as e:
-            log.warn('unable to save minute perfs to disk: {}'.format(e))
-
-        try:
-            save_algo_object(
-                algo_name=self.algo_namespace,
-                key='portfolio_{}'.format(self.exchange.name),
-                obj=self.exchange.portfolio
-            )
-        except Exception as e:
-            log.warn('unable to save portfolio to disk: {}'.format(e))
-
-    def _order(self,
-               asset,
-               amount,
-               limit_price=None,
-               stop_price=None,
-               style=None,
-               attempt_index=0):
-        try:
-            return self.exchange.order(asset, amount, limit_price,
-                                       stop_price,
-                                       style)
-        except ExchangeRequestError as e:
-            log.warn(
-                'order attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_order:
-                sleep(self.retry_delay)
-                return self._order(
-                    asset, amount, limit_price, stop_price, style,
-                    attempt_index + 1)
-            else:
-                raise ExchangeTransactionError(
-                    transaction_type='order',
-                    attempts=attempt_index,
-                    error=e
-                )
-
-    @api_method
-    @disallowed_in_before_trading_start(OrderInBeforeTradingStart())
-    def order(self,
-              asset,
-              amount,
-              limit_price=None,
-              stop_price=None,
-              style=None):
-        amount, style = self._calculate_order(asset, amount,
-                                              limit_price, stop_price,
-                                              style)
-
-        order_id = self._order(asset, amount, limit_price, stop_price, style)
-        order = self.portfolio.open_orders[order_id]
-
-        self.perf_tracker.process_order(order)
-        return order
-
-    def round_order(self, amount):
-        """
-        We need fractions with cryptocurrencies
-
-        :param amount:
-        :return:
-        """
-        return amount
-
-    @api_method
-    def batch_market_order(self, share_counts):
-        raise NotImplementedError()
-
-    def _get_open_orders(self, asset=None, attempt_index=0):
-        try:
-            return self.exchange.get_open_orders(asset)
-        except ExchangeRequestError as e:
-            log.warn(
-                'open orders attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_open_orders:
-                sleep(self.retry_delay)
-                return self._get_open_orders(asset, attempt_index + 1)
-            else:
-                raise ExchangePortfolioDataError(
-                    data_type='open-orders',
-                    attempts=attempt_index,
-                    error=e
-                )
-
-    @error_keywords(sid='Keyword argument `sid` is no longer supported for '
-                        'get_open_orders. Use `asset` instead.')
-    @api_method
-    def get_open_orders(self, asset=None):
-        return self._get_open_orders(asset)
-
-    @api_method
-    def get_order(self, order_id):
-        return self.exchange.get_order(order_id)
-
-    @api_method
-    def cancel_order(self, order_param):
-        order_id = order_param
-        if isinstance(order_param, zp.Order):
-            order_id = order_param.id
-        self.exchange.cancel_order(order_id)

catalyst/exchange/asset_finder_exchange.py

@@ -1,91 +0,0 @@
-from logbook import Logger
-
-log = Logger('AssetFinderExchange')
-
-
-class AssetFinderExchange(object):
-    def __init__(self, exchange):
-        self.exchange = exchange
-        self._asset_cache = {}
-
-    @property
-    def sids(self):
-        """
-        This seems to be used to pre-fetch assets.
-        I don't think that we need this for live-trading.
-        Leaving the list empty.
-        """
-        return list()
-
-    def retrieve_all(self, sids, default_none=False):
-        """
-        Retrieve all assets in `sids`.
-
-        Parameters
-        ----------
-        sids : iterable of int
-            Assets to retrieve.
-        default_none : bool
-            If True, return None for failed lookups.
-            If False, raise `SidsNotFound`.
-
-        Returns
-        -------
-        assets : list[Asset or None]
-            A list of the same length as `sids` containing Assets (or Nones)
-            corresponding to the requested sids.
-
-        Raises
-        ------
-        SidsNotFound
-            When a requested sid is not found and default_none=False.
-        """
-        for sid in sids:
-            if sid in self._asset_cache:
-                log.info('got asset from cache: {}'.format(sid))
-            else:
-                log.info('fetching asset: {}'.format(sid))
-        return list()
-
-    def lookup_symbol(self, symbol, as_of_date, fuzzy=False):
-        """Lookup an asset by symbol.
-
-        Parameters
-        ----------
-        symbol : str
-            The ticker symbol to resolve.
-        as_of_date : datetime or None
-            Look up the last owner of this symbol as of this datetime.
-            If ``as_of_date`` is None, then this can only resolve the equity
-            if exactly one equity has ever owned the ticker.
-        fuzzy : bool, optional
-            Should fuzzy symbol matching be used? Fuzzy symbol matching
-            attempts to resolve differences in representations for
-            shareclasses. For example, some people may represent the ``A``
-            shareclass of ``BRK`` as ``BRK.A``, where others could write
-            ``BRK_A``.
-
-        Returns
-        -------
-        equity : Asset
-            The equity that held ``symbol`` on the given ``as_of_date``, or the
-            only equity to hold ``symbol`` if ``as_of_date`` is None.
-
-        Raises
-        ------
-        SymbolNotFound
-            Raised when no equity has ever held the given symbol.
-        MultipleSymbolsFound
-            Raised when no ``as_of_date`` is given and more than one equity
-            has held ``symbol``. This is also raised when ``fuzzy=True`` and
-            there are multiple candidates for the given ``symbol`` on the
-            ``as_of_date``.
-        """
-        log.info('looking up symbol: {}'.format(symbol))
-
-        if symbol in self._asset_cache:
-            return self._asset_cache[symbol]
-        else:
-            asset = self.exchange.get_asset(symbol)
-            self._asset_cache[symbol] = asset
-            return asset

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,647 +0,0 @@
-import base64
-import numpy as np
-import hashlib
-import hmac
-import json
-import re
-import time
-
-import pandas as pd
-import pytz
-import requests
-import six
-from catalyst.assets._assets import Asset
-from logbook import Logger
-
-# from websocket import create_connection
-from catalyst.exchange.exchange import Exchange
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    InvalidHistoryFrequencyError
-)
-from catalyst.finance.execution import (MarketOrder,
-                                        LimitOrder,
-                                        StopOrder,
-                                        StopLimitOrder)
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
-
-# Trying to account for REST api instability
-# https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
-requests.adapters.DEFAULT_RETRIES = 20
-
-BITFINEX_URL = 'https://api.bitfinex.com'
-
-log = Logger('Bitfinex')
-warning_logger = Logger('AlgoWarning')
-
-
-class Bitfinex(Exchange):
-    def __init__(self, key, secret, base_currency, portfolio=None):
-        self.url = BITFINEX_URL
-        self.key = key
-        self.secret = secret
-        self.id = 'b'
-        self.name = 'bitfinex'
-        self.assets = {}
-        self.load_assets()
-        self.base_currency = base_currency
-        self._portfolio = portfolio
-        self.minute_writer = None
-        self.minute_reader = None
-
-    def _request(self, operation, data, version='v1'):
-        payload_object = {
-            'request': '/{}/{}'.format(version, operation),
-            'nonce': '{0:f}'.format(time.time() * 1000000),
-            # convert to string
-            'options': {}
-        }
-
-        if data is None:
-            payload_dict = payload_object
-        else:
-            payload_dict = payload_object.copy()
-            payload_dict.update(data)
-
-        payload_json = json.dumps(payload_dict)
-        if six.PY3:
-            payload = base64.b64encode(bytes(payload_json, 'utf-8'))
-        else:
-            payload = base64.b64encode(payload_json)
-
-        m = hmac.new(self.secret, payload, hashlib.sha384)
-        m = m.hexdigest()
-
-        # headers
-        headers = {
-            'X-BFX-APIKEY': self.key,
-            'X-BFX-PAYLOAD': payload,
-            'X-BFX-SIGNATURE': m
-        }
-
-        if data is None:
-            request = requests.get(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ), data={},
-                headers=headers)
-        else:
-            request = requests.post(
-                '{url}/{version}/{operation}'.format(
-                    url=self.url,
-                    version=version,
-                    operation=operation
-                ),
-                headers=headers)
-
-        return request
-
-    def _get_v2_symbol(self, asset):
-        pair = asset.symbol.split('_')
-        symbol = 't' + pair[0].upper() + pair[1].upper()
-        return symbol
-
-    def _get_v2_symbols(self, assets):
-        """
-        Workaround to support Bitfinex v2
-        TODO: Might require a separate asset dictionary
-
-        :param assets:
-        :return:
-        """
-
-        v2_symbols = []
-        for asset in assets:
-            v2_symbols.append(self._get_v2_symbol(asset))
-
-        return v2_symbols
-
-    def _create_order(self, order_status):
-        """
-        Create a Catalyst order object from a Bitfinex order dictionary
-        :param order_status:
-        :return: Order
-        """
-        if order_status['is_cancelled']:
-            status = ORDER_STATUS.CANCELLED
-        elif not order_status['is_live']:
-            log.info('found executed order {}'.format(order_status))
-            status = ORDER_STATUS.FILLED
-        else:
-            status = ORDER_STATUS.OPEN
-
-        amount = float(order_status['original_amount'])
-        filled = float(order_status['executed_amount'])
-        is_buy = (amount > 0)
-
-        price = float(order_status['price'])
-        order_type = order_status['type']
-
-        stop_price = None
-        limit_price = None
-
-        # TODO: is this comprehensive enough?
-        if order_type.endswith('limit'):
-            limit_price = price
-        elif order_type.endswith('stop'):
-            stop_price = price
-
-        executed_price = float(order_status['avg_execution_price'])
-
-        # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
-        commission = None
-
-        # TODO: zipline likes rounded dates to match statistics, is this ok?
-        date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
-        date = pytz.utc.localize(date)
-        order = Order(
-            dt=date,
-            asset=self.assets[order_status['symbol']],
-            amount=amount,
-            stop=stop_price,
-            limit=limit_price,
-            filled=filled,
-            id=order_status['id'],
-            commission=commission
-        )
-        order.status = status
-
-        return order, executed_price
-
-    def update_portfolio(self):
-        """
-        Update the portfolio cash and position balances based on the
-        latest ticker prices.
-
-        :return:
-        """
-        try:
-            response = self._request('balances', None)
-            balances = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in balances:
-            raise ExchangeRequestError(
-                error='unable to fetch balance {}'.format(balances['message'])
-            )
-
-        base_position = None
-        for position in balances:
-            if not base_position and position['type'] == 'exchange' \
-                    and position['currency'] == self.base_currency:
-                base_position = position
-
-        if position is None:
-            raise ValueError(
-                error='Base currency %s not found in portfolio' % self.base_currency
-            )
-
-        portfolio = self._portfolio
-        portfolio.cash = float(base_position['available'])
-        if portfolio.starting_cash is None:
-            portfolio.starting_cash = portfolio.cash
-
-        if portfolio.positions:
-            assets = portfolio.positions.keys()
-            tickers = self.tickers(assets)
-            portfolio.positions_value = 0.0
-            for ticker in tickers:
-                # TODO: convert if the position is not in the base currency
-                position = portfolio.positions[ticker['asset']]
-                position.last_sale_price = ticker['last_price']
-                position.last_sale_date = ticker['timestamp']
-
-                portfolio.positions_value += \
-                    position.amount * position.last_sale_price
-                portfolio.portfolio_value = \
-                    portfolio.positions_value + portfolio.cash
-
-    @property
-    def portfolio(self):
-        """
-        Return the Portfolio
-
-        :return:
-        """
-        # if self._portfolio is None:
-        #     portfolio = ExchangePortfolio(
-        #         start_date=pd.Timestamp.utcnow()
-        #     )
-        #     self.store.portfolio = portfolio
-        #     self.update_portfolio()
-        #
-        #     portfolio.starting_cash = portfolio.cash
-        # else:
-        #     portfolio = self.store.portfolio
-
-        return self._portfolio
-
-    @property
-    def account(self):
-        account = Account()
-
-        account.settled_cash = None
-        account.accrued_interest = None
-        account.buying_power = None
-        account.equity_with_loan = None
-        account.total_positions_value = None
-        account.total_positions_exposure = None
-        account.regt_equity = None
-        account.regt_margin = None
-        account.initial_margin_requirement = None
-        account.maintenance_margin_requirement = None
-        account.available_funds = None
-        account.excess_liquidity = None
-        account.cushion = None
-        account.day_trades_remaining = None
-        account.leverage = None
-        account.net_leverage = None
-        account.net_liquidation = None
-
-        return account
-
-    @property
-    def positions(self):
-        return self.portfolio.positions
-
-    @property
-    def time_skew(self):
-        # TODO: research the time skew conditions
-        return pd.Timedelta('0s')
-
-    def subscribe_to_market_data(self, symbol):
-        pass
-
-    def get_candles(self, data_frequency, assets, bar_count=None):
-        """
-        Retrieve OHLVC candles from Bitfinex
-
-        :param data_frequency:
-        :param assets:
-        :param bar_count:
-        :return:
-
-        Available Frequencies
-        ---------------------
-        '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
-         '1M'
-        """
-
-        # TODO: use BcolzMinuteBarReader to read from cache
-        freq_match = re.match(r'([0-9].*)(m|h|d)', data_frequency, re.M | re.I)
-        if freq_match:
-            number = int(freq_match.group(1))
-            unit = freq_match.group(2)
-
-            if unit == 'd':
-                converted_unit = 'D'
-            else:
-                converted_unit = unit
-
-            frequency = '{}{}'.format(number, converted_unit)
-            allowed_frequencies = ['1m', '5m', '15m', '30m', '1h', '3h', '6h',
-                                   '12h', '1D', '7D', '14D', '1M']
-
-            if frequency not in allowed_frequencies:
-                raise InvalidHistoryFrequencyError(
-                    frequency=data_frequency
-                )
-        elif data_frequency == 'minute':
-            frequency = '1m'
-        elif data_frequency == 'daily':
-            frequency = '1D'
-        else:
-            raise InvalidHistoryFrequencyError(
-                frequency=data_frequency
-            )
-
-        # Making sure that assets are iterable
-        asset_list = [assets] if isinstance(assets, Asset) else assets
-        ohlc_list = dict()
-        for asset in asset_list:
-            symbol = self._get_v2_symbol(asset)
-            url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
-                url=self.url,
-                frequency=frequency,
-                symbol=symbol
-            )
-
-            if bar_count:
-                is_list = True
-                url += '/hist?limit={}'.format(int(bar_count))
-            else:
-                is_list = False
-                url += '/last'
-
-            try:
-                response = requests.get(url)
-            except Exception as e:
-                raise ExchangeRequestError(error=e)
-
-            if 'error' in response.content:
-                raise ExchangeRequestError(
-                    error='Unable to retrieve candles: {}'.format(
-                        response.content)
-                )
-
-            candles = response.json()
-
-            def ohlc_from_candle(candle):
-                ohlc = dict(
-                    open=np.float64(candle[1]),
-                    high=np.float64(candle[3]),
-                    low=np.float64(candle[4]),
-                    close=np.float64(candle[2]),
-                    volume=np.float64(candle[5]),
-                    price=np.float64(candle[2]),
-                    last_traded=pd.Timestamp.utcfromtimestamp(
-                        candle[0] / 1000.0),
-                    minute_dt=pd.Timestamp.utcnow().floor('1 min')
-                )
-                return ohlc
-
-            if is_list:
-                ohlc_bars = []
-                # We can to list candles from old to new
-                for candle in reversed(candles):
-                    ohlc = ohlc_from_candle(candle)
-                    ohlc_bars.append(ohlc)
-
-                ohlc_list[asset] = ohlc_bars
-
-            else:
-                ohlc = ohlc_from_candle(candles)
-                ohlc_list[asset] = ohlc
-
-        return ohlc_list[assets] \
-            if isinstance(assets, Asset) else ohlc_list
-
-    def order(self, asset, amount, limit_price, stop_price, style):
-        """Place an order.
-
-        Parameters
-        ----------
-        asset : Asset
-            The asset that this order is for.
-        amount : int
-            The amount of shares to order. If ``amount`` is positive, this is
-            the number of shares to buy or cover. If ``amount`` is negative,
-            this is the number of shares to sell or short.
-        limit_price : float, optional
-            The limit price for the order.
-        stop_price : float, optional
-            The stop price for the order.
-        style : ExecutionStyle, optional
-            The execution style for the order.
-
-        Returns
-        -------
-        order_id : str or None
-            The unique identifier for this order, or None if no order was
-            placed.
-
-        Notes
-        -----
-        The ``limit_price`` and ``stop_price`` arguments provide shorthands for
-        passing common execution styles. Passing ``limit_price=N`` is
-        equivalent to ``style=LimitOrder(N)``. Similarly, passing
-        ``stop_price=M`` is equivalent to ``style=StopOrder(M)``, and passing
-        ``limit_price=N`` and ``stop_price=M`` is equivalent to
-        ``style=StopLimitOrder(N, M)``. It is an error to pass both a ``style``
-        and ``limit_price`` or ``stop_price``.
-
-        Bitfinex Order Types
-        --------------------
-        LIMIT, MARKET, STOP, TRAILING STOP,
-        EXCHANGE MARKET, EXCHANGE LIMIT, EXCHANGE STOP,
-        EXCHANGE TRAILING STOP, FOK, EXCHANGE FOK.
-
-        See Also
-        --------
-        :class:`catalyst.finance.execution.ExecutionStyle`
-        :func:`catalyst.api.order_value`
-        :func:`catalyst.api.order_percent`
-        """
-        if amount == 0:
-            log.warn('skipping order amount of 0')
-            return None
-
-        base_currency = asset.symbol.split('_')[1]
-        if base_currency.lower() != self.base_currency.lower():
-            raise NotImplementedError(
-                'Currency pairs must share their base with the exchange.'
-            )
-
-        is_buy = (amount > 0)
-
-        if isinstance(style, MarketOrder):
-            order_type = 'market'
-        elif isinstance(style, LimitOrder):
-            order_type = 'limit'
-            price = limit_price
-        elif isinstance(style, StopOrder):
-            order_type = 'stop'
-            price = stop_price
-        elif isinstance(style, StopLimitOrder):
-            log.warn('using limit order instead of stop/limit')
-            # TODO: Not sure how to do this with the api. Investigate.
-            order_type = 'limit'
-            price = limit_price
-        else:
-            raise NotImplementedError('%s orders not available' % style)
-
-        log.debug(
-            'ordering {amount} {symbol} for {price}'.format(
-                amount=amount,
-                symbol=asset.symbol,
-                price=price
-            )
-        )
-
-        exchange_symbol = self.get_symbol(asset)
-        req = dict(
-            symbol=exchange_symbol,
-            amount=str(float(abs(amount))),
-            price=str(float(price)),
-            side='buy' if is_buy else 'sell',
-            type='exchange ' + order_type,  # TODO: support margin trades
-            exchange=self.name,
-            is_hidden=False,
-            is_postonly=False,
-            use_all_available=0,
-            ocoorder=False,
-            buy_price_oco=0,
-            sell_price_oco=0
-        )
-
-        date = pd.Timestamp.utcnow()
-        try:
-            response = self._request('order/new', req)
-            exchange_order = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in exchange_order:
-            raise ExchangeRequestError(
-                error='unable to create Bitfinex order {}'.format(
-                    exchange_order['message'])
-            )
-
-        order_id = exchange_order['id']
-        order = Order(
-            dt=date,
-            asset=asset,
-            amount=amount,
-            stop=style.get_stop_price(is_buy),
-            limit=style.get_limit_price(is_buy),
-            id=order_id
-        )
-        # TODO: is this required?
-        order.broker_order_id = order_id
-
-        self.portfolio.create_order(order)
-
-        return order_id
-
-    def get_open_orders(self, asset=None):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not None, return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If no asset is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-        try:
-            response = self._request('orders', None)
-            order_statuses = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_statuses:
-            raise ExchangeRequestError(
-                error='Unable to retrieve open orders: {}'.format(
-                    order_statuses['message'])
-            )
-
-        orders = list()
-        for order_status in order_statuses:
-            order, = self._create_order(order_status)
-            if asset is None or asset == order.sid:
-                orders.append(order)
-
-        return orders
-
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        """
-        try:
-            response = self._request(
-                'order/status', {'order_id': int(order_id)})
-            order_status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in order_status:
-            raise ExchangeRequestError(
-                error='Unable to retrieve order status: {}'.format(
-                    order_status['message'])
-            )
-        return self._create_order(order_status)
-
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-        order_id = order_param.id \
-            if isinstance(order_param, Order) else order_param
-
-        try:
-            response = self._request('order/cancel', {'order_id': order_id})
-            status = response.json()
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'message' in status:
-            raise ExchangeRequestError(
-                error='Unable to cancel order: {} {}'.format(
-                    order_id, status['message'])
-            )
-
-    def tickers(self, assets):
-        """
-        Fetch ticket data for assets
-        https://docs.bitfinex.com/v2/reference#rest-public-tickers
-
-        :param assets:
-        :return:
-        """
-        symbols = self._get_v2_symbols(assets)
-        log.debug('fetching tickers {}'.format(symbols))
-
-        try:
-            response = requests.get(
-                '{url}/v2/tickers?symbols={symbols}'.format(
-                    url=self.url,
-                    symbols=','.join(symbols),
-                )
-            )
-        except Exception as e:
-            raise ExchangeRequestError(error=e)
-
-        if 'error' in response.content:
-            raise ExchangeRequestError(
-                error='Unable to retrieve tickers: {}'.format(
-                    response.content)
-            )
-
-        tickers = response.json()
-
-        formatted_tickers = []
-        for index, ticker in enumerate(tickers):
-            if not len(ticker) == 11:
-                raise ExchangeRequestError(
-                    error='Invalid ticker in response: {}'.format(ticker)
-                )
-
-            tick = dict(
-                asset=assets[index],
-                timestamp=pd.Timestamp.utcnow(),
-                bid=ticker[1],
-                ask=ticker[3],
-                last_price=ticker[7],
-                low=ticker[10],
-                high=ticker[9],
-                volume=ticker[8],
-            )
-            formatted_tickers.append(tick)
-
-        log.debug('got tickers {}'.format(formatted_tickers))
-        return formatted_tickers

catalyst/exchange/data_portal_exchange.py

@@ -1,121 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from time import sleep
-
-from logbook import Logger
-
-from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    ExchangeBarDataError
-)
-
-log = Logger('DataPortalExchange')
-
-
-class DataPortalExchange(DataPortal):
-    def __init__(self, exchange, *args, **kwargs):
-        self.exchange = exchange
-
-        # TODO: put somewhere accessible by each algo
-        self.retry_get_history_window = 5
-        self.retry_get_spot_value = 5
-        self.retry_delay = 5
-
-        super(DataPortalExchange, self).__init__(*args, **kwargs)
-
-    def _get_history_window(self,
-                            assets,
-                            end_dt,
-                            bar_count,
-                            frequency,
-                            field,
-                            data_frequency,
-                            ffill=True,
-                            attempt_index=0):
-        try:
-            return self.exchange.get_history_window(
-                assets,
-                end_dt,
-                bar_count,
-                frequency,
-                field,
-                data_frequency,
-                ffill)
-        except ExchangeRequestError as e:
-            log.warn(
-                'get history attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_history_window:
-                sleep(self.retry_delay)
-                return self._get_history_window(assets,
-                                                end_dt,
-                                                bar_count,
-                                                frequency,
-                                                field,
-                                                data_frequency,
-                                                ffill,
-                                                attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='history',
-                    attempts=attempt_index,
-                    error=e
-                )
-
-    def get_history_window(self,
-                           assets,
-                           end_dt,
-                           bar_count,
-                           frequency,
-                           field,
-                           data_frequency,
-                           ffill=True):
-        return self._get_history_window(assets,
-                                        end_dt,
-                                        bar_count,
-                                        frequency,
-                                        field,
-                                        data_frequency,
-                                        ffill)
-
-    def _get_spot_value(self, assets, field, dt, data_frequency,
-                        attempt_index=0):
-        try:
-            return self.exchange.get_spot_value(assets, field, dt,
-                                                data_frequency)
-        except ExchangeRequestError as e:
-            log.warn(
-                'get spot value attempt {}: {}'.format(attempt_index, e)
-            )
-            if attempt_index < self.retry_get_spot_value:
-                sleep(self.retry_delay)
-                return self._get_spot_value(assets, field, dt, data_frequency,
-                                            attempt_index + 1)
-            else:
-                raise ExchangeBarDataError(
-                    data_type='spot',
-                    attempts=attempt_index,
-                    error=e
-                )
-
-    def get_spot_value(self, assets, field, dt, data_frequency):
-        return self._get_spot_value(assets, field, dt, data_frequency)
-
-    def get_adjusted_value(self, asset, field, dt,
-                           perspective_dt,
-                           data_frequency,
-                           spot_value=None):
-        # TODO: does this pertain to cryptocurrencies?
-        raise NotImplementedError("get_adjusted_value is not implemented yet!")

catalyst/exchange/exchange.py

@@ -1,489 +0,0 @@
-import abc
-import random
-from time import sleep
-import collections
-from abc import ABCMeta, abstractmethod, abstractproperty
-from datetime import timedelta
-
-import numpy as np
-import pandas as pd
-from catalyst.assets._assets import Asset
-from logbook import Logger
-
-from catalyst.data.data_portal import BASE_FIELDS
-from catalyst.errors import (
-    SymbolNotFound,
-)
-from catalyst.finance.order import ORDER_STATUS
-from catalyst.finance.transaction import Transaction
-from catalyst.exchange.exchange_utils import get_exchange_symbols
-
-log = Logger('Exchange')
-
-
-class Exchange:
-    __metaclass__ = ABCMeta
-
-    def __init__(self):
-        self.name = None
-        self.trading_pairs = None
-        self.assets = {}
-        self._portfolio = None
-        self.minute_writer = None
-        self.minute_reader = None
-
-    @abstractmethod
-    def subscribe_to_market_data(self, symbol):
-        pass
-
-    @abstractproperty
-    def positions(self):
-        pass
-
-    @abstractproperty
-    def update_portfolio(self):
-        pass
-
-    @abstractproperty
-    def portfolio(self):
-        pass
-
-    @abstractproperty
-    def account(self):
-        pass
-
-    @abstractproperty
-    def time_skew(self):
-        pass
-
-    def get_symbol(self, asset):
-        """
-        Get the exchange specific symbol of the given asset.
-
-        :param asset: Asset
-        :return: symbol: str
-        """
-        symbol = None
-
-        for key in self.assets:
-            if not symbol and self.assets[key].symbol == asset.symbol:
-                symbol = key
-
-        if not symbol:
-            raise ValueError('Currency %s not supported by exchange %s' %
-                             (asset['symbol'], self.name))
-
-        return symbol
-
-    def get_symbols(self, assets):
-        """
-        Get a list of symbols corresponding to each given asset.
-
-        :param assets: Asset[]
-        :return:
-        """
-        symbols = []
-
-        for asset in assets:
-            symbols.append(self.get_symbol(asset))
-
-        return symbols
-
-    def get_asset(self, symbol):
-        """
-        Find an Asset on the current exchange based on its Catalyst symbol
-        :param symbol: the [target]_[base] currency pair symbol
-        :return: Asset
-        """
-        asset = None
-
-        for key in self.assets:
-            if not asset and self.assets[key].symbol.lower() == symbol.lower():
-                asset = self.assets[key]
-
-        if not asset:
-            raise SymbolNotFound('Asset not found: %s' % symbol)
-
-        return asset
-
-    def load_assets(self):
-        """
-        Populate the 'assets' attribute with a dictionary of Assets.
-        The key of the resulting dictionary is the exchange specific
-        currency pair symbol. The universal symbol is contained in the
-        'symbol' attribute of each asset.
-
-
-        Notes
-        -----
-        The sid of each asset is calculated based on a numeric hash of the
-        universal symbol. This simple approach avoids maintaining a mapping
-        of sids.
-
-        This method can be overridden if an exchange offers equivalent data
-        via its api.
-        """
-
-        symbol_map = get_exchange_symbols(self.name)
-        for exchange_symbol in symbol_map:
-            asset = symbol_map[exchange_symbol]
-            symbol = asset['symbol']
-            asset_name = ' / '.join(symbol.split('_')).upper()
-
-            asset_obj = Asset(
-                symbol=symbol,
-                asset_name=asset_name,
-                sid=abs(hash(symbol)) % (10 ** 4),
-                exchange=self.name,
-                start_date=pd.to_datetime(asset['start_date'], utc=True),
-                end_date=pd.Timestamp.utcnow() + timedelta(minutes=300000),
-            )
-
-            self.assets[exchange_symbol] = asset_obj
-
-    def check_open_orders(self):
-        """
-        Loop through the list of open orders in the Portfolio object.
-        For each executed order found, create a transaction and apply to the
-        Portfolio.
-
-        :return:
-        transactions: Transaction[]
-        """
-        transactions = list()
-        if self.portfolio.open_orders:
-            for order_id in list(self.portfolio.open_orders):
-                log.debug('found open order: {}'.format(order_id))
-
-                order, executed_price = self.get_order(order_id)
-                log.debug('got updated order {} {}'.format(
-                    order, executed_price))
-
-                if order.status == ORDER_STATUS.FILLED:
-                    transaction = Transaction(
-                        asset=order.asset,
-                        amount=order.amount,
-                        dt=pd.Timestamp.utcnow(),
-                        price=executed_price,
-                        order_id=order.id,
-                        commission=order.commission
-                    )
-                    transactions.append(transaction)
-
-                    self.portfolio.execute_order(order, transaction)
-
-                elif order.status == ORDER_STATUS.CANCELLED:
-                    self.portfolio.remove_order(order)
-
-                else:
-                    delta = pd.Timestamp.utcnow() - order.dt
-                    log.info(
-                        'order {order_id} still open after {delta}'.format(
-                            order_id=order_id,
-                            delta=delta
-                        )
-                    )
-        return transactions
-
-    def get_spot_value(self, assets, field, dt=None, data_frequency='minute'):
-        """
-        Public API method that returns a scalar value representing the value
-        of the desired asset's field at either the given dt.
-
-        Parameters
-        ----------
-        assets : Asset, ContinuousFuture, or iterable of same.
-            The asset or assets whose data is desired.
-        field : {'open', 'high', 'low', 'close', 'volume',
-                 'price', 'last_traded'}
-            The desired field of the asset.
-        dt : pd.Timestamp
-            The timestamp for the desired value.
-        data_frequency : str
-            The frequency of the data to query; i.e. whether the data is
-            'daily' or 'minute' bars
-
-        Returns
-        -------
-        value : float, int, or pd.Timestamp
-            The spot value of ``field`` for ``asset`` The return type is based
-            on the ``field`` requested. If the field is one of 'open', 'high',
-            'low', 'close', or 'price', the value will be a float. If the
-            ``field`` is 'volume' the value will be a int. If the ``field`` is
-            'last_traded' the value will be a Timestamp.
-
-        Bitfinex timeframes
-        -------------------
-        Available values: '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h',
-         '1D', '7D', '14D', '1M'
-        """
-        if field not in BASE_FIELDS:
-            raise KeyError('Invalid column: ' + str(field))
-
-        if isinstance(assets, collections.Iterable):
-            values = list()
-            for asset in assets:
-                value = self.get_single_spot_value(
-                    asset, field, data_frequency)
-                values.append(value)
-
-            return values
-        else:
-            return self.get_single_spot_value(
-                assets, field, data_frequency)
-
-    def get_single_spot_value(self, asset, field, data_frequency):
-        """
-        Similar to 'get_spot_value' but for a single asset
-
-        Note
-        ----
-        We're writing each minute bar to disk using zipline's machinery.
-        This is especially useful when running multiple algorithms
-        concurrently. By using local data when possible, we try to reaching
-        request limits on exchanges.
-
-        :param asset:
-        :param field:
-        :param data_frequency:
-        :return value: The spot value of the given asset / field
-        """
-        log.debug(
-            'fetching spot value {field} for symbol {symbol}'.format(
-                symbol=asset.symbol,
-                field=field
-            )
-        )
-
-        if field == 'price':
-            field = 'close'
-
-        # Don't use a timezone here
-        dt = pd.Timestamp.utcnow().floor('1 min')
-        value = None
-        if self.minute_reader is not None:
-            try:
-                # Slight delay to minimize the chances that multiple algos
-                # might try to hit the cache at the exact same time.
-                sleep_time = random.uniform(0.5, 0.8)
-                sleep(sleep_time)
-                # TODO: This does not always! Why is that? Open an issue with zipline.
-                # See: https://github.com/zipline-live/zipline/issues/26
-                value = self.minute_reader.get_value(
-                    sid=asset.sid,
-                    dt=dt,
-                    field=field
-                )
-            except Exception as e:
-                log.warn('minute data not found: {}'.format(e))
-
-        if value is None or np.isnan(value):
-            ohlc = self.get_candles(data_frequency, asset)
-            if field not in ohlc:
-                raise KeyError('Invalid column: %s' % field)
-
-            if self.minute_writer is not None:
-                df = pd.DataFrame(
-                    [ohlc],
-                    index=pd.DatetimeIndex([dt]),
-                    columns=['open', 'high', 'low', 'close', 'volume']
-                )
-
-                try:
-                    self.minute_writer.write_sid(
-                        sid=asset.sid,
-                        df=df
-                    )
-                    log.debug('wrote minute data: {}'.format(dt))
-                except Exception as e:
-                    log.warn(
-                        'unable to write minute data: {} {}'.format(dt, e))
-
-            value = ohlc[field]
-            log.debug('got spot value: {}'.format(value))
-        else:
-            log.debug('got spot value from cache: {}'.format(value))
-
-        return value
-
-    def get_history_window(self,
-                           assets,
-                           end_dt,
-                           bar_count,
-                           frequency,
-                           field,
-                           data_frequency,
-                           ffill=True):
-
-        """
-        Public API method that returns a dataframe containing the requested
-        history window.  Data is fully adjusted.
-
-        Parameters
-        ----------
-        assets : list of catalyst.data.Asset objects
-            The assets whose data is desired.
-
-        end_dt: not applicable to cryptocurrencies
-
-        bar_count: int
-            The number of bars desired.
-
-        frequency: string
-            "1d" or "1m"
-
-        field: string
-            The desired field of the asset.
-
-        data_frequency: string
-            The frequency of the data to query; i.e. whether the data is
-            'daily' or 'minute' bars.
-
-        # TODO: fill how?
-        ffill: boolean
-            Forward-fill missing values. Only has effect if field
-            is 'price'.
-
-        Returns
-        -------
-        A dataframe containing the requested data.
-        """
-
-        candles = self.get_candles(
-            data_frequency=frequency,
-            assets=assets,
-            bar_count=bar_count,
-        )
-
-        frames = []
-        for asset in assets:
-            asset_candles = candles[asset]
-
-            asset_data = dict()
-            asset_data[asset] = map(lambda candle: candle[field],
-                                    asset_candles)
-
-            dates = map(lambda candle: candle['last_traded'],
-                        asset_candles)
-
-            df = pd.DataFrame(asset_data, index=dates)
-            frames.append(df)
-
-        return pd.concat(frames)
-
-    @abstractmethod
-    def order(self, asset, amount, limit_price, stop_price, style):
-        """Place an order.
-
-        Parameters
-        ----------
-        asset : Asset
-            The asset that this order is for.
-        amount : int
-            The amount of shares to order. If ``amount`` is positive, this is
-            the number of shares to buy or cover. If ``amount`` is negative,
-            this is the number of shares to sell or short.
-        limit_price : float, optional
-            The limit price for the order.
-        stop_price : float, optional
-            The stop price for the order.
-        style : ExecutionStyle, optional
-            The execution style for the order.
-
-        Returns
-        -------
-        order_id : str or None
-            The unique identifier for this order, or None if no order was
-            placed.
-
-        Notes
-        -----
-        The ``limit_price`` and ``stop_price`` arguments provide shorthands for
-        passing common execution styles. Passing ``limit_price=N`` is
-        equivalent to ``style=LimitOrder(N)``. Similarly, passing
-        ``stop_price=M`` is equivalent to ``style=StopOrder(M)``, and passing
-        ``limit_price=N`` and ``stop_price=M`` is equivalent to
-        ``style=StopLimitOrder(N, M)``. It is an error to pass both a ``style``
-        and ``limit_price`` or ``stop_price``.
-
-        See Also
-        --------
-        :class:`catalyst.finance.execution.ExecutionStyle`
-        :func:`catalyst.api.order_value`
-        :func:`catalyst.api.order_percent`
-        """
-        pass
-
-    @abstractmethod
-    def get_open_orders(self, asset):
-        """Retrieve all of the current open orders.
-
-        Parameters
-        ----------
-        asset : Asset
-            If passed and not None, return only the open orders for the given
-            asset instead of all open orders.
-
-        Returns
-        -------
-        open_orders : dict[list[Order]] or list[Order]
-            If no asset is passed this will return a dict mapping Assets
-            to a list containing all the open orders for the asset.
-            If an asset is passed then this will return a list of the open
-            orders for this asset.
-        """
-        pass
-
-    @abstractmethod
-    def get_order(self, order_id):
-        """Lookup an order based on the order id returned from one of the
-        order functions.
-
-        Parameters
-        ----------
-        order_id : str
-            The unique identifier for the order.
-
-        Returns
-        -------
-        order : Order
-            The order object.
-        execution_price: float
-            The execution price per share of the order
-        """
-        pass
-
-    @abstractmethod
-    def cancel_order(self, order_param):
-        """Cancel an open order.
-
-        Parameters
-        ----------
-        order_param : str or Order
-            The order_id or order object to cancel.
-        """
-        pass
-
-    @abstractmethod
-    def get_candles(self, data_frequency, assets, bar_count=None):
-        """
-        Retrieve OHLCV candles for the given assets
-
-        :param data_frequency:
-        :param assets:
-        :param end_dt:
-        :param bar_count:
-        :param limit:
-        :return:
-        """
-        pass
-
-    @abc.abstractmethod
-    def tickers(self, assets):
-        """
-        Retrieve current tick data for the given assets
-
-        :param assets:
-        :return:
-        """
-        return

catalyst/exchange/exchange_clock.py

@@ -1,60 +0,0 @@
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from time import sleep
-
-import pandas as pd
-from catalyst.gens.sim_engine import (
-    BAR,
-    SESSION_START,
-    MINUTE_END,
-    SESSION_END
-)
-from logbook import Logger
-
-log = Logger('ExchangeClock')
-
-
-class ExchangeClock(object):
-    """Realtime clock for live trading.
-
-    This class is a drop-in replacement for
-    :class:`zipline.gens.sim_engine.MinuteSimulationClock`.
-
-    This is a stripped down version because crypto exchanges run around the clock.
-
-    The :param:`time_skew` parameter represents the time difference between
-    the Broker and the live trading machine's clock.
-    """
-
-    def __init__(self, sessions, time_skew=pd.Timedelta("0s")):
-
-        self.sessions = sessions
-        self.time_skew = time_skew
-        self._last_emit = None
-        self._before_trading_start_bar_yielded = True
-
-    def __iter__(self):
-        yield pd.Timestamp.utcnow(), SESSION_START
-
-        while True:
-            current_time = pd.Timestamp.utcnow()
-            current_minute = current_time.floor('1 min')
-
-            if self._last_emit is None or current_minute > self._last_emit:
-                log.debug('emitting minutely bar: {}'.format(current_minute))
-
-                self._last_emit = current_minute
-                yield current_minute, BAR
-            else:
-                sleep(1)

catalyst/exchange/exchange_errors.py

@@ -1,60 +0,0 @@
-from catalyst.errors import ZiplineError
-
-
-class ExchangeRequestError(ZiplineError):
-    msg = (
-        'Request failed: {error}'
-    ).strip()
-
-
-class ExchangeRequestErrorTooManyAttempts(ZiplineError):
-    msg = (
-        'Request failed: {error}, giving up after {attempts} attempts'
-    ).strip()
-
-
-class ExchangeBarDataError(ZiplineError):
-    msg = (
-        'Unable to retrieve bar data: {data_type}, ' +
-        'giving up after {attempts} attempts: {error}'
-    ).strip()
-
-
-class ExchangePortfolioDataError(ZiplineError):
-    msg = (
-        'Unable to retrieve portfolio data: {data_type}, ' +
-        'giving up after {attempts} attempts: {error}'
-    ).strip()
-
-
-class ExchangeTransactionError(ZiplineError):
-    msg = (
-        'Unable to execute transaction: {transaction_type}, ' +
-        'giving up after {attempts} attempts: {error}'
-    ).strip()
-
-
-class ExchangeAuthNotFound(ZiplineError):
-    msg = (
-        'Please create an auth.json file containing the api token and key for '
-        'exchange {exchange}. Place the file here: {filename}'
-    ).strip()
-
-
-class ExchangeSymbolsNotFound(ZiplineError):
-    msg = (
-        'Unable to download or find a local copy of symbols.json for exchange '
-        '{exchange}. The file should be here: {filename}'
-    ).strip()
-
-
-class AlgoPickleNotFound(ZiplineError):
-    msg = (
-        'Pickle not found for algo {algo} in path {filename}'
-    ).strip()
-
-
-class InvalidHistoryFrequencyError(ZiplineError):
-    msg = (
-        'History frequency {frequency} not supported by the exchange.'
-    ).strip()

catalyst/exchange/exchange_portfolio.py

@@ -1,87 +0,0 @@
-import numpy as np
-from logbook import Logger
-
-from catalyst.protocol import Portfolio, Positions, Position
-
-log = Logger('ExchangePortfolio')
-
-
-class ExchangePortfolio(Portfolio):
-    """
-    Since the goal is to support multiple exchanges, it makes sense to
-    include additional stats in the portfolio object.
-
-    Instead of relying on the performance tracker, each exchange portfolio
-    tracks its own holding. This offers a separation between tracking an
-    exchange and the statistics of the algorithm.
-    """
-
-    def __init__(self, start_date, starting_cash=None):
-        self.capital_used = 0.0
-        self.starting_cash = starting_cash
-        self.portfolio_value = starting_cash
-        self.pnl = 0.0
-        self.returns = 0.0
-        self.cash = starting_cash
-        self.positions = Positions()
-        self.start_date = start_date
-        self.positions_value = 0.0
-        self.open_orders = dict()
-
-    def calculate_pnl(self):
-        log.debug('calculating pnl')
-
-    def create_order(self, order):
-        log.debug('creating order {}'.format(order.id))
-        self.open_orders[order.id] = order
-
-        order_position = self.positions[order.asset] \
-            if order.asset in self.positions else None
-
-        if order_position is None:
-            order_position = Position(order.asset)
-            self.positions[order.asset] = order_position
-
-        order_position.amount += order.amount
-        log.debug('open order added to portfolio')
-
-    def execute_order(self, order, transaction):
-        log.debug('executing order {}'.format(order.id))
-        del self.open_orders[order.id]
-
-        order_position = self.positions[order.asset] \
-            if order.asset in self.positions else None
-
-        if order_position is None:
-            raise ValueError(
-                'Trying to execute order for a position not held: %s' % order.id
-            )
-
-        self.capital_used += order.amount * transaction.price
-
-        if order.amount > 0:
-            if order_position.cost_basis > 0:
-                order_position.cost_basis = np.average(
-                    [order_position.cost_basis, transaction.price],
-                    weights=[order_position.amount, order.amount]
-                )
-            else:
-                order_position.cost_basis = transaction.price
-
-        log.debug('updated portfolio with executed order')
-
-    def remove_order(self, order):
-        log.info('removing cancelled order {}'.format(order.id))
-        del self.open_orders[order.id]
-
-        order_position = self.positions[order.asset] \
-            if order.asset in self.positions else None
-
-        if order_position is None:
-            raise ValueError(
-                'Trying to remove order for a position not held: %s' % order.id
-            )
-
-        order_position.amount -= order.amount
-
-        log.debug('removed order from portfolio')

catalyst/exchange/exchange_utils.py

@@ -1,133 +0,0 @@
-import json
-import os
-import pickle
-import urllib
-from datetime import date, datetime
-
-from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
-    ExchangeSymbolsNotFound
-from catalyst.utils.paths import data_root, ensure_directory
-
-SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
-              'live-trading/catalyst/exchange/symbols/{exchange}.json'
-
-
-def get_exchange_folder(exchange_name, environ=None):
-    if not environ:
-        environ = os.environ
-
-    root = data_root(environ)
-    exchange_folder = os.path.join(root, 'exchanges', exchange_name)
-    ensure_directory(exchange_folder)
-
-    return exchange_folder
-
-
-def download_exchange_symbols(exchange_name, environ=None):
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'symbols.json')
-
-    url = SYMBOLS_URL.format(exchange=exchange_name)
-    response = urllib.urlretrieve(url=url, filename=filename)
-    return response
-
-
-def get_exchange_symbols(exchange_name, environ=None):
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'symbols.json')
-
-    if not os.path.isfile(filename):
-        download_exchange_symbols(exchange_name, environ)
-
-    if os.path.isfile(filename):
-        with open(filename) as data_file:
-            data = json.load(data_file)
-            return data
-    else:
-        raise ExchangeSymbolsNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
-
-
-def get_exchange_auth(exchange_name, environ=None):
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'auth.json')
-
-    if os.path.isfile(filename):
-        with open(filename) as data_file:
-            data = json.load(data_file)
-            return data
-    else:
-        raise ExchangeAuthNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
-
-
-def get_algo_folder(algo_name, environ=None):
-    if not environ:
-        environ = os.environ
-
-    root = data_root(environ)
-    algo_folder = os.path.join(root, 'live_algos', algo_name)
-    ensure_directory(algo_folder)
-
-    return algo_folder
-
-
-def get_algo_object(algo_name, key, environ=None, rel_path=None):
-    folder = get_algo_folder(algo_name, environ)
-
-    if rel_path is not None:
-        folder = os.path.join(folder, rel_path)
-
-    filename = os.path.join(folder, key + '.p')
-
-    if os.path.isfile(filename):
-        try:
-            with open(filename, 'rb') as handle:
-                return pickle.load(handle)
-        except Exception as e:
-            return None
-    else:
-        return None
-
-
-def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
-    folder = get_algo_folder(algo_name, environ)
-
-    if rel_path is not None:
-        folder = os.path.join(folder, rel_path)
-        ensure_directory(folder)
-
-    filename = os.path.join(folder, key + '.p')
-
-    with open(filename, 'wb') as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-
-def append_algo_object(algo_name, key, obj, environ=None):
-    algo_folder = get_algo_folder(algo_name, environ)
-    filename = os.path.join(algo_folder, key + '.p')
-
-    mode = 'a+b' if os.path.isfile(filename) else 'wb'
-    with open(filename, mode) as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-
-def get_exchange_minute_writer_root(exchange_name, environ=None):
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-
-    minute_data_folder = os.path.join(exchange_folder, 'minute_data')
-    ensure_directory(minute_data_folder)
-
-    return minute_data_folder
-
-
-def perf_serial(obj):
-    """JSON serializer for objects not serializable by default json code"""
-
-    if isinstance(obj, (datetime, date)):
-        return obj.isoformat()
-    raise TypeError("Type %s not serializable" % type(obj))

catalyst/exchange/symbols/bitfinex.json

@@ -1,110 +0,0 @@
-{
-  "btcusd": {
-    "symbol": "btc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcusd": {
-    "symbol": "ltc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcbtc": {
-    "symbol": "ltc_btc",
-    "start_date": "2010-01-01"
-  },
-  "ethusd": {
-    "symbol": "eth_usd",
-    "start_date": "2010-01-01"
-  },
-  "ethbtc": {
-    "symbol": "eth_btc",
-    "start_date": "2010-01-01"
-  },
-  "etcbtc": {
-    "symbol": "etc_btc",
-    "start_date": "2010-01-01"
-  },
-  "etcusd": {
-    "symbol": "etc_usd",
-    "start_date": "2010-01-01"
-  },
-  "rrtusd": {
-    "symbol": "rrt_usd",
-    "start_date": "2010-01-01"
-  },
-  "rrtbtc": {
-    "symbol": "rrt_btc",
-    "start_date": "2010-01-01"
-  },
-  "zecusd": {
-    "symbol": "zec_usd",
-    "start_date": "2010-01-01"
-  },
-  "zecbtc": {
-    "symbol": "zec_btc",
-    "start_date": "2010-01-01"
-  },
-  "xmrusd": {
-    "symbol": "xmr_usd",
-    "start_date": "2010-01-01"
-  },
-  "xmrbtc": {
-    "symbol": "xmr_btc",
-    "start_date": "2010-01-01"
-  },
-  "dshusd": {
-    "symbol": "dsh_usd",
-    "start_date": "2010-01-01"
-  },
-  "dshbtc": {
-    "symbol": "dsh_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccbtc": {
-    "symbol": "bcc_btc",
-    "start_date": "2010-01-01"
-  },
-  "bcubtc": {
-    "symbol": "bcu_btc",
-    "start_date": "2010-01-01"
-  },
-  "bccusd": {
-    "symbol": "bcc_usd",
-    "start_date": "2010-01-01"
-  },
-  "bcuusd": {
-    "symbol": "bcu_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpusd": {
-    "symbol": "xrp_usd",
-    "start_date": "2010-01-01"
-  },
-  "xrpbtc": {
-    "symbol": "xrp_btc",
-    "start_date": "2010-01-01"
-  },
-  "iotusd": {
-    "symbol": "iot_usd",
-    "start_date": "2010-01-01"
-  },
-  "iotbtc": {
-    "symbol": "iot_btc",
-    "start_date": "2010-01-01"
-  },
-  "ioteth": {
-    "symbol": "iot_eth",
-    "start_date": "2010-01-01"
-  },
-  "eosusd": {
-    "symbol": "eos_usd",
-    "start_date": "2010-01-01"
-  },
-  "eosbtc": {
-    "symbol": "eos_btc",
-    "start_date": "2010-01-01"
-  },
-  "eoseth": {
-    "symbol": "eos_eth",
-    "start_date": "2010-01-01"
-  }
-}

catalyst/utils/run_algo.py

@@ -3,18 +3,12 @@ import re
 from runpy import run_path
 import sys
 import warnings
-from time import sleep
-from datetime import timedelta
-
-import pandas as pd
 
 import click
-
 try:
     from pygments import highlight
     from pygments.lexers import PythonLexer
     from pygments.formatters import TerminalFormatter
-
     PYGMENTS = True
 except:
     PYGMENTS = False
@@ -35,21 +29,6 @@ from catalyst.utils.calendars import get_calendar
 from catalyst.utils.factory import create_simulation_parameters
 import catalyst.utils.paths as pth
 
-from catalyst.exchange.algorithm_exchange import ExchangeTradingAlgorithm
-from catalyst.exchange.data_portal_exchange import DataPortalExchange
-from catalyst.exchange.bitfinex.bitfinex import Bitfinex
-from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
-from catalyst.exchange.exchange_portfolio import ExchangePortfolio
-from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
-    ExchangeRequestErrorTooManyAttempts
-)
-from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object
-from logbook import Logger
-
-log = Logger('run_algo')
-
 
 class _RunAlgoError(click.ClickException, ValueError):
     """Signal an error that should have a different message if invoked from
@@ -89,11 +68,7 @@ def _run(handle_data,
          output,
          print_algo,
          local_namespace,
-         environ,
+         environ):
-         live,
-         exchange,
-         algo_namespace,
-         base_currency):
     """Run a backtest for the given algorithm.
 
     This is shared between the cli and :func:`catalyst.run_algo`.
@@ -142,18 +117,6 @@ def _run(handle_data,
         else:
             click.echo(algotext)
 
-    if exchange is not None:
-        start = pd.Timestamp.utcnow()
-        end = start + timedelta(minutes=1439)
-
-    open_calendar = get_calendar('OPEN')
-    sim_params = create_simulation_parameters(
-        start=start,
-        end=end,
-        capital_base=capital_base,
-        data_frequency=data_frequency,
-        emission_rate=data_frequency,
-    )
     if bundle is not None:
         bundles = bundle.split(',')
 
@@ -183,8 +146,10 @@ def _run(handle_data,
                     str(bundle_data.asset_finder.engine.url),
                 )
 
+            open_calendar = get_calendar('OPEN')
+
             env = TradingEnvironment(
-                load=partial(load_crypto_market_data, environ=environ),
+                load=partial(load_crypto_market_data, bundle=b, bundle_data=bundle_data, environ=environ),
                 bm_symbol='USDT_BTC',
                 trading_calendar=open_calendar,
                 asset_db_path=connstr,
@@ -214,16 +179,16 @@ def _run(handle_data,
 
             if b == 'poloniex':
                 return CryptoPricingLoader(
-                    bundle_data,
+                           bundle_data,
-                    data_frequency,
+                           data_frequency,
-                    CryptoPricing,
+                           CryptoPricing,
-                )
+                       )
             elif b == 'quandl':
                 return USEquityPricingLoader(
-                    bundle_data,
+                           bundle_data,
-                    data_frequency,
+                           data_frequency,
-                    USEquityPricing,
+                           USEquityPricing,
-                )
+                       )
             raise ValueError(
                 "No PipelineLoader registered for bundle %s." % b
             )
@@ -240,65 +205,20 @@ def _run(handle_data,
             )
 
     else:
-        if live and exchange is not None:
+        env = TradingEnvironment(environ=environ)
-            env = TradingEnvironment(
+        choose_loader = None
-                environ=environ,
-                exchange_tz="UTC",
-                asset_db_path=None
-            )
-            env.asset_finder = AssetFinderExchange(exchange)
 
-            data = DataPortalExchange(
+    perf = TradingAlgorithm(
-                exchange=exchange,
-                asset_finder=env.asset_finder,
-                trading_calendar=open_calendar,
-                first_trading_day=pd.to_datetime('today', utc=True)
-            )
-            choose_loader = None
-
-            def update_portfolio(attempt_index=0):
-                """
-                Fetch the portfolio for the exchange
-                We can't continue on error because it is required to bootstrap
-                the algorithm.
-                :param attempt_index:
-                :return:
-                """
-                try:
-                    exchange.update_portfolio()
-                    return exchange.portfolio
-                except ExchangeRequestError as e:
-                    if attempt_index < 20:
-                        sleep(5)
-                        return update_portfolio(attempt_index + 1)
-                    else:
-                        raise ExchangeRequestErrorTooManyAttempts(
-                            attempts=attempt_index,
-                            error=e
-                        )
-
-            portfolio = update_portfolio()
-            sim_params = create_simulation_parameters(
-                start=start,
-                end=end,
-                capital_base=portfolio.starting_cash,
-                emission_rate='minute',
-                data_frequency='minute'
-            )
-        else:
-            env = TradingEnvironment(environ=environ)
-            choose_loader = None
-
-    TradingAlgorithmClass = (
-        partial(ExchangeTradingAlgorithm, exchange=exchange,
-                algo_namespace=algo_namespace)
-        if live and exchange else TradingAlgorithm)
-
-    perf = TradingAlgorithmClass(
         namespace=namespace,
         env=env,
         get_pipeline_loader=choose_loader,
-        sim_params=sim_params,
+        sim_params=create_simulation_parameters(
+            start=start,
+            end=end,
+            capital_base=capital_base,
+            data_frequency=data_frequency,
+            emission_rate=data_frequency,
+        ),
         **{
             'initialize': initialize,
             'handle_data': handle_data,
@@ -374,10 +294,10 @@ def load_extensions(default, extensions, strict, environ, reload=False):
             _loaded_extensions.add(ext)
 
 
-def run_algorithm(initialize,
+def run_algorithm(start,
-                  capital_base=None,
+                  end,
-                  start=None,
+                  initialize,
-                  end=None,
+                  capital_base,
                   handle_data=None,
                   before_trading_start=None,
                   analyze=None,
@@ -388,11 +308,7 @@ def run_algorithm(initialize,
                   default_extension=True,
                   extensions=(),
                   strict_extensions=True,
-                  environ=os.environ,
+                  environ=os.environ):
-                  live=False,
-                  exchange_name=None,
-                  base_currency=None,
-                  algo_namespace=None):
     """Run a trading algorithm.
 
     Parameters
@@ -446,12 +362,6 @@ def run_algorithm(initialize,
     environ : mapping[str -> str], optional
         The os environment to use. Many extensions use this to get parameters.
         This defaults to ``os.environ``.
-    live: execute live trading
-    exchange_conn: The exchange connection parameters
-
-    Supported Exchanges
-    -------------------
-    bitfinex
 
     Returns
     -------
@@ -462,53 +372,26 @@ def run_algorithm(initialize,
     --------
     catalyst.data.bundles.bundles : The available data bundles.
     """
-    mode = 'live' if live else 'backtest'
-    log.info('running algo in {mode} mode'.format(mode=mode))
     load_extensions(default_extension, extensions, strict_extensions, environ)
 
-    exchange = None
+    non_none_data = valfilter(bool, {
-    if mode == 'backtest':
+        'data': data is not None,
-        non_none_data = valfilter(bool, {
+        'bundle': bundle is not None,
-            'data': data is not None,
+    })
-            'bundle': bundle is not None,
+    if not non_none_data:
-        })
+        # if neither data nor bundle are passed use 'quantopian-quandl'
-        if not non_none_data:
+        bundle = 'quantopian-quandl'
-            # if neither data nor bundle are passed use 'quantopian-quandl'
-            bundle = 'quantopian-quandl'
 
-        elif len(non_none_data) != 1:
+    elif len(non_none_data) != 1:
-            raise ValueError(
+        raise ValueError(
-                'must specify one of `data`, `data_portal`, or `bundle`,'
+            'must specify one of `data`, `data_portal`, or `bundle`,'
-                ' got: %r' % non_none_data,
+            ' got: %r' % non_none_data,
-            )
+        )
 
-        elif 'bundle' not in non_none_data and bundle_timestamp is not None:
+    elif 'bundle' not in non_none_data and bundle_timestamp is not None:
-            raise ValueError(
+        raise ValueError(
-                'cannot specify `bundle_timestamp` without passing `bundle`',
+            'cannot specify `bundle_timestamp` without passing `bundle`',
-            )
+        )
-    else:
-        if exchange_name is not None:
-            portfolio = get_algo_object(
-                algo_name=algo_namespace,
-                key='portfolio_{}'.format(exchange_name),
-                environ=environ
-            )
-            if portfolio is None:
-                portfolio = ExchangePortfolio(
-                    start_date=pd.Timestamp.utcnow()
-                )
-
-            exchange_auth = get_exchange_auth(exchange_name)
-            if exchange_name == 'bitfinex':
-                exchange = Bitfinex(
-                    key=exchange_auth['key'],
-                    secret=exchange_auth['secret'].encode('UTF-8'),
-                    base_currency=base_currency,
-                    portfolio=portfolio
-                )
-            else:
-                raise NotImplementedError(
-                    'exchange not supported: %s' % exchange_name)
 
     return _run(
         handle_data=handle_data,
@@ -529,8 +412,4 @@ def run_algorithm(initialize,
         print_algo=False,
         local_namespace=False,
         environ=environ,
-        live=live,
-        exchange=exchange,
-        algo_namespace=algo_namespace,
-        base_currency=base_currency
     )

docs/source/live-trading-blueprint.md

@@ -1,207 +0,0 @@
-<h1>Live Trading Blueprint</h1>
-The purpose of this document is to allow project contributors navigate
-through the ongoing live trading implementation.
-
-<h2>Components</h2>
-At a high level, the following components have been implemented to coerce
-zipline into live trading.
-
-<h3>Exchange</h3>
-
-*catalyst/exchange*
-
-Exchange is a new package introducing cryptocurrency
-exchanges to zipline. The package contains mostly new implementations
-of existing components, adapted to characteristics of exchanges.
-
-Here are some key characteristics which make cryptocurrency exchanges
-exchanges different compared to equity brokers.
-* They trade around the clock.
-* Currency symbols are inconsistent across exchanges.
-* They trade currency pairs (i.e. the base currency is not always be USD).
-This is a paradigm shift in context of zipline. Additional
-business logic will be required to manage the portfolio data and orders.
-* The price of a single asset might vary across exchanges. This means
-arbitrage opportunities. Consequently, to extract maximum alpha, the
-platform should not only support multiple exchanges, but also multiple
-exchanges per algorithm.
-* The fee model is usually more complex than that of an equity broker.
-It can vary drastically between exchanges.
-* There are no splits, mergers, etc. to worry about.
-* A complete order book is usually available, the platform should
-offer access to it order to help traders reduce slippage.
-
-<h3>New Components</h3>
-These components of the exchange package were added to the zipline
-sources.
-
-<h4>Exchange</h4>
-
-*catalyst/exchange/exchange.py*
-
-Abstract class which acts as an interface for the implementation of
-various exchanges. It also contains logic common to all exchanges.
-
-<h4>Bitfinex</h4>
-
-*catalyst/exchange/bitfinex.py*
-
-The Bitfinex exchange implementation. It extends the Exchange class.
-
-<h4>DataPortalExchange</h4>
-
-*catalyst/exchange/data_portal_exchange.py*
-
-Extends the zipline DataPortal to route spot data to the exchange.
-This is critical because it allows the algoritm to request data in
-real-time.
-
-For example, `data.current(asset, 'price')` retrieves the current price
-of the asset, not the price at the time of yielding the bar this
-is critical to minimize slippage.
-
-At the time of writing, it only supports spot data but I believe that
-it should be extended to historical data as well. Some exchanges
-have better historical data APIs than others. This will need to
-be considered during each individual implementation.
-
-<h4>ExchangeClock</h4>
-
-*catalyst/exchange/exchange_clock.py*
-
-An implementation to the zipline Clock which runs 24/7. It yields a
-bar every minute.
-
-<h4>AssetFinderExchange</h4>
-
-*catalyst/exchange/asset_finder_exchange.py*
-
-An alternate implementation of AssetFinder which locates each asset
-against the exchanges instead of bundle databases.
-
-For example, `symbol('eth_usd')` should return an Ethereum/USD asset
-regardless of currency notation of the target exchange.
-
-To acheive this, I have created a dictionary of currencies for the
-Bitfinex exchange. Here is what it looks like.
-* Each key represents the exchange specific symbol.
-* The symbol attribute represents the abstract symbol common across
-all exchanges for the given currency.
-* The start_date attribute should correspond to its first trading day
-on the exchange.
-
-```json
-{
-  "btcusd": {
-    "symbol": "btc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcusd": {
-    "symbol": "ltc_usd",
-    "start_date": "2010-01-01"
-  },
-  "ltcbtc": {
-    "symbol": "ltc_btc",
-    "start_date": "2010-01-01"
-  },
-  "ethusd": {
-    "symbol": "eth_usd",
-    "start_date": "2010-01-01"
-  },
-  "ethbtc": {
-    "symbol": "eth_btc",
-    "start_date": "2010-01-01"
-  }
-}
-```
-
-<h4>ExchangeTradingAlgorithm</h4>
-
-*catalyst/exchange/algorithm_exchange.py*
-
-Extends the TradingAlgorithm class which orchestrates the api
-operations. This class brings together most of the components
-described above.
-
-<h3>Modified Components</h3>
-
-The following components have been modified to include conditional
-business logic to enable live trading.
-
-<h4>run_algorithm</h4>
-
-*catalyst/utils/run_algo.py*
-
-The run_algorithm interface is an entry point to execute an
-algorithm in zipline. This component was already modified for
-the catalyst concurrency bundles. I added conditional logic
-which should not interfere with backtesting.
-
-In a nutshell, the run_algorithm method now contains three additional
-parameters:
-* live: If True, zipline will attempt to trade live. If False or not
-specified, it will run a backtest as normal.
-* algo_namespace: An arbitrary namespace for the current algorithm.
-It will be used to persist data between runs.
-* exchange_conn: A dictionary containing the attributes required
-to instantiate an exchange. Here is an example for Bitfinex:
-
-```python
-exchange_conn = dict(
-    name='bitfinex',
-    key='',
-    secret=b'',
-    base_currency='usd'
-)
-```
-
-The following sample algorithm uses the run_algorithm interface:
-
-*catalyst/examples/buy_and_hold_live.py*
-
-<h2>Portfolio Management</h2>
-
-Zipline has a Portfolio class containing key metrics used by zipline
-for, but not only, these reasons:
-
-* Placing orders: When placing orders (e.g. order_target_percent),
-zipline queries the portfolio to assess the size of current positions,
-cash available, etc.
-* Measuring performance: The portfolio contains attributes like
-cost basis of each asset, p&l, etc. which zipline uses to compute all
-of its performance criteria.
-
-When backtesting, zipline automatically updates the Portfolio object
-of its corresponding algorithm. When live trading, these updates should
-be the responsibility of the exchange as it holds the truth for:
-
-* Executed price of each order (including fees and slippage)
-* Partial / failed orders
-* Cash (i.e. base currency) available
-* Cost basis of each position
-
-If each exchange account had a one-to-one relationship with an
-algorithm, portfolio metrics could be retrieved directly from the
-exchange without persisting any data to the algorithm. However,
-doing this would have at least the following drawbacks:
-
-* It may not be reasonable to ask users to dedicate an
-exchange account to a single algorithm. Exchanges are not easy
-to partition.
-* If an exchange account contains existing positions, the calculated
-cost basis would correspond to all positions, not just those
-initiated by the algorithm.
-* It would not be possible impose trading limits on algorithms.
-
-It follows that Portfolio metrics should be calculated using a strategic
-combination of the exchange data and algorithm activity. While tracking
-the activity of an algorithm works well in backtesting, it is more
-challenging during live trading. A live algorithm might run over
-several months. It might have to stop and start for many reasons.
-This means that the platform should have the ability to persist
-algorithm activity in order to be reliable.
-
-In the interest of time, I will start by persisting algorithm
-activity in memory. Data will be lost when the algorithm execution stops.
-The intent it to offer a simple basis from which to implement data
-persistence strategies in the future.

setup.py

@@ -212,7 +212,7 @@ def read_requirements(path,
                       conda_format=False,
                       filter_names=None):
     """
-    Read a requirements.txt file, expressed as a path relative to Zipline root.
+    Read a requirements.txt file, expressed as a path relative to Catalyst root.
 
     Returns requirements with the pinned versions as lower bounds
     if `strict_bounds` is falsey.

tests/exchange/base.py

@@ -1,46 +0,0 @@
-import unittest
-from abc import ABCMeta, abstractmethod
-
-
-class BaseExchangeTestCase():
-    __metaclass__ = ABCMeta
-
-    @abstractmethod
-    def test_positions(self):
-        pass
-
-    @abstractmethod
-    def test_portfolio(self):
-        pass
-
-    @abstractmethod
-    def test_account(self):
-        pass
-
-    @abstractmethod
-    def test_time_skew(self):
-        pass
-
-    @abstractmethod
-    def test_get_open_orders(self):
-        pass
-
-    @abstractmethod
-    def test_order(self):
-        pass
-
-    @abstractmethod
-    def test_get_order(self):
-        pass
-
-    @abstractmethod
-    def test_cancel_order(self):
-        pass
-
-    @abstractmethod
-    def test_get_spot_value(self):
-        pass
-
-    @abstractmethod
-    def test_tickers(self):
-        pass

tests/exchange/test_bitfinex.py

@@ -1,94 +0,0 @@
-from catalyst.exchange.bitfinex import Bitfinex
-from .base import BaseExchangeTestCase
-from logbook import Logger
-import pandas as pd
-from catalyst.finance.execution import (MarketOrder,
-                                        LimitOrder,
-                                        StopOrder,
-                                        StopLimitOrder)
-
-log = Logger('BitfinexTestCase')
-
-
-class BitfinexTestCase(BaseExchangeTestCase):
-    def test_positions(self):
-        log.info('querying positions from bitfinex')
-        bitfinex = Bitfinex()
-        balance = bitfinex.positions()
-        log.info('the balance: {}'.format(balance))
-        pass
-
-    def test_portfolio(self):
-        log.info('fetching portfolio data')
-        pass
-
-    def test_account(self):
-        log.info('fetching account data')
-        pass
-
-    def test_time_skew(self):
-        log.info('time skew not implemented')
-        pass
-
-    def test_get_open_orders(self):
-        log.info('fetching open orders')
-        bitfinex = Bitfinex()
-        order_id = bitfinex.get_open_orders()
-        log.info('open orders: {}'.format(order_id))
-        pass
-
-    def test_order(self):
-        log.info('ordering from bitfinex')
-        bitfinex = Bitfinex()
-        order_id = bitfinex.order(
-            asset=bitfinex.get_asset('eth_usd'),
-            style=LimitOrder(limit_price=200),
-            limit_price=200,
-            amount=0.5,
-            stop_price=None
-        )
-        log.info('order created {}'.format(order_id))
-        pass
-
-    def test_get_order(self):
-        log.info('querying orders from bitfinex')
-        bitfinex = Bitfinex()
-        response = bitfinex.get_order(order_id=3361248395)
-        log.info('the order: {}'.format(response))
-        pass
-
-    def test_cancel_order(self):
-        log.info('canceling order from bitfinex')
-        bitfinex = Bitfinex()
-        response = bitfinex.cancel_order(order_id=3330847408)
-        log.info('canceled order: {}'.format(response))
-        pass
-
-    def test_get_spot_value(self):
-        log.info('spot value not implemented')
-        bitfinex = Bitfinex()
-        assets = [
-            bitfinex.get_asset('eth_usd'),
-            bitfinex.get_asset('etc_usd'),
-            bitfinex.get_asset('eos_usd'),
-        ]
-        # assets = bitfinex.get_asset('eth_usd')
-        value = bitfinex.get_spot_value(
-            assets=assets,
-            field='close',
-            data_frequency='minute'
-        )
-        pass
-
-    def test_tickers(self):
-        log.info('fetching ticker from bitfinex')
-        bitfinex = Bitfinex()
-        current_date = pd.Timestamp.utcnow()
-        assets = [
-            bitfinex.get_asset('eth_usd'),
-            bitfinex.get_asset('etc_usd'),
-            bitfinex.get_asset('eos_usd'),
-        ]
-        tickers = bitfinex.tickers(date=current_date, assets=assets)
-        log.info('got tickers {}'.format(tickers))
-        pass

tests/exchange/test_clock.py

@@ -1,50 +0,0 @@
-from unittest import TestCase
-from logbook import Logger
-from mock import patch, sentinel
-from catalyst.exchange.exchange_clock import ExchangeClock
-from catalyst.utils.calendars.trading_calendar import days_at_time
-from datetime import time
-from collections import defaultdict
-from catalyst.utils.calendars import get_calendar
-import pandas as pd
-
-log = Logger('ExchangeClockTestCase')
-
-
-class ExchangeClockTestCase(TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.open_calendar = get_calendar("OPEN")
-
-        cls.sessions = pd.Timestamp.utcnow()
-
-    def setUp(self):
-        self.internal_clock = None
-        self.events = defaultdict(list)
-
-    def advance_clock(self, x):
-        """Mock function for sleep. Advances the internal clock by 1 min"""
-        # The internal clock advance time must be 1 minute to match
-        # MinutesSimulationClock's update frequency
-        self.internal_clock += pd.Timedelta('1 min')
-
-    def get_clock(self, arg, *args, **kwargs):
-        """Mock function for pandas.to_datetime which is used to query the
-        current time in RealtimeClock"""
-        assert arg == "now"
-        return self.internal_clock
-
-    def test_clock(self):
-        with patch('catalyst.exchange.exchange_clock.pd.to_datetime') as to_dt, \
-                patch('catalyst.exchange.exchange_clock.sleep') as sleep:
-            clock = ExchangeClock(sessions=self.sessions)
-            to_dt.side_effect = self.get_clock
-            sleep.side_effect = self.advance_clock
-            start_time = pd.Timestamp.utcnow()
-            self.internal_clock = start_time
-
-            events = list(clock)
-
-            # Event 0 is SESSION_START which always happens at 00:00.
-            ts, event_type = events[1]
-        pass