Improved frequency support for data.history() in backtest, standardized class names, improved unit tests and working on new sample algo.

catalyst/__main__.py

@@ -9,7 +9,7 @@ from six import text_type
 
 from catalyst.data import bundles as bundles_module
 from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.init_utils import get_exchange
+from catalyst.exchange.factory import get_exchange
 from catalyst.utils.cli import Date, Timestamp
 from catalyst.utils.run_algo import _run, load_extensions
 

catalyst/exchange/bundle_utils.py

@@ -103,8 +103,6 @@ def get_start_dt(end_dt, bar_count, data_frequency):
     return start_dt
 
 
-
-
 def get_month_start_end(dt):
     """
     Returns the first and last day of the month for the specified date.
@@ -215,4 +213,3 @@ def find_most_recent_time(bundle_name):
         return list(most_recent_bundle.keys())[0]
     else:
         return None
-

catalyst/exchange/exchange.py

@@ -20,7 +20,8 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
-from catalyst.exchange.exchange_utils import get_exchange_symbols
+from catalyst.exchange.exchange_utils import get_exchange_symbols, \
+    get_frequency, resample_history_df
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 
@@ -393,7 +394,7 @@ class Exchange:
         )
         series = pd.Series(values, index=dates)
 
-        #TODO: ensure that this working as expected, if not use fillna
+        # TODO: ensure that this working as expected, if not use fillna
         series.reindex(periods, method='ffill', fill_value=previous_value)
 
         return series
@@ -441,25 +442,9 @@ class Exchange:
         A dataframe containing the requested data.
         """
 
-        freq_match = re.match(r'([0-9].*)(m|M|d|D)', frequency, re.M | re.I)
-        if freq_match:
-            candle_size = int(freq_match.group(1))
-            unit = freq_match.group(2)
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency)
-
-        if unit.lower() == 'd':
-            if data_frequency == 'minute':
-                data_frequency = 'daily'
-
-        elif unit.lower() == 'm':
-            if data_frequency == 'daily':
-                data_frequency = 'minute'
-
-        else:
-            raise InvalidHistoryFrequencyError(frequency)
-
+        candle_size, unit, data_frequency = get_frequency(
+            frequency, data_frequency
+        )
         adj_bar_count = candle_size * bar_count
         try:
             series = self.bundle.get_history_window_series_and_load(
@@ -512,23 +497,7 @@ class Exchange:
                 else:
                     series[asset] = candle_series
 
-        df = pd.DataFrame(series)
-
-        if candle_size > 1:
-            if field == 'open':
-                agg = 'first'
-            elif field == 'high':
-                agg = 'max'
-            elif field == 'low':
-                agg = 'min'
-            elif field == 'close':
-                agg = 'last'
-            elif field == 'volume':
-                agg = 'sum'
-            else:
-                raise ValueError('Invalid field.')
-
-            df = df.resample('{}T'.format(candle_size)).agg(agg)
+        df = resample_history_df(pd.DataFrame(series), candle_size, field)
 
         return df
 

catalyst/exchange/exchange_data_portal.py

@@ -26,6 +26,7 @@ from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
     PricingDataNotLoadedError)
+from catalyst.exchange.exchange_utils import get_frequency, resample_history_df
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
@@ -300,16 +301,23 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         :param ffill:
         :return:
         """
-
         bundle = self.exchange_bundles[exchange.name]
+
+        candle_size, unit, data_frequency = get_frequency(
+            frequency, data_frequency
+        )
+        adj_bar_count = candle_size * bar_count
+
         series = bundle.get_history_window_series_and_load(
             assets=assets,
             end_dt=end_dt,
-            bar_count=bar_count,
+            bar_count=adj_bar_count,
             field=field,
             data_frequency=data_frequency
         )
-        return pd.DataFrame(series)
+
+        df = resample_history_df(pd.DataFrame(series), candle_size, field)
+        return df
 
     def get_exchange_spot_value(self, exchange, assets, field, dt,
                                 data_frequency):

catalyst/exchange/exchange_utils.py

@@ -1,6 +1,7 @@
 import json
 import os
 import pickle
+import re
 
 from catalyst.assets._assets import TradingPair
 from six.moves.urllib import request
@@ -8,7 +9,8 @@ from datetime import date, datetime
 
 import pandas as pd
 
-from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound
+from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \
+    InvalidHistoryFrequencyError
 from catalyst.utils.paths import data_root, ensure_directory, \
     last_modified_time
 
@@ -17,6 +19,13 @@ SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
 
 
 def get_exchange_folder(exchange_name, environ=None):
+    """
+    The root path of an exchange folder.
+
+    :param exchange_name:
+    :param environ:
+    :return:
+    """
     if not environ:
         environ = os.environ
 
@@ -28,11 +37,25 @@ def get_exchange_folder(exchange_name, environ=None):
 
 
 def get_exchange_symbols_filename(exchange_name, environ=None):
+    """
+    The absolute path of the exchange's symbol.json file.
+
+    :param exchange_name:
+    :param environ:
+    :return:
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
     return os.path.join(exchange_folder, 'symbols.json')
 
 
 def download_exchange_symbols(exchange_name, environ=None):
+    """
+    Downloads the exchange's symbols.json from the repository.
+
+    :param exchange_name:
+    :param environ:
+    :return: response
+    """
     filename = get_exchange_symbols_filename(exchange_name)
     url = SYMBOLS_URL.format(exchange=exchange_name)
     response = request.urlretrieve(url=url, filename=filename)
@@ -40,6 +63,13 @@ def download_exchange_symbols(exchange_name, environ=None):
 
 
 def get_exchange_symbols(exchange_name, environ=None):
+    """
+    The de-serialized content of the exchange's symbols.json.
+
+    :param exchange_name:
+    :param environ:
+    :return:
+    """
     filename = get_exchange_symbols_filename(exchange_name)
 
     if not os.path.isfile(filename) or \
@@ -60,11 +90,24 @@ def get_exchange_symbols(exchange_name, environ=None):
 
 
 def get_symbols_string(assets):
+    """
+    A concatenated string of symbols from a list of assets.
+
+    :param assets:
+    :return:
+    """
     array = [assets] if isinstance(assets, TradingPair) else assets
     return ', '.join([asset.symbol for asset in array])
 
 
 def get_exchange_auth(exchange_name, environ=None):
+    """
+    The de-serialized contend of the exchange's auth.json file.
+
+    :param exchange_name:
+    :param environ:
+    :return:
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
     filename = os.path.join(exchange_folder, 'auth.json')
 
@@ -81,6 +124,13 @@ def get_exchange_auth(exchange_name, environ=None):
 
 
 def get_algo_folder(algo_name, environ=None):
+    """
+    The algorithm root folder of the algorithm.
+
+    :param algo_name:
+    :param environ:
+    :return:
+    """
     if not environ:
         environ = os.environ
 
@@ -92,6 +142,15 @@ def get_algo_folder(algo_name, environ=None):
 
 
 def get_algo_object(algo_name, key, environ=None, rel_path=None):
+    """
+    The de-serialized object of the algo name and key.
+
+    :param algo_name:
+    :param key:
+    :param environ:
+    :param rel_path:
+    :return:
+    """
     if algo_name is None:
         return None
 
@@ -113,6 +172,16 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None):
 
 
 def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
+    """
+    Serialize and save an object by algo name and key.
+
+    :param algo_name:
+    :param key:
+    :param obj:
+    :param environ:
+    :param rel_path:
+    :return:
+    """
     folder = get_algo_folder(algo_name, environ)
 
     if rel_path is not None:
@@ -125,16 +194,16 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None):
         pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
 
 
-def append_algo_object(algo_name, key, obj, environ=None):
-    algo_folder = get_algo_folder(algo_name, environ)
-    filename = os.path.join(algo_folder, key + '.p')
-
-    mode = 'a+b' if os.path.isfile(filename) else 'wb'
-    with open(filename, mode) as handle:
-        pickle.dump(obj, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-
 def get_algo_df(algo_name, key, environ=None, rel_path=None):
+    """
+    The de-serialized DataFrame of an algo name and key.
+
+    :param algo_name:
+    :param key:
+    :param environ:
+    :param rel_path:
+    :return:
+    """
     folder = get_algo_folder(algo_name, environ)
 
     if rel_path is not None:
@@ -153,6 +222,16 @@ def get_algo_df(algo_name, key, environ=None, rel_path=None):
 
 
 def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
+    """
+    Serialize to csv and save a DataFrame by algo name and key.
+
+    :param algo_name:
+    :param key:
+    :param df:
+    :param environ:
+    :param rel_path:
+    :return:
+    """
     folder = get_algo_folder(algo_name, environ)
 
     if rel_path is not None:
@@ -166,6 +245,13 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None):
 
 
 def get_exchange_minute_writer_root(exchange_name, environ=None):
+    """
+    The minute writer folder for the exchange.
+
+    :param exchange_name:
+    :param environ:
+    :return:
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
 
     minute_data_folder = os.path.join(exchange_folder, 'minute_data')
@@ -175,6 +261,13 @@ def get_exchange_minute_writer_root(exchange_name, environ=None):
 
 
 def get_exchange_bundles_folder(exchange_name, environ=None):
+    """
+    The temp folder for bundle downloads by algo name.
+
+    :param exchange_name:
+    :param environ:
+    :return:
+    """
     exchange_folder = get_exchange_folder(exchange_name, environ)
 
     temp_bundles = os.path.join(exchange_folder, 'temp_bundles')
@@ -184,8 +277,86 @@ def get_exchange_bundles_folder(exchange_name, environ=None):
 
 
 def perf_serial(obj):
-    """JSON serializer for objects not serializable by default json code"""
+    """
+    JSON serializer for objects not serializable by default json code
 
+    :param obj:
+    :return:
+    """
     if isinstance(obj, (datetime, date)):
         return obj.isoformat()
+
     raise TypeError("Type %s not serializable" % type(obj))
+
+
+def get_common_assets(exchanges):
+    """
+    The assets available in all specified exchanges.
+
+    :param exchanges:
+    :return:
+    """
+    symbols = []
+    for exchange_name in exchanges:
+        s = [asset.symbol for asset in exchanges[exchange_name].get_assets()]
+        symbols.append(s)
+
+    inter_symbols = set.intersection(*map(set, symbols))
+
+    assets = []
+    for symbol in inter_symbols:
+        for exchange_name in exchanges:
+            asset = exchanges[exchange_name].get_asset(symbol)
+            assets.append(asset)
+
+    return assets
+
+
+def get_frequency(freq, data_frequency):
+    if freq == 'daily':
+        freq = '1d'
+    elif freq == 'minute':
+        freq = '1m'
+
+    freq_match = re.match(r'([0-9].*)(m|M|d|D)', freq, re.M | re.I)
+    if freq_match:
+        candle_size = int(freq_match.group(1))
+        unit = freq_match.group(2)
+
+    else:
+        raise InvalidHistoryFrequencyError(freq)
+
+    if unit.lower() == 'd':
+        if data_frequency == 'minute':
+            data_frequency = 'daily'
+
+    elif unit.lower() == 'm':
+        if data_frequency == 'daily':
+            data_frequency = 'minute'
+
+    else:
+        raise InvalidHistoryFrequencyError(freq)
+
+    return candle_size, unit, data_frequency
+
+
+def resample_history_df(df, candle_size, field):
+    if candle_size > 1:
+        if field == 'open':
+            agg = 'first'
+        elif field == 'high':
+            agg = 'max'
+        elif field == 'low':
+            agg = 'min'
+        elif field == 'close':
+            agg = 'last'
+        elif field == 'volume':
+            agg = 'sum'
+        else:
+            raise ValueError('Invalid field.')
+
+        # TODO: pad with nan?
+        return df.resample('{}T'.format(candle_size)).agg(agg)
+
+    else:
+        return df

catalyst/exchange/factory.py

@@ -5,28 +5,39 @@ from catalyst.exchange.exchange_utils import get_exchange_auth
 from catalyst.exchange.poloniex.poloniex import Poloniex
 
 
-def get_exchange(exchange_name):
+def get_exchange(exchange_name, base_currency=None):
     exchange_auth = get_exchange_auth(exchange_name)
     if exchange_name == 'bitfinex':
         return Bitfinex(
             key=exchange_auth['key'],
             secret=exchange_auth['secret'],
-            base_currency=None,  # TODO: make optional at the exchange
+            base_currency=base_currency,
             portfolio=None
         )
+
     elif exchange_name == 'bittrex':
         return Bittrex(
             key=exchange_auth['key'],
             secret=exchange_auth['secret'],
-            base_currency=None,
+            base_currency=base_currency,
             portfolio=None
         )
+
     elif exchange_name == 'poloniex':
         return Poloniex(
             key=exchange_auth['key'],
             secret=exchange_auth['secret'],
-            base_currency=None,
+            base_currency=base_currency,
             portfolio=None
         )
+
     else:
         raise ExchangeNotFoundError(exchange_name=exchange_name)
+
+
+def get_exchanges(exchange_names):
+    exchanges = dict()
+    for exchange_name in exchange_names:
+        exchanges[exchange_name] = get_exchange(exchange_name)
+
+    return exchanges

catalyst/support/issue_44.py

@@ -0,0 +1,109 @@
+import pandas as pd
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+
+from catalyst.api import (
+    symbols,
+)
+
+
+def initialize(context):
+    context.i = -1
+    context.base_currency = 'btc'
+
+
+def handle_data(context, data):
+    lookback = 60 * 24 * 7  # (minutes, hours, days)
+    context.i += 1
+    if context.i < lookback:
+        return
+
+    today = context.blotter.current_dt.strftime('%Y-%m-%d %H:%M:%S')
+
+    try:
+        # update universe everyday
+        new_day = 60 * 24
+        if not context.i % new_day:
+            context.universe = universe(context, today)
+
+        # get data every 30 minutes
+        minutes = 30
+        if not context.i % minutes and context.universe:
+            for coin in context.coins:
+                pair = str(coin.symbol)
+
+                # ohlcv data
+                open = data.history(coin, 'open', lookback,
+                                    '1m').ffill().bfill().resample(
+                    '30T').first()
+                high = data.history(coin, 'high', lookback,
+                                    '1m').ffill().bfill().resample('30T').max()
+                low = data.history(coin, 'low', lookback,
+                                   '1m').ffill().bfill().resample('30T').min()
+                close = data.history(coin, 'price', lookback,
+                                     '1m').ffill().bfill().resample(
+                    '30T').last()
+                volume = data.history(coin, 'volume', lookback,
+                                      '1m').ffill().bfill().resample(
+                    '30T').sum()
+
+                print(today, pair, close[-1])
+
+    except Exception as e:
+        print(e)
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+def universe(context, today):
+    json_symbols = get_exchange_symbols('poloniex')
+    poloniex_universe_df = pd.DataFrame.from_dict(
+        json_symbols).transpose().astype(str)
+    poloniex_universe_df['base_currency'] = poloniex_universe_df.apply(
+        lambda row: row.symbol.split('_')[1],
+        axis=1)
+    poloniex_universe_df['market_currency'] = poloniex_universe_df.apply(
+        lambda row: row.symbol.split('_')[0],
+        axis=1)
+    poloniex_universe_df = poloniex_universe_df[
+        poloniex_universe_df['base_currency'] == context.base_currency]
+    poloniex_universe_df = poloniex_universe_df[
+        poloniex_universe_df.symbol != 'gas_btc']
+
+    # Markets currently not working on Catalyst 0.3.1
+    # 2017-01-01
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'bcn_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'burst_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'dgb_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'doge_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'emc2_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'pink_btc']
+    # poloniex_universe_df = poloniex_universe_df[poloniex_universe_df.symbol != 'sc_btc']
+    print(poloniex_universe_df.head())
+
+    date = str(today).split(' ')[0]
+
+    poloniex_universe_df = poloniex_universe_df[
+        poloniex_universe_df.start_date < date]
+    context.coins = symbols(*poloniex_universe_df.symbol)
+    print(len(poloniex_universe_df))
+    return poloniex_universe_df.symbol.tolist()
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-01-01', utc=True)
+    end_date = pd.to_datetime('2017-10-15', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=10000.0,
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='poloniex',
+                                data_frequency='minute',
+                                base_currency='btc',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='test')

catalyst/utils/run_algo.py

@@ -31,7 +31,7 @@ import catalyst.utils.paths as pth
 
 from catalyst.exchange.exchange_algorithm import ExchangeTradingAlgorithmLive, \
     ExchangeTradingAlgorithmBacktest
-from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \
     DataPortalExchangeBacktest
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio

conatus_strategies/neo_rsi.py

@@ -0,0 +1,197 @@
+import talib
+import pandas as pd
+from logbook import Logger
+
+from catalyst.api import (
+    order,
+    order_target_percent,
+    symbol,
+    record,
+    get_open_orders,
+)
+from catalyst.exchange.stats_utils import get_pretty_stats
+from catalyst.utils.run_algo import run_algorithm
+
+algo_namespace = 'neo_rsi'
+log = Logger(algo_namespace)
+
+
+def initialize(context):
+    log.info('initializing algo')
+    context.asset = symbol('neo_usd', 'bitfinex')
+
+    context.BUY_SIGNAL = 30
+    context.SELL_SIGNAL = 70
+    context.SLIPPAGE_ALLOWED = 0.02
+
+    context.errors = []
+    pass
+
+
+def _handle_data(context, data):
+    dt = data.current_dt
+    log.info('BAR {}'.format(dt))
+
+    price = data.current(context.asset, 'close')
+    log.info('got price {price}'.format(price=price))
+
+    if price is None:
+        log.warn('no pricing data')
+        return
+
+    try:
+        prices = data.history(
+            context.asset,
+            fields='price',
+            bar_count=15,
+            frequency='15m'
+        )
+    except Exception as e:
+        log.warn('historical data not available: '.format(e))
+        return
+
+    rsi = talib.RSI(prices.values, timeperiod=14)[-1]
+    log.info('got rsi: {}'.format(rsi))
+
+    cash = context.portfolio.cash
+    log.info('base currency available: {cash}'.format(cash=cash))
+
+
+    orders = get_open_orders(context.asset)
+    # if len(orders) > 0:
+    #     log.info('skipping bar until all open orders execute')
+    #     return
+
+    if context.asset in context.portfolio.positions:
+        if rsi >= context.SELL_SIGNAL:
+            position = context.portfolio.positions[context.asset]
+            log.info('closing position')
+            amount = -position.amount
+            expected_proceeds = -(amount * price)
+            order(
+                asset=context.asset,
+                amount=amount,
+                limit_price=price * (1 - context.SLIPPAGE_ALLOWED),
+            )
+    else:
+        if rsi <= context.BUY_SIGNAL:
+            log.info('opening position')
+            order(
+                asset=context.asset,
+                amount=cash / price,
+                limit_price=price * (1 + context.SLIPPAGE_ALLOWED),
+            )
+
+    volume = data.current(context.asset, 'volume')
+    record(
+        price=price,
+        volume=volume,
+        cash=cash,
+        starting_cash=context.portfolio.starting_cash,
+        leverage=context.account.leverage,
+        rsi=rsi
+    )
+
+
+def handle_data(context, data):
+    try:
+        _handle_data(context, data)
+    except Exception as e:
+        log.warn('aborting the bar on error {}'.format(e))
+        context.errors.append(e)
+
+    log.debug('completed bar {}, total execution errors {}'.format(
+        data.current_dt,
+        len(context.errors)
+    ))
+
+    if len(context.errors) > 0:
+        log.info('the errors:\n{}'.format(context.errors))
+
+
+def analyze(context=None, results=None):
+    import matplotlib.pyplot as plt
+
+    # Plot the portfolio and asset data.
+    ax1 = plt.subplot(611)
+    results[['portfolio_value']].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio Value (USD)')
+
+    ax2 = plt.subplot(612, sharex=ax1)
+    ax2.set_ylabel('{asset} (USD)'.format(asset=context.asset.symbol))
+    (results[['price']]).plot(ax=ax2)
+
+    trans = results.ix[[t != [] for t in results.transactions]]
+    buys = trans.ix[
+        [t[0]['amount'] > 0 for t in trans.transactions]
+    ]
+    ax2.plot(
+        buys.index,
+        results.price[buys.index],
+        '^',
+        markersize=10,
+        color='g',
+    )
+
+    ax3 = plt.subplot(613, sharex=ax1)
+    results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
+    ax3.set_ylabel('Leverage ')
+
+    ax4 = plt.subplot(614, sharex=ax1)
+    results[['starting_cash', 'cash']].plot(ax=ax4)
+    ax4.set_ylabel('Cash (USD)')
+
+    results[[
+        'treasury',
+        'algorithm',
+        'benchmark',
+    ]] = results[[
+        'treasury_period_return',
+        'algorithm_period_return',
+        'benchmark_period_return',
+    ]]
+
+    ax5 = plt.subplot(615, sharex=ax1)
+    results[[
+        'treasury',
+        'algorithm',
+        'benchmark',
+    ]].plot(ax=ax5)
+    ax5.set_ylabel('Percent Change')
+
+    ax6 = plt.subplot(616, sharex=ax1)
+    results[['volume']].plot(ax=ax6)
+    ax6.set_ylabel('Volume (mCoins/5min)')
+
+    plt.legend(loc=3)
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+# run_algorithm(
+#     initialize=initialize,
+#     handle_data=handle_data,
+#     analyze=analyze,
+#     exchange_name='bitfinex',
+#     live=True,
+#     algo_namespace=algo_namespace,
+#     base_currency='btc',
+#     live_graph=False
+# )
+
+# Backtest
+run_algorithm(
+    capital_base=10000,
+    data_frequency='minute',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=analyze,
+    exchange_name='bitfinex',
+    algo_namespace=algo_namespace,
+    base_currency='usd',
+    start=pd.to_datetime('2017-9-10', utc=True),
+    end=pd.to_datetime('2017-10-22', utc=True),
+)

tests/exchange/base.py

@@ -1,4 +1,3 @@
-import unittest
 from abc import ABCMeta, abstractmethod
 
 

tests/exchange/test_bundle.py

@@ -137,10 +137,10 @@ class TestExchangeBundle:
         log.info('ingesting exchange bundle {}'.format(exchange_name))
         exchange_bundle.ingest(
             data_frequency=data_frequency,
-            include_symbols=include_symbols,
+            include_symbols=None,
             exclude_symbols=None,
-            start=start,
-            end=end,
+            start=None,
+            end=None,
             show_progress=True
         )
 

tests/exchange/test_data_portal.py

@@ -1,47 +1,37 @@
 import pandas as pd
+from catalyst.exchange.exchange_data_portal import DataPortalExchangeBacktest, \
+    DataPortalExchangeLive
 from logbook import Logger
+from test_utils import rnd_history_date_days, rnd_bar_count
 
 from catalyst import get_calendar
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.bitfinex.bitfinex import Bitfinex
 from catalyst.exchange.bittrex.bittrex import Bittrex
-from catalyst.exchange.data_portal_exchange import DataPortalExchangeBacktest, \
-    DataPortalExchangeLive
-from catalyst.exchange.exchange_utils import get_exchange_auth
+from catalyst.exchange.exchange_utils import get_exchange_auth, \
+    get_common_assets
+from catalyst.exchange.factory import get_exchange, get_exchanges
 
 log = Logger('test_bitfinex')
 
 
-class TestExchangeDataPortalTestCase:
+class TestExchangeDataPortal:
     @classmethod
     def setup(self):
         log.info('creating bitfinex exchange')
-        auth_bitfinex = get_exchange_auth('bitfinex')
-        self.bitfinex = Bitfinex(
-            key=auth_bitfinex['key'],
-            secret=auth_bitfinex['secret'],
-            base_currency='usd'
-        )
-
-        log.info('creating bittrex exchange')
-        auth_bitfinex = get_exchange_auth('bittrex')
-        self.bittrex = Bittrex(
-            key=auth_bitfinex['key'],
-            secret=auth_bitfinex['secret'],
-            base_currency='usd'
-        )
-
+        exchanges = get_exchanges(['bitfinex', 'bittrex', 'poloniex'])
         open_calendar = get_calendar('OPEN')
         asset_finder = AssetFinderExchange()
 
         self.data_portal_live = DataPortalExchangeLive(
-            exchanges=dict(bitfinex=self.bitfinex, bittrex=self.bittrex),
+            exchanges=exchanges,
             asset_finder=asset_finder,
             trading_calendar=open_calendar,
             first_trading_day=pd.to_datetime('today', utc=True)
         )
+
         self.data_portal_backtest = DataPortalExchangeBacktest(
-            exchanges=dict(bitfinex=self.bitfinex),
+            exchanges=exchanges,
             asset_finder=asset_finder,
             trading_calendar=open_calendar,
             first_trading_day=None  # will set dynamically based on assets
@@ -106,3 +96,20 @@ class TestExchangeDataPortalTestCase:
             assets, 'close', date, 'minute')
         log.info('found spot value {}'.format(value))
         pass
+
+    def test_history_compare_exchanges(self):
+        exchanges = get_exchanges(['bittrex', 'bitfinex', 'poloniex'])
+        assets = get_common_assets(exchanges)
+
+        date = rnd_history_date_days()
+        bar_count = rnd_bar_count()
+        data = self.data_portal_backtest.get_history_window(
+            assets=assets,
+            end_dt=date,
+            bar_count=bar_count,
+            frequency='1d',
+            field='close',
+            data_frequency='daily'
+        )
+
+        log.info('found history window: {}'.format(data))

tests/exchange/test_utils.py

@@ -0,0 +1,17 @@
+from datetime import timedelta
+from random import randint
+
+import pandas as pd
+
+
+def rnd_history_date_days(max_days=30):
+    now = pd.Timestamp.utcnow()
+    days = randint(0, max_days)
+
+    return now - timedelta(days=days)
+
+
+def rnd_bar_count(max_bars=21):
+    now = pd.Timestamp.utcnow()
+
+    return randint(0, max_bars)