BLD: working on unit tests.

catalyst/data/loader.py

@@ -146,6 +146,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None,
         exchange = get_exchange(
             exchange_name='poloniex', base_currency='usdt'
         )
+        exchange.init()
 
     benchmark_asset = exchange.get_asset(bm_symbol)
 

catalyst/examples/mean_reversion_simple.py

@@ -37,14 +37,14 @@ def initialize(context):
     context.base_price = None
     context.current_day = None
 
-    context.RSI_OVERSOLD = 50
+    context.RSI_OVERSOLD = 55
     context.RSI_OVERBOUGHT = 60
     context.CANDLE_SIZE = '5T'
 
     context.start_time = time.time()
 
-    # context.set_commission(maker=0.1, taker=0.2)
-    context.set_slippage(spread=0.0001)
+    # context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
 
 
 def handle_data(context, data):
@@ -248,7 +248,7 @@ if __name__ == '__main__':
 
     if live:
         run_algorithm(
-            capital_base=0.025,
+            capital_base=0.1,
             initialize=initialize,
             handle_data=handle_data,
             analyze=analyze,
@@ -280,7 +280,7 @@ if __name__ == '__main__':
             analyze=analyze,
             exchange_name='bitfinex',
             algo_namespace=NAMESPACE,
-            base_currency='eth',
+            base_currency='btc',
             start=pd.to_datetime('2017-10-01', utc=True),
             end=pd.to_datetime('2017-11-10', utc=True),
             output=out

catalyst/examples/mean_reversion_simple_custom_fees.py

@@ -0,0 +1,288 @@
+# For this example, we're going to write a simple momentum script.  When the
+# stock goes up quickly, we're going to buy; when it goes down quickly, we're
+# going to sell.  Hopefully we'll ride the waves.
+import os
+import tempfile
+import time
+
+import numpy as np
+import pandas as pd
+import talib
+from logbook import Logger
+
+from catalyst import run_algorithm
+from catalyst.api import symbol, record, order_target_percent, get_open_orders
+from catalyst.exchange.utils.stats_utils import extract_transactions
+# We give a name to the algorithm which Catalyst will use to persist its state.
+# In this example, Catalyst will create the `.catalyst/data/live_algos`
+# directory. If we stop and start the algorithm, Catalyst will resume its
+# state using the files included in the folder.
+from catalyst.utils.paths import ensure_directory
+
+NAMESPACE = 'mean_reversion_simple'
+log = Logger(NAMESPACE)
+
+
+# To run an algorithm in Catalyst, you need two functions: initialize and
+# handle_data.
+
+def initialize(context):
+    # This initialize function sets any data or variables that you'll use in
+    # your algorithm.  For instance, you'll want to define the trading pair (or
+    # trading pairs) you want to backtest.  You'll also want to define any
+    # parameters or values you're going to use.
+
+    # In our example, we're looking at Neo in Ether.
+    context.market = symbol('eth_btc')
+    context.base_price = None
+    context.current_day = None
+
+    context.RSI_OVERSOLD = 50
+    context.RSI_OVERBOUGHT = 60
+    context.CANDLE_SIZE = '5T'
+
+    context.start_time = time.time()
+
+    context.set_commission(maker=0.001, taker=0.002)
+    # context.set_slippage(spread=0.001)
+
+
+def handle_data(context, data):
+    # This handle_data function is where the real work is done.  Our data is
+    # minute-level tick data, and each minute is called a frame.  This function
+    # runs on each frame of the data.
+
+    # We flag the first period of each day.
+    # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle
+    # would only execute once. This method works with minute and daily
+    # frequencies.
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
+    # We're computing the volume-weighted-average-price of the security
+    # defined above, in the context.market variable.  For this example, we're
+    # using three bars on the 15 min bars.
+
+    # The frequency attribute determine the bar size. We use this convention
+    # for the frequency alias:
+    # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases
+    prices = data.history(
+        context.market,
+        fields='close',
+        bar_count=50,
+        frequency=context.CANDLE_SIZE
+    )
+
+    # Ta-lib calculates various technical indicator based on price and
+    # volume arrays.
+
+    # In this example, we are comp
+    rsi = talib.RSI(prices.values, timeperiod=14)
+
+    # We need a variable for the current price of the security to compare to
+    # the average. Since we are requesting two fields, data.current()
+    # returns a DataFrame with
+    current = data.current(context.market, fields=['close', 'volume'])
+    price = current['close']
+
+    # If base_price is not set, we use the current value. This is the
+    # price at the first bar which we reference to calculate price_change.
+    if context.base_price is None:
+        context.base_price = price
+
+    price_change = (price - context.base_price) / context.base_price
+    cash = context.portfolio.cash
+
+    # Now that we've collected all current data for this frame, we use
+    # the record() method to save it. This data will be available as
+    # a parameter of the analyze() function for further analysis.
+
+    record(
+        volume=current['volume'],
+        price=price,
+        price_change=price_change,
+        rsi=rsi[-1],
+        cash=cash
+    )
+    # We are trying to avoid over-trading by limiting our trades to
+    # one per day.
+    if context.traded_today:
+        return
+
+    # TODO: retest with open orders
+    # Since we are using limit orders, some orders may not execute immediately
+    # we wait until all orders are executed before considering more trades.
+    orders = get_open_orders(context.market)
+    if len(orders) > 0:
+        log.info('exiting because orders are open: {}'.format(orders))
+        return
+
+    # Exit if we cannot trade
+    if not data.can_trade(context.market):
+        return
+
+    # Another powerful built-in feature of the Catalyst backtester is the
+    # portfolio object.  The portfolio object tracks your positions, cash,
+    # cost basis of specific holdings, and more.  In this line, we calculate
+    # how long or short our position is at this minute.
+    pos_amount = context.portfolio.positions[context.market].amount
+
+    if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0:
+        log.info(
+            '{}: buying - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        # Set a style for limit orders,
+        limit_price = price * 1.005
+        order_target_percent(
+            context.market, 1, limit_price=limit_price
+        )
+        context.traded_today = True
+
+    elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0:
+        log.info(
+            '{}: selling - price: {}, rsi: {}'.format(
+                data.current_dt, price, rsi[-1]
+            )
+        )
+        limit_price = price * 0.995
+        order_target_percent(
+            context.market, 0, limit_price=limit_price
+        )
+        context.traded_today = True
+
+
+def analyze(context=None, perf=None):
+    end = time.time()
+    log.info('elapsed time: {}'.format(end - context.start_time))
+
+    import matplotlib.pyplot as plt
+    # The base currency of the algo exchange
+    base_currency = context.exchanges.values()[0].base_currency.upper()
+
+    # Plot the portfolio value over time.
+    ax1 = plt.subplot(611)
+    perf.loc[:, 'portfolio_value'].plot(ax=ax1)
+    ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency))
+
+    # Plot the price increase or decrease over time.
+    ax2 = plt.subplot(612, sharex=ax1)
+    perf.loc[:, 'price'].plot(ax=ax2, label='Price')
+
+    ax2.set_ylabel('{asset}\n({base})'.format(
+        asset=context.market.symbol, base=base_currency
+    ))
+
+    transaction_df = extract_transactions(perf)
+    if not transaction_df.empty:
+        buy_df = transaction_df[transaction_df['amount'] > 0]
+        sell_df = transaction_df[transaction_df['amount'] < 0]
+        ax2.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'price'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax2.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'price'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+
+    ax4 = plt.subplot(613, sharex=ax1)
+    perf.loc[:, 'cash'].plot(
+        ax=ax4, label='Base Currency ({})'.format(base_currency)
+    )
+    ax4.set_ylabel('Cash\n({})'.format(base_currency))
+
+    perf['algorithm'] = perf.loc[:, 'algorithm_period_return']
+
+    ax5 = plt.subplot(614, sharex=ax1)
+    perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5)
+    ax5.set_ylabel('Percent\nChange')
+
+    ax6 = plt.subplot(615, sharex=ax1)
+    perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI')
+    ax6.set_ylabel('RSI')
+    ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod')
+    ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod')
+
+    if not transaction_df.empty:
+        ax6.scatter(
+            buy_df.index.to_pydatetime(),
+            perf.loc[buy_df.index.floor('1 min'), 'rsi'],
+            marker='^',
+            s=100,
+            c='green',
+            label=''
+        )
+        ax6.scatter(
+            sell_df.index.to_pydatetime(),
+            perf.loc[sell_df.index.floor('1 min'), 'rsi'],
+            marker='v',
+            s=100,
+            c='red',
+            label=''
+        )
+    plt.legend(loc=3)
+    start, end = ax6.get_ylim()
+    ax6.yaxis.set_ticks(np.arange(0, end, end / 5))
+
+    # Show the plot.
+    plt.gcf().set_size_inches(18, 8)
+    plt.show()
+    pass
+
+
+if __name__ == '__main__':
+    # The execution mode: backtest or live
+    live = False
+
+    if live:
+        run_algorithm(
+            capital_base=0.025,
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='poloniex',
+            live=True,
+            algo_namespace=NAMESPACE,
+            base_currency='btc',
+            live_graph=False,
+            simulate_orders=False,
+            stats_output=None,
+        )
+
+    else:
+        folder = os.path.join(
+            tempfile.gettempdir(), 'catalyst', NAMESPACE
+        )
+        ensure_directory(folder)
+
+        timestr = time.strftime('%Y%m%d-%H%M%S')
+        out = os.path.join(folder, '{}.p'.format(timestr))
+        # catalyst run -f catalyst/examples/mean_reversion_simple.py \
+        #    -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \
+        #   --data-frequency minute --capital-base 10000
+        run_algorithm(
+            capital_base=0.1,
+            data_frequency='minute',
+            initialize=initialize,
+            handle_data=handle_data,
+            analyze=analyze,
+            exchange_name='bitfinex',
+            algo_namespace=NAMESPACE,
+            base_currency='eth',
+            start=pd.to_datetime('2017-10-01', utc=True),
+            end=pd.to_datetime('2017-11-10', utc=True),
+            output=out
+        )
+        log.info('saved perf stats: {}'.format(out))

catalyst/exchange/exchange_algorithm.py

@@ -587,7 +587,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
             orders = []
             for asset in self.blotter.open_orders:
                 asset_orders = self.blotter.open_orders[asset]
-                orders += asset_orders
+                if asset_orders:
+                    orders += asset_orders
 
             required_cash = self.portfolio.cash if not orders else None
             cash, positions_value = exchange.sync_positions(

catalyst/exchange/exchange_blotter.py

@@ -10,6 +10,7 @@ from catalyst.finance.transaction import create_transaction, Transaction
 from catalyst.utils.input_validation import expect_types
 from logbook import Logger
 from redo import retry
+from six import iteritems
 
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
@@ -41,6 +42,11 @@ class TradingPairFeeSchedule(CommissionModel):
             )
         )
 
+    def get_maker_taker(self, asset):
+        maker = self.maker if self.maker is not None else asset.maker
+        taker = self.taker if self.taker is not None else asset.taker
+        return maker, taker
+
     def calculate(self, order, transaction):
         """
         Calculate the final fee based on the order parameters.
@@ -54,8 +60,7 @@ class TradingPairFeeSchedule(CommissionModel):
         cost = abs(transaction.amount) * transaction.price
 
         asset = order.asset
-        maker = self.maker if self.maker is not None else asset.maker
-        taker = self.taker if self.taker is not None else asset.taker
+        maker, taker = self.get_maker_taker(asset)
 
         multiplier = taker
         if order.limit is not None:
@@ -250,6 +255,7 @@ class ExchangeBlotter(Blotter):
         for order, txn in self.check_open_orders():
             order.dt = txn.dt
 
+            # TODO: is the commission already on the order object?
             transactions.append(txn)
 
             if not order.open:

catalyst/exchange/utils/stats_utils.py

@@ -10,6 +10,7 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from catalyst.exchange.utils.exchange_utils import get_algo_folder
 from catalyst.utils.paths import data_root, ensure_directory
+from operator import itemgetter
 
 s3_conn = []
 mailgun = []
@@ -260,7 +261,14 @@ def prepare_stats(stats, recorded_cols=list()):
     return df, columns
 
 
-def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
+def set_print_settings():
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 8)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+
+def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True):
     """
     Format and print the last few rows of a statistics DataFrame.
     See the pyfolio project for the data structure.
@@ -280,17 +288,17 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     """
     if isinstance(stats, pd.DataFrame):
         stats = stats.T.to_dict().values()
+        stats.sort(key=itemgetter('period_close'))
+
+    if len(stats) > num_rows:
+        display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows]
+    else:
+        display_stats = stats
 
-    display_stats = stats[-num_rows:] if len(stats) > num_rows else stats
     df, columns = prepare_stats(
         display_stats, recorded_cols=recorded_cols
     )
-
-    pd.set_option('display.expand_frame_repr', False)
-    pd.set_option('precision', 8)
-    pd.set_option('display.width', 1000)
-    pd.set_option('display.max_colwidth', 1000)
-
+    set_print_settings()
     return df.to_string(columns=columns)
 
 
@@ -438,6 +446,17 @@ def df_to_string(df):
     return df.to_string()
 
 
+def extract_orders(perf):
+    order_list = perf.orders.values
+    all_orders = [t for sublist in order_list for t in sublist]
+    all_orders.sort(key=lambda o: o['dt'])
+
+    orders = pd.DataFrame(all_orders)
+    if not orders.empty:
+        orders.set_index('dt', inplace=True, drop=True)
+    return orders
+
+
 def extract_transactions(perf):
     """
     Compute indexes for buy and sell transactions

catalyst/utils/run_algo.py

@@ -143,7 +143,7 @@ def _run(handle_data,
     log.warn(
         'Catalyst is currently in ALPHA. It is going through rapid '
         'development and it is subject to errors. Please use carefully. '
-        'We encourage your to report any issue on GitHub: '
+        'We encourage you to report any issue on GitHub: '
         'https://github.com/enigmampc/catalyst/issues'
     )
     sleep(3)

tests/exchange/test_suites/test_suite_algo.py

@@ -0,0 +1,72 @@
+import importlib
+from os.path import join, isfile
+
+import pandas as pd
+import os
+
+from catalyst import run_algorithm
+from catalyst.exchange.utils.stats_utils import get_pretty_stats, \
+    extract_transactions, set_print_settings, extract_orders
+from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
+from logbook import TestHandler, WARNING
+from pathtools.path import listdir
+
+filter_algos = [
+    'mean_reversion_simple_custom_fees.py',
+]
+
+
+class TestSuiteAlgo(WithLogger, ZiplineTestCase):
+    @staticmethod
+    def analyze(context, perf):
+        set_print_settings()
+
+        transaction_df = extract_transactions(perf)
+        print('the transactions:\n{}'.format(transaction_df))
+
+        orders_df = extract_orders(perf)
+        print('the orders:\n{}'.format(orders_df))
+
+        stats = get_pretty_stats(perf, show_tail=False, num_rows=5)
+        print('the stats:\n{}'.format(stats))
+        pass
+
+    def test_run_examples(self):
+        folder = join('..', '..', '..', 'catalyst', 'examples')
+        files = [f for f in listdir(folder) if isfile(join(folder, f))]
+
+        algo_list = []
+        for filename in files:
+            name = os.path.basename(filename)
+            if filter_algos and name not in filter_algos:
+                continue
+
+            module_name = 'catalyst.examples.{}'.format(
+                name.replace('.py', '')
+            )
+            algo_list.append(module_name)
+
+        for module_name in algo_list:
+            algo = importlib.import_module(module_name)
+            namespace = module_name.replace('.', '_')
+
+            log_catcher = TestHandler()
+            with log_catcher:
+                run_algorithm(
+                    capital_base=0.1,
+                    data_frequency='minute',
+                    initialize=algo.initialize,
+                    handle_data=algo.handle_data,
+                    analyze=TestSuiteAlgo.analyze,
+                    exchange_name='bitfinex',
+                    algo_namespace='test_{}'.format(namespace),
+                    base_currency='eth',
+                    start=pd.to_datetime('2017-10-01', utc=True),
+                    end=pd.to_datetime('2017-10-02', utc=True),
+                    # output=out
+                )
+                warnings = [record for record in log_catcher.records if
+                            record.level == WARNING]
+                self.assertEqual(0, len(warnings))
+
+            pass