Bug fixes and polishing stats

catalyst/assets/_assets.pyx

@@ -37,7 +37,7 @@ import warnings
 cimport numpy as np
 
 from catalyst.utils.calendars import get_calendar
-from catalyst.exchange.exchange_errors import InvalidSymbolError
+from catalyst.exchange.exchange_errors import InvalidSymbolError, SidHashError
 
 # IMPORTANT NOTE: You must change this template if you change
 # Asset.__reduce__, or else we'll attempt to unpickle an old version of this
@@ -477,8 +477,11 @@ cdef class TradingPair(Asset):
         except Exception as e:
             raise InvalidSymbolError(symbol=symbol, error=e)
 
-        if sid == 0:
-            sid = abs(hash(symbol)) % (10 ** 4)
+        if sid == 0 or sid is None:
+            try:
+                sid = abs(hash(symbol)) % (10 ** 4)
+            except Exception as e:
+                raise SidHashError(symbol=symbol)
 
         if asset_name is None:
             asset_name = ' / '.join(symbol.split('_')).upper()
@@ -518,6 +521,24 @@ cdef class TradingPair(Asset):
             leverage=self.leverage
         )
 
+    cpdef __reduce__(self):
+        """
+        Function used by pickle to determine how to serialize/deserialize this
+        class.  Should return a tuple whose first element is self.__class__,
+        and whose second element is a tuple of all the attributes that should
+        be serialized/deserialized during pickling.
+        """
+        return (self.__class__, (self.symbol,
+                                 self.exchange,
+                                 self.start_date,
+                                 self.asset_name,
+                                 self.sid,
+                                 self.leverage,
+                                 self.end_date,
+                                 self.first_traded,
+                                 self.auto_close_date,
+                                 self.exchange_full))
+
 def make_asset_array(int size, Asset asset):
     cdef np.ndarray out = np.empty([size], dtype=object)
     out.fill(asset)

catalyst/examples/buy_the_dip_live.py

@@ -8,6 +8,7 @@ from catalyst.api import (
     record,
     get_open_orders,
 )
+from catalyst.exchange.stats_utils import get_pretty_stats
 from catalyst.utils.run_algo import run_algorithm
 
 algo_namespace = 'buy_the_dip_live'
@@ -140,7 +141,7 @@ def handle_data(context, data):
 
 
 def analyze(context, stats):
-    log.info('the full stats:\n{}'.format(stats.head()))
+    log.info('the daily stats:\n{}'.format(get_pretty_stats(stats)))
     pass
 
 

catalyst/exchange/algorithm_exchange.py

@@ -18,6 +18,7 @@ from datetime import timedelta
 from time import sleep
 from os import listdir
 from os.path import isfile, join
+from collections import deque
 
 import logbook
 import pandas as pd
@@ -35,6 +36,7 @@ from catalyst.exchange.exchange_errors import (
 )
 from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root, \
     save_algo_object, get_algo_object, get_algo_folder
+from catalyst.exchange.stats_utils import get_pretty_stats
 from catalyst.finance.performance.period import calc_period_stats
 from catalyst.gens.tradesimulation import AlgorithmSimulator
 from catalyst.utils.api_support import (
@@ -55,6 +57,7 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
         self.exchange = kwargs.pop('exchange', None)
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.orders = {}
+        self.minute_stats = deque(maxlen=60)
         self.is_running = True
 
         self.retry_check_open_orders = 5
@@ -63,6 +66,8 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
         self.retry_order = 2
         self.retry_delay = 5
 
+        self.stats_minutes = 5
+
         super(self.__class__, self).__init__(*args, **kwargs)
         self._create_minute_writer()
 
@@ -93,10 +98,14 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
     def signal_handler(self, signal, frame):
         self.is_running = False
 
-        log.info('You pressed Ctrl+C!')
+        if self._analyze is None:
+            log.info('Interruption signal detected {}, exiting the '
+                     'algorithm'.format(signal))
+
+        else:
+            log.info('Interruption signal detected {}, calling `analyze()` '
+                     'before exiting the algorithm'.format(signal))
 
-        stats = None
-        try:
             algo_folder = get_algo_folder(self.algo_namespace)
             folder = join(algo_folder, 'daily_perf')
             files = [f for f in listdir(folder) if isfile(join(folder, f))]
@@ -108,12 +117,9 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
                     daily_perf_list.append(pickle.load(handle))
 
             stats = pd.DataFrame(daily_perf_list)
-            stats.set_index('period_close', drop=True, inplace=True)
 
-        except Exception as e:
-            log.warn('Unable to compute daily stats: {}'.format(e))
+            self.analyze(stats)
 
-        self.analyze(stats)
         sys.exit(0)
 
     def _create_clock(self):
@@ -306,10 +312,17 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
             # Performance tracker and keep only minute and cumulative
             self.perf_tracker.update_performance()
 
-            # TODO: save for future use?
             minute_stats = self.prepare_period_stats(
                 data.current_dt, data.current_dt + timedelta(minutes=1))
-            log.debug('the minute performance:\n{}'.format(minute_stats))
+            # Saving the last hour in memory
+            self.minute_stats.append(minute_stats)
+
+            print_df = pd.DataFrame(list(self.minute_stats))
+            log.debug(
+                'statistics for the last {stats_minutes} minutes:\n{stats}'.format(
+                    stats_minutes=self.stats_minutes,
+                    stats=get_pretty_stats(print_df, self.stats_minutes)
+                ))
 
             today = pd.to_datetime('today', utc=True)
             daily_stats = self.prepare_period_stats(

catalyst/exchange/bitfinex/bitfinex.py

@@ -134,7 +134,6 @@ class Bitfinex(Exchange):
 
         amount = float(order_status['original_amount'])
         filled = float(order_status['executed_amount'])
-        is_buy = (amount > 0)
 
         price = float(order_status['price'])
         order_type = order_status['type']
@@ -153,7 +152,6 @@ class Bitfinex(Exchange):
         # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
         commission = None
 
-        # TODO: zipline likes rounded dates to match statistics, is this ok?
         date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
         date = pytz.utc.localize(date)
         order = Order(
@@ -451,7 +449,7 @@ class Bitfinex(Exchange):
 
         orders = list()
         for order_status in order_statuses:
-            order, = self._create_order(order_status)
+            order, executed_price = self._create_order(order_status)
             if asset is None or asset == order.sid:
                 orders.append(order)
 

catalyst/exchange/bitfinex/symbols.json

@@ -3,6 +3,10 @@
     "symbol": "btc_usd",
     "start_date": "2010-01-01"
   },
+  "bchusd": {
+    "symbol": "bch_usd",
+    "start_date": "2010-01-01"
+  },
   "ltcusd": {
     "symbol": "ltc_usd",
     "start_date": "2010-01-01"

catalyst/exchange/exchange.py

@@ -368,21 +368,18 @@ class Exchange:
             bar_count=bar_count,
         )
 
-        frames = []
+        series = dict()
         for asset in assets:
             asset_candles = candles[asset]
 
-            asset_data = dict()
-            asset_data[asset] = map(lambda candle: candle[field],
-                                    asset_candles)
+            values = map(lambda candle: candle[field], asset_candles)
+            dates = map(lambda candle: candle['last_traded'], asset_candles)
 
-            dates = map(lambda candle: candle['last_traded'],
-                        asset_candles)
+            value_series = pd.Series(values, index=dates)
+            series[asset] = value_series
 
-            df = pd.DataFrame(asset_data, index=dates)
-            frames.append(df)
-
-        return pd.concat(frames)
+        df = pd.concat(series)
+        return df
 
     @abstractmethod
     def create_order(self, asset, amount, is_buy, style):

catalyst/exchange/exchange_errors.py

@@ -73,3 +73,9 @@ class InvalidOrderType(ZiplineError):
     msg = (
         'Order type not found.'
     ).strip()
+
+
+class SidHashError(ZiplineError):
+    msg = (
+        'Unable to hash sid from symbol {symbol}.'
+    ).strip()

catalyst/exchange/stats_utils.py

@@ -0,0 +1,47 @@
+import pandas as pd
+
+
+def get_pretty_stats(stats_df, num_rows=10):
+    """
+    Format and print the last few rows of a statistics DataFrame.
+    See the pyfolio project for the data structure.
+
+    :param stats_df:
+    :param num_rows:
+    :return:
+    """
+    stats_df.set_index('period_close', drop=True, inplace=True)
+    stats_df.dropna(axis=1, how='all', inplace=True)
+
+    pd.set_option('display.expand_frame_repr', False)
+    pd.set_option('precision', 3)
+    pd.set_option('display.width', 1000)
+    pd.set_option('display.max_colwidth', 1000)
+
+    columns = ['starting_cash', 'ending_cash', 'portfolio_value',
+               'pnl', 'long_exposure', 'short_exposure', 'orders',
+               'transactions', 'positions']
+
+    def format_positions(positions):
+        parts = []
+        for position in positions:
+            msg = '{amount:.2f}{market} cost basis {cost_basis:.4f}{base}'.format(
+                amount=position['amount'],
+                market=position['sid'].market_currency,
+                cost_basis=position['cost_basis'],
+                base=position['sid'].base_currency
+            )
+            parts.append(msg)
+        return ', '.join(parts)
+
+    formatters = {
+        'orders': lambda orders: len(orders),
+        'transactions': lambda transactions: len(transactions),
+        'returns': lambda returns: "{0:.4f}".format(returns),
+        'positions': format_positions
+    }
+
+    return stats_df.tail(num_rows).to_string(
+        columns=columns,
+        formatters=formatters
+    )