BLD: Live chart refactoring

catalyst/exchange/exchange_algorithm.py

@@ -330,6 +330,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         self.algo_namespace = kwargs.pop('algo_namespace', None)
         self.live_graph = kwargs.pop('live_graph', None)
         self.stats_output = kwargs.pop('stats_output', None)
+        self._analyze_live = kwargs.pop('analyze_live', None)
 
         self._clock = None
         self.frame_stats = list()
@@ -421,10 +422,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase):
         # TODO: should we apply time skew? not sure to understand the utility.
 
         log.debug('creating clock')
-        if self.live_graph:
+        if self.live_graph or self._analyze_live is not None:
             self._clock = LiveGraphClock(
                 self.sim_params.sessions,
-                context=self
+                context=self,
+                callback=self._analyze_live,
             )
         else:
             self._clock = SimpleClock(

catalyst/exchange/live_graph_clock.py

@@ -4,10 +4,8 @@ from catalyst.gens.sim_engine import (
     SESSION_START
 )
 from logbook import Logger
-
 from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.exchange_errors import \
-    MismatchingBaseCurrenciesExchanges
+from catalyst.exchange.utils.stats_utils import prepare_stats
 
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
@@ -38,177 +36,23 @@ class LiveGraphClock(object):
     the exchange and the live trading machine's clock. It's not used currently.
     """
 
-    def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')):
-
-        global mdates, plt  # TODO: Could be cleaner
-        import matplotlib.dates as mdates
-        from matplotlib import pyplot as plt
-        from matplotlib import style
+    def __init__(self, sessions, context, callback=None,
+                 time_skew=pd.Timedelta('0s')):
 
         self.sessions = sessions
         self.time_skew = time_skew
         self._last_emit = None
         self._before_trading_start_bar_yielded = True
         self.context = context
-        self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
-
-        style.use('dark_background')
-
-        fig = plt.figure()
-        fig.canvas.set_window_title('Enigma Catalyst: {}'.format(
-            self.context.algo_namespace))
-
-        self.ax_pnl = fig.add_subplot(311)
-
-        self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl)
-
-        self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl)
-
-        if len(context.minute_stats) > 0:
-            self.draw_pnl()
-            self.draw_custom_signals()
-            self.draw_exposure()
-
-        # rotates and right aligns the x labels, and moves the bottom of the
-        # axes up to make room for them
-        fig.autofmt_xdate()
-        fig.subplots_adjust(hspace=0.5)
-
-        plt.tight_layout()
-        plt.ion()
-        plt.show()
-
-    def format_ax(self, ax):
-        """
-        Trying to assign reasonable parameters to the time axis.
-
-        Parameters
-        ----------
-        ax:
-
-        """
-        # TODO: room for improvement
-        ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
-        ax.xaxis.set_major_formatter(self.fmt)
-
-        locator = mdates.HourLocator(interval=4)
-        locator.MAXTICKS = 5000
-        ax.xaxis.set_minor_locator(locator)
-
-        datemin = pd.Timestamp.utcnow()
-        ax.set_xlim(datemin)
-
-        ax.grid(True)
-
-    def set_legend(self, ax):
-        """
-        Set legend on the chart.
-
-        Parameters
-        ----------
-        ax
-
-        """
-        ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
-
-    def draw_pnl(self):
-        """
-        Draw p&l line on the chart.
-
-        """
-        ax = self.ax_pnl
-        df = self.context.pnl_stats
-
-        ax.clear()
-        ax.set_title('Performance')
-        ax.plot(df.index, df['performance'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Performance'
-                )
-
-        def perc(val):
-            return '{:2f}'.format(val)
-
-        ax.format_ydata = perc
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_custom_signals(self):
-        """
-        Draw custom signals on the chart.
-
-        """
-        ax = self.ax_custom_signals
-        df = self.context.custom_signals_stats
-
-        colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
-
-        ax.clear()
-        ax.set_title('Custom Signals')
-        for index, column in enumerate(df.columns.values.tolist()):
-            ax.plot(df.index, df[column], '-',
-                    color=colors[index],
-                    linewidth=1.0,
-                    label=column
-                    )
-
-        self.set_legend(ax)
-        self.format_ax(ax)
-
-    def draw_exposure(self):
-        """
-        Draw exposure line on the chart.
-
-        """
-        ax = self.ax_exposure
-        context = self.context
-        df = context.exposure_stats
-
-        # TODO: list exchanges in graph
-        base_currency = None
-        positions = []
-        for exchange_name in context.exchanges:
-            exchange = context.exchanges[exchange_name]
-
-            if not base_currency:
-                base_currency = exchange.base_currency
-            elif base_currency != exchange.base_currency:
-                raise MismatchingBaseCurrenciesExchanges(
-                    base_currency=base_currency,
-                    exchange_name=exchange.name,
-                    exchange_currency=exchange.base_currency
-                )
-
-            positions += exchange.portfolio.positions
-
-        ax.clear()
-        ax.set_title('Exposure')
-        ax.plot(df.index, df['base_currency'], '-',
-                color='green',
-                linewidth=1.0,
-                label='Base Currency: {}'.format(base_currency.upper())
-                )
-
-        symbols = []
-        for position in positions:
-            symbols.append(position.symbol)
-
-        ax.plot(df.index, df['long_exposure'], '-',
-                color='blue',
-                linewidth=1.0,
-                label='Long Exposure: {}'.format(', '.join(symbols).upper()))
-
-        self.set_legend(ax)
-        self.format_ax(ax)
+        self.callback = callback
 
     def __iter__(self):
+        from matplotlib import pyplot as plt
         yield pd.Timestamp.utcnow(), SESSION_START
 
         while True:
             current_time = pd.Timestamp.utcnow()
-            current_minute = current_time.floor('1 min')
+            current_minute = current_time.floor('1T')
 
             if self._last_emit is None or current_minute > self._last_emit:
                 log.debug('emitting minutely bar: {}'.format(current_minute))
@@ -216,14 +60,11 @@ class LiveGraphClock(object):
                 self._last_emit = current_minute
                 yield current_minute, BAR
 
-                try:
-                    self.draw_pnl()
-                    self.draw_custom_signals()
-                    self.draw_exposure()
-
-                    plt.draw()
-                except Exception as e:
-                    log.warn('Unable to update the graph: {}'.format(e))
+                recorded_cols = list(self.context.recorded_vars.keys())
+                df, _ = prepare_stats(
+                    self.context.frame_stats, recorded_cols=recorded_cols
+                )
+                self.callback(self.context, df)
 
             else:
                 # I can't use the "animate" reactive approach here because

catalyst/exchange/utils/live_chart_utils.py

@@ -0,0 +1,130 @@
+from catalyst.exchange.exchange_errors import \
+    MismatchingBaseCurrenciesExchanges
+import matplotlib.dates as mdates
+import pandas as pd
+
+fmt = mdates.DateFormatter('%Y-%m-%d %H:%M')
+
+
+def format_ax(ax):
+    """
+    Trying to assign reasonable parameters to the time axis.
+
+    Parameters
+    ----------
+    ax:
+
+    """
+    # TODO: room for improvement
+    ax.xaxis.set_major_locator(mdates.DayLocator(interval=1))
+    ax.xaxis.set_major_formatter(fmt)
+
+    locator = mdates.HourLocator(interval=4)
+    locator.MAXTICKS = 5000
+    ax.xaxis.set_minor_locator(locator)
+
+    datemin = pd.Timestamp.utcnow()
+    ax.set_xlim(datemin)
+
+    ax.grid(True)
+
+
+def set_legend(ax):
+    """
+    Set legend on the chart.
+
+    Parameters
+    ----------
+    ax
+
+    """
+    ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1)
+
+
+def draw_pnl(ax, df):
+    """
+    Draw p&l line on the chart.
+
+    """
+    ax.clear()
+    ax.set_title('Performance')
+    index = df.index.unique()
+    dt = index.get_level_values(level=0)
+    pnl = index.get_level_values(level=4)
+    ax.plot(
+        dt, pnl, '-',
+        color='green',
+        linewidth=1.0,
+        label='Performance'
+    )
+
+    def perc(val):
+        return '{:2f}'.format(val)
+
+    ax.format_ydata = perc
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_custom_signals(ax, df):
+    """
+    Draw custom signals on the chart.
+
+    """
+    colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink']
+
+    ax.clear()
+    ax.set_title('Custom Signals')
+    for index, column in enumerate(df.columns.values.tolist()):
+        ax.plot(df.index, df[column], '-',
+                color=colors[index],
+                linewidth=1.0,
+                label=column
+                )
+
+    set_legend(ax)
+    format_ax(ax)
+
+
+def draw_exposure(ax, df, context):
+    """
+    Draw exposure line on the chart.
+
+    """
+    # TODO: list exchanges in graph
+    base_currency = None
+    positions = []
+    for exchange_name in context.exchanges:
+        exchange = context.exchanges[exchange_name]
+
+        if not base_currency:
+            base_currency = exchange.base_currency
+        elif base_currency != exchange.base_currency:
+            raise MismatchingBaseCurrenciesExchanges(
+                base_currency=base_currency,
+                exchange_name=exchange.name,
+                exchange_currency=exchange.base_currency
+            )
+
+        positions += exchange.portfolio.positions
+
+    ax.clear()
+    ax.set_title('Exposure')
+    ax.plot(df.index, df['base_currency'], '-',
+            color='green',
+            linewidth=1.0,
+            label='Base Currency: {}'.format(base_currency.upper())
+            )
+
+    symbols = []
+    for position in positions:
+        symbols.append(position.symbol)
+
+    ax.plot(df.index, df['long_exposure'], '-',
+            color='blue',
+            linewidth=1.0,
+            label='Long Exposure: {}'.format(', '.join(symbols).upper()))
+
+    set_legend(ax)
+    format_ax(ax)

catalyst/exchange/utils/stats_utils.py

@@ -279,8 +279,9 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10):
     if isinstance(stats, pd.DataFrame):
         stats = stats.T.to_dict().values()
 
+    display_stats = stats[-num_rows:] if len(stats) > num_rows else stats
     df, columns = prepare_stats(
-        stats[-num_rows:], recorded_cols=recorded_cols
+        display_stats, recorded_cols=recorded_cols
     )
 
     pd.set_option('display.expand_frame_repr', False)

catalyst/utils/run_algo.py

@@ -93,6 +93,7 @@ def _run(handle_data,
          algo_namespace,
          base_currency,
          live_graph,
+         analyze_live,
          simulate_orders,
          stats_output):
     """Run a backtest for the given algorithm.
@@ -276,6 +277,7 @@ def _run(handle_data,
             live_graph=live_graph,
             simulate_orders=simulate_orders,
             stats_output=stats_output,
+            analyze_live=analyze_live,
         )
     elif exchanges:
         # Removed the existing Poloniex fork to keep things simple
@@ -437,6 +439,7 @@ def run_algorithm(initialize,
                   base_currency=None,
                   algo_namespace=None,
                   live_graph=False,
+                  analyze_live=None,
                   simulate_orders=True,
                   stats_output=None,
                   output=os.devnull):
@@ -569,6 +572,7 @@ def run_algorithm(initialize,
         algo_namespace=algo_namespace,
         base_currency=base_currency,
         live_graph=live_graph,
+        analyze_live=analyze_live,
         simulate_orders=simulate_orders,
         stats_output=stats_output
     )