From 8e232ac5efd660798e6649401a45501b49beb6c1 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 29 Dec 2017 15:54:44 -0500 Subject: [PATCH] BLD: Live chart refactoring --- catalyst/exchange/exchange_algorithm.py | 6 +- catalyst/exchange/live_graph_clock.py | 181 ++------------------ catalyst/exchange/utils/live_chart_utils.py | 130 ++++++++++++++ catalyst/exchange/utils/stats_utils.py | 3 +- catalyst/utils/run_algo.py | 4 + 5 files changed, 151 insertions(+), 173 deletions(-) create mode 100644 catalyst/exchange/utils/live_chart_utils.py diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index fe1f3679..3e26c4d2 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -330,6 +330,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.algo_namespace = kwargs.pop('algo_namespace', None) self.live_graph = kwargs.pop('live_graph', None) self.stats_output = kwargs.pop('stats_output', None) + self._analyze_live = kwargs.pop('analyze_live', None) self._clock = None self.frame_stats = list() @@ -421,10 +422,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # TODO: should we apply time skew? not sure to understand the utility. log.debug('creating clock') - if self.live_graph: + if self.live_graph or self._analyze_live is not None: self._clock = LiveGraphClock( self.sim_params.sessions, - context=self + context=self, + callback=self._analyze_live, ) else: self._clock = SimpleClock( diff --git a/catalyst/exchange/live_graph_clock.py b/catalyst/exchange/live_graph_clock.py index 6f674455..8fea4de5 100644 --- a/catalyst/exchange/live_graph_clock.py +++ b/catalyst/exchange/live_graph_clock.py @@ -4,10 +4,8 @@ from catalyst.gens.sim_engine import ( SESSION_START ) from logbook import Logger - from catalyst.constants import LOG_LEVEL -from catalyst.exchange.exchange_errors import \ - MismatchingBaseCurrenciesExchanges +from catalyst.exchange.utils.stats_utils import prepare_stats log = Logger('LiveGraphClock', level=LOG_LEVEL) @@ -38,177 +36,23 @@ class LiveGraphClock(object): the exchange and the live trading machine's clock. It's not used currently. """ - def __init__(self, sessions, context, time_skew=pd.Timedelta('0s')): - - global mdates, plt # TODO: Could be cleaner - import matplotlib.dates as mdates - from matplotlib import pyplot as plt - from matplotlib import style + def __init__(self, sessions, context, callback=None, + time_skew=pd.Timedelta('0s')): self.sessions = sessions self.time_skew = time_skew self._last_emit = None self._before_trading_start_bar_yielded = True self.context = context - self.fmt = mdates.DateFormatter('%Y-%m-%d %H:%M') - - style.use('dark_background') - - fig = plt.figure() - fig.canvas.set_window_title('Enigma Catalyst: {}'.format( - self.context.algo_namespace)) - - self.ax_pnl = fig.add_subplot(311) - - self.ax_custom_signals = fig.add_subplot(312, sharex=self.ax_pnl) - - self.ax_exposure = fig.add_subplot(313, sharex=self.ax_pnl) - - if len(context.minute_stats) > 0: - self.draw_pnl() - self.draw_custom_signals() - self.draw_exposure() - - # rotates and right aligns the x labels, and moves the bottom of the - # axes up to make room for them - fig.autofmt_xdate() - fig.subplots_adjust(hspace=0.5) - - plt.tight_layout() - plt.ion() - plt.show() - - def format_ax(self, ax): - """ - Trying to assign reasonable parameters to the time axis. - - Parameters - ---------- - ax: - - """ - # TODO: room for improvement - ax.xaxis.set_major_locator(mdates.DayLocator(interval=1)) - ax.xaxis.set_major_formatter(self.fmt) - - locator = mdates.HourLocator(interval=4) - locator.MAXTICKS = 5000 - ax.xaxis.set_minor_locator(locator) - - datemin = pd.Timestamp.utcnow() - ax.set_xlim(datemin) - - ax.grid(True) - - def set_legend(self, ax): - """ - Set legend on the chart. - - Parameters - ---------- - ax - - """ - ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1) - - def draw_pnl(self): - """ - Draw p&l line on the chart. - - """ - ax = self.ax_pnl - df = self.context.pnl_stats - - ax.clear() - ax.set_title('Performance') - ax.plot(df.index, df['performance'], '-', - color='green', - linewidth=1.0, - label='Performance' - ) - - def perc(val): - return '{:2f}'.format(val) - - ax.format_ydata = perc - - self.set_legend(ax) - self.format_ax(ax) - - def draw_custom_signals(self): - """ - Draw custom signals on the chart. - - """ - ax = self.ax_custom_signals - df = self.context.custom_signals_stats - - colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink'] - - ax.clear() - ax.set_title('Custom Signals') - for index, column in enumerate(df.columns.values.tolist()): - ax.plot(df.index, df[column], '-', - color=colors[index], - linewidth=1.0, - label=column - ) - - self.set_legend(ax) - self.format_ax(ax) - - def draw_exposure(self): - """ - Draw exposure line on the chart. - - """ - ax = self.ax_exposure - context = self.context - df = context.exposure_stats - - # TODO: list exchanges in graph - base_currency = None - positions = [] - for exchange_name in context.exchanges: - exchange = context.exchanges[exchange_name] - - if not base_currency: - base_currency = exchange.base_currency - elif base_currency != exchange.base_currency: - raise MismatchingBaseCurrenciesExchanges( - base_currency=base_currency, - exchange_name=exchange.name, - exchange_currency=exchange.base_currency - ) - - positions += exchange.portfolio.positions - - ax.clear() - ax.set_title('Exposure') - ax.plot(df.index, df['base_currency'], '-', - color='green', - linewidth=1.0, - label='Base Currency: {}'.format(base_currency.upper()) - ) - - symbols = [] - for position in positions: - symbols.append(position.symbol) - - ax.plot(df.index, df['long_exposure'], '-', - color='blue', - linewidth=1.0, - label='Long Exposure: {}'.format(', '.join(symbols).upper())) - - self.set_legend(ax) - self.format_ax(ax) + self.callback = callback def __iter__(self): + from matplotlib import pyplot as plt yield pd.Timestamp.utcnow(), SESSION_START while True: current_time = pd.Timestamp.utcnow() - current_minute = current_time.floor('1 min') + current_minute = current_time.floor('1T') if self._last_emit is None or current_minute > self._last_emit: log.debug('emitting minutely bar: {}'.format(current_minute)) @@ -216,14 +60,11 @@ class LiveGraphClock(object): self._last_emit = current_minute yield current_minute, BAR - try: - self.draw_pnl() - self.draw_custom_signals() - self.draw_exposure() - - plt.draw() - except Exception as e: - log.warn('Unable to update the graph: {}'.format(e)) + recorded_cols = list(self.context.recorded_vars.keys()) + df, _ = prepare_stats( + self.context.frame_stats, recorded_cols=recorded_cols + ) + self.callback(self.context, df) else: # I can't use the "animate" reactive approach here because diff --git a/catalyst/exchange/utils/live_chart_utils.py b/catalyst/exchange/utils/live_chart_utils.py new file mode 100644 index 00000000..036fa67a --- /dev/null +++ b/catalyst/exchange/utils/live_chart_utils.py @@ -0,0 +1,130 @@ +from catalyst.exchange.exchange_errors import \ + MismatchingBaseCurrenciesExchanges +import matplotlib.dates as mdates +import pandas as pd + +fmt = mdates.DateFormatter('%Y-%m-%d %H:%M') + + +def format_ax(ax): + """ + Trying to assign reasonable parameters to the time axis. + + Parameters + ---------- + ax: + + """ + # TODO: room for improvement + ax.xaxis.set_major_locator(mdates.DayLocator(interval=1)) + ax.xaxis.set_major_formatter(fmt) + + locator = mdates.HourLocator(interval=4) + locator.MAXTICKS = 5000 + ax.xaxis.set_minor_locator(locator) + + datemin = pd.Timestamp.utcnow() + ax.set_xlim(datemin) + + ax.grid(True) + + +def set_legend(ax): + """ + Set legend on the chart. + + Parameters + ---------- + ax + + """ + ax.legend(loc='upper left', ncol=1, fontsize=10, numpoints=1) + + +def draw_pnl(ax, df): + """ + Draw p&l line on the chart. + + """ + ax.clear() + ax.set_title('Performance') + index = df.index.unique() + dt = index.get_level_values(level=0) + pnl = index.get_level_values(level=4) + ax.plot( + dt, pnl, '-', + color='green', + linewidth=1.0, + label='Performance' + ) + + def perc(val): + return '{:2f}'.format(val) + + ax.format_ydata = perc + + set_legend(ax) + format_ax(ax) + + +def draw_custom_signals(ax, df): + """ + Draw custom signals on the chart. + + """ + colors = ['blue', 'green', 'red', 'black', 'orange', 'yellow', 'pink'] + + ax.clear() + ax.set_title('Custom Signals') + for index, column in enumerate(df.columns.values.tolist()): + ax.plot(df.index, df[column], '-', + color=colors[index], + linewidth=1.0, + label=column + ) + + set_legend(ax) + format_ax(ax) + + +def draw_exposure(ax, df, context): + """ + Draw exposure line on the chart. + + """ + # TODO: list exchanges in graph + base_currency = None + positions = [] + for exchange_name in context.exchanges: + exchange = context.exchanges[exchange_name] + + if not base_currency: + base_currency = exchange.base_currency + elif base_currency != exchange.base_currency: + raise MismatchingBaseCurrenciesExchanges( + base_currency=base_currency, + exchange_name=exchange.name, + exchange_currency=exchange.base_currency + ) + + positions += exchange.portfolio.positions + + ax.clear() + ax.set_title('Exposure') + ax.plot(df.index, df['base_currency'], '-', + color='green', + linewidth=1.0, + label='Base Currency: {}'.format(base_currency.upper()) + ) + + symbols = [] + for position in positions: + symbols.append(position.symbol) + + ax.plot(df.index, df['long_exposure'], '-', + color='blue', + linewidth=1.0, + label='Long Exposure: {}'.format(', '.join(symbols).upper())) + + set_legend(ax) + format_ax(ax) diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py index fd68964b..c31dcd5d 100644 --- a/catalyst/exchange/utils/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -279,8 +279,9 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10): if isinstance(stats, pd.DataFrame): stats = stats.T.to_dict().values() + display_stats = stats[-num_rows:] if len(stats) > num_rows else stats df, columns = prepare_stats( - stats[-num_rows:], recorded_cols=recorded_cols + display_stats, recorded_cols=recorded_cols ) pd.set_option('display.expand_frame_repr', False) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 7f1068c7..015f91e4 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -93,6 +93,7 @@ def _run(handle_data, algo_namespace, base_currency, live_graph, + analyze_live, simulate_orders, stats_output): """Run a backtest for the given algorithm. @@ -276,6 +277,7 @@ def _run(handle_data, live_graph=live_graph, simulate_orders=simulate_orders, stats_output=stats_output, + analyze_live=analyze_live, ) elif exchanges: # Removed the existing Poloniex fork to keep things simple @@ -437,6 +439,7 @@ def run_algorithm(initialize, base_currency=None, algo_namespace=None, live_graph=False, + analyze_live=None, simulate_orders=True, stats_output=None, output=os.devnull): @@ -569,6 +572,7 @@ def run_algorithm(initialize, algo_namespace=algo_namespace, base_currency=base_currency, live_graph=live_graph, + analyze_live=analyze_live, simulate_orders=simulate_orders, stats_output=stats_output )