Performance tracking improvements

catalyst/algorithm.py

@@ -1135,7 +1135,7 @@ class TradingAlgorithm(object):
                           'date_rule. You should use keyword argument '
                           'time_rule= when calling schedule_function without '
                           'specifying a date_rule', stacklevel=3)
-        
+
         freq = self.sim_params.data_frequency
 
         date_rule = date_rule or date_rules.every_day()

catalyst/examples/buy_the_dip_live.py

@@ -8,6 +8,7 @@ from catalyst.api import (
     get_open_orders,
 )
 from catalyst.exchange.exchange_errors import ExchangeRequestError
+import matplotlib.pyplot as plt
 
 algo_namespace = 'buy_the_dip_live'
 log = Logger(algo_namespace)
@@ -15,7 +16,9 @@ log = Logger(algo_namespace)
 
 def initialize(context):
     log.info('initializing algo')
-    context.asset = symbol('eos_usd')
+    context.ASSET_NAME = 'EOS_USD'
+    context.TICK_SIZE = 1000.0
+    context.asset = symbol(context.ASSET_NAME)
 
     context.TARGET_POSITIONS = 100
     context.BUY_INCREMENT = 1
@@ -37,18 +40,18 @@ def handle_data(context, data):
     #                     bar_count=20,
     #                     frequency='1d'
     #                     )
-    # ohlc = data.history(context.asset,
-    #                     fields=['price', 'volume'],
-    #                     bar_count=120,
-    #                     frequency='1m'
-    #                     )
+    ohlc = data.history(context.asset,
+                        fields=['price', 'volume'],
+                        bar_count=120,
+                        frequency='1m'
+                        )
     # ohlc = data.history([context.asset, symbol('iot_usd')],
     #                     fields=['price', 'volume'],
     #                     bar_count=20,
     #                     frequency='1d'
     #                     )
 
-    # hist_price = ohlc['price']
+    hist_price = ohlc['price']
 
     cash = context.portfolio.cash
     log.info('base currency available: {cash}'.format(cash=cash))
@@ -110,16 +113,20 @@ def handle_data(context, data):
         leverage=context.account.leverage,
     )
 
-    try:
-        context.perf_tracker.update_performance()
-        log.info('the performance:\n{}'.format(
-            context.perf_tracker.to_dict('minute')
-        ))
-    except Exception as e:
-        log.warn('unable to calculate performance: {}'.format(e))
     pass
 
 
+def analyze(context, stats):
+    pnl, = plt.plot(stats.index, stats['pnl'], '-',
+                         color='blue',
+                         linewidth=1.0,
+                         label='P&L',
+                         )
+
+    plt.legend(handles=[pnl])
+    plt.show()
+    pass
+
 exchange_conn = dict(
     name='bitfinex',
     key='',
@@ -129,6 +136,7 @@ exchange_conn = dict(
 run_algorithm(
     initialize=initialize,
     handle_data=handle_data,
+    analyze=analyze,
     capital_base=100000,
     exchange_conn=exchange_conn,
     live=True,

catalyst/exchange/algorithm_exchange.py

@@ -13,6 +13,9 @@
 from datetime import time
 from time import sleep
 import logbook
+import signal
+import sys
+import pandas as pd
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
@@ -41,9 +44,12 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
     def __init__(self, *args, **kwargs):
         self.exchange = kwargs.pop('exchange', None)
         self.orders = {}
+        self.minute_perfs = []
+        self.is_running = True
 
         self.retry_check_open_orders = 5
         self.retry_update_portfolio = 5
+        self.retry_get_open_orders = 5
         self.retry_order = 1
         self.retry_delay = 5
 
@@ -51,6 +57,16 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
 
         log.info('exchange trading algorithm successfully initialized')
 
+    def signal_handler(self, signal, frame):
+        self.is_running = False
+
+        log.info('You pressed Ctrl+C!')
+        stats = pd.DataFrame(self.minute_perfs)
+        stats.set_index('period_close', drop=True, inplace=True)
+        self.analyze(stats)
+
+        sys.exit(0)
+
     def _create_clock(self):
         # This method is taken from TradingAlgorithm.
         # The clock has been replaced to use RealtimeClock
@@ -86,6 +102,8 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
             "US/Eastern"
         )
 
+        signal.signal(signal.SIGINT, self.signal_handler)
+
         return ExchangeClock(
             self.sim_params.sessions,
             execution_opens,
@@ -109,6 +127,9 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
             universe_func=self._calculate_universe
         )
 
+        # self.perf_tracker.cumulative_performance.keep_transactions = True
+        # self.perf_tracker.cumulative_performance.keep_orders = True
+
         return self.trading_client.transform()
 
     def updated_portfolio(self):
@@ -145,6 +166,9 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
                 return self._check_open_orders(attempt_index + 1)
 
     def handle_data(self, data):
+        if not self.is_running:
+            return
+
         self._update_portfolio()
 
         transactions = self._check_open_orders()
@@ -159,6 +183,25 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
         # every bar no matter if the algorithm places an order or not.
         self.validate_account_controls()
 
+        try:
+            # Since the clock runs 24/7, I trying to disable the daily
+            # Performance tracker and keep only minute and cumulative
+            self.perf_tracker.update_performance()
+            perf_dict = self.perf_tracker.to_dict('minute')
+
+            # Weird messy part of zipline
+            # I derived the logic from: catalyst.algorithm.TradingAlgorithm#_create_daily_stats
+            minute_perf = perf_dict['minute_perf']
+            minute_perf.update(perf_dict['cumulative_risk_metrics'])
+
+            log.debug('the minute performance:\n{}'.format(
+                minute_perf
+            ))
+            self.minute_perfs.append(minute_perf)
+
+        except Exception as e:
+            log.warn('unable to calculate performance: {}'.format(e))
+
     def _order(self,
                asset,
                amount,
@@ -202,11 +245,22 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
     def batch_market_order(self, share_counts):
         raise NotImplementedError()
 
+    def _get_open_orders(self, asset=None, attempt_index=0):
+        try:
+            return self.exchange.get_open_orders(asset)
+        except ExchangeRequestError as e:
+            log.warn(
+                'open orders attempt {}: {}'.format(attempt_index, e)
+            )
+            if attempt_index < self.retry_get_open_orders:
+                sleep(self.retry_delay)
+                return self._get_open_orders(asset, attempt_index + 1)
+
     @error_keywords(sid='Keyword argument `sid` is no longer supported for '
                         'get_open_orders. Use `asset` instead.')
     @api_method
     def get_open_orders(self, asset=None):
-        return self.exchange.get_open_orders(asset)
+        return self._get_open_orders(asset)
 
     @api_method
     def get_order(self, order_id):

catalyst/exchange/bitfinex.py

@@ -54,7 +54,8 @@ class Bitfinex(Exchange):
     def _request(self, operation, data, version='v1'):
         payload_object = {
             'request': '/{}/{}'.format(version, operation),
-            'nonce': '{0:f}'.format(time.time() * 100000),  # convert to string
+            'nonce': '{0:f}'.format(time.time() * 1000000),
+            # convert to string
             'options': {}
         }
 
@@ -154,8 +155,10 @@ class Bitfinex(Exchange):
         # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it.
         commission = None
 
+        # TODO: zipline likes rounded dates to match statistics, is this ok?
         date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
         date = pytz.utc.localize(date)
+        date = date.floor('1 min')
         order = ExchangeOrder(
             dt=date,
             asset=self.assets[order_status['symbol']],
@@ -451,6 +454,7 @@ class Bitfinex(Exchange):
             sell_price_oco=0
         )
 
+        date = pd.Timestamp.utcnow().floor('1 min')  # Making zipline happy
         try:
             response = self._request('order/new', req)
             exchange_order = response.json()
@@ -465,7 +469,7 @@ class Bitfinex(Exchange):
 
         order_id = exchange_order['id']
         order = ExchangeOrder(
-            dt=pd.Timestamp.utcnow(),
+            dt=date,
             asset=asset,
             amount=amount,
             stop=style.get_stop_price(is_buy),

catalyst/exchange/exchange.py

@@ -101,7 +101,7 @@ class Exchange:
                     transaction = Transaction(
                         asset=order.asset,
                         amount=order.amount,
-                        dt=pd.Timestamp.utcnow(),
+                        dt=pd.Timestamp.utcnow().floor('1 min'),
                         price=order.executed_price,
                         order_id=order.id,
                         commission=order.commission
@@ -334,7 +334,10 @@ class Exchange:
         if field not in ohlc:
             raise KeyError('Invalid column: %s' % field)
 
-        return ohlc[field]
+        value = ohlc[field]
+        log.debug('got spot value: {}'.format(value))
+
+        return value
 
     def get_history_window(self,
                            assets,

catalyst/exchange/exchange_clock.py

@@ -15,8 +15,6 @@ from time import sleep
 
 from logbook import Logger
 import pandas as pd
-import signal
-import sys
 
 from catalyst.gens.sim_engine import (
     BAR,
@@ -59,13 +57,6 @@ class ExchangeClock(object):
         self._before_trading_start_bar_yielded = True
 
     def __iter__(self):
-        def signal_handler(signal, frame):
-            log.info('You pressed Ctrl+C!')
-            # yield pd.Timestamp.utcnow(), BAR
-            sys.exit(0)
-
-        signal.signal(signal.SIGINT, signal_handler)
-
         yield pd.Timestamp.utcnow(), SESSION_START
 
         while True:

catalyst/utils/run_algo.py

@@ -184,9 +184,6 @@ def _run(handle_data,
 
             first_trading_day = bundle_data.minute_bar_reader.first_trading_day
 
-            # DataPortalClass = (partial(DataPortalExchange, exchange)
-            #                    if exchange
-            #                    else DataPortal)
             data = DataPortal(
                 env.asset_finder,
                 open_calendar,