Bug fixes and housekeeping

catalyst/exchange/algorithm_exchange.py

@@ -10,31 +10,30 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from datetime import time, timedelta
-from time import sleep
-import logbook
 import signal
 import sys
+from datetime import timedelta
+from time import sleep
+
+import logbook
 import pandas as pd
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
-from catalyst.exchange.exchange_clock import ExchangeClock
-from catalyst.gens.tradesimulation import AlgorithmSimulator
 from catalyst.errors import OrderInBeforeTradingStart
-from catalyst.utils.input_validation import error_keywords
-from catalyst.utils.api_support import (
-    api_method,
-    disallowed_in_before_trading_start)
-
-from catalyst.utils.calendars.trading_calendar import days_at_time
+from catalyst.exchange.exchange_clock import ExchangeClock
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangePortfolioDataError,
     ExchangeTransactionError
 )
-from catalyst.finance.performance.period import calc_period_stats
 from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object
+from catalyst.finance.performance.period import calc_period_stats
+from catalyst.gens.tradesimulation import AlgorithmSimulator
+from catalyst.utils.api_support import (
+    api_method,
+    disallowed_in_before_trading_start)
+from catalyst.utils.input_validation import error_keywords
 
 log = logbook.Logger("ExchangeTradingAlgorithm")
 
@@ -104,20 +103,13 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
         execution_closes = \
             self.trading_calendar.execution_time_from_close(market_closes)
 
-        # FIXME generalize these values
-        before_trading_start_minutes = days_at_time(
-            self.sim_params.sessions,
-            time(8, 45),
-            "US/Eastern"
-        )
-
         signal.signal(signal.SIGINT, self.signal_handler)
 
         return ExchangeClock(
             self.sim_params.sessions,
             execution_opens,
             execution_closes,
-            before_trading_start_minutes,
+            None,
             minute_emission=minutely_emission,
             time_skew=self.exchange.time_skew
         )
@@ -136,9 +128,6 @@ class ExchangeTradingAlgorithm(TradingAlgorithm):
             universe_func=self._calculate_universe
         )
 
-        # self.perf_tracker.cumulative_performance.keep_transactions = True
-        # self.perf_tracker.cumulative_performance.keep_orders = True
-
         return self.trading_client.transform()
 
     def updated_portfolio(self):

catalyst/exchange/bitfinex.py

@@ -1,34 +1,30 @@
-import re
-import pytz
-import six
 import base64
 import hashlib
 import hmac
 import json
+import re
 import time
-import requests
+
 import pandas as pd
-from datetime import timedelta, datetime
-from catalyst.protocol import Portfolio, Account
+import pytz
+import requests
+import six
+from catalyst.assets._assets import Asset
+from logbook import Logger
+
 # from websocket import create_connection
 from catalyst.exchange.exchange import Exchange
-from logbook import Logger
-from catalyst.assets._assets import Asset
-from catalyst.finance.order import ORDER_STATUS
-from catalyst.exchange.exchange_order import ExchangeOrder
-from catalyst.finance.execution import (MarketOrder,
-                                        LimitOrder,
-                                        StopOrder,
-                                        StopLimitOrder)
-from catalyst.exchange.exchange_portfolio import ExchangePortfolio
-from catalyst.exchange.exchange_utils import get_exchange_symbols
-from catalyst.errors import (
-    IncompatibleHistoryFrequency,
-)
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError
 )
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+from catalyst.finance.execution import (MarketOrder,
+                                        LimitOrder,
+                                        StopOrder,
+                                        StopLimitOrder)
+from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.protocol import Account
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
@@ -159,7 +155,7 @@ class Bitfinex(Exchange):
         # TODO: zipline likes rounded dates to match statistics, is this ok?
         date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp']))
         date = pytz.utc.localize(date)
-        order = ExchangeOrder(
+        order = Order(
             dt=date,
             asset=self.assets[order_status['symbol']],
             amount=amount,
@@ -170,9 +166,8 @@ class Bitfinex(Exchange):
             commission=commission
         )
         order.status = status
-        order.executed_price = executed_price
 
-        return order
+        return order, executed_price
 
     def update_portfolio(self):
         """
@@ -494,7 +489,7 @@ class Bitfinex(Exchange):
             )
 
         order_id = exchange_order['id']
-        order = ExchangeOrder(
+        order = Order(
             dt=date,
             asset=asset,
             amount=amount,
@@ -540,8 +535,7 @@ class Bitfinex(Exchange):
 
         orders = list()
         for order_status in order_statuses:
-            # TODO: filter by asset
-            order = self._create_order(order_status)
+            order, = self._create_order(order_status)
             if asset is None or asset == order.sid:
                 orders.append(order)
 
@@ -584,7 +578,7 @@ class Bitfinex(Exchange):
             The order_id or order object to cancel.
         """
         order_id = order_param.id \
-            if isinstance(order_param, ExchangeOrder) else order_param
+            if isinstance(order_param, Order) else order_param
 
         try:
             response = self._request('order/cancel', {'order_id': order_id})

catalyst/exchange/data_portal_exchange.py

@@ -11,12 +11,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import pandas as pd
-
 from time import sleep
-from catalyst.data.data_portal import DataPortal
 
 from logbook import Logger
+
+from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError

catalyst/exchange/exchange.py

@@ -1,20 +1,18 @@
 import abc
-import json
-from abc import ABCMeta, abstractmethod, abstractproperty
 import collections
+from abc import ABCMeta, abstractmethod, abstractproperty
+from datetime import timedelta
 
 import pandas as pd
 from catalyst.assets._assets import Asset
-from catalyst.finance.order import ORDER_STATUS
-from catalyst.finance.transaction import Transaction
-from catalyst.data.data_portal import BASE_FIELDS
+from logbook import Logger
 
+from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.errors import (
-    MultipleSymbolsFound,
     SymbolNotFound,
 )
-from datetime import timedelta
-from logbook import Logger
+from catalyst.finance.order import ORDER_STATUS
+from catalyst.finance.transaction import Transaction
 
 log = Logger('Exchange')
 
@@ -90,7 +88,7 @@ class Exchange:
         if self.portfolio.open_orders:
             for order_id in list(self.portfolio.open_orders):
                 log.debug('found open order: {}'.format(order_id))
-                order = self.get_order(order_id)
+                order, executed_price = self.get_order(order_id)
                 log.debug('got updated order {}'.format(order))
 
                 if order.status == ORDER_STATUS.FILLED:
@@ -98,14 +96,13 @@ class Exchange:
                         asset=order.asset,
                         amount=order.amount,
                         dt=pd.Timestamp.utcnow(),
-                        price=order.executed_price,
+                        price=executed_price,
                         order_id=order.id,
                         commission=order.commission
                     )
                     transactions.append(transaction)
 
-                    # TODO: use the transaction to pass the executed price
-                    self.portfolio.execute_order(order)
+                    self.portfolio.execute_order(order, transaction)
                 elif order.status == ORDER_STATUS.CANCELLED:
                     self.portfolio.remove_order(order)
                 else:
@@ -219,6 +216,8 @@ class Exchange:
         -------
         order : Order
             The order object.
+        execution_price: float
+            The execution price per share of the order
         """
         pass
 

catalyst/exchange/exchange_clock.py

@@ -13,16 +13,13 @@
 
 from time import sleep
 
-from logbook import Logger
 import pandas as pd
-
 from catalyst.gens.sim_engine import (
     BAR,
     SESSION_START,
-    SESSION_END,
-    MINUTE_END,
-    BEFORE_TRADING_START_BAR
+    MINUTE_END
 )
+from logbook import Logger
 
 log = Logger('ExchangeClock')
 

catalyst/exchange/exchange_order.py

@@ -1,39 +0,0 @@
-import math
-
-import catalyst.protocol as zp
-from catalyst.assets import Asset
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.utils.input_validation import expect_types
-
-
-class ExchangeOrder(Order):
-    @expect_types(asset=Asset)
-    def __init__(self, dt, asset, amount, stop=None, limit=None, filled=0,
-                 commission=0, id=None, executed_price=None):
-        """
-        @dt - datetime.datetime that the order was placed
-        @asset - asset for the order.
-        @amount - the number of shares to buy/sell
-                  a positive sign indicates a buy
-                  a negative sign indicates a sell
-        @filled - how many shares of the order have been filled so far
-        """
-
-        # get a string representation of the uuid.
-        self.id = self.make_id() if id is None else id
-        self.dt = dt
-        self.reason = None
-        self.created = dt
-        self.asset = asset
-        self.amount = amount
-        self.filled = filled
-        self.commission = commission
-        self._status = ORDER_STATUS.OPEN
-        self.stop = stop
-        self.limit = limit
-        self.stop_reached = False
-        self.limit_reached = False
-        self.direction = math.copysign(1, self.amount)
-        self.type = zp.DATASOURCE_TYPE.ORDER
-        self.broker_order_id = None
-        self.executed_price = executed_price

catalyst/exchange/exchange_portfolio.py

@@ -1,8 +1,8 @@
 import numpy as np
-from catalyst.protocol import Portfolio, Positions, Position
-from catalyst.finance.order import BUY
 from logbook import Logger
 
+from catalyst.protocol import Portfolio, Positions, Position
+
 log = Logger('ExchangePortfolio')
 
 
@@ -36,7 +36,7 @@ class ExchangePortfolio(Portfolio):
         order_position.amount += order.amount
         log.debug('open order added to portfolio')
 
-    def execute_order(self, order):
+    def execute_order(self, order, transaction):
         log.debug('executing order {}'.format(order.id))
         del self.open_orders[order.id]
 
@@ -48,16 +48,16 @@ class ExchangePortfolio(Portfolio):
                 'Trying to execute order for a position not held: %s' % order.id
             )
 
-        self.capital_used += order.amount * order.executed_price
+        self.capital_used += order.amount * transaction.price
 
         if order.amount > 0:
             if order_position.cost_basis > 0:
                 order_position.cost_basis = np.average(
-                    [order_position.cost_basis, order.executed_price],
+                    [order_position.cost_basis, transaction.price],
                     weights=[order_position.amount, order.amount]
                 )
             else:
-                order_position.cost_basis = order.executed_price
+                order_position.cost_basis = transaction.price
 
         log.debug('updated portfolio with executed order')
 

catalyst/exchange/exchange_utils.py

@@ -1,10 +1,11 @@
-import os
-import urllib
 import json
+import os
 import pickle
-from catalyst.utils.paths import data_root, ensure_directory
+import urllib
+
 from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
-    ExchangeSymbolsNotFound, AlgoPickleNotFound
+    ExchangeSymbolsNotFound
+from catalyst.utils.paths import data_root, ensure_directory
 
 SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
               'live-trading/catalyst/exchange/symbols/{exchange}.json'