catalyst/zipline/gens/tradesimulation.py

import logbook

from datetime import datetime, timedelta
from numbers import Integral

from zipline import ndict

from zipline.gens.transform import StatefulTransform
from zipline.finance.trading import TransactionSimulator
from zipline.finance.performance import PerformanceTracker

class TradeSimulationClient(object):
    """
    Generator that takes the expected output of a merge, a user
    algorithm, a trading environment, and a simulator style as
    arguments.  Pipes the merge stream through a TransactionSimulator
    and a PerformanceTracker, which keep track of the current state of
    our algorithm's simulated universe. Results are fed to the user's
    algorithm, which directly inserts transactions into the
    TransactionSimulator's order book.

    TransactionSimulator maintains a dictionary from sids to the
    unfulfilled orders placed by the user's algorithm.  As trade
    events arrive, if the algorithm has open orders against the
    trade's sid, the simulator will fill orders up to 25% of market
    cap.  Applied transactions are added to a txn field on the event
    and forwarded to PerformanceTracker. The txn field is set to None
    on non-trade events and events that do not match any open orders.

    PerformanceTracker receives the updated event messages from
    TransactionSimulator, maintaining a set of daily and cumulative
    performance metrics for the algorithm.  The tracker removes the
    txn field from each event it receives, replacing it with a
    portfolio field to be fed into the user algo. At the end of each
    trading day, the PerformanceTracker also generates a daily
    performance report, which is appended to event's perf_report
    field.

    Fully processed events are run through a batcher generator, which
    batches together events with the same dt field into a single event
    to be fed to the algo. The portfolio object is repeatedly
    overwritten so that only the most recent snapshot of the universe
    is sent to the algo.
    """
    
    def __init__(self, algo, environment, sim_style):

        self.algo = algo
        self.sids = algo.get_sid_filter()
        self.environment = environment
        self.style = sim_style
        
    def get_hash(self):
        """
        There should only ever be one TSC in the system.
        """
        return self.__class__.__name__ + hash_args()

    def simulate(self, stream_in):
        """
        Main generator work loop.
        """

        # Simulate filling any open orders made by the previous run of
        # the user's algorithm.  Sets the txn field to true on any
        # event that results in a filled order.
        ordering_client = StatefulTransform(
            TransactionSimulator,
            self.sids,
            style = self.style
        )
        with_filled_orders = ordering_client.transform(stream_in)

        # Pipe the events with transactions to perf. This will remove
        # the txn field added by TransactionSimulator and replace it
        # with a portfolio object to be passed to the user's
        # algorithm. Also adds a PERF_MESSAGE field which is usually
        # none, but contains an update message once per day.
        perf_tracker = StatefulTransform(
            PerformanceTracker,
            self.environment,
            self.sids
        )
        with_portfolio = perf_tracker.transform(with_filled_orders)
        
        # Pass the messages from perf along with the trading client's
        # state into the algorithm for simulation. We provide the
        # trading client so that the algorithm can place new orders
        # into the client's order book.
        algo_results = AlgorithmSimulator(
            with_portfolio,
            ordering_client.state,
            self.algo, 
        )

        # The algorithm will yield a daily_results message (as
        # calculated by the performance tracker) at the end of each
        # day.  It will also yield a risk report at the end of the
        # simulation.
        for message in algo_results:        
            yield message


class AlgorithmSimulator(object):
    
    def __init__(self, stream_in, order_book, algo): 
    
        self.stream_in = stream_in

        # We extract the order book from the txn client so that 
        # the algo can place new orders.
        self.order_book = order_book

        self.algo = algo
        self.sids = algo.get_sid_filter()

        # Monkey patch the user algorithm to place orders in the
        # TransactionSimulator's order book.
        self.algo.set_order(self.order)
        self.algo.set_logger(logbook.Logger("Algolog"))

        # Call the user's initialize method.
        self.algo.initialize()

        # The algorithm's universe as of our most recent event.
        self.universe = ndict()
        
        for sid in self.sids:
            self.universe[sid] = ndict()
        self.universe.portfolio = None

        # We don't have a datetime for the current snapshot until we
        # receive a message.
        self.simulation_dt = None
        self.this_snapshot_dt = None

        self.__generator = None

    def __iter__(self):
        return self
    
    def next(self):
        if self.__generator:
            return self.__generator.next()
        else:
            self.__generator = self._gen()
            return self.__generator.next()

    def order(self, sid, amount):
        """
        Closure to pass into the user's algo to allow placing orders
        into the txn_sim's dict of open orders.
        """
        assert sid in self.sids, "Order on invalid sid: %i" % sid
        order = ndict({
            'dt'     : self.simulation_dt,
            'sid'    : sid,
            'amount' : int(amount),
            'filled' : 0
        })

        # Tell the user if they try to buy 0 shares of something.
        if order.amount == 0:
            zero_message = "Requested to trade zero shares of {sid}".format(
                sid=event.sid
            )
            log.debug(zero_message)
            # Don't bother placing orders for 0 shares.
            return 

        # Add non-zero orders to the order book.  
        # !!!IMPORTANT SIDE-EFFECT!!!
        # This modifies the internal state of the transaction
        # simulator so that it can fill the placed order when it
        # receives its next message.
        self.order_book.place_order(order)

    def _gen(self):
        """
        Internal generator work loop.
        """
        for event in self.stream_in:
            # Yield any perf messages received to be relayed back to the browser.
            if event.perf_message:
                yield event.perf_message
                del event['perf_message']
                if event.dt == "DONE":
                    break
                
            # This should only happen for the first event we run.
            if self.simulation_dt == None:
                self.simulation_dt = event.dt
            
            # ======================
            # Time Compression Logic
            # ======================
        
            if self.this_snapshot_dt != None:
                self.update_current_snapshot(event)

            # The algorithm has been missing events because it took
            # too long processing.  Update the universe with data from
            # this event, then check if enough time has passed that we
            # can start a new snapshot.
            else: 
                self.update_universe(event)
                if event.dt >= self.simulation_dt:
                    self.this_snapshot_dt = event.dt

    def update_current_snapshot(self, event):
        """
        Update our current snapshot of the universe. Call handle_data if 
        """
        # The new event matches our snapshot dt. Just update the
        # universe and move on.
        if event.dt == self.this_snapshot_dt:
            self.update_universe(event)
            
        # The new event does not match our snapshot.  
        else:
            self.simulate_current_snapshot()
            
            # Once we've finished simulating the old snapshot,
            # we can update the universe with the new event.
            self.update_universe(event)
            
            # The current event is later than the simulation time,
            # which means the algorithm finished quickly enough to
            # receive the new event.  Start a new snapshot with this
            # event's dt.
            if event.dt >= self.simulation_dt:
                self.this_snapshot_dt = event.dt
                
            # The algorithm spent enough time processing that it
            # missed the new event. Wait to start a new snapshot until
            # the events catch up to the algo's simulated dt.
            else:
                self.this_snapshot_dt = None
                
    def simulate_current_snapshot(self):
        """
        Run the user's algo against our current snapshot and update the algo's
        simulated time.
        """        
        start_tic = datetime.now()
        self.algo.handle_data(self.universe)
        stop_tic = datetime.now()
            
        # How long did you take?
        delta = stop_tic - start_tic
            
        # Update the simulation time.
        self.simulation_dt = self.this_snapshot_dt + delta
        
    def update_universe(self, event):
        """
        Update the universe with new event information.
        """
        # Update our portfolio.
        self.universe.portfolio = event.portfolio
    
        # Update our knowledge of this event's sid
        for field in event.keys():
            self.universe[event.sid][field] = event[field]