Merge pull request #941 from quantopian/futures-tick-size

ENH: Adds tick_size and renames futures multiplier

docs/source/whatsnew/0.8.4.txt

@@ -186,3 +186,4 @@ Miscellaneous
 * Treasury and benchmark downloads will now wait up to an hour to download
   again if data returned from a remote source does not extend to the date
   expected. (:issue:`841`).
+* Added a tool to downgrade the assets db to previous versions (:issue:`941`).

etc/requirements_dev.txt

@@ -57,3 +57,6 @@ Markdown==2.6.2
 futures==3.0.3
 requests-futures==0.9.5
 piprot==0.9.1
+
+# For asset db management
+alembic==0.7.7

tests/test_algorithm.py

@@ -649,7 +649,7 @@ class TestTransformAlgorithm(TestCase):
         cls.env = TradingEnvironment()
         cls.env.write_data(equities_identifiers=[0, 1, 133])
 
-        futures_metadata = {0: {'contract_multiplier': 10}}
+        futures_metadata = {0: {'multiplier': 10}}
         cls.futures_env = TradingEnvironment()
         cls.futures_env.write_data(futures_data=futures_metadata)
 
@@ -1916,7 +1916,7 @@ class TestFutureFlip(TestCase):
     def test_flip_algo(self):
         metadata = {1: {'symbol': 'TEST',
                         'end_date': self.days[3],
-                        'contract_multiplier': 5}}
+                        'multiplier': 5}}
         self.env.write_data(futures_data=metadata)
 
         algo = FutureFlipAlgo(sid=1, amount=1, env=self.env,

tests/test_assets.py

@@ -50,11 +50,15 @@ from zipline.assets.futures import (
 from zipline.assets.asset_writer import (
     check_version_info,
     write_version_info,
+    _futures_defaults,
 )
 from zipline.assets.asset_db_schema import (
     ASSET_DB_VERSION,
     _version_table_schema,
 )
+from zipline.assets.asset_db_migrations import (
+    downgrade
+)
 from zipline.errors import (
     EquitiesNotFound,
     FutureContractsNotFound,
@@ -64,6 +68,7 @@ from zipline.errors import (
     SidAssignmentError,
     SidsNotFound,
     SymbolNotFound,
+    AssetDBImpossibleDowngrade,
 )
 from zipline.finance.trading import TradingEnvironment, noop_load
 from zipline.utils.test_utils import (
@@ -280,7 +285,8 @@ class TestFuture(TestCase):
             notice_date=pd.Timestamp('2014-01-20', tz='UTC'),
             expiration_date=pd.Timestamp('2014-02-20', tz='UTC'),
             auto_close_date=pd.Timestamp('2014-01-18', tz='UTC'),
-            contract_multiplier=500
+            tick_size=.01,
+            multiplier=500
         )
         cls.future2 = Future(
             0,
@@ -311,7 +317,8 @@ class TestFuture(TestCase):
                         in reprd)
         self.assertTrue("auto_close_date=Timestamp('2014-01-18 00:00:00+0000'"
                         in reprd)
-        self.assertTrue("contract_multiplier=500" in reprd)
+        self.assertTrue("tick_size=0.01" in reprd)
+        self.assertTrue("multiplier=500" in reprd)
 
     def test_reduce(self):
         reduced = self.future.__reduce__()
@@ -319,11 +326,8 @@ class TestFuture(TestCase):
 
     def test_to_and_from_dict(self):
         dictd = self.future.to_dict()
-        self.assertTrue('root_symbol' in dictd)
-        self.assertTrue('notice_date' in dictd)
-        self.assertTrue('expiration_date' in dictd)
-        self.assertTrue('auto_close_date' in dictd)
-        self.assertTrue('contract_multiplier' in dictd)
+        for field in _futures_defaults.keys():
+            self.assertTrue(field in dictd)
 
         from_dict = Future.from_dict(dictd)
         self.assertTrue(isinstance(from_dict, Future))
@@ -1362,3 +1366,30 @@ class TestAssetDBVersioning(TestCase):
 
         # Now that the versions match, this Finder should succeed
         AssetFinder(engine=env.engine)
+
+    def test_downgrade(self):
+        # Attempt to downgrade a current assets db all the way down to v0
+        env = TradingEnvironment(load=noop_load)
+        conn = env.engine.connect()
+        downgrade(env.engine, 0)
+
+        # Verify that the db version is now 0
+        metadata = sa.MetaData(conn)
+        metadata.reflect(bind=env.engine)
+        version_table = metadata.tables['version_info']
+        check_version_info(version_table, 0)
+
+        # Check some of the v1-to-v0 downgrades
+        self.assertTrue('futures_contracts' in metadata.tables)
+        self.assertTrue('version_info' in metadata.tables)
+        self.assertFalse('tick_size' in
+                         metadata.tables['futures_contracts'].columns)
+        self.assertTrue('contract_multiplier' in
+                        metadata.tables['futures_contracts'].columns)
+
+    def test_impossible_downgrade(self):
+        # Attempt to downgrade a current assets db to a
+        # higher-than-current version
+        env = TradingEnvironment(load=noop_load)
+        with self.assertRaises(AssetDBImpossibleDowngrade):
+            downgrade(env.engine, ASSET_DB_VERSION + 5)

tests/test_perf_tracking.py

@@ -1007,7 +1007,7 @@ class TestPositionPerformance(unittest.TestCase):
         cls.env = TradingEnvironment()
         # Sids 1 and 2 are equities, Sid 3 is a future
         cls.env.write_data(equities_identifiers=[1, 2],
-                           futures_data={3: {'contract_multiplier': 100}})
+                           futures_data={3: {'multiplier': 100}})
 
     @classmethod
     def tearDownClass(cls):
@@ -2551,8 +2551,8 @@ class TestPositionTracker(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.env = TradingEnvironment()
-        futures_metadata = {3: {'contract_multiplier': 1000},
-                            4: {'contract_multiplier': 1000}}
+        futures_metadata = {3: {'multiplier': 1000},
+                            4: {'multiplier': 1000}}
         cls.env.write_data(equities_identifiers=[1, 2],
                            futures_data=futures_metadata)
 

zipline/algorithm.py

@@ -855,7 +855,7 @@ class TradingAlgorithm(object):
             return 0
 
         if isinstance(asset, Future):
-            value_multiplier = asset.contract_multiplier
+            value_multiplier = asset.multiplier
         else:
             value_multiplier = 1
 

zipline/assets/_assets.pyx

@@ -37,7 +37,7 @@ cimport numpy as np
 # IMPORTANT NOTE: You must change this template if you change
 # Asset.__reduce__, or else we'll attempt to unpickle an old version of this
 # class
-CACHE_FILE_TEMPLATE = '/tmp/.%s-%s.v4.cache'
+CACHE_FILE_TEMPLATE = '/tmp/.%s-%s.v5.cache'
 
 cdef class Asset:
 
@@ -228,7 +228,8 @@ cdef class Future(Asset):
     cdef readonly object notice_date
     cdef readonly object expiration_date
     cdef readonly object auto_close_date
-    cdef readonly float contract_multiplier
+    cdef readonly object tick_size
+    cdef readonly float multiplier
 
     def __cinit__(self,
                   int sid, # sid is required
@@ -242,13 +243,15 @@ cdef class Future(Asset):
                   object auto_close_date=None,
                   object first_traded=None,
                   object exchange="",
-                  float contract_multiplier=1):
+                  object tick_size="",
+                  float multiplier=1):
 
-        self.root_symbol         = root_symbol
-        self.notice_date         = notice_date
-        self.expiration_date     = expiration_date
-        self.auto_close_date     = auto_close_date
-        self.contract_multiplier = contract_multiplier
+        self.root_symbol     = root_symbol
+        self.notice_date     = notice_date
+        self.expiration_date = expiration_date
+        self.auto_close_date = auto_close_date
+        self.tick_size       = tick_size
+        self.multiplier      = multiplier
 
     def __str__(self):
         if self.symbol:
@@ -259,7 +262,8 @@ cdef class Future(Asset):
     def __repr__(self):
         attrs = ('symbol', 'root_symbol', 'asset_name', 'exchange',
                  'start_date', 'end_date', 'first_traded', 'notice_date',
-                 'expiration_date', 'auto_close_date', 'contract_multiplier')
+                 'expiration_date', 'auto_close_date', 'tick_size',
+                 'multiplier')
         tuples = ((attr, repr(getattr(self, attr, None)))
                   for attr in attrs)
         strings = ('%s=%s' % (t[0], t[1]) for t in tuples)
@@ -284,7 +288,8 @@ cdef class Future(Asset):
                                  self.auto_close_date,
                                  self.first_traded,
                                  self.exchange,
-                                 self.contract_multiplier,))
+                                 self.tick_size,
+                                 self.multiplier,))
 
     cpdef to_dict(self):
         """
@@ -295,7 +300,8 @@ cdef class Future(Asset):
         super_dict['notice_date'] = self.notice_date
         super_dict['expiration_date'] = self.expiration_date
         super_dict['auto_close_date'] = self.auto_close_date
-        super_dict['contract_multiplier'] = self.contract_multiplier
+        super_dict['tick_size'] = self.tick_size
+        super_dict['multiplier'] = self.multiplier
         return super_dict
 
 

zipline/assets/asset_db_migrations.py

@@ -0,0 +1,109 @@
+import sqlalchemy as sa
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+
+from zipline.assets.asset_writer import write_version_info
+from zipline.errors import AssetDBImpossibleDowngrade
+
+
+def downgrade(engine, desired_version):
+    """Downgrades the assets db at the given engine to the desired version.
+
+    Parameters
+    ----------
+    engine : Engine
+        An SQLAlchemy engine to the assets database.
+    desired_version : int
+        The desired resulting version for the assets database.
+    """
+
+    # Check the version of the db at the engine
+    conn = engine.connect()
+    metadata = sa.MetaData(conn)
+    metadata.reflect(bind=engine)
+    version_info_table = metadata.tables['version_info']
+    starting_version = sa.select((version_info_table.c.version,)).scalar()
+
+    # Check for accidental upgrade
+    if starting_version < desired_version:
+        raise AssetDBImpossibleDowngrade(db_version=starting_version,
+                                         desired_version=desired_version)
+
+    # Check if the desired version is already the db version
+    if starting_version == desired_version:
+        # No downgrade needed
+        return
+
+    # Create alembic context
+    ctx = MigrationContext.configure(conn)
+    op = Operations(ctx)
+
+    # Integer keys of downgrades to run
+    # E.g.: [5, 4, 3, 2] would downgrade v6 to v2
+    downgrade_keys = range(desired_version, starting_version)[::-1]
+
+    # Disable foreign keys until all downgrades are complete
+    _pragma_foreign_keys(conn, False)
+
+    # Execute the downgrades in order
+    for downgrade_key in downgrade_keys:
+        _downgrade_methods[downgrade_key](op, version_info_table)
+
+    # Re-enable foreign keys
+    _pragma_foreign_keys(conn, True)
+
+
+def _pragma_foreign_keys(connection, on):
+    """Sets the PRAGMA foreign_keys state of the SQLite database. Disabling
+    the pragma allows for batch modification of tables with foreign keys.
+
+    Parameters
+    ----------
+    connection : Connection
+        A SQLAlchemy connection to the db
+    on : bool
+        If true, PRAGMA foreign_keys will be set to ON. Otherwise, the PRAGMA
+        foreign_keys will be set to OFF.
+    """
+    connection.execute("PRAGMA foreign_keys=%s" % ("ON" if on else "OFF"))
+
+
+def _downgrade_v1_to_v0(op, version_info_table):
+    """
+    Downgrade assets db by removing the 'tick_size' column and renaming the
+    'multiplier' column.
+    """
+    version_info_table.delete().execute()
+
+    # Drop indices before batch
+    # This is to prevent index collision when creating the temp table
+    op.drop_index('ix_futures_contracts_root_symbol')
+    op.drop_index('ix_futures_contracts_symbol')
+
+    # Execute batch op to allow column modification in SQLite
+    with op.batch_alter_table('futures_contracts') as batch_op:
+
+        # Rename 'multiplier'
+        batch_op.alter_column(column_name='multiplier',
+                              new_column_name='contract_multiplier')
+
+        # Delete 'tick_size'
+        batch_op.drop_column('tick_size')
+
+    # Recreate indices after batch
+    op.create_index('ix_futures_contracts_root_symbol',
+                    table_name='futures_contracts',
+                    columns=['root_symbol'])
+    op.create_index('ix_futures_contracts_symbol',
+                    table_name='futures_contracts',
+                    columns=['symbol'],
+                    unique=True)
+
+    write_version_info(version_info_table, 0)
+
+# This dict contains references to downgrade methods that can be applied to an
+# assets db. The resulting db's version is the key.
+# e.g. The method at key '0' is the downgrade method from v1 to v0
+_downgrade_methods = {
+    0: _downgrade_v1_to_v0,
+}

zipline/assets/asset_db_schema.py

@@ -4,7 +4,7 @@ import sqlalchemy as sa
 # Define a version number for the database generated by these writers
 # Increment this version number any time a change is made to the schema of the
 # assets database
-ASSET_DB_VERSION = 0
+ASSET_DB_VERSION = 1
 
 
 def generate_asset_db_metadata(bind=None):
@@ -120,7 +120,8 @@ def _futures_contracts_schema(metadata):
         sa.Column('notice_date', sa.Integer, nullable=False),
         sa.Column('expiration_date', sa.Integer, nullable=False),
         sa.Column('auto_close_date', sa.Integer, nullable=False),
-        sa.Column('contract_multiplier', sa.Float),
+        sa.Column('multiplier', sa.Float),
+        sa.Column('tick_size', sa.Float),
     )
 
 

zipline/assets/asset_writer.py

@@ -60,7 +60,8 @@ _futures_defaults = {
     'notice_date': None,
     'expiration_date': None,
     'auto_close_date': None,
-    'contract_multiplier': 1,
+    'tick_size': None,
+    'multiplier': 1,
 }
 
 # Default values for the exchanges DataFrame

zipline/errors.py

@@ -507,3 +507,10 @@ class AssetDBVersionError(ZiplineError):
         "Expected version: {expected_version}. Try rebuilding your asset "
         "database or updating your version of Zipline."
     )
+
+
+class AssetDBImpossibleDowngrade(ZiplineError):
+    msg = (
+        "The existing Asset database is version: {db_version} which is lower "
+        "than the desired downgrade version: {desired_version}."
+    )

zipline/finance/performance/period.py

@@ -126,8 +126,8 @@ def calc_period_stats(pos_stats, ending_cash):
         net_leverage=net_leverage)
 
 
-def calc_payout(contract_multiplier, amount, old_price, price):
-    return (price - old_price) * contract_multiplier * amount
+def calc_payout(multiplier, amount, old_price, price):
+    return (price - old_price) * multiplier * amount
 
 
 class PerformancePeriod(object):
@@ -235,7 +235,7 @@ class PerformancePeriod(object):
             pos = positions[asset]
             amount = pos.amount
             payout = calc_payout(
-                asset.contract_multiplier,
+                asset.multiplier,
                 amount,
                 old_price,
                 pos.last_sale_price)
@@ -288,7 +288,7 @@ class PerformancePeriod(object):
                 amount = pos.amount
                 price = txn.price
                 cash_adj = calc_payout(
-                    asset.contract_multiplier, amount, old_price, price)
+                    asset.multiplier, amount, old_price, price)
                 self.adjust_cash(cash_adj)
                 if amount + txn.amount == 0:
                     del self._payout_last_sale_prices[asset]

zipline/finance/performance/position_tracker.py

@@ -156,8 +156,7 @@ class PositionTracker(object):
                 self._position_exposure_multipliers[sid] = 1
             if isinstance(asset, Future):
                 self._position_value_multipliers[sid] = 0
-                self._position_exposure_multipliers[sid] = \
-                    asset.contract_multiplier
+                self._position_exposure_multipliers[sid] = asset.multiplier
                 # Futures auto-close timing is controlled by the Future's
                 # auto_close_date property
                 self._insert_auto_close_position_date(

zipline/test_algorithms.py

@@ -343,7 +343,7 @@ class TestOrderValueAlgorithm(TradingAlgorithm):
 
         multiplier = 2.
         if isinstance(self.sid(0), Future):
-            multiplier *= self.sid(0).contract_multiplier
+            multiplier *= self.sid(0).multiplier
 
         self.order_value(self.sid(0), data[0].price * multiplier)
 
@@ -391,7 +391,7 @@ class TestOrderPercentAlgorithm(TradingAlgorithm):
         if isinstance(self.sid(0), Future):
             self.target_shares += np.floor(
                 (.001 * self.portfolio.portfolio_value) /
-                (data[0].price * self.sid(0).contract_multiplier)
+                (data[0].price * self.sid(0).multiplier)
             )
 
 
@@ -439,7 +439,7 @@ class TestTargetValueAlgorithm(TradingAlgorithm):
             self.target_shares = np.round(20 / data[0].price)
         if isinstance(self.sid(0), Future):
             self.target_shares = np.round(
-                20 / (data[0].price * self.sid(0).contract_multiplier))
+                20 / (data[0].price * self.sid(0).multiplier))
 
 
 class FutureFlipAlgo(TestAlgorithm):

zipline/utils/test_utils.py

@@ -388,7 +388,7 @@ def make_future_info(first_sid,
             'start_date': start_date_func(month_begin),
             'notice_date': notice_date_func(month_begin),
             'expiration_date': notice_date_func(month_begin),
-            'contract_multiplier': 500,
+            'multiplier': 500,
         })
     return pd.DataFrame.from_records(contracts, index='sid').convert_objects()