From db1e62971a2ccfe80aa690011aa3b46d86c3ae99 Mon Sep 17 00:00:00 2001 From: jfkirk Date: Tue, 5 Jan 2016 14:57:07 -0500 Subject: [PATCH 1/6] ENH: Adds tick_size and renames futures multiplier --- tests/test_algorithm.py | 4 +-- tests/test_assets.py | 9 ++++-- tests/test_perf_tracking.py | 4 +-- zipline/algorithm.py | 2 +- zipline/assets/_assets.pyx | 28 +++++++++++-------- zipline/assets/asset_db_schema.py | 5 ++-- zipline/assets/asset_writer.py | 3 +- .../finance/performance/position_tracker.py | 3 +- zipline/test_algorithms.py | 6 ++-- zipline/utils/test_utils.py | 2 +- 10 files changed, 38 insertions(+), 28 deletions(-) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index d1f3ac76..5b0bc476 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -649,7 +649,7 @@ class TestTransformAlgorithm(TestCase): cls.env = TradingEnvironment() cls.env.write_data(equities_identifiers=[0, 1, 133]) - futures_metadata = {0: {'contract_multiplier': 10}} + futures_metadata = {0: {'multiplier': 10}} cls.futures_env = TradingEnvironment() cls.futures_env.write_data(futures_data=futures_metadata) @@ -1916,7 +1916,7 @@ class TestFutureFlip(TestCase): def test_flip_algo(self): metadata = {1: {'symbol': 'TEST', 'end_date': self.days[3], - 'contract_multiplier': 5}} + 'multiplier': 5}} self.env.write_data(futures_data=metadata) algo = FutureFlipAlgo(sid=1, amount=1, env=self.env, diff --git a/tests/test_assets.py b/tests/test_assets.py index b88fb104..a8339542 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -280,7 +280,8 @@ class TestFuture(TestCase): notice_date=pd.Timestamp('2014-01-20', tz='UTC'), expiration_date=pd.Timestamp('2014-02-20', tz='UTC'), auto_close_date=pd.Timestamp('2014-01-18', tz='UTC'), - contract_multiplier=500 + tick_size=.01, + multiplier=500 ) cls.future2 = Future( 0, @@ -311,7 +312,8 @@ class TestFuture(TestCase): in reprd) self.assertTrue("auto_close_date=Timestamp('2014-01-18 00:00:00+0000'" in reprd) - self.assertTrue("contract_multiplier=500" in reprd) + self.assertTrue("tick_size=0.01" in reprd) + self.assertTrue("multiplier=500" in reprd) def test_reduce(self): reduced = self.future.__reduce__() @@ -323,7 +325,8 @@ class TestFuture(TestCase): self.assertTrue('notice_date' in dictd) self.assertTrue('expiration_date' in dictd) self.assertTrue('auto_close_date' in dictd) - self.assertTrue('contract_multiplier' in dictd) + self.assertTrue('tick_size' in dictd) + self.assertTrue('multiplier' in dictd) from_dict = Future.from_dict(dictd) self.assertTrue(isinstance(from_dict, Future)) diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py index 1d82b1c2..a08db78c 100644 --- a/tests/test_perf_tracking.py +++ b/tests/test_perf_tracking.py @@ -2551,8 +2551,8 @@ class TestPositionTracker(unittest.TestCase): @classmethod def setUpClass(cls): cls.env = TradingEnvironment() - futures_metadata = {3: {'contract_multiplier': 1000}, - 4: {'contract_multiplier': 1000}} + futures_metadata = {3: {'multiplier': 1000}, + 4: {'multiplier': 1000}} cls.env.write_data(equities_identifiers=[1, 2], futures_data=futures_metadata) diff --git a/zipline/algorithm.py b/zipline/algorithm.py index bf11cae6..60f572a7 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -855,7 +855,7 @@ class TradingAlgorithm(object): return 0 if isinstance(asset, Future): - value_multiplier = asset.contract_multiplier + value_multiplier = asset.multiplier else: value_multiplier = 1 diff --git a/zipline/assets/_assets.pyx b/zipline/assets/_assets.pyx index dd3c0434..608b5252 100644 --- a/zipline/assets/_assets.pyx +++ b/zipline/assets/_assets.pyx @@ -37,7 +37,7 @@ cimport numpy as np # IMPORTANT NOTE: You must change this template if you change # Asset.__reduce__, or else we'll attempt to unpickle an old version of this # class -CACHE_FILE_TEMPLATE = '/tmp/.%s-%s.v4.cache' +CACHE_FILE_TEMPLATE = '/tmp/.%s-%s.v5.cache' cdef class Asset: @@ -228,7 +228,8 @@ cdef class Future(Asset): cdef readonly object notice_date cdef readonly object expiration_date cdef readonly object auto_close_date - cdef readonly float contract_multiplier + cdef readonly object tick_size + cdef readonly float multiplier def __cinit__(self, int sid, # sid is required @@ -242,13 +243,15 @@ cdef class Future(Asset): object auto_close_date=None, object first_traded=None, object exchange="", - float contract_multiplier=1): + object tick_size="", + float multiplier=1): - self.root_symbol = root_symbol - self.notice_date = notice_date - self.expiration_date = expiration_date - self.auto_close_date = auto_close_date - self.contract_multiplier = contract_multiplier + self.root_symbol = root_symbol + self.notice_date = notice_date + self.expiration_date = expiration_date + self.auto_close_date = auto_close_date + self.tick_size = tick_size + self.multiplier = multiplier def __str__(self): if self.symbol: @@ -259,7 +262,8 @@ cdef class Future(Asset): def __repr__(self): attrs = ('symbol', 'root_symbol', 'asset_name', 'exchange', 'start_date', 'end_date', 'first_traded', 'notice_date', - 'expiration_date', 'auto_close_date', 'contract_multiplier') + 'expiration_date', 'auto_close_date', 'tick_size', + 'multiplier') tuples = ((attr, repr(getattr(self, attr, None))) for attr in attrs) strings = ('%s=%s' % (t[0], t[1]) for t in tuples) @@ -284,7 +288,8 @@ cdef class Future(Asset): self.auto_close_date, self.first_traded, self.exchange, - self.contract_multiplier,)) + self.tick_size, + self.multiplier,)) cpdef to_dict(self): """ @@ -295,7 +300,8 @@ cdef class Future(Asset): super_dict['notice_date'] = self.notice_date super_dict['expiration_date'] = self.expiration_date super_dict['auto_close_date'] = self.auto_close_date - super_dict['contract_multiplier'] = self.contract_multiplier + super_dict['tick_size'] = self.tick_size + super_dict['multiplier'] = self.multiplier return super_dict diff --git a/zipline/assets/asset_db_schema.py b/zipline/assets/asset_db_schema.py index 29f6b568..3e0b4f15 100644 --- a/zipline/assets/asset_db_schema.py +++ b/zipline/assets/asset_db_schema.py @@ -4,7 +4,7 @@ import sqlalchemy as sa # Define a version number for the database generated by these writers # Increment this version number any time a change is made to the schema of the # assets database -ASSET_DB_VERSION = 0 +ASSET_DB_VERSION = 1 def generate_asset_db_metadata(bind=None): @@ -120,7 +120,8 @@ def _futures_contracts_schema(metadata): sa.Column('notice_date', sa.Integer, nullable=False), sa.Column('expiration_date', sa.Integer, nullable=False), sa.Column('auto_close_date', sa.Integer, nullable=False), - sa.Column('contract_multiplier', sa.Float), + sa.Column('multiplier', sa.Float), + sa.Column('tick_size', sa.Float), ) diff --git a/zipline/assets/asset_writer.py b/zipline/assets/asset_writer.py index 593a0ea0..96d1c022 100644 --- a/zipline/assets/asset_writer.py +++ b/zipline/assets/asset_writer.py @@ -60,7 +60,8 @@ _futures_defaults = { 'notice_date': None, 'expiration_date': None, 'auto_close_date': None, - 'contract_multiplier': 1, + 'tick_size': None, + 'multiplier': 1, } # Default values for the exchanges DataFrame diff --git a/zipline/finance/performance/position_tracker.py b/zipline/finance/performance/position_tracker.py index 9534d487..8489ccb6 100644 --- a/zipline/finance/performance/position_tracker.py +++ b/zipline/finance/performance/position_tracker.py @@ -156,8 +156,7 @@ class PositionTracker(object): self._position_exposure_multipliers[sid] = 1 if isinstance(asset, Future): self._position_value_multipliers[sid] = 0 - self._position_exposure_multipliers[sid] = \ - asset.contract_multiplier + self._position_exposure_multipliers[sid] = asset.multiplier # Futures auto-close timing is controlled by the Future's # auto_close_date property self._insert_auto_close_position_date( diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index ffe39226..6d8d399b 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -343,7 +343,7 @@ class TestOrderValueAlgorithm(TradingAlgorithm): multiplier = 2. if isinstance(self.sid(0), Future): - multiplier *= self.sid(0).contract_multiplier + multiplier *= self.sid(0).multiplier self.order_value(self.sid(0), data[0].price * multiplier) @@ -391,7 +391,7 @@ class TestOrderPercentAlgorithm(TradingAlgorithm): if isinstance(self.sid(0), Future): self.target_shares += np.floor( (.001 * self.portfolio.portfolio_value) / - (data[0].price * self.sid(0).contract_multiplier) + (data[0].price * self.sid(0).multiplier) ) @@ -439,7 +439,7 @@ class TestTargetValueAlgorithm(TradingAlgorithm): self.target_shares = np.round(20 / data[0].price) if isinstance(self.sid(0), Future): self.target_shares = np.round( - 20 / (data[0].price * self.sid(0).contract_multiplier)) + 20 / (data[0].price * self.sid(0).multiplier)) class FutureFlipAlgo(TestAlgorithm): diff --git a/zipline/utils/test_utils.py b/zipline/utils/test_utils.py index cbcd3fda..8ea3aa00 100644 --- a/zipline/utils/test_utils.py +++ b/zipline/utils/test_utils.py @@ -388,7 +388,7 @@ def make_future_info(first_sid, 'start_date': start_date_func(month_begin), 'notice_date': notice_date_func(month_begin), 'expiration_date': notice_date_func(month_begin), - 'contract_multiplier': 500, + 'multiplier': 500, }) return pd.DataFrame.from_records(contracts, index='sid').convert_objects() From ece9e59ef94cb0e63fb2e6e57e2bf813e074df24 Mon Sep 17 00:00:00 2001 From: jfkirk Date: Wed, 6 Jan 2016 13:31:25 -0500 Subject: [PATCH 2/6] ENH: Adds asset db downgrade management and tests --- etc/requirements_dev.txt | 3 + tests/test_assets.py | 31 ++++++++ zipline/assets/asset_db_migrations.py | 109 ++++++++++++++++++++++++++ zipline/errors.py | 7 ++ zipline/finance/performance/period.py | 8 +- 5 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 zipline/assets/asset_db_migrations.py diff --git a/etc/requirements_dev.txt b/etc/requirements_dev.txt index a796b340..30add841 100644 --- a/etc/requirements_dev.txt +++ b/etc/requirements_dev.txt @@ -57,3 +57,6 @@ Markdown==2.6.2 futures==3.0.3 requests-futures==0.9.5 piprot==0.9.1 + +# For asset db management +alembic==0.7.7 diff --git a/tests/test_assets.py b/tests/test_assets.py index a8339542..10734a53 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -55,6 +55,9 @@ from zipline.assets.asset_db_schema import ( ASSET_DB_VERSION, _version_table_schema, ) +from zipline.assets.asset_db_migrations import ( + downgrade +) from zipline.errors import ( EquitiesNotFound, FutureContractsNotFound, @@ -64,6 +67,7 @@ from zipline.errors import ( SidAssignmentError, SidsNotFound, SymbolNotFound, + AssetDBImpossibleDowngrade, ) from zipline.finance.trading import TradingEnvironment, noop_load from zipline.utils.test_utils import ( @@ -1365,3 +1369,30 @@ class TestAssetDBVersioning(TestCase): # Now that the versions match, this Finder should succeed AssetFinder(engine=env.engine) + + def test_downgrade(self): + # Attempt to downgrade a current assets db all the way down to v0 + env = TradingEnvironment(load=noop_load) + conn = env.engine.connect() + downgrade(env.engine, 0) + + # Verify that the db version is now 0 + metadata = sa.MetaData(conn) + metadata.reflect(bind=env.engine) + version_table = metadata.tables['version_info'] + check_version_info(version_table, 0) + + # Check some of the v1-to-v0 downgrades + self.assertTrue('futures_contracts' in metadata.tables) + self.assertTrue('version_info' in metadata.tables) + self.assertFalse('tick_size' in + metadata.tables['futures_contracts'].columns) + self.assertTrue('contract_multiplier' in + metadata.tables['futures_contracts'].columns) + + def test_impossible_downgrade(self): + # Attempt to downgrade a current assets db to a + # higher-than-current version + env = TradingEnvironment(load=noop_load) + with self.assertRaises(AssetDBImpossibleDowngrade): + downgrade(env.engine, ASSET_DB_VERSION + 5) diff --git a/zipline/assets/asset_db_migrations.py b/zipline/assets/asset_db_migrations.py new file mode 100644 index 00000000..af32e4c2 --- /dev/null +++ b/zipline/assets/asset_db_migrations.py @@ -0,0 +1,109 @@ +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from zipline.assets.asset_writer import write_version_info +from zipline.errors import AssetDBImpossibleDowngrade + + +def downgrade(engine, desired_version): + """Downgrades the assets db at the given engine to the desired version. + + Parameters + ---------- + engine : Engine + An SQLAlchemy engine to the assets database. + desired_version : int + The desired resulting version for the assets database. + """ + + # Check the version of the db at the engine + conn = engine.connect() + metadata = sa.MetaData(conn) + metadata.reflect(bind=engine) + version_info_table = metadata.tables['version_info'] + starting_version = sa.select((version_info_table.c.version,)).scalar() + + # Check for accidental upgrade + if starting_version < desired_version: + raise AssetDBImpossibleDowngrade(db_version=starting_version, + desired_version=desired_version) + + # Check if the desired version is already the db version + if starting_version == desired_version: + # No downgrade needed + return + + # Create alembic context + ctx = MigrationContext.configure(conn) + op = Operations(ctx) + + # Integer keys of downgrades to run + # E.g.: [5, 4, 3, 2] would downgrade v6 to v2 + downgrade_keys = range(desired_version, starting_version)[::-1] + + # Disable foreign keys until all downgrades are complete + _pragma_foreign_keys(conn, False) + + # Execute the downgrades in order + for downgrade_key in downgrade_keys: + _downgrade_methods[downgrade_key](op, version_info_table) + + # Re-enable foreign keys + _pragma_foreign_keys(conn, True) + + +def _pragma_foreign_keys(connection, on): + """Sets the PRAGMA foreign_keys state of the SQLLite database. Disabling + the pragma allows for batch modification of tables with foreign keys. + + Parameters + ---------- + connection : Connection + A SQLAlchemy connection to the db + on : bool + If true, PRAGMA foreign_keys will be set to ON. Otherwise, the PRAGMA + foreign_keys will be set to OFF. + """ + connection.execute("PRAGMA foreign_keys=%s" % ("ON" if on else "OFF")) + + +def _downgrade_v1_to_v0(op, version_info_table): + """ + Downgrade assets db by removing the 'tick_size' column and renaming the + 'multiplier' column. + """ + version_info_table.delete().execute() + + # Drop indices before batch + # This is to prevent index collision when creating the temp table + op.drop_index('ix_futures_contracts_root_symbol') + op.drop_index('ix_futures_contracts_symbol') + + # Execute batch op to allow column modification in SQLLite + with op.batch_alter_table('futures_contracts') as batch_op: + + # Rename 'multiplier' + batch_op.alter_column(column_name='multiplier', + new_column_name='contract_multiplier') + + # Delete 'tick_size' + batch_op.drop_column('tick_size') + + # Recreate indices after batch + op.create_index('ix_futures_contracts_root_symbol', + table_name='futures_contracts', + columns=['root_symbol']) + op.create_index('ix_futures_contracts_symbol', + table_name='futures_contracts', + columns=['symbol'], + unique=True) + + write_version_info(version_info_table, 0) + +# This dict contains references to downgrade methods that can be applied to an +# assets db. The resulting db's version is the key. +# e.g. The method at key '0' is the downgrade method from v1 to v0 +_downgrade_methods = { + 0: _downgrade_v1_to_v0, +} diff --git a/zipline/errors.py b/zipline/errors.py index 78d6f9d8..57e2df7d 100644 --- a/zipline/errors.py +++ b/zipline/errors.py @@ -507,3 +507,10 @@ class AssetDBVersionError(ZiplineError): "Expected version: {expected_version}. Try rebuilding your asset " "database or updating your version of Zipline." ) + + +class AssetDBImpossibleDowngrade(ZiplineError): + msg = ( + "The existing Asset database is version: {db_version} which is lower " + "than the desired downgrade version: {desired_version}." + ) diff --git a/zipline/finance/performance/period.py b/zipline/finance/performance/period.py index 22a8158a..132bb8f6 100644 --- a/zipline/finance/performance/period.py +++ b/zipline/finance/performance/period.py @@ -126,8 +126,8 @@ def calc_period_stats(pos_stats, ending_cash): net_leverage=net_leverage) -def calc_payout(contract_multiplier, amount, old_price, price): - return (price - old_price) * contract_multiplier * amount +def calc_payout(multiplier, amount, old_price, price): + return (price - old_price) * multiplier * amount class PerformancePeriod(object): @@ -235,7 +235,7 @@ class PerformancePeriod(object): pos = positions[asset] amount = pos.amount payout = calc_payout( - asset.contract_multiplier, + asset.multiplier, amount, old_price, pos.last_sale_price) @@ -288,7 +288,7 @@ class PerformancePeriod(object): amount = pos.amount price = txn.price cash_adj = calc_payout( - asset.contract_multiplier, amount, old_price, price) + asset.multiplier, amount, old_price, price) self.adjust_cash(cash_adj) if amount + txn.amount == 0: del self._payout_last_sale_prices[asset] From c9c2e3e60e3a82f2c355e9815be0c03d7d9ea98d Mon Sep 17 00:00:00 2001 From: jfkirk Date: Fri, 22 Jan 2016 11:14:45 -0500 Subject: [PATCH 3/6] TST: Updates TestPositionPerformance --- tests/test_perf_tracking.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_perf_tracking.py b/tests/test_perf_tracking.py index a08db78c..a2b38fe1 100644 --- a/tests/test_perf_tracking.py +++ b/tests/test_perf_tracking.py @@ -1007,7 +1007,7 @@ class TestPositionPerformance(unittest.TestCase): cls.env = TradingEnvironment() # Sids 1 and 2 are equities, Sid 3 is a future cls.env.write_data(equities_identifiers=[1, 2], - futures_data={3: {'contract_multiplier': 100}}) + futures_data={3: {'multiplier': 100}}) @classmethod def tearDownClass(cls): From d82bd5c05d57dab77e512a18e6df692c86b5be26 Mon Sep 17 00:00:00 2001 From: jfkirk Date: Fri, 22 Jan 2016 15:26:14 -0500 Subject: [PATCH 4/6] TST: Cleans up test_to_and_from_dict --- tests/test_assets.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_assets.py b/tests/test_assets.py index 10734a53..a9bbcf72 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -50,6 +50,7 @@ from zipline.assets.futures import ( from zipline.assets.asset_writer import ( check_version_info, write_version_info, + _futures_defaults, ) from zipline.assets.asset_db_schema import ( ASSET_DB_VERSION, @@ -325,12 +326,8 @@ class TestFuture(TestCase): def test_to_and_from_dict(self): dictd = self.future.to_dict() - self.assertTrue('root_symbol' in dictd) - self.assertTrue('notice_date' in dictd) - self.assertTrue('expiration_date' in dictd) - self.assertTrue('auto_close_date' in dictd) - self.assertTrue('tick_size' in dictd) - self.assertTrue('multiplier' in dictd) + for field in _futures_defaults.keys(): + self.assertTrue(field in dictd) from_dict = Future.from_dict(dictd) self.assertTrue(isinstance(from_dict, Future)) From 5d0ffd0db4f075c43c58d30a557dc755692d6851 Mon Sep 17 00:00:00 2001 From: jfkirk Date: Fri, 22 Jan 2016 15:31:41 -0500 Subject: [PATCH 5/6] DOC: SQLLite -> SQLite --- zipline/assets/asset_db_migrations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zipline/assets/asset_db_migrations.py b/zipline/assets/asset_db_migrations.py index af32e4c2..b1c5aa2f 100644 --- a/zipline/assets/asset_db_migrations.py +++ b/zipline/assets/asset_db_migrations.py @@ -54,7 +54,7 @@ def downgrade(engine, desired_version): def _pragma_foreign_keys(connection, on): - """Sets the PRAGMA foreign_keys state of the SQLLite database. Disabling + """Sets the PRAGMA foreign_keys state of the SQLite database. Disabling the pragma allows for batch modification of tables with foreign keys. Parameters @@ -80,7 +80,7 @@ def _downgrade_v1_to_v0(op, version_info_table): op.drop_index('ix_futures_contracts_root_symbol') op.drop_index('ix_futures_contracts_symbol') - # Execute batch op to allow column modification in SQLLite + # Execute batch op to allow column modification in SQLite with op.batch_alter_table('futures_contracts') as batch_op: # Rename 'multiplier' From f15245924dcb182b458f3da7f65740534d0abd0e Mon Sep 17 00:00:00 2001 From: jfkirk Date: Fri, 22 Jan 2016 15:48:31 -0500 Subject: [PATCH 6/6] DOC: Whatsnew entry for assets db downgrade --- docs/source/whatsnew/0.8.4.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/whatsnew/0.8.4.txt b/docs/source/whatsnew/0.8.4.txt index adc5e9ef..fe0df981 100644 --- a/docs/source/whatsnew/0.8.4.txt +++ b/docs/source/whatsnew/0.8.4.txt @@ -186,3 +186,4 @@ Miscellaneous * Treasury and benchmark downloads will now wait up to an hour to download again if data returned from a remote source does not extend to the date expected. (:issue:`841`). +* Added a tool to downgrade the assets db to previous versions (:issue:`941`).