From 1b17296efda87a0132ceba5ab62f731c897e43c9 Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Wed, 10 Aug 2016 13:27:28 -0400 Subject: [PATCH 1/3] BUG: fix the v5-v4 downgrade path to properly take the most recently held ticker --- zipline/assets/asset_db_migrations.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/zipline/assets/asset_db_migrations.py b/zipline/assets/asset_db_migrations.py index 2c56badd..72d24b78 100644 --- a/zipline/assets/asset_db_migrations.py +++ b/zipline/assets/asset_db_migrations.py @@ -272,14 +272,22 @@ def _downgrade_v5(op): from equities inner join + -- Nested select here to take the most recently held ticker + -- for each sid. The group by with no aggregation function will + -- take the last element in the group, so we first order by + -- the end date ascending to ensure that the groupby takes + -- the last ticker. (select * from - equity_symbol_mappings + (select + * + from + equity_symbol_mappings + order by + equity_symbol_mappings.end_date asc) group by - equity_symbol_mappings.sid - order by - equity_symbol_mappings.end_date desc) sym + sid) sym on equities.sid == sym.sid """, From ea80b8892c290f4da774ac8d115647144d897e5f Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Wed, 10 Aug 2016 15:32:36 -0400 Subject: [PATCH 2/3] TST: Adds dispatch for assert_equal(set, set) --- zipline/testing/predicates.py | 72 ++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/zipline/testing/predicates.py b/zipline/testing/predicates.py index 4a0f1919..3dd00c1d 100644 --- a/zipline/testing/predicates.py +++ b/zipline/testing/predicates.py @@ -30,7 +30,6 @@ from nose.tools import ( # noqa assert_raises_regexp, assert_regexp_matches, assert_sequence_equal, - assert_set_equal, assert_true, assert_tuple_equal, ) @@ -302,37 +301,54 @@ def assert_float_equal(result, ) -@assert_equal.register(dict, dict) -def assert_dict_equal(result, expected, path=(), msg='', **kwargs): - if path is None: - path = () +def _check_sets(result, expected, msg, path, type_): + """Compare two sets. This is used to check dictionary keys and sets. - result_keys = viewkeys(result) - expected_keys = viewkeys(expected) - if result_keys != expected_keys: - if result_keys > expected_keys: - diff = result_keys - expected_keys - msg = 'extra %s in result: %r' % (_s('key', diff), diff) - elif result_keys < expected_keys: - diff = expected_keys - result_keys - msg = 'result is missing %s: %r' % (_s('key', diff), diff) + Parameters + ---------- + result : set + expected : set + msg : str + path : tuple + type : str + The type of an element. For dict we use ``'key'`` and for set we use + ``'element'``. + """ + if result != expected: + if result > expected: + diff = result - expected + msg = 'extra %s in result: %r' % (_s(type_, diff), diff) + elif result < expected: + diff = expected - result + msg = 'result is missing %s: %r' % (_s(type_, diff), diff) else: - sym = result_keys ^ expected_keys - in_result = sym - expected_keys - in_expected = sym - result_keys + in_result = result - expected + in_expected = expected - result msg = '%s only in result: %s\n%s only in expected: %s' % ( - _s('key', in_result), + _s(type_, in_result), in_result, - _s('key', in_expected), + _s(type_, in_expected), in_expected, ) raise AssertionError( - '%sdict keys do not match\n%s' % ( + '%s%ss do not match\n%s' % ( _fmt_msg(msg), - _fmt_path(path + ('.%s()' % ('viewkeys' if PY2 else 'keys'),)), + type_, + _fmt_path(path), ), ) + +@assert_equal.register(dict, dict) +def assert_dict_equal(result, expected, path=(), msg='', **kwargs): + _check_sets( + viewkeys(result), + viewkeys(expected), + msg, + path + ('.%s()' % ('viewkeys' if PY2 else 'keys'),), + 'key', + ) + failures = [] for k, (resultv, expectedv) in iteritems(dzip_exact(result, expected)): try: @@ -350,7 +366,7 @@ def assert_dict_equal(result, expected, path=(), msg='', **kwargs): raise AssertionError('\n'.join(failures)) -@assert_equal.register(list, list) # noqa +@assert_equal.register(list, list) def assert_list_equal(result, expected, path=(), msg='', **kwargs): result_len = len(result) expected_len = len(expected) @@ -362,7 +378,6 @@ def assert_list_equal(result, expected, path=(), msg='', **kwargs): _fmt_path(path), ) ) - for n, (resultv, expectedv) in enumerate(zip(result, expected)): assert_equal( resultv, @@ -373,6 +388,17 @@ def assert_list_equal(result, expected, path=(), msg='', **kwargs): ) +@assert_equal.register(set, set) +def assert_set_equal(result, expected, path=(), msg='', **kwargs): + _check_sets( + result, + expected, + msg, + path, + 'element', + ) + + @assert_equal.register(np.ndarray, np.ndarray) def assert_array_equal(result, expected, From 23363ee45a7cc31d18eab39257073ecb6e22b53a Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Wed, 10 Aug 2016 15:32:49 -0400 Subject: [PATCH 3/3] TST: test the v5->v4 downgrade --- tests/test_assets.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_assets.py b/tests/test_assets.py index 8fb0646b..a500687b 100644 --- a/tests/test_assets.py +++ b/tests/test_assets.py @@ -1470,3 +1470,36 @@ class TestAssetDBVersioning(ZiplineTestCase): # higher-than-current version with self.assertRaises(AssetDBImpossibleDowngrade): downgrade(self.engine, ASSET_DB_VERSION + 5) + + def test_v5_to_v4_selects_most_recent_ticker(self): + T = pd.Timestamp + AssetDBWriter(self.engine).write( + equities=pd.DataFrame( + [['A', 'A', T('2014-01-01'), T('2014-01-02')], + ['B', 'B', T('2014-01-01'), T('2014-01-02')], + # these two are both ticker sid 2 + ['B', 'C', T('2014-01-03'), T('2014-01-04')], + ['C', 'C', T('2014-01-01'), T('2014-01-02')]], + index=[0, 1, 2, 2], + columns=['symbol', 'asset_name', 'start_date', 'end_date'], + ), + ) + + downgrade(self.engine, 4) + metadata = sa.MetaData(self.engine) + metadata.reflect() + + def select_fields(r): + return r.sid, r.symbol, r.asset_name, r.start_date, r.end_date + + expected_data = { + (0, 'A', 'A', T('2014-01-01').value, T('2014-01-02').value), + (1, 'B', 'B', T('2014-01-01').value, T('2014-01-02').value), + (2, 'B', 'C', T('2014-01-01').value, T('2014-01-04').value), + } + actual_data = set(map( + select_fields, + sa.select(metadata.tables['equities'].c).execute(), + )) + + assert_equal(expected_data, actual_data)