From 52f78fcbc76245b0988808bc32f4849116e4a7bc Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 23 Jan 2015 23:18:13 -0500 Subject: [PATCH 1/7] restricted list trading control added. --- tests/test_algorithm.py | 22 ++++++++++++++++++++++ zipline/algorithm.py | 9 +++++++++ zipline/finance/controls.py | 24 ++++++++++++++++++++++++ zipline/test_algorithms.py | 6 ++++++ 4 files changed, 61 insertions(+) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index b9c874e7..403144d2 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -55,6 +55,7 @@ from zipline.test_algorithms import ( SetMaxPositionSizeAlgorithm, SetMaxOrderCountAlgorithm, SetMaxOrderSizeAlgorithm, + SetDoNotOrderListAlgorithm, api_algo, api_get_environment_algo, api_symbol_algo, @@ -955,6 +956,27 @@ class TestTradingControls(TestCase): algo = SetMaxPositionSizeAlgorithm(max_shares=10, max_notional=61.0) self.check_algo_fails(algo, handle_data, 0) + def test_set_do_not_order_list(self): + # set the restricted list to be the sid, and fail. + algo = SetDoNotOrderListAlgorithm(sid=self.sid, + restricted_list=[self.sid]) + + def handle_data(algo, data): + algo.order(self.sid, 100) + algo.order_count += 1 + + self.check_algo_fails(algo, handle_data, 0) + + # set the restricted list to exclude the sid, and succeed + algo = SetDoNotOrderListAlgorithm(sid=self.sid, + restricted_list=[134, 135, 136]) + + def handle_data(algo, data): + algo.order(self.sid, 100) + algo.order_count += 1 + + self.check_algo_succeeds(algo, handle_data) + def test_set_max_order_size(self): # Buy one share. diff --git a/zipline/algorithm.py b/zipline/algorithm.py index 64e9b9a0..9a705d86 100644 --- a/zipline/algorithm.py +++ b/zipline/algorithm.py @@ -48,6 +48,7 @@ from zipline.finance.controls import ( MaxOrderCount, MaxOrderSize, MaxPositionSize, + RestrictedListOrder ) from zipline.finance.execution import ( LimitOrder, @@ -1080,6 +1081,14 @@ class TradingAlgorithm(object): control = MaxOrderCount(max_count) self.register_trading_control(control) + @api_method + def set_do_not_order_list(self, restricted_list): + """ + Set a restriction on which sids can be ordered. + """ + control = RestrictedListOrder(restricted_list) + self.register_trading_control(control) + @api_method def set_long_only(self): """ diff --git a/zipline/finance/controls.py b/zipline/finance/controls.py index 4e5f6511..d02f4fcf 100644 --- a/zipline/finance/controls.py +++ b/zipline/finance/controls.py @@ -99,6 +99,30 @@ class MaxOrderCount(TradingControl): self.orders_placed += 1 +class RestrictedListOrder(TradingControl): + """ + TradingControl representing a restricted list of securities that + cannot be ordered by the algorithm. + """ + + def __init__(self, restricted_list): + + super(RestrictedListOrder, self).__init__() + self.restricted_list = set(restricted_list) + + def validate(self, + sid, + amount, + _portfolio, + _algo_datetime, + _algo_current_data): + """ + Fail if the sid is in the restricted_list. + """ + if sid in self.restricted_list: + self.fail(sid, amount) + + class MaxOrderSize(TradingControl): """ TradingControl representing a limit on the magnitude of any single order diff --git a/zipline/test_algorithms.py b/zipline/test_algorithms.py index d25adbc3..132b6bb9 100644 --- a/zipline/test_algorithms.py +++ b/zipline/test_algorithms.py @@ -430,6 +430,12 @@ class SetMaxOrderSizeAlgorithm(TradingAlgorithm): max_notional=max_notional) +class SetDoNotOrderListAlgorithm(TradingAlgorithm): + def initialize(self, sid=None, restricted_list=None): + self.order_count = 0 + self.set_do_not_order_list(restricted_list) + + class SetMaxOrderCountAlgorithm(TradingAlgorithm): def initialize(self, count): self.order_count = 0 From 909b412e9bfc04ca69c0dc4e7b8ac20dd12131a5 Mon Sep 17 00:00:00 2001 From: fawce Date: Wed, 28 Jan 2015 22:16:29 -0500 Subject: [PATCH 2/7] modified do not order guard to take an iteratble or a container container allows for dynamic restrictions, necessary for a point in time implementation of the restricted list. --- tests/test_algorithm.py | 8 +++++--- zipline/finance/controls.py | 7 ++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_algorithm.py b/tests/test_algorithm.py index 403144d2..ab275647 100644 --- a/tests/test_algorithm.py +++ b/tests/test_algorithm.py @@ -958,17 +958,19 @@ class TestTradingControls(TestCase): def test_set_do_not_order_list(self): # set the restricted list to be the sid, and fail. - algo = SetDoNotOrderListAlgorithm(sid=self.sid, + algo = SetDoNotOrderListAlgorithm( + sid=self.sid, restricted_list=[self.sid]) def handle_data(algo, data): algo.order(self.sid, 100) algo.order_count += 1 - + self.check_algo_fails(algo, handle_data, 0) # set the restricted list to exclude the sid, and succeed - algo = SetDoNotOrderListAlgorithm(sid=self.sid, + algo = SetDoNotOrderListAlgorithm( + sid=self.sid, restricted_list=[134, 135, 136]) def handle_data(algo, data): diff --git a/zipline/finance/controls.py b/zipline/finance/controls.py index d02f4fcf..d3d77486 100644 --- a/zipline/finance/controls.py +++ b/zipline/finance/controls.py @@ -106,9 +106,14 @@ class RestrictedListOrder(TradingControl): """ def __init__(self, restricted_list): + """ + restricted list can be an iterable, or + an object that implements __contains__ for dynamic + restrictions. + """ super(RestrictedListOrder, self).__init__() - self.restricted_list = set(restricted_list) + self.restricted_list = restricted_list def validate(self, sid, From 536ace94b8b3c8eeaaa2d55361fec8fa67a16ec9 Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 3 Feb 2015 18:41:21 -0500 Subject: [PATCH 3/7] new security list class and tests --- tests/test_security_list.py | 323 +++++++++++++++++++++++++++++++++ zipline/utils/security_list.py | 134 ++++++++++++++ 2 files changed, 457 insertions(+) create mode 100644 tests/test_security_list.py create mode 100644 zipline/utils/security_list.py diff --git a/tests/test_security_list.py b/tests/test_security_list.py new file mode 100644 index 00000000..667b684f --- /dev/null +++ b/tests/test_security_list.py @@ -0,0 +1,323 @@ +import pytz +import os.path +import shutil +from datetime import datetime, timedelta +from unittest import TestCase + + +from zipline.algorithm import TradingAlgorithm +from zipline.errors import TradingControlViolation +from zipline.sources import SpecificEquityTrades +from zipline.utils.test_utils import setup_logger +from zipline.utils import factory +from zipline.utils.security_list import ( + SecurityListSet, load_from_directory, SECURITY_LISTS_DIR) + +LEVERAGED_ETFS = load_from_directory('leveraged_etf_list') + + +class RestrictedAlgoWithCheck(TradingAlgorithm): + def initialize(self, sid): + self.rl = SecurityListSet(self.get_datetime) + self.set_do_not_order_list(self.rl.LEVERAGED_ETF_LIST) + self.order_count = 0 + self.sid = sid + + def handle_data(self, data): + if not self.order_count: + if self.sid not in \ + self.rl.LEVERAGED_ETF_LIST: + self.order(self.sid, 100) + self.order_count += 1 + + +class RestrictedAlgoWithoutCheck(TradingAlgorithm): + def initialize(self, sid): + self.rl = SecurityListSet(self.get_datetime) + self.set_do_not_order_list(self.rl.LEVERAGED_ETF_LIST) + self.order_count = 0 + self.sid = sid + + def handle_data(self, data): + self.order(self.sid, 100) + self.order_count += 1 + + +class IterateRLAlgo(TradingAlgorithm): + def initialize(self, sid): + self.rl = SecurityListSet(self.get_datetime) + self.set_do_not_order_list(self.rl.LEVERAGED_ETF_LIST) + self.order_count = 0 + self.sid = sid + self.found = False + + def handle_data(self, data): + for stock in self.rl.LEVERAGED_ETF_LIST: + if stock == self.sid: + self.found = True + + +class SecurityListTestCase(TestCase): + + def setUp(self): + self.extra_knowledge_date = \ + datetime(2015, 1, 27, 0, 0, tzinfo=pytz.utc) + self.trading_day_before_first_kd = datetime( + 2015, 1, 23, 0, 0, tzinfo=pytz.utc) + + setup_logger(self) + + def test_iterate_over_rl(self): + sim_params = factory.create_simulation_parameters( + start=LEVERAGED_ETFS.keys()[0], num_days=4) + + trade_history = factory.create_trade_history( + 'BZQ', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + algo = IterateRLAlgo(sid='BZQ', sim_params=sim_params) + algo.run(self.source) + self.assertTrue(algo.found) + + def test_security_list(self): + + # set the knowledge date to the first day of the + # leveraged etf knowledge date. + def get_datetime(): + return LEVERAGED_ETFS.keys()[0] + + rl = SecurityListSet(get_datetime) + # assert that a sample from the leveraged list are in restricted + + self.assertIn("BZQ", rl.LEVERAGED_ETF_LIST) + self.assertIn("URTY", rl.LEVERAGED_ETF_LIST) + + # assert that a sample of allowed stocks are not in restricted + # AAPL + self.assertNotIn("AAPL", rl.LEVERAGED_ETF_LIST) + # GOOG + self.assertNotIn("GOOG", rl.LEVERAGED_ETF_LIST) + + def test_security_add(self): + def get_datetime(): + return datetime(2015, 1, 27, tzinfo=pytz.utc) + try: + add_data(['AAPL', 'GOOG'], []) + rl = SecurityListSet(get_datetime) + self.assertIn("AAPL", rl.LEVERAGED_ETF_LIST) + self.assertIn("GOOG", rl.LEVERAGED_ETF_LIST) + self.assertIn("BZQ", rl.LEVERAGED_ETF_LIST) + self.assertIn("URTY", rl.LEVERAGED_ETF_LIST) + finally: + remove_data_directory() + + def test_security_add_delete(self): + try: + def get_datetime(): + return datetime(2015, 1, 27, tzinfo=pytz.utc) + add_data([], ['BZQ', 'URTY']) + rl = SecurityListSet(get_datetime) + self.assertNotIn("BZQ", rl.LEVERAGED_ETF_LIST) + self.assertNotIn("URTY", rl.LEVERAGED_ETF_LIST) + finally: + remove_data_directory() + + def test_algo_without_rl_violation_via_check(self): + sim_params = factory.create_simulation_parameters( + start=LEVERAGED_ETFS.keys()[0], num_days=4) + + trade_history = factory.create_trade_history( + 'BZQ', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + + algo = RestrictedAlgoWithCheck(sid='BZQ', sim_params=sim_params) + algo.run(self.source) + + def test_algo_without_rl_violation(self): + sim_params = factory.create_simulation_parameters( + start=LEVERAGED_ETFS.keys()[0], num_days=4) + + trade_history = factory.create_trade_history( + 'AAPL', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + algo = RestrictedAlgoWithoutCheck(sid='AAPL', sim_params=sim_params) + algo.run(self.source) + + def test_algo_with_rl_violation(self): + sim_params = factory.create_simulation_parameters( + start=LEVERAGED_ETFS.keys()[0], num_days=4) + + trade_history = factory.create_trade_history( + 'BZQ', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + + self.df_source, self.df = \ + factory.create_test_df_source(sim_params) + + algo = RestrictedAlgoWithoutCheck(sid='BZQ', sim_params=sim_params) + with self.assertRaises(TradingControlViolation) as ctx: + algo.run(self.source) + + self.check_algo_exception(algo, ctx, 0) + + def test_algo_with_rl_violation_on_knowledge_date(self): + + sim_params = factory.create_simulation_parameters( + start=self.trading_day_before_first_kd, num_days=4) + trade_history = factory.create_trade_history( + 'BZQ', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + + algo = RestrictedAlgoWithoutCheck(sid='BZQ', sim_params=sim_params) + with self.assertRaises(TradingControlViolation) as ctx: + algo.run(self.source) + + self.check_algo_exception(algo, ctx, 1) + + def test_algo_with_rl_violation_after_knowledge_date(self): + sim_params = factory.create_simulation_parameters( + start=LEVERAGED_ETFS.keys()[0] + timedelta(days=7), num_days=5) + + trade_history = factory.create_trade_history( + 'BZQ', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + algo = RestrictedAlgoWithoutCheck(sid='BZQ', sim_params=sim_params) + with self.assertRaises(TradingControlViolation) as ctx: + algo.run(self.source) + + self.check_algo_exception(algo, ctx, 0) + + def test_algo_with_rl_violation_cumulative(self): + """ + Add a new restriction, run a test long after both + knowledge dates, make sure stock from original restriction + set is still disallowed. + """ + sim_params = factory.create_simulation_parameters( + start=LEVERAGED_ETFS.keys()[0] + timedelta(days=7), num_days=4) + + try: + add_data(['AAPL'], []) + trade_history = factory.create_trade_history( + 'BZQ', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + algo = RestrictedAlgoWithoutCheck( + sid='BZQ', sim_params=sim_params) + with self.assertRaises(TradingControlViolation) as ctx: + algo.run(self.source) + + self.check_algo_exception(algo, ctx, 0) + finally: + remove_data_directory() + + def test_algo_without_rl_violation_after_delete(self): + try: + # add a delete statement removing bzq + # write a new delete statement file to disk + add_data([], ['BZQ']) + + sim_params = factory.create_simulation_parameters( + start=self.extra_knowledge_date, num_days=3) + trade_history = factory.create_trade_history( + 'BZQ', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + algo = RestrictedAlgoWithoutCheck( + sid='BZQ', sim_params=sim_params) + algo.run(self.source) + finally: + remove_data_directory() + + def test_algo_with_rl_violation_after_add(self): + try: + add_data(['AAPL'], []) + sim_params = factory.create_simulation_parameters( + start=self.trading_day_before_first_kd, num_days=4) + trade_history = factory.create_trade_history( + 'AAPL', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + algo = RestrictedAlgoWithoutCheck( + sid='AAPL', sim_params=sim_params) + with self.assertRaises(TradingControlViolation) as ctx: + algo.run(self.source) + + self.check_algo_exception(algo, ctx, 2) + finally: + remove_data_directory() + + def check_algo_exception(self, algo, ctx, expected_order_count): + self.assertEqual(algo.order_count, expected_order_count) + exc = ctx.exception + self.assertEqual(TradingControlViolation, type(exc)) + exc_msg = str(ctx.exception) + self.assertTrue("RestrictedListOrder" in exc_msg) + + +def add_data(adds, deletes): + directory = os.path.join( + SECURITY_LISTS_DIR, + "leveraged_etf_list/20150127/20150125" + ) + if not os.path.exists(directory): + os.makedirs(directory) + del_path = os.path.join(directory, "delete.txt") + with open(del_path, 'w') as f: + for sym in deletes: + f.write(sym) + f.write('\n') + add_path = os.path.join(directory, "add.txt") + with open(add_path, 'w') as f: + for sym in adds: + f.write(sym) + f.write('\n') + + +def remove_data_directory(): + directory = os.path.join( + SECURITY_LISTS_DIR, + "leveraged_etf_list/20150127/" + ) + shutil.rmtree(directory) diff --git a/zipline/utils/security_list.py b/zipline/utils/security_list.py new file mode 100644 index 00000000..9d242008 --- /dev/null +++ b/zipline/utils/security_list.py @@ -0,0 +1,134 @@ +import os.path +import pytz +import pandas as pd +from datetime import datetime +from os import listdir + +DATE_FORMAT = "%Y%m%d" +import zipline +zipline_dir = os.path.join(*zipline.__path__) +SECURITY_LISTS_DIR = os.path.join(zipline_dir, '..', 'security_lists') + + +def loopback(symbol, *args, **kwargs): + return symbol + + +class SecurityListSet(object): + + def __init__(self, current_date_func, lookup_func=None): + if lookup_func is None: + self.lookup_func = loopback + else: + self.lookup_func = lookup_func + self.current_date_func = current_date_func + self._leveraged_etf = None + + @property + def LEVERAGED_ETF_LIST(self): + if self._leveraged_etf is None: + self._leveraged_etf = SecurityList( + self.lookup_func, + load_from_directory('leveraged_etf_list'), + self.current_date_func + ) + return self._leveraged_etf + + +class SecurityList(object): + + def __init__(self, lookup_func, data, current_date_func): + """ + lookup_func: function that takes a string symbol and a date and + returns a Security object. + data: a nested dictionary: + knowledge_date -> lookup_date -> + {add: [symbol list], 'delete': []}, delete: [symbol list]} + current_date_func: function taking no parameters, returning + current datetime + """ + self.lookup_func = lookup_func + self.data = data + self._cache = {} + self._knowledge_dates = self.make_knowledge_dates(self.data) + self.current_date = current_date_func + self.count = 0 + self._list = set() + + def make_knowledge_dates(self, data): + knowledge_dates = sorted( + [pd.Timestamp(k) for k in data.keys()]) + return knowledge_dates + + def __iter__(self): + return iter(self.get_restricted_list()) + + def __contains__(self, item): + rl = self.get_restricted_list() + return rl.__contains__(item) + + def get_restricted_list(self): + + cd = self.current_date() + for kd in self._knowledge_dates: + if cd < kd: + break + if kd in self._cache: + self._list = self._cache[kd] + continue + + for effective_date, changes in self.data[kd].iteritems(): + for symbol in changes['add']: + sid = self.lookup_func( + symbol, + as_of_date=effective_date + ) + self._list.add(sid) + for symbol in changes['delete']: + sid = self.lookup_func( + symbol, + as_of_date=effective_date + ) + if sid in self._list: + self._list.remove(sid) + + self._cache[kd] = self._list + return self._list + + +def load_from_directory(list_name): + """ + To resolve the symbol in the LEVERAGED_ETF list, + the date on which the symbol was in effect is needed. + + Furthermore, to maintain a point in time record of our own maintenance + of the restricted list, we need a knowledge date. Thus, restricted lists + are dictionaries of datetime->symbol lists. + new symbols should be entered as a new knowledge date entry. + + This method assumes a directory structure of: + SECURITY_LISTS_DIR/listname/knowledge_date/lookup_date/add.txt + SECURITY_LISTS_DIR/listname/knowledge_date/lookup_date/delete.txt + + The return value is a dictionary with: + knowledge_date -> lookup_date -> + {add: [symbol list], 'delete': [symbol list]} + """ + data = {} + dir_path = SECURITY_LISTS_DIR + "/" + list_name + for kd_name in listdir(dir_path): + kd = datetime.strptime(kd_name, DATE_FORMAT).replace( + tzinfo=pytz.utc) + data[kd] = {} + kd_path = os.path.join(dir_path, kd_name) + for ld_name in listdir(dir_path + '/' + kd_name): + ld = datetime.strptime(kd_name, DATE_FORMAT).replace( + tzinfo=pytz.utc) + data[kd][ld] = {} + ld_path = os.path.join(kd_path, ld_name) + for fname in listdir(ld_path): + fpath = os.path.join(ld_path, fname) + with open(fpath) as f: + symbols = f.read().splitlines() + data[kd][ld][fname.split('.')[0]] = symbols + return data From 1ab082102a659be08f750b9e93fd100bbf5a9740 Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 3 Feb 2015 18:41:41 -0500 Subject: [PATCH 4/7] first edition of leveraged etf list --- tests/test_security_list.py | 84 +++---- zipline/finance/controls.py | 4 +- .../leveraged_etf_list/20150126/20120913/add | 2 + .../20150126/20120913/delete | 0 .../leveraged_etf_list/20150126/20120919/add | 1 + .../20150126/20120919/delete | 0 .../leveraged_etf_list/20150126/20121012/add | 1 + .../20150126/20121012/delete | 0 .../leveraged_etf_list/20150126/20130605/add | 2 + .../20150126/20130605/delete | 0 .../leveraged_etf_list/20150126/20130916/add | 1 + .../20150126/20130916/delete | 0 .../leveraged_etf_list/20150126/20131002/add | 2 + .../20150126/20131002/delete | 0 .../leveraged_etf_list/20150126/20131009/add | 1 + .../20150126/20131009/delete | 0 .../leveraged_etf_list/20150126/20131121/add | 4 + .../20150126/20131121/delete | 0 .../leveraged_etf_list/20150126/20131227/add | 2 + .../20150126/20131227/delete | 0 .../leveraged_etf_list/20150126/20140410/add | 1 + .../20150126/20140410/delete | 0 .../leveraged_etf_list/20150126/20140923/add | 1 + .../20150126/20140923/delete | 0 .../leveraged_etf_list/20150126/20141119/add | 2 + .../20150126/20141119/delete | 0 .../leveraged_etf_list/20150126/20141226/add | 1 + .../20150126/20141226/delete | 0 .../leveraged_etf_list/20150126/20150123/add | 207 ++++++++++++++++++ .../20150126/20150123/delete | 0 zipline/utils/security_list.py | 59 ++--- zipline/utils/test_utils.py | 30 +++ 32 files changed, 321 insertions(+), 84 deletions(-) create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20120913/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20120913/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20120919/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20120919/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20121012/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20121012/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20130605/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20130605/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20130916/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20130916/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131002/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131002/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131009/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131009/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131121/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131121/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131227/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20131227/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20140410/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20140410/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20140923/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20140923/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20141119/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20141119/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20141226/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20141226/delete create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20150123/add create mode 100644 zipline/resources/security_lists/leveraged_etf_list/20150126/20150123/delete diff --git a/tests/test_security_list.py b/tests/test_security_list.py index 667b684f..2a18216f 100644 --- a/tests/test_security_list.py +++ b/tests/test_security_list.py @@ -1,6 +1,4 @@ import pytz -import os.path -import shutil from datetime import datetime, timedelta from unittest import TestCase @@ -8,10 +6,11 @@ from unittest import TestCase from zipline.algorithm import TradingAlgorithm from zipline.errors import TradingControlViolation from zipline.sources import SpecificEquityTrades -from zipline.utils.test_utils import setup_logger +from zipline.utils.test_utils import ( + setup_logger, add_security_data, remove_security_data_directory) from zipline.utils import factory from zipline.utils.security_list import ( - SecurityListSet, load_from_directory, SECURITY_LISTS_DIR) + SecurityListSet, load_from_directory) LEVERAGED_ETFS = load_from_directory('leveraged_etf_list') @@ -19,14 +18,14 @@ LEVERAGED_ETFS = load_from_directory('leveraged_etf_list') class RestrictedAlgoWithCheck(TradingAlgorithm): def initialize(self, sid): self.rl = SecurityListSet(self.get_datetime) - self.set_do_not_order_list(self.rl.LEVERAGED_ETF_LIST) + self.set_do_not_order_list(self.rl.leveraged_etf_list) self.order_count = 0 self.sid = sid def handle_data(self, data): if not self.order_count: if self.sid not in \ - self.rl.LEVERAGED_ETF_LIST: + self.rl.leveraged_etf_list: self.order(self.sid, 100) self.order_count += 1 @@ -34,7 +33,7 @@ class RestrictedAlgoWithCheck(TradingAlgorithm): class RestrictedAlgoWithoutCheck(TradingAlgorithm): def initialize(self, sid): self.rl = SecurityListSet(self.get_datetime) - self.set_do_not_order_list(self.rl.LEVERAGED_ETF_LIST) + self.set_do_not_order_list(self.rl.leveraged_etf_list) self.order_count = 0 self.sid = sid @@ -46,13 +45,13 @@ class RestrictedAlgoWithoutCheck(TradingAlgorithm): class IterateRLAlgo(TradingAlgorithm): def initialize(self, sid): self.rl = SecurityListSet(self.get_datetime) - self.set_do_not_order_list(self.rl.LEVERAGED_ETF_LIST) + self.set_do_not_order_list(self.rl.leveraged_etf_list) self.order_count = 0 self.sid = sid self.found = False def handle_data(self, data): - for stock in self.rl.LEVERAGED_ETF_LIST: + for stock in self.rl.leveraged_etf_list: if stock == self.sid: self.found = True @@ -93,38 +92,38 @@ class SecurityListTestCase(TestCase): rl = SecurityListSet(get_datetime) # assert that a sample from the leveraged list are in restricted - self.assertIn("BZQ", rl.LEVERAGED_ETF_LIST) - self.assertIn("URTY", rl.LEVERAGED_ETF_LIST) + self.assertIn("BZQ", rl.leveraged_etf_list) + self.assertIn("URTY", rl.leveraged_etf_list) # assert that a sample of allowed stocks are not in restricted # AAPL - self.assertNotIn("AAPL", rl.LEVERAGED_ETF_LIST) + self.assertNotIn("AAPL", rl.leveraged_etf_list) # GOOG - self.assertNotIn("GOOG", rl.LEVERAGED_ETF_LIST) + self.assertNotIn("GOOG", rl.leveraged_etf_list) def test_security_add(self): def get_datetime(): return datetime(2015, 1, 27, tzinfo=pytz.utc) try: - add_data(['AAPL', 'GOOG'], []) + add_security_data(['AAPL', 'GOOG'], []) rl = SecurityListSet(get_datetime) - self.assertIn("AAPL", rl.LEVERAGED_ETF_LIST) - self.assertIn("GOOG", rl.LEVERAGED_ETF_LIST) - self.assertIn("BZQ", rl.LEVERAGED_ETF_LIST) - self.assertIn("URTY", rl.LEVERAGED_ETF_LIST) + self.assertIn("AAPL", rl.leveraged_etf_list) + self.assertIn("GOOG", rl.leveraged_etf_list) + self.assertIn("BZQ", rl.leveraged_etf_list) + self.assertIn("URTY", rl.leveraged_etf_list) finally: - remove_data_directory() + remove_security_data_directory() def test_security_add_delete(self): try: def get_datetime(): return datetime(2015, 1, 27, tzinfo=pytz.utc) - add_data([], ['BZQ', 'URTY']) + add_security_data([], ['BZQ', 'URTY']) rl = SecurityListSet(get_datetime) - self.assertNotIn("BZQ", rl.LEVERAGED_ETF_LIST) - self.assertNotIn("URTY", rl.LEVERAGED_ETF_LIST) + self.assertNotIn("BZQ", rl.leveraged_etf_list) + self.assertNotIn("URTY", rl.leveraged_etf_list) finally: - remove_data_directory() + remove_security_data_directory() def test_algo_without_rl_violation_via_check(self): sim_params = factory.create_simulation_parameters( @@ -226,7 +225,7 @@ class SecurityListTestCase(TestCase): start=LEVERAGED_ETFS.keys()[0] + timedelta(days=7), num_days=4) try: - add_data(['AAPL'], []) + add_security_data(['AAPL'], []) trade_history = factory.create_trade_history( 'BZQ', [10.0, 10.0, 11.0, 11.0], @@ -242,13 +241,13 @@ class SecurityListTestCase(TestCase): self.check_algo_exception(algo, ctx, 0) finally: - remove_data_directory() + remove_security_data_directory() def test_algo_without_rl_violation_after_delete(self): try: # add a delete statement removing bzq # write a new delete statement file to disk - add_data([], ['BZQ']) + add_security_data([], ['BZQ']) sim_params = factory.create_simulation_parameters( start=self.extra_knowledge_date, num_days=3) @@ -264,11 +263,11 @@ class SecurityListTestCase(TestCase): sid='BZQ', sim_params=sim_params) algo.run(self.source) finally: - remove_data_directory() + remove_security_data_directory() def test_algo_with_rl_violation_after_add(self): try: - add_data(['AAPL'], []) + add_security_data(['AAPL'], []) sim_params = factory.create_simulation_parameters( start=self.trading_day_before_first_kd, num_days=4) trade_history = factory.create_trade_history( @@ -286,7 +285,7 @@ class SecurityListTestCase(TestCase): self.check_algo_exception(algo, ctx, 2) finally: - remove_data_directory() + remove_security_data_directory() def check_algo_exception(self, algo, ctx, expected_order_count): self.assertEqual(algo.order_count, expected_order_count) @@ -294,30 +293,3 @@ class SecurityListTestCase(TestCase): self.assertEqual(TradingControlViolation, type(exc)) exc_msg = str(ctx.exception) self.assertTrue("RestrictedListOrder" in exc_msg) - - -def add_data(adds, deletes): - directory = os.path.join( - SECURITY_LISTS_DIR, - "leveraged_etf_list/20150127/20150125" - ) - if not os.path.exists(directory): - os.makedirs(directory) - del_path = os.path.join(directory, "delete.txt") - with open(del_path, 'w') as f: - for sym in deletes: - f.write(sym) - f.write('\n') - add_path = os.path.join(directory, "add.txt") - with open(add_path, 'w') as f: - for sym in adds: - f.write(sym) - f.write('\n') - - -def remove_data_directory(): - directory = os.path.join( - SECURITY_LISTS_DIR, - "leveraged_etf_list/20150127/" - ) - shutil.rmtree(directory) diff --git a/zipline/finance/controls.py b/zipline/finance/controls.py index d3d77486..d38953e8 100644 --- a/zipline/finance/controls.py +++ b/zipline/finance/controls.py @@ -107,8 +107,8 @@ class RestrictedListOrder(TradingControl): def __init__(self, restricted_list): """ - restricted list can be an iterable, or - an object that implements __contains__ for dynamic + restricted list can be an iterable or a + container (implements __contains__) for dynamic restrictions. """ diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20120913/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20120913/add new file mode 100644 index 00000000..2680b739 --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20120913/add @@ -0,0 +1,2 @@ +ROSA +SFSA diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20120913/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20120913/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20120919/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20120919/add new file mode 100644 index 00000000..87935918 --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20120919/add @@ -0,0 +1 @@ +RTSA diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20120919/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20120919/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20121012/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20121012/add new file mode 100644 index 00000000..e2f8ae1a --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20121012/add @@ -0,0 +1 @@ +VZZB diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20121012/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20121012/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20130605/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20130605/add new file mode 100644 index 00000000..aefbe5eb --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20130605/add @@ -0,0 +1,2 @@ +LSKY +SSDL diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20130605/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20130605/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20130916/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20130916/add new file mode 100644 index 00000000..e446ea27 --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20130916/add @@ -0,0 +1 @@ +MFSA diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20130916/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20130916/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131002/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131002/add new file mode 100644 index 00000000..7a55ca5a --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131002/add @@ -0,0 +1,2 @@ +DSTJ +DSXJ diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131002/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131002/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131009/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131009/add new file mode 100644 index 00000000..7c5e71b0 --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131009/add @@ -0,0 +1 @@ +JFT diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131009/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131009/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131121/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131121/add new file mode 100644 index 00000000..0d50b50c --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131121/add @@ -0,0 +1,4 @@ +FSA +FSE +FSG +FSU diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131121/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131121/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131227/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131227/add new file mode 100644 index 00000000..e14acb0c --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131227/add @@ -0,0 +1,2 @@ +IPLT +LPLT diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20131227/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20131227/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20140410/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20140410/add new file mode 100644 index 00000000..b403214d --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20140410/add @@ -0,0 +1 @@ +BXDB diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20140410/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20140410/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20140923/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20140923/add new file mode 100644 index 00000000..ba1b836d --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20140923/add @@ -0,0 +1 @@ +BRZS diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20140923/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20140923/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20141119/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20141119/add new file mode 100644 index 00000000..eff691f2 --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20141119/add @@ -0,0 +1,2 @@ +BXUB +BXUC diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20141119/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20141119/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20141226/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20141226/add new file mode 100644 index 00000000..e397a94f --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20141226/add @@ -0,0 +1 @@ +BARS diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20141226/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20141226/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20150123/add b/zipline/resources/security_lists/leveraged_etf_list/20150126/20150123/add new file mode 100644 index 00000000..35f60e65 --- /dev/null +++ b/zipline/resources/security_lists/leveraged_etf_list/20150126/20150123/add @@ -0,0 +1,207 @@ +AGA +AGQ +BAR +BDCL +BDD +BIB +BIS +BOIL +BOM +BRZU +BUNT +BZQ +CEFL +CMD +CROC +CSMB +CURE +DAG +DDM +DEE +DGAZ +DGLD +DGP +DGZ +DIG +DPK +DRN +DRR +DRV +DSLV +DTO +DUG +DUST +DVHL +DVYL +DWTI +DXD +DYY +DZK +DZZ +EDC +EDZ +EET +EEV +EFO +EFU +EMLB +EMSA +EPV +ERX +ERY +EUO +EURL +EWV +EZJ +FAS +FAZ +FBG +FBGX +FEEU +FIBG +FIEG +FIEU +FIGY +FINU +FINZ +FLGE +FXP +GASL +GDAY +GLL +HDLV +IGU +INDL +ITLT +JDST +JGBD +JGBT +JNUG +JPNL +JPX +KOLD +KORU +KRU +LBJ +LBND +LLDM +LLSC +LLSP +LMLP +LTL +MATL +MDLL +MFLA +MIDU +MIDZ +MLPL +MORL +MVV +MZZ +NUGT +PST +QID +QLD +RETL +REW +RGRA +RGRC +RGRE +RGRI +RGRP +ROLA +ROM +RTLA +RUSL +RUSS +RWXL +RXD +RXL +SAA +SBND +SCC +SCO +SDD +SDOW +SDP +SDS +SDYL +SFLA +SIJ +SKF +SMDD +SMK +SMLL +SMN +SOXL +SOXS +SPLX +SPUU +SPXL +SPXS +SPXU +SQQQ +SRS +SRTY +SSG +SSO +SYTL +SZK +TBAR +TBT +TBZ +TECL +TECS +TLL +TMF +TMV +TNA +TPS +TQQQ +TTT +TVIX +TVIZ +TWM +TYD +TYO +TZA +UBR +UBT +UCC +UCD +UCI +UCO +UDNT +UDOW +UGAZ +UGE +UGL +UGLD +UJB +ULE +UMDD +UMX +UPRO +UPV +UPW +URE +URR +URTY +USD +USLV +UST +USV +UUPT +UVXY +UWM +UWTI +UXI +UXJ +UYG +UYM +XPP +YANG +YCL +YCS +YINN +ZSL diff --git a/zipline/resources/security_lists/leveraged_etf_list/20150126/20150123/delete b/zipline/resources/security_lists/leveraged_etf_list/20150126/20150123/delete new file mode 100644 index 00000000..e69de29b diff --git a/zipline/utils/security_list.py b/zipline/utils/security_list.py index 9d242008..725d8321 100644 --- a/zipline/utils/security_list.py +++ b/zipline/utils/security_list.py @@ -7,7 +7,7 @@ from os import listdir DATE_FORMAT = "%Y%m%d" import zipline zipline_dir = os.path.join(*zipline.__path__) -SECURITY_LISTS_DIR = os.path.join(zipline_dir, '..', 'security_lists') +SECURITY_LISTS_DIR = os.path.join(zipline_dir, 'resources', 'security_lists') def loopback(symbol, *args, **kwargs): @@ -25,7 +25,7 @@ class SecurityListSet(object): self._leveraged_etf = None @property - def LEVERAGED_ETF_LIST(self): + def leveraged_etf_list(self): if self._leveraged_etf is None: self._leveraged_etf = SecurityList( self.lookup_func, @@ -53,7 +53,7 @@ class SecurityList(object): self._knowledge_dates = self.make_knowledge_dates(self.data) self.current_date = current_date_func self.count = 0 - self._list = set() + self._current_set = set() def make_knowledge_dates(self, data): knowledge_dates = sorted( @@ -61,39 +61,45 @@ class SecurityList(object): return knowledge_dates def __iter__(self): - return iter(self.get_restricted_list()) + return iter(self.restricted_list) def __contains__(self, item): - rl = self.get_restricted_list() - return rl.__contains__(item) + return item in self.restricted_list - def get_restricted_list(self): + @property + def restricted_list(self): cd = self.current_date() for kd in self._knowledge_dates: if cd < kd: break if kd in self._cache: - self._list = self._cache[kd] + self._current_set = self._cache[kd] continue for effective_date, changes in self.data[kd].iteritems(): - for symbol in changes['add']: - sid = self.lookup_func( - symbol, - as_of_date=effective_date - ) - self._list.add(sid) - for symbol in changes['delete']: - sid = self.lookup_func( - symbol, - as_of_date=effective_date - ) - if sid in self._list: - self._list.remove(sid) + self.update_current( + effective_date, + changes['add'], + self._current_set.add + ) - self._cache[kd] = self._list - return self._list + self.update_current( + effective_date, + changes['delete'], + self._current_set.remove + ) + + self._cache[kd] = self._current_set + return self._current_set + + def update_current(self, effective_date, symbols, change_func): + for symbol in symbols: + sid = self.lookup_func( + symbol, + as_of_date=effective_date + ) + change_func(sid) def load_from_directory(list_name): @@ -115,13 +121,13 @@ def load_from_directory(list_name): {add: [symbol list], 'delete': [symbol list]} """ data = {} - dir_path = SECURITY_LISTS_DIR + "/" + list_name + dir_path = os.path.join(SECURITY_LISTS_DIR, list_name) for kd_name in listdir(dir_path): kd = datetime.strptime(kd_name, DATE_FORMAT).replace( tzinfo=pytz.utc) data[kd] = {} kd_path = os.path.join(dir_path, kd_name) - for ld_name in listdir(dir_path + '/' + kd_name): + for ld_name in listdir(os.path.join(dir_path, kd_name)): ld = datetime.strptime(kd_name, DATE_FORMAT).replace( tzinfo=pytz.utc) data[kd][ld] = {} @@ -130,5 +136,6 @@ def load_from_directory(list_name): fpath = os.path.join(ld_path, fname) with open(fpath) as f: symbols = f.read().splitlines() - data[kd][ld][fname.split('.')[0]] = symbols + data[kd][ld][fname] = symbols + return data diff --git a/zipline/utils/test_utils.py b/zipline/utils/test_utils.py index 451590f7..e29f5f7c 100644 --- a/zipline/utils/test_utils.py +++ b/zipline/utils/test_utils.py @@ -1,10 +1,13 @@ from contextlib import contextmanager from logbook import FileHandler from zipline.finance.blotter import ORDER_STATUS +from zipline.utils.security_list import SECURITY_LISTS_DIR from six import itervalues +import os import pandas as pd +import shutil def to_utc(time_str): @@ -129,3 +132,30 @@ def nullctx(): do_stuff() """ yield + + +def add_security_data(adds, deletes): + directory = os.path.join( + SECURITY_LISTS_DIR, + "leveraged_etf_list/20150127/20150125" + ) + if not os.path.exists(directory): + os.makedirs(directory) + del_path = os.path.join(directory, "delete") + with open(del_path, 'w') as f: + for sym in deletes: + f.write(sym) + f.write('\n') + add_path = os.path.join(directory, "add") + with open(add_path, 'w') as f: + for sym in adds: + f.write(sym) + f.write('\n') + + +def remove_security_data_directory(): + directory = os.path.join( + SECURITY_LISTS_DIR, + "leveraged_etf_list/20150127/" + ) + shutil.rmtree(directory) From 2f7983fa35dd8acac65b223b2d21132a307ff59e Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 5 Feb 2015 12:13:39 -0500 Subject: [PATCH 5/7] adding data files to the egg for pip distribution. --- MANIFEST.in | 2 ++ setup.py | 1 + 2 files changed, 3 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index 04f196ac..f93ec43c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,4 @@ include README.md include LICENSE + +recursive-include zipline/resources *.* \ No newline at end of file diff --git a/setup.py b/setup.py index 1ca18e20..464a2e7a 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( author_email='opensource@quantopian.com', packages=find_packages(), scripts=['scripts/run_algo.py'], + include_package_data=True, long_description=LONG_DESCRIPTION, license='Apache 2.0', classifiers=[ From 08a0d1b604c427b6d219e3a7bcf9efb619b0958c Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 5 Feb 2015 13:04:05 -0500 Subject: [PATCH 6/7] python 3.3 compatible iterators --- tests/test_security_list.py | 16 +++++++++------- zipline/utils/security_list.py | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_security_list.py b/tests/test_security_list.py index 2a18216f..2b65fc9e 100644 --- a/tests/test_security_list.py +++ b/tests/test_security_list.py @@ -68,7 +68,7 @@ class SecurityListTestCase(TestCase): def test_iterate_over_rl(self): sim_params = factory.create_simulation_parameters( - start=LEVERAGED_ETFS.keys()[0], num_days=4) + start=list(LEVERAGED_ETFS.keys())[0], num_days=4) trade_history = factory.create_trade_history( 'BZQ', @@ -87,7 +87,7 @@ class SecurityListTestCase(TestCase): # set the knowledge date to the first day of the # leveraged etf knowledge date. def get_datetime(): - return LEVERAGED_ETFS.keys()[0] + return list(LEVERAGED_ETFS.keys())[0] rl = SecurityListSet(get_datetime) # assert that a sample from the leveraged list are in restricted @@ -127,7 +127,7 @@ class SecurityListTestCase(TestCase): def test_algo_without_rl_violation_via_check(self): sim_params = factory.create_simulation_parameters( - start=LEVERAGED_ETFS.keys()[0], num_days=4) + start=list(LEVERAGED_ETFS.keys())[0], num_days=4) trade_history = factory.create_trade_history( 'BZQ', @@ -143,7 +143,7 @@ class SecurityListTestCase(TestCase): def test_algo_without_rl_violation(self): sim_params = factory.create_simulation_parameters( - start=LEVERAGED_ETFS.keys()[0], num_days=4) + start=list(LEVERAGED_ETFS.keys())[0], num_days=4) trade_history = factory.create_trade_history( 'AAPL', @@ -158,7 +158,7 @@ class SecurityListTestCase(TestCase): def test_algo_with_rl_violation(self): sim_params = factory.create_simulation_parameters( - start=LEVERAGED_ETFS.keys()[0], num_days=4) + start=list(LEVERAGED_ETFS.keys())[0], num_days=4) trade_history = factory.create_trade_history( 'BZQ', @@ -199,7 +199,8 @@ class SecurityListTestCase(TestCase): def test_algo_with_rl_violation_after_knowledge_date(self): sim_params = factory.create_simulation_parameters( - start=LEVERAGED_ETFS.keys()[0] + timedelta(days=7), num_days=5) + start=list( + LEVERAGED_ETFS.keys())[0] + timedelta(days=7), num_days=5) trade_history = factory.create_trade_history( 'BZQ', @@ -222,7 +223,8 @@ class SecurityListTestCase(TestCase): set is still disallowed. """ sim_params = factory.create_simulation_parameters( - start=LEVERAGED_ETFS.keys()[0] + timedelta(days=7), num_days=4) + start=list( + LEVERAGED_ETFS.keys())[0] + timedelta(days=7), num_days=4) try: add_security_data(['AAPL'], []) diff --git a/zipline/utils/security_list.py b/zipline/utils/security_list.py index 725d8321..6fb046e7 100644 --- a/zipline/utils/security_list.py +++ b/zipline/utils/security_list.py @@ -77,7 +77,7 @@ class SecurityList(object): self._current_set = self._cache[kd] continue - for effective_date, changes in self.data[kd].iteritems(): + for effective_date, changes in iter(self.data[kd].items()): self.update_current( effective_date, changes['add'], From 412baa3c3f756a385c993afdcf1ba4a7525bf5ce Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 5 Feb 2015 13:47:39 -0500 Subject: [PATCH 7/7] fixed catastrophic bug in load_from_directory and added a new test case was not iterating over lookup date directory names, and therefore mising all by one list of stocks. discovered because of differing sort orders between my local machine, other devs, and travis ci. --- tests/test_security_list.py | 21 +++++++++++++++++++++ zipline/utils/security_list.py | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/test_security_list.py b/tests/test_security_list.py index 2b65fc9e..bb54868f 100644 --- a/tests/test_security_list.py +++ b/tests/test_security_list.py @@ -94,6 +94,7 @@ class SecurityListTestCase(TestCase): self.assertIn("BZQ", rl.leveraged_etf_list) self.assertIn("URTY", rl.leveraged_etf_list) + self.assertIn("JFT", rl.leveraged_etf_list) # assert that a sample of allowed stocks are not in restricted # AAPL @@ -178,6 +179,26 @@ class SecurityListTestCase(TestCase): self.check_algo_exception(algo, ctx, 0) + # repeat with a symbol from a different lookup date + + trade_history = factory.create_trade_history( + 'JFT', + [10.0, 10.0, 11.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + sim_params + ) + self.source = SpecificEquityTrades(event_list=trade_history) + + self.df_source, self.df = \ + factory.create_test_df_source(sim_params) + + algo = RestrictedAlgoWithoutCheck(sid='JFT', sim_params=sim_params) + with self.assertRaises(TradingControlViolation) as ctx: + algo.run(self.source) + + self.check_algo_exception(algo, ctx, 0) + def test_algo_with_rl_violation_on_knowledge_date(self): sim_params = factory.create_simulation_parameters( diff --git a/zipline/utils/security_list.py b/zipline/utils/security_list.py index 6fb046e7..4de9dbaf 100644 --- a/zipline/utils/security_list.py +++ b/zipline/utils/security_list.py @@ -127,8 +127,8 @@ def load_from_directory(list_name): tzinfo=pytz.utc) data[kd] = {} kd_path = os.path.join(dir_path, kd_name) - for ld_name in listdir(os.path.join(dir_path, kd_name)): - ld = datetime.strptime(kd_name, DATE_FORMAT).replace( + for ld_name in listdir(kd_path): + ld = datetime.strptime(ld_name, DATE_FORMAT).replace( tzinfo=pytz.utc) data[kd][ld] = {} ld_path = os.path.join(kd_path, ld_name)