wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-27 11:20:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

MAINT: One way to set sim_params and data_frequency.

Browse Source 
There were sevaral places you could supply sim_params
in TradingAlgorithm (__init__, run). This got confusing
as its not clear who updated what and which one was the
correct one to use at each time.

Then there were to ways to define data_frequency, one in
__init__() and one in the sim_params which also added code
complexity.

This refactor makes it explicit that sim_params are to be
passed to __init__() only. Moreover, data_frequency is
only stored in sim_params. For backwards compatibility,
it can still be supplied separately but will link to
the one in sim_params.

For example, you could create new sim params via:

sim_params = create_simulation_parameters(data_frequency='minute')
algo = MyAlgo(sim_params)
algo.run(data)

In addition, perf_tracker only gets initialized in one place:
_create_generator() which should also make the various ways
of running an algorithm more deterministic.

This also fixes a bug with SimulationParameters where
you could not change the period_start. Unfortunately, the
current implementation still requieres an implicit call to
update the internal variables.
This commit is contained in:
Thomas Wiecki
2014-06-30 17:28:02 +02:00
parent 4c9cf1321d
commit 10885e1b77
9 changed files with 95 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_algorithm.py
+17 -27
View File
@@ -93,8 +93,7 @@ class TestRecordAlgorithm(TestCase):
def test_record_incr(self):
algo = RecordAlgorithm(
sim_params=self.sim_params,
data_frequency='daily')
sim_params=self.sim_params)
output = algo.run(self.source)
np.testing.assert_array_equal(output['incr'].values,
@@ -161,8 +160,9 @@ class TestMiscellaneousAPI(TestCase):
algo.minute += 1
algo = TradingAlgorithm(initialize=initialize,
handle_data=handle_data)
algo.run(self.source, sim_params=self.sim_params)
handle_data=handle_data,
sim_params=self.sim_params)
algo.run(self.source)
class TestTransformAlgorithm(TestCase):
@@ -194,14 +194,6 @@ class TestTransformAlgorithm(TestCase):
self.assertEqual(len(algo.sources), 1)
assert isinstance(algo.sources[0], SpecificEquityTrades)
def test_multi_source_as_input_no_start_end(self):
algo = TestRegisterTransformAlgorithm(
sids=[133]
)
with self.assertRaises(AssertionError):
algo.run([self.source, self.df_source])
def test_invalid_order_parameters(self):
algo = InvalidOrderAlgorithm(
sids=[133],
@@ -261,26 +253,26 @@ class TestTransformAlgorithm(TestCase):
assert algo.registered_transforms['mavg']['class'] is MovingAverage
def test_data_frequency_setting(self):
self.sim_params.data_frequency = 'daily'
algo = TestRegisterTransformAlgorithm(
sim_params=self.sim_params,
data_frequency='daily'
)
self.assertEqual(algo.data_frequency, 'daily')
self.assertEqual(algo.sim_params.data_frequency, 'daily')
self.assertEqual(algo.annualizer, 250)
self.sim_params.data_frequency = 'minute'
algo = TestRegisterTransformAlgorithm(
sim_params=self.sim_params,
data_frequency='minute'
)
self.assertEqual(algo.data_frequency, 'minute')
self.assertEqual(algo.sim_params.data_frequency, 'minute')
self.assertEqual(algo.annualizer, 250 * 6 * 60)
self.sim_params.data_frequency = 'minute'
algo = TestRegisterTransformAlgorithm(
sim_params=self.sim_params,
data_frequency='minute',
annualizer=10
)
self.assertEqual(algo.data_frequency, 'minute')
self.assertEqual(algo.sim_params.data_frequency, 'minute')
self.assertEqual(algo.annualizer, 10)
def test_order_methods(self):
@@ -294,7 +286,6 @@ class TestTransformAlgorithm(TestCase):
for AlgoClass in AlgoClasses:
algo = AlgoClass(
sim_params=self.sim_params,
data_frequency='daily'
)
algo.run(self.df)
@@ -310,7 +301,6 @@ class TestTransformAlgorithm(TestCase):
for name in method_names_to_test:
algo = TestOrderStyleForwardingAlgorithm(
sim_params=self.sim_params,
data_frequency='daily',
instant_fill=False,
method_name=name
)
@@ -318,19 +308,18 @@ class TestTransformAlgorithm(TestCase):
def test_order_instant(self):
algo = TestOrderInstantAlgorithm(sim_params=self.sim_params,
data_frequency='daily',
instant_fill=True)
algo.run(self.df)
def test_minute_data(self):
source = RandomWalkSource(freq='minute',
start=pd.Timestamp('2000-1-1',
start=pd.Timestamp('2000-1-3',
tz='UTC'),
end=pd.Timestamp('2000-1-1',
end=pd.Timestamp('2000-1-4',
tz='UTC'))
self.sim_params.data_frequency = 'minute'
algo = TestOrderInstantAlgorithm(sim_params=self.sim_params,
data_frequency='minute',
instant_fill=True)
algo.run(source)
@@ -355,8 +344,7 @@ class TestPositions(TestCase):
factory.create_test_df_source(self.sim_params)
def test_empty_portfolio(self):
algo = EmptyPositionsAlgorithm(sim_params=self.sim_params,
data_frequency='daily')
algo = EmptyPositionsAlgorithm(sim_params=self.sim_params)
daily_stats = algo.run(self.df)
expected_position_count = [
@@ -634,7 +622,9 @@ def handle_data(context, data):
end = pd.Timestamp('1991-01-15', tz='UTC')
source = RandomWalkSource(start=start,
end=end)
algo = TradingAlgorithm(script=history_algo, data_frequency='minute')
sim_params = factory.create_simulation_parameters(
data_frequency='minute')
algo = TradingAlgorithm(script=history_algo, sim_params=sim_params)
output = algo.run(source)
self.assertIsNot(output, None)
tests/test_algorithm_gen.py
+4 -2
View File
@@ -27,6 +27,7 @@ from zipline.finance import trading
from zipline.algorithm import TradingAlgorithm
from zipline.finance import slippage
from zipline.utils import factory
from zipline.utils.factory import create_simulation_parameters
from zipline.utils.test_utils import (
setup_logger,
teardown_logger
@@ -222,7 +223,8 @@ class AlgorithmGeneratorTestCase(TestCase):
algo.datetime should be equal to the last benchmark time.
See https://github.com/quantopian/zipline/issues/241
"""
sim_params = factory.create_simulation_parameters(num_days=1)
algo = TestAlgo(self, sim_params=sim_params, data_frequency='minute')
sim_params = create_simulation_parameters(num_days=1,
data_frequency='minute')
algo = TestAlgo(self, sim_params=sim_params)
algo.run(source=[])
self.assertEqual(algo.datetime, sim_params.last_close)
tests/test_events_through_risk.py
+5 -6
View File
@@ -63,12 +63,12 @@ class TestEventsThroughRisk(unittest.TestCase):
sim_params = SimulationParameters(
period_start=start_date,
period_end=end_date,
data_frequency='daily',
emission_rate='daily'
)
algo = BuyAndHoldAlgorithm(
sim_params=sim_params,
data_frequency='daily')
sim_params=sim_params)
first_date = datetime.datetime(2006, 1, 3, tzinfo=pytz.utc)
second_date = datetime.datetime(2006, 1, 4, tzinfo=pytz.utc)
@@ -128,7 +128,7 @@ class TestEventsThroughRisk(unittest.TestCase):
]
algo.benchmark_return_source = benchmark_data
algo.sources = list([trade_bar_data])
algo.set_sources(list([trade_bar_data]))
gen = algo._create_generator(sim_params)
# TODO: Hand derive these results.
@@ -188,8 +188,7 @@ class TestEventsThroughRisk(unittest.TestCase):
data_frequency='minute')
algo = BuyAndHoldAlgorithm(
sim_params=sim_params,
data_frequency='minute')
sim_params=sim_params)
first_date = datetime.datetime(2006, 1, 3, tzinfo=pytz.utc)
first_open, first_close = \
@@ -288,7 +287,7 @@ class TestEventsThroughRisk(unittest.TestCase):
]
algo.benchmark_return_source = benchmark_data
algo.sources = list([trade_bar_data])
algo.set_sources(list([trade_bar_data]))
gen = algo._create_generator(sim_params)
crm = algo.perf_tracker.cumulative_risk_metrics
tests/test_tradesimulation.py
+5 -5
View File
@@ -21,9 +21,9 @@ from zipline.utils import factory
class TestTradeSimulation(TestCase):
def test_minutely_emissions_generate_performance_stats_for_last_day(self):
params = factory.create_simulation_parameters(num_days=1)
params.data_frequency = 'minute'
params.emission_rate = 'minute'
algo = NoopAlgorithm()
algo.run(source=[], sim_params=params)
params = factory.create_simulation_parameters(num_days=1,
data_frequency='minute',
emission_rate='minute')
algo = NoopAlgorithm(sim_params=params)
algo.run(source=[])
self.assertEqual(algo.perf_tracker.day_count, 1.0)
zipline/__init__.py
+1 -1
View File
@@ -32,7 +32,7 @@ from . algorithm import TradingAlgorithm
from . import api
try:
ip = get_ipython() # flake8: noqa
ip = get_ipython() # flake8: noqa
ip.register_magic_function(utils.parse_cell_magic, "line_cell", "zipline")
except:
pass
zipline/algorithm.py
+51 -54
View File
@@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from copy import copy
import warnings
import pytz
import pandas as pd
@@ -147,11 +148,6 @@ class TradingAlgorithm(object):
self.slippage = VolumeShareSlippage()
self.commission = PerShare()
if 'data_frequency' in kwargs:
self.set_data_frequency(kwargs.pop('data_frequency'))
else:
self.data_frequency = None
self.instant_fill = kwargs.pop('instant_fill', False)
# Override annualizer if set
@@ -162,13 +158,13 @@ class TradingAlgorithm(object):
self.capital_base = kwargs.pop('capital_base', DEFAULT_CAPITAL_BASE)
self.sim_params = kwargs.pop('sim_params', None)
if self.sim_params:
if self.data_frequency is None:
self.data_frequency = self.sim_params.data_frequency
else:
self.sim_params.data_frequency = self.data_frequency
if self.sim_params is None:
self.sim_params = create_simulation_parameters(
capital_base=self.capital_base
)
self.perf_tracker = PerformanceTracker(self.sim_params)
# perf_tacker gets instantiated in ._create_generator()
self.perf_tracker = None
self.blotter = kwargs.pop('blotter', None)
if not self.blotter:
@@ -209,6 +205,11 @@ class TradingAlgorithm(object):
if self._initialize is None:
self._initialize = lambda x: None
# Alternative way of setting data_frequency for backwards
# compatibility.
if 'data_frequency' in kwargs:
self.data_frequency = kwargs.pop('data_frequency')
# an algorithm subclass needs to set initialized to True when
# it is fully initialized.
self.initialized = False
@@ -261,7 +262,7 @@ class TradingAlgorithm(object):
blotter=repr(self.blotter),
recorded_vars=repr(self.recorded_vars))
def _create_data_generator(self, source_filter, sim_params):
def _create_data_generator(self, source_filter, sim_params=None):
"""
Create a merged data generator using the sources and
transforms attached to this algorithm.
@@ -271,9 +272,12 @@ class TradingAlgorithm(object):
processed by the zipline, and False for those that should be
skipped.
"""
if sim_params is None:
sim_params = self.sim_params
if self.benchmark_return_source is None:
env = trading.environment
if (self.data_frequency == 'minute'
if (sim_params.data_frequency == 'minute'
or sim_params.emission_rate == 'minute'):
update_time = lambda date: env.get_open_and_close(date)[1]
else:
@@ -315,15 +319,11 @@ class TradingAlgorithm(object):
processed by the zipline, and False for those that should be
skipped.
"""
sim_params.data_frequency = self.data_frequency
# perf_tracker will be instantiated in __init__ if a sim_params
# is passed to the constructor. If not, we instantiate here.
# Instantiate perf_tracker
if self.perf_tracker is None:
self.perf_tracker = PerformanceTracker(sim_params)
self.data_gen = self._create_data_generator(source_filter,
sim_params)
self.data_gen = self._create_data_generator(source_filter, sim_params)
self.trading_client = AlgorithmSimulator(self, sim_params)
@@ -343,14 +343,15 @@ class TradingAlgorithm(object):
# TODO: make a new subclass, e.g. BatchAlgorithm, and move
# the run method to the subclass, and refactor to put the
# generator creation logic into get_generator.
def run(self, source, sim_params=None, benchmark_return_source=None):
def run(self, source, overwrite_sim_params=True,
benchmark_return_source=None):
"""Run the algorithm.
:Arguments:
source : can be either:
- pandas.DataFrame
- zipline source
- list of zipline sources
- list of sources
If pandas.DataFrame is provided, it must have the
following structure:
@@ -364,44 +365,34 @@ class TradingAlgorithm(object):
Daily performance metrics such as returns, alpha etc.
"""
if isinstance(source, (list, tuple)):
assert self.sim_params is not None or sim_params is not None, \
"""When providing a list of sources, \
sim_params have to be specified as a parameter
or in the constructor."""
if isinstance(source, list):
if overwrite_sim_params:
warnings.warn("""List of sources passed, will not attempt to extract sids, and start and end
dates. Make sure to set the correct fields in sim_params passed to
__init__().""", UserWarning)
overwrite_sim_params = False
elif isinstance(source, pd.DataFrame):
# if DataFrame provided, wrap in DataFrameSource
source = DataFrameSource(source)
elif isinstance(source, pd.Panel):
source = DataPanelSource(source)
if not isinstance(source, (list, tuple)):
self.sources = [source]
if isinstance(source, list):
self.set_sources(source)
else:
self.sources = source
self.set_sources([source])
# Check for override of sim_params.
# If it isn't passed to this function,
# use the default params set with the algorithm.
# Else, we create simulation parameters using the start and end of the
# source provided.
if sim_params is None:
if self.sim_params is None:
start = source.start
end = source.end
sim_params = create_simulation_parameters(
start=start,
end=end,
capital_base=self.capital_base,
)
else:
sim_params = self.sim_params
# update sim params to ensure it's set
self.sim_params = sim_params
if self.sim_params.sids is None:
# Override sim_params if params are provided by the source.
if overwrite_sim_params:
if hasattr(source, 'start'):
self.sim_params.period_start = source.start
if hasattr(source, 'end'):
self.sim_params.period_end = source.end
all_sids = [sid for s in self.sources for sid in s.sids]
self.sim_params.sids = set(all_sids)
# Changing period_start and period_close might require updating
# of first_open and last_close.
self.sim_params._update_internal()
# Create history containers
if len(self.history_specs) != 0:
@@ -427,7 +418,7 @@ class TradingAlgorithm(object):
self.perf_tracker = None
# create transforms and zipline
self.gen = self._create_generator(sim_params)
self.gen = self._create_generator(self.sim_params)
with ZiplineAPI(self):
# loop through simulated_trading, each iteration returns a
@@ -677,10 +668,16 @@ class TradingAlgorithm(object):
assert isinstance(transforms, list)
self.transforms = transforms
def set_data_frequency(self, data_frequency):
assert data_frequency in ('daily', 'minute')
self.data_frequency = data_frequency
self.annualizer = ANNUALIZER[self.data_frequency]
# Remain backwards compatibility
@property
def data_frequency(self):
return self.sim_params.data_frequency
@data_frequency.setter
def data_frequency(self, value):
assert value in ('daily', 'minute')
self.sim_params.data_frequency = value
self.annualizer = ANNUALIZER[self.sim_params.data_frequency]
@api_method
def order_percent(self, sid, percent,
zipline/finance/trading.py
+6 -3
View File
@@ -354,9 +354,6 @@ class SimulationParameters(object):
data_frequency='daily',
sids=None):
# This is the global environment for trading simulation.
environment = TradingEnvironment.instance()
self.period_start = period_start
self.period_end = period_end
self.capital_base = capital_base
@@ -365,6 +362,12 @@ class SimulationParameters(object):
self.data_frequency = data_frequency
self.sids = sids
self._update_internal()
def _update_internal(self):
# This is the global environment for trading simulation.
environment = TradingEnvironment.instance()
assert self.period_start <= self.period_end, \
"Period start falls after period end."
zipline/utils/cli.py
+2 -2
View File
@@ -119,7 +119,7 @@ def parse_cell_magic(line, cell):
local_namespace = args.pop('local_namespace', False)
# By default, execute inside IPython namespace
if not local_namespace:
args['namespace'] = get_ipython().user_ns # flake8: noqa
args['namespace'] = get_ipython().user_ns # flake8: noqa
# If we are running inside NB, do not output to file but create a
# variable instead
@@ -128,7 +128,7 @@ def parse_cell_magic(line, cell):
perf = run_pipeline(print_algo=False, algo_text=cell, **args)
if output_var_name is not None:
get_ipython().user_ns[output_var_name] = perf # flake8: noqa
get_ipython().user_ns[output_var_name] = perf # flake8: noqa
def run_pipeline(print_algo=True, **kwargs):
zipline/utils/factory.py
+4 -1
View File
@@ -43,7 +43,8 @@ __all__ = ['load_from_yahoo', 'load_bars_from_yahoo']
def create_simulation_parameters(year=2006, start=None, end=None,
capital_base=float("1.0e5"),
num_days=None, load=None,
sids=None):
sids=None, data_frequency='daily',
emission_rate='daily'):
"""Construct a complete environment with reasonable defaults"""
if start is None:
start = datetime(year, 1, 1, tzinfo=pytz.utc)
@@ -60,6 +61,8 @@ def create_simulation_parameters(year=2006, start=None, end=None,
period_end=end,
capital_base=capital_base,
sids=sids,
data_frequency=data_frequency,
emission_rate=emission_rate,
)
return sim_params