From 6630917da88df2a0610f167ecc5311a0f1fec7d0 Mon Sep 17 00:00:00 2001 From: Stephen Diehl Date: Thu, 15 Mar 2012 16:08:01 -0400 Subject: [PATCH 1/8] working on frame argument sent to algorithm. --- zipline/finance/performance.py | 28 ++++++--------- zipline/finance/risk.py | 17 +++++++-- zipline/finance/trading.py | 41 +++++++++++++--------- zipline/test/test_finance.py | 32 +++++++++-------- zipline/test/test_perf_tracking.py | 56 +++++++++++++++++++----------- 5 files changed, 101 insertions(+), 73 deletions(-) diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index f5f675ee..0f5b7b56 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -12,11 +12,12 @@ import zipline.finance.risk as risk class PerformanceTracker(): - def __init__(self, period_start, period_end, capital_base, trading_environment): + def __init__(self, trading_environment): + self.trading_environment = trading_environment self.trading_day = datetime.timedelta(hours=6, minutes=30) self.calendar_day = datetime.timedelta(hours=24) - self.period_start = period_start - self.period_end = period_end + self.period_start = self.trading_environment.period_start + self.period_end = self.trading_environment.period_end self.market_open = self.period_start self.market_close = self.market_open + self.trading_day self.progress = 0.0 @@ -24,21 +25,20 @@ class PerformanceTracker(): self.day_count = 0 self.cumulative_capital_used= 0.0 self.max_capital_used = 0.0 - self.capital_base = capital_base - self.trading_environment = trading_environment + self.capital_base = self.trading_environment.capital_base self.returns = [] self.txn_count = 0 self.event_count = 0 self.cumulative_performance = PerformancePeriod( {}, - capital_base, - starting_cash = capital_base + self.capital_base, + starting_cash = self.capital_base ) self.todays_performance = PerformancePeriod( {}, - capital_base, - starting_cash = capital_base + self.capital_base, + starting_cash = self.capital_base ) def to_dict(self): @@ -121,16 +121,8 @@ class PerformanceTracker(): 'todays_perf' : self.todays_perf.to_dict(), 'cumulative_risk_metrics' : self.cumulative_risk_metrics.to_dict() } - - def update(self, event_frame): - for dt, event_series in event_frame.iteritems(): - data = {} - data.update(event_series) - event = zp.namedict(data) - self.process_event(event) - + def process_event(self, event): - qutil.LOGGER.debug("series is " + str(event)) self.event_count += 1 if(event.dt >= self.market_close): self.handle_market_close() diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index cfca36bf..cdb3eaa6 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -5,7 +5,6 @@ import numpy as np import numpy.linalg as la import zipline.util as qutil import zipline.protocol as zp -from pymongo import ASCENDING, DESCENDING class DailyReturn(): @@ -137,7 +136,6 @@ class RiskMetrics(): return period_returns, returns def calculate_volatility(self, daily_returns): - #qutil.LOGGER.debug("trading days {td}".format(td=self.trading_days)) return np.std(daily_returns, ddof=1) * math.sqrt(self.trading_days) def calculate_sharpe(self): @@ -326,11 +324,24 @@ def advance_by_months(dt, jump_in_months): class TradingEnvironment(object): - def __init__(self, benchmark_returns, treasury_curves): + def __init__( + self, + benchmark_returns, + treasury_curves, + period_start=None, + period_end=None, + capital_base=None, + frame_index=None + ): + self.trading_days = [] self.trading_day_map = {} self.treasury_curves = treasury_curves self.benchmark_returns = benchmark_returns + self.frame_index = frame_index + self.period_start = period_start + self.period_end = period_end + self.capital_base = capital_base for bm in benchmark_returns: self.trading_days.append(bm.date) self.trading_day_map[bm.date] = bm diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 0dd77da4..6c3d1318 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -8,19 +8,27 @@ from zmq.core.poll import select import zipline.messaging as qmsg import zipline.util as qutil import zipline.protocol as zp +import zipline.finance.performance as perf class TradeSimulationClient(qmsg.Component): - def __init__(self, simulation_dt): + def __init__(self, trading_environment): qmsg.Component.__init__(self) - self.received_count = 0 - self.prev_dt = None - self.event_queue = None - self.event_callbacks = [] - self.txn_count = 0 - self.current_dt = simulation_dt - self.last_iteration_duration = datetime.timedelta(seconds=0) - self.event_frame = None + self.received_count = 0 + self.prev_dt = None + self.event_queue = None + self.event_callbacks = [] + self.txn_count = 0 + self.trading_environment = trading_environment + self.current_dt = trading_environment.period_start + self.last_iteration_dur = datetime.timedelta(seconds=0) + + assert self.trading_environment.frame_index != None + self.event_frame = pandas.DataFrame( + index=self.trading_environment.frame_index + ) + + self.perf = perf.PerformanceTracker(self.trading_environment) @property def get_id(self): @@ -67,9 +75,9 @@ class TradeSimulationClient(qmsg.Component): self.run_callbacks() #update time based on receipt of the order - self.last_iteration_duration = datetime.datetime.utcnow() - event_start + self.last_iteration_dur = datetime.datetime.utcnow() - event_start - self.current_dt = self.current_dt + self.last_iteration_duration + self.current_dt = self.current_dt + self.last_iteration_dur #signal done to order source. self.order_socket.send(str(zp.ORDER_PROTOCOL.BREAK)) @@ -95,15 +103,16 @@ class TradeSimulationClient(qmsg.Component): self.order_socket.send(str(zp.ORDER_PROTOCOL.DONE)) def queue_event(self, event): + self.perf.process_event(event) if self.event_queue == None: - self.event_queue = {} + self.event_queue = [] series = event.as_series() - self.event_queue[event.dt] = series + self.event_queue.append(series) def get_frame(self): - frame = pandas.DataFrame(self.event_queue) - self.event_queue = None - return frame + for event in self.event_queue: + self.event_frame[event['sid']] = event + return self.event_frame class OrderDataSource(qmsg.DataSource): """DataSource that relays orders from the client""" diff --git a/zipline/test/test_finance.py b/zipline/test/test_finance.py index 2542aa2e..028d53a0 100644 --- a/zipline/test/test_finance.py +++ b/zipline/test/test_finance.py @@ -207,7 +207,11 @@ class FinanceTestCase(TestCase): set1 = SpecificEquityTrades("flat-133", trade_history) - trading_client = TradeSimulationClient(start_date) + self.trading_environment.period_start = trade_history[0].dt + self.trading_environment.period_end = trade_history[-1].dt + self.trading_environment.capital_base = 10000 + + trading_client = TradeSimulationClient(self.trading_environment) #client will send 10 orders for 100 shares of 133 test_algo = TestAlgorithm(133, 100, 10, trading_client) @@ -280,25 +284,23 @@ class FinanceTestCase(TestCase): volume, start_date, trade_time_increment, - self.trading_environment ) - + self.trading_environment + ) + + + self.trading_environment.period_start = trade_history[0].dt + self.trading_environment.period_end = trade_history[-1].dt + self.trading_environment.capital_base = 10000 + set1 = SpecificEquityTrades("flat-133", trade_history) #client sill send 10 orders for 100 shares of 133 - trading_client = TradeSimulationClient(start_date) + trading_client = TradeSimulationClient(self.trading_environment) test_algo = TestAlgorithm(133, 100, 10, trading_client) order_source = OrderDataSource() transaction_sim = TransactionSimulator() - perf_tracker = perf.PerformanceTracker( - trade_history[0]['dt'], - trade_history[-1]['dt'], - 1000000.0, - self.trading_environment) - #register perf_tracker to receive callbacks from the client. - trading_client.add_event_callback(perf_tracker.update) - sim.register_components([ trading_client, order_source, @@ -339,19 +341,19 @@ class FinanceTestCase(TestCase): self.assertEqual( transaction_sim.txn_count, - perf_tracker.txn_count, + trading_client.perf.txn_count, "The perf tracker should handle the same number of transactions \ as the simulator emits." ) self.assertEqual( - len(perf_tracker.cumulative_performance.positions), + len(trading_client.perf.cumulative_performance.positions), 1, "Portfolio should have one position." ) self.assertEqual( - perf_tracker.cumulative_performance.positions[133].sid, + trading_client.perf.cumulative_performance.positions[133].sid, 133, "Portfolio should have one position in 133." ) diff --git a/zipline/test/test_perf_tracking.py b/zipline/test/test_perf_tracking.py index 218f3113..36a99ae8 100644 --- a/zipline/test/test_perf_tracking.py +++ b/zipline/test/test_perf_tracking.py @@ -506,34 +506,46 @@ shares in position" trade_count = 100 sid = 133 - price = [10.1] * trade_count + price = 10.1 + price_list = [price] * trade_count volume = [100] * trade_count start_date = datetime.datetime.strptime("01/01/2011","%m/%d/%Y") start_date = start_date.replace(tzinfo=pytz.utc) trade_time_increment = datetime.timedelta(days=1) trade_history = factory.create_trade_history( sid, - price, + price_list, volume, start_date, trade_time_increment, self.trading_environment ) - trade_client = TradeSimulationClient(start_date) - start = trade_history[0].dt - end = trade_history[-1].dt - tracker = perf.PerformanceTracker( - start, - end, - 1000.0, - self.trading_environment + sid2 = 134 + price2 = 12.12 + price2_list = [price2] * trade_count + trade_history2 = factory.create_trade_history( + sid2, + price2_list, + volume, + start_date, + trade_time_increment, + self.trading_environment ) + trade_history.extend(trade_history2) + + self.trading_environment.period_start = trade_history[0].dt + self.trading_environment.period_end = trade_history[-1].dt + self.trading_environment.capital_base = 1000.0 + self.trading_environment.frame_index = ['sid', 'volume', 'dt', \ + 'price', 'changed'] + client = TradeSimulationClient(self.trading_environment) + for event in trade_history: #create a transaction for all but - #one trade, to simulate None transaction - if(event.dt != start): + #first trade in each sid, to simulate None transaction + if(event.dt != self.trading_environment.period_start): txn = zp.namedict({ 'sid' : event.sid, 'amount' : -25, @@ -543,17 +555,19 @@ shares in position" }) else: txn = None - event[zp.TRANSFORM_TYPE.TRANSACTION] = txn - trade_client.queue_event(event) + event[zp.TRANSFORM_TYPE.TRANSACTION] = txn + client.queue_event(event) - df = trade_client.get_frame() - tracker.update(df) + df = client.get_frame() - #we skip one trade, to test case of None transaction - txn_count = len(trade_history) - 1 - self.assertEqual(tracker.txn_count, txn_count) + self.assertEqual(df[133]['price'], price) + self.assertEqual(df[134]['price'], price2) - cumulative_pos = tracker.cumulative_performance.positions[sid] - expected_size = txn_count * -25 + #we skip two trades, to test case of None transaction + txn_count = len(trade_history) - 2 + self.assertEqual(client.perf.txn_count, txn_count) + + cumulative_pos = client.perf.cumulative_performance.positions[sid] + expected_size = txn_count / 2 * -25 self.assertEqual(cumulative_pos.amount, expected_size) \ No newline at end of file From 930ec57269334c05f878bf2d0dac073b9347f947 Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 15 Mar 2012 16:52:44 -0400 Subject: [PATCH 2/8] fixed bug with date handling in risk report --- zipline/finance/performance.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index ada03fcd..3a80171d 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -133,13 +133,15 @@ class PerformanceTracker(): """ - def __init__(self, period_start, period_end, capital_base, trading_environment): - + def __init__(self, trading_environment): + + + self.trading_environment = trading_environment self.trading_day = datetime.timedelta(hours = 6, minutes = 30) self.calendar_day = datetime.timedelta(hours = 24) - self.period_start = period_start - self.period_end = period_end + self.period_start = self.trading_environment.period_start + self.period_end = self.trading_environment.period_end self.market_open = self.period_start self.market_close = self.market_open + self.trading_day self.progress = 0.0 @@ -147,8 +149,7 @@ class PerformanceTracker(): self.day_count = 0 self.cumulative_capital_used = 0.0 self.max_capital_used = 0.0 - self.capital_base = capital_base - self.trading_environment = trading_environment + self.capital_base = self.trading_environment.capital_base self.returns = [] self.txn_count = 0 self.event_count = 0 @@ -156,14 +157,14 @@ class PerformanceTracker(): self.cumulative_performance = PerformancePeriod( {}, - capital_base, - starting_cash = capital_base + self.capital_base, + starting_cash = self.capital_base ) self.todays_performance = PerformancePeriod( {}, - capital_base, - starting_cash = capital_base + self.capital_base, + starting_cash = self.capital_base ) From a35c702528b768515eb33b0127878211de6e6550 Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 15 Mar 2012 17:37:26 -0400 Subject: [PATCH 3/8] fixed tests net of merge of dataflow. --- setup.cfg | 12 ++++++------ zipline/protocol.py | 1 - zipline/protocol_utils.py | 6 ++++++ zipline/test/client.py | 5 ++--- zipline/test/test_finance.py | 4 ++++ zipline/test/test_risk.py | 7 +++++-- 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/setup.cfg b/setup.cfg index d14c258b..159f83a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,12 +2,12 @@ verbosity=2 detailed-errors=1 -with-xcoverage=1 -cover-package=zipline -cover-erase=1 -cover-html=1 -cover-html-dir=docs/_build/html/cover -with-xunit=1 +#with-xcoverage=1 +#cover-package=zipline +#cover-erase=1 +#cover-html=1 +#cover-html-dir=docs/_build/html/cover +#with-xunit=1 # Drop into debugger on failure diff --git a/zipline/protocol.py b/zipline/protocol.py index 9fc98238..e60177aa 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -119,7 +119,6 @@ import numbers import datetime import pytz import copy -import pandas from collections import namedtuple from protocol_utils import Enum, FrameExceptionFactory, namedict diff --git a/zipline/protocol_utils.py b/zipline/protocol_utils.py index 0e6cb858..ba805b3b 100644 --- a/zipline/protocol_utils.py +++ b/zipline/protocol_utils.py @@ -1,4 +1,5 @@ import copy +import pandas from ctypes import Structure, c_ubyte def Enum(*options): @@ -85,3 +86,8 @@ class namedict(object): def has_attr(self, name): return self.__dict__.has_key(name) + + def as_series(self): + s = pandas.Series(self.__dict__) + s.name = self.sid + return s diff --git a/zipline/test/client.py b/zipline/test/client.py index f9ddf443..1ebf22f3 100644 --- a/zipline/test/client.py +++ b/zipline/test/client.py @@ -84,9 +84,8 @@ class TestAlgorithm(): event = zp.namedict(data) #place an order for 100 shares of sid:133 if self.incr < self.count: - if event.source_id != zp.FINANCE_COMPONENT.ORDER_SOURCE: - self.trading_client.order(self.sid, self.amount) - self.incr += 1 + self.trading_client.order(self.sid, self.amount) + self.incr += 1 elif not self.done: self.trading_client.signal_order_done() self.done = True diff --git a/zipline/test/test_finance.py b/zipline/test/test_finance.py index 028d53a0..b1413800 100644 --- a/zipline/test/test_finance.py +++ b/zipline/test/test_finance.py @@ -210,6 +210,8 @@ class FinanceTestCase(TestCase): self.trading_environment.period_start = trade_history[0].dt self.trading_environment.period_end = trade_history[-1].dt self.trading_environment.capital_base = 10000 + self.trading_environment.frame_index = ['sid', 'volume', 'dt', \ + 'price', 'changed'] trading_client = TradeSimulationClient(self.trading_environment) #client will send 10 orders for 100 shares of 133 @@ -291,6 +293,8 @@ class FinanceTestCase(TestCase): self.trading_environment.period_start = trade_history[0].dt self.trading_environment.period_end = trade_history[-1].dt self.trading_environment.capital_base = 10000 + self.trading_environment.frame_index = ['sid', 'volume', 'dt', \ + 'price', 'changed'] set1 = SpecificEquityTrades("flat-133", trade_history) diff --git a/zipline/test/test_risk.py b/zipline/test/test_risk.py index 0d293b5e..57264ad6 100644 --- a/zipline/test/test_risk.py +++ b/zipline/test/test_risk.py @@ -11,6 +11,9 @@ class Risk(unittest.TestCase): def setUp(self): qutil.configure_logging() + start_date = datetime.datetime(year=2006, month=1, day=1, tzinfo=pytz.utc) + end_date = datetime.datetime(year=2006, month=12, day=31, tzinfo=pytz.utc) + self.benchmark_returns, self.treasury_curves = \ factory.load_market_data() @@ -23,9 +26,9 @@ class Risk(unittest.TestCase): self.oneday = datetime.timedelta(days=1) self.tradingday = datetime.timedelta(hours=6, minutes=30) self.dt = datetime.datetime.utcnow() - start_date = datetime.datetime(year=2006, month=1, day=1, tzinfo=pytz.utc) + self.algo_returns_06 = factory.create_returns_from_list(RETURNS, start_date, self.trading_calendar) - end_date = datetime.datetime(year=2006, month=12, day=31, tzinfo=pytz.utc) + self.metrics_06 = risk.RiskReport(self.algo_returns_06, self.trading_calendar) def tearDown(self): From 62d0422eb4c4566e3b8813a05e529dfd0e0261ff Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 15 Mar 2012 17:38:22 -0400 Subject: [PATCH 4/8] a notebook to help explore the dataframe produced for the algo client. --- notebooks/Experimenting with Frames.ipynb | 352 ++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 notebooks/Experimenting with Frames.ipynb diff --git a/notebooks/Experimenting with Frames.ipynb b/notebooks/Experimenting with Frames.ipynb new file mode 100644 index 00000000..50e5b5a5 --- /dev/null +++ b/notebooks/Experimenting with Frames.ipynb @@ -0,0 +1,352 @@ +{ + "metadata": { + "name": "Experimenting with Frames" + }, + "nbformat": 3, + "worksheets": [ + { + "cells": [ + { + "cell_type": "heading", + "source": [ + "Performance Tracking" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import datetime", + "import pandas", + "import pytz", + "", + "import zipline.test.factory as factory", + "import zipline.finance.performance as perf", + "import zipline.protocol as zp", + "import zipline.finance.risk as risk", + "import zipline.finance.trading as trading" + ], + "language": "python", + "outputs": [], + "prompt_number": 38 + }, + { + "cell_type": "heading", + "source": [ + "Create a simulated trade history using the test factory" + ] + }, + { + "cell_type": "markdown", + "source": [ + "For any backtesting, zipline relies on a TradingEnvironment object. Trading environment holds essential facts: ", + " ", + " - start and end times for the simulation.", + " - historical daily returns for your benchmark.", + " - historical treasury curves", + " - an assumed capital base for your portfolio", + " - a calendar of trading days based on your benchmark", + "", + "zipline ships with a compressed archives of the S&P daily returns, and US treasury curves to facilitate standalone development and testing. In the next cell we instantiate the environment using these defaults. You can see more of this in zipline/test/test_perf_tracking.py" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "benchmark_returns, treasury_curves = factory.load_market_data()", + " ", + "trading_environment = risk.TradingEnvironment(benchmark_returns, treasury_curves)" + ], + "language": "python", + "outputs": [], + "prompt_number": 39 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "trade_count = 100", + "sid = 133", + "price = 10.1 ", + "price_list = [price] * trade_count", + "volume = [100] * trade_count", + "start_date = datetime.datetime.strptime(\"01/01/2011\",\"%m/%d/%Y\")", + "start_date = start_date.replace(tzinfo=pytz.utc)", + "trade_time_increment = datetime.timedelta(days=1)", + "", + "trade_history = factory.create_trade_history( ", + " sid, ", + " price_list, ", + " volume, ", + " start_date, ", + " trade_time_increment, ", + " trading_environment ", + ")", + "", + "sid2 = 134", + "price2 = 12.12", + "price2_list = [price2] * trade_count ", + "trade_history2 = factory.create_trade_history( ", + " sid2, ", + " price2_list, ", + " volume, ", + " start_date, ", + " trade_time_increment, ", + " trading_environment ", + ")", + " ", + "trade_history.extend(trade_history2) ", + "trade_history = sorted(trade_history, key=lambda x: x.dt)" + ], + "language": "python", + "outputs": [], + "prompt_number": 40 + }, + { + "cell_type": "markdown", + "source": [ + "Now that we have a simulated history of trades for two companies and a corresponding trading environment, we can create a dataframe of trades." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df = pandas.DataFrame(index = ['price', 'volume', 'dt'])", + "for event in trade_history:", + " series = event.as_series()", + " #df.index = df.index.tolist().append(event.sid)", + " #series.name = event.sid", + " df[event.sid] = series" + ], + "language": "python", + "outputs": [], + "prompt_number": 92 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 93, + "text": [ + " 133 134", + "price 10.1 12.12", + "volume 100 100", + "dt 2011-04-08 00:00:00+00:00 2011-04-08 00:00:00+00:00" + ] + } + ], + "prompt_number": 93 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df_t = df.transpose()", + "df_t" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 94, + "text": [ + " price volume dt", + "133 10.1 100 2011-04-08 00:00:00+00:00", + "134 12.12 100 2011-04-08 00:00:00+00:00" + ] + } + ], + "prompt_number": 94 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df[133]" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 56, + "text": [ + "sid 133", + "volume 100", + "dt 2011-04-08 00:00:00+00:00", + "price 10.1", + "changed NaN", + "Name: 133" + ] + } + ], + "prompt_number": 56 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df_t['price']" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 57, + "text": [ + "133 10.1", + "134 12.12", + "Name: price" + ] + } + ], + "prompt_number": 57 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df_t['price'].max()" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 50, + "text": [ + "12.12" + ] + } + ], + "prompt_number": 50 + }, + { + "cell_type": "code", + "collapsed": true, + "input": [ + "last = trade_history[23].dt" + ], + "language": "python", + "outputs": [], + "prompt_number": 51 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df_t['changed'] = df_t['dt'] > last" + ], + "language": "python", + "outputs": [], + "prompt_number": 53 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df_t" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 54, + "text": [ + " sid volume dt price changed", + "133 133 100 2011-04-08 00:00:00+00:00 10.1 True", + "134 134 100 2011-04-08 00:00:00+00:00 12.12 True" + ] + } + ], + "prompt_number": 54 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df_t.index" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 59, + "text": [ + "Int64Index([133, 134])" + ] + } + ], + "prompt_number": 59 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df.index" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 60, + "text": [ + "Index([sid, volume, dt, price, changed], dtype=object)" + ] + } + ], + "prompt_number": 60 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df.columns" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 61, + "text": [ + "Int64Index([133, 134])" + ] + } + ], + "prompt_number": 61 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "df_t.columns" + ], + "language": "python", + "outputs": [ + { + "output_type": "pyout", + "prompt_number": 62, + "text": [ + "Index([sid, volume, dt, price, changed], dtype=object)" + ] + } + ], + "prompt_number": 62 + }, + { + "cell_type": "code", + "collapsed": true, + "input": [], + "language": "python", + "outputs": [] + } + ] + } + ] +} \ No newline at end of file From c42ffb5e1abd203a6a5d25f001962c906cdc2ae6 Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 15 Mar 2012 19:21:03 -0400 Subject: [PATCH 5/8] splitting out coverage config for jenkins, as per @sdiehl comments on PR --- etc/jenkins.sh | 2 +- jenkins_setup.cfg | 12 ++++++++++++ setup.cfg | 9 --------- 3 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 jenkins_setup.cfg diff --git a/etc/jenkins.sh b/etc/jenkins.sh index 5b304092..96d4745f 100755 --- a/etc/jenkins.sh +++ b/etc/jenkins.sh @@ -29,7 +29,7 @@ pip freeze paver apidocs html #run all the tests in test. see setup.cfg for flags. -nosetests +nosetests --config=jenkins_setup.cfg #run pylint checks cp ./pylint.rcfile /mnt/jenkins/.pylintrc #default location for config file... diff --git a/jenkins_setup.cfg b/jenkins_setup.cfg new file mode 100644 index 00000000..a6f24289 --- /dev/null +++ b/jenkins_setup.cfg @@ -0,0 +1,12 @@ +[nosetests] +verbosity=2 +detailed-errors=1 + +with-xcoverage=1 +cover-package=zipline +cover-erase=1 +cover-html=1 +cover-html-dir=docs/_build/html/cover +with-xunit=1 + + diff --git a/setup.cfg b/setup.cfg index 159f83a4..740ef396 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,15 +2,6 @@ verbosity=2 detailed-errors=1 -#with-xcoverage=1 -#cover-package=zipline -#cover-erase=1 -#cover-html=1 -#cover-html-dir=docs/_build/html/cover -#with-xunit=1 - - # Drop into debugger on failure #pdb=0 #pdb-failures=0 - From 1ab84ae2ca5e3da653eecce34604aed604c4328c Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 15 Mar 2012 22:52:49 -0400 Subject: [PATCH 6/8] extended namedict and performance tracker to allow portfolio calculations to be accessed during a live test as a namedict. --- zipline/finance/performance.py | 38 +++++++++++++++++++++++++++++----- zipline/protocol_utils.py | 3 +++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index 3a80171d..3458dbab 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -79,7 +79,7 @@ Position Tracking | last_sale_date | datetime of the last trade of the position's | | | security on the exchange | +-----------------+----------------------------------------------------+ - | timestamp | System time evevent occurs in zipilne | + | timestamp | System time event occurs in zipilne | +-----------------+----------------------------------------------------+ Performance Period @@ -154,6 +154,7 @@ class PerformanceTracker(): self.txn_count = 0 self.event_count = 0 self.result_stream = None + self.last_dict = None self.cumulative_performance = PerformancePeriod( {}, @@ -167,6 +168,8 @@ class PerformanceTracker(): starting_cash = self.capital_base ) + def get_portfolio(self): + return self.cumulative_performance.to_namedict() def publish_to(self, zmq_socket, context=None): """ @@ -400,10 +403,12 @@ class PerformancePeriod(): def to_dict(self): """ - Creates a dictionary representing the state of this performance period - Returns a dict object of the form: - + Creates a dictionary representing the state of this performance + period. """ + positions = {} + for sid, pos in self.positions.iteritems(): + positions[sid] = pos.to_dict() return { 'ending_value' : self.ending_value, @@ -411,6 +416,29 @@ class PerformancePeriod(): 'starting_value' : self.starting_value, 'starting_cash' : self.starting_cash, 'ending_cash' : self.ending_cash, - 'positions' : self.positions, + 'positions' : positions, 'timestamp' : datetime.datetime.now(), } + + def to_namedict(self): + """ + Creates a namedict representing the state of this perfomance period. + + """ + positions = {} + for sid, pos in self.positions.iteritems(): + positions[sid] = zp.namedict(pos.to_dict()) + + positions = zp.namedict(positions) + + return zp.namedict({ + 'ending_value' : self.ending_value, + 'capital_used' : self.period_capital_used, + 'starting_value' : self.starting_value, + 'starting_cash' : self.starting_cash, + 'ending_cash' : self.ending_cash, + 'positions' : positions + }) + + + diff --git a/zipline/protocol_utils.py b/zipline/protocol_utils.py index ba805b3b..62ab340c 100644 --- a/zipline/protocol_utils.py +++ b/zipline/protocol_utils.py @@ -86,6 +86,9 @@ class namedict(object): def has_attr(self, name): return self.__dict__.has_key(name) + + def __getitem__(self, key): + return self.__dict__[key] def as_series(self): s = pandas.Series(self.__dict__) From 77993ee6ca1557931668dbca9436f61a34021aa3 Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 16 Mar 2012 09:52:08 -0400 Subject: [PATCH 7/8] trivial mod to force push to happen. --- zipline/finance/performance.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index 3458dbab..b07dc081 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -404,7 +404,7 @@ class PerformancePeriod(): def to_dict(self): """ Creates a dictionary representing the state of this performance - period. + period. See header comments for a detailed description. """ positions = {} for sid, pos in self.positions.iteritems(): @@ -422,7 +422,9 @@ class PerformancePeriod(): def to_namedict(self): """ - Creates a namedict representing the state of this perfomance period. + Creates a namedict representing the state of this perfomance period. + Properties are the same as the results of to_dict. See header comments + for a detailed description. """ positions = {} From 65253737cd9e32d207d4a552dbfb5dbb15c896ea Mon Sep 17 00:00:00 2001 From: fawce Date: Fri, 16 Mar 2012 11:47:06 -0400 Subject: [PATCH 8/8] taking code recommendations from @sdiehl PR notes... as always :) --- zipline/finance/performance.py | 20 ++++++++++++++------ zipline/protocol_utils.py | 3 --- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index b07dc081..346aa205 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -406,9 +406,7 @@ class PerformancePeriod(): Creates a dictionary representing the state of this performance period. See header comments for a detailed description. """ - positions = {} - for sid, pos in self.positions.iteritems(): - positions[sid] = pos.to_dict() + positions = self.get_positions() return { 'ending_value' : self.ending_value, @@ -427,9 +425,7 @@ class PerformancePeriod(): for a detailed description. """ - positions = {} - for sid, pos in self.positions.iteritems(): - positions[sid] = zp.namedict(pos.to_dict()) + positions = self.get_positions(namedicted=True) positions = zp.namedict(positions) @@ -442,5 +438,17 @@ class PerformancePeriod(): 'positions' : positions }) + def get_positions(self, namedicted=False): + positions = {} + for sid, pos in self.positions.iteritems(): + cur = pos.to_dict() + if namedicted: + positions[sid] = zp.namedict(cur) + else: + positions[sid] = cur + + return positions + + diff --git a/zipline/protocol_utils.py b/zipline/protocol_utils.py index 62ab340c..ba805b3b 100644 --- a/zipline/protocol_utils.py +++ b/zipline/protocol_utils.py @@ -86,9 +86,6 @@ class namedict(object): def has_attr(self, name): return self.__dict__.has_key(name) - - def __getitem__(self, key): - return self.__dict__[key] def as_series(self): s = pandas.Series(self.__dict__)