From 2742ffcc473b36dcbf5317367d670481e89545eb Mon Sep 17 00:00:00 2001 From: fawce Date: Wed, 14 Mar 2012 15:13:32 -0400 Subject: [PATCH] using pandas for the dataframe relayed to the algorithm. all unit tests are passing. --- zipline/finance/performance.py | 10 ++- zipline/finance/trading.py | 3 +- zipline/protocol.py | 2 +- zipline/test/client.py | 24 +++++--- zipline/test/test_perf_tracking.py | 98 ++++++++++++++++++++---------- zipline/test/test_risk.py | 25 ++++---- 6 files changed, 101 insertions(+), 61 deletions(-) diff --git a/zipline/finance/performance.py b/zipline/finance/performance.py index 79e10309..f5f675ee 100644 --- a/zipline/finance/performance.py +++ b/zipline/finance/performance.py @@ -1,6 +1,7 @@ import datetime import pytz import math +import pandas from zmq.core.poll import select @@ -123,7 +124,10 @@ class PerformanceTracker(): def update(self, event_frame): for dt, event_series in event_frame.iteritems(): - self.process_event(event_series) + data = {} + data.update(event_series) + event = zp.namedict(data) + self.process_event(event) def process_event(self, event): qutil.LOGGER.debug("series is " + str(event)) @@ -131,7 +135,7 @@ class PerformanceTracker(): if(event.dt >= self.market_close): self.handle_market_close() - if event.TRANSACTION != None: + if not pandas.isnull(event.TRANSACTION): self.txn_count += 1 self.cumulative_performance.execute_transaction(event.TRANSACTION) self.todays_performance.execute_transaction(event.TRANSACTION) @@ -238,7 +242,7 @@ class Position(): self.amount = self.amount + txn.amount def currentValue(self): - return self.amount * self.last_sale + return self.amount * self.last_sale_price def __repr__(self): diff --git a/zipline/finance/trading.py b/zipline/finance/trading.py index 4319be6e..0dd77da4 100644 --- a/zipline/finance/trading.py +++ b/zipline/finance/trading.py @@ -97,7 +97,8 @@ class TradeSimulationClient(qmsg.Component): def queue_event(self, event): if self.event_queue == None: self.event_queue = {} - self.event_queue[event.dt] = event.as_series() + series = event.as_series() + self.event_queue[event.dt] = series def get_frame(self): frame = pandas.DataFrame(self.event_queue) diff --git a/zipline/protocol.py b/zipline/protocol.py index 4d13a0b0..74fbfbba 100644 --- a/zipline/protocol.py +++ b/zipline/protocol.py @@ -166,7 +166,7 @@ class namedict(object): """ def __init__(self, dct=None): - if(dct): + if dct: self.__dict__.update(dct) def __setitem__(self, key, value): diff --git a/zipline/test/client.py b/zipline/test/client.py index c63f68b8..f9ddf443 100644 --- a/zipline/test/client.py +++ b/zipline/test/client.py @@ -70,19 +70,23 @@ class TestAlgorithm(): def __init__(self, sid, amount, order_count, trading_client): self.trading_client = trading_client - self.trading_client.add_event_callback(self.handle_event) + self.trading_client.add_event_callback(self.handle_frame) self.count = order_count self.sid = sid self.amount = amount self.incr = 0 self.done = False - def handle_event(self, event): - #place an order for 100 shares of sid:133 - if self.incr < self.count: - if event.source_id != zp.FINANCE_COMPONENT.ORDER_SOURCE: - self.trading_client.order(self.sid, self.amount) - self.incr += 1 - elif not self.done: - self.trading_client.signal_order_done() - self.done = True + def handle_frame(self, frame): + for dt, s in frame.iteritems(): + data = {} + data.update(s) + event = zp.namedict(data) + #place an order for 100 shares of sid:133 + if self.incr < self.count: + if event.source_id != zp.FINANCE_COMPONENT.ORDER_SOURCE: + self.trading_client.order(self.sid, self.amount) + self.incr += 1 + elif not self.done: + self.trading_client.signal_order_done() + self.done = True diff --git a/zipline/test/test_perf_tracking.py b/zipline/test/test_perf_tracking.py index b8ab30c0..b8a4567c 100644 --- a/zipline/test/test_perf_tracking.py +++ b/zipline/test/test_perf_tracking.py @@ -2,11 +2,14 @@ import unittest import copy import random import datetime +import pytz import zipline.test.factory as factory import zipline.util as qutil import zipline.finance.performance as perf import zipline.finance.risk as risk +import zipline.protocol as zp +from zipline.finance.trading import TradeSimulationClient class PerformanceTestCase(unittest.TestCase): def setUp(self): @@ -85,12 +88,12 @@ single buy transaction ) self.assertEqual( - pp.positions[1].last_sale, + pp.positions[1].last_sale_price, trades[-1]['price'], "last sale should be same as last trade. \ expected {exp} actual {act}".format( exp=trades[-1]['price'], - act=pp.positions[1].last_sale + act=pp.positions[1].last_sale_price ) ) @@ -155,7 +158,7 @@ single short-sale transaction""" ) self.assertEqual( - pp.positions[1].last_sale, + pp.positions[1].last_sale_price, trades_1[-1]['price'], "last sale should be price of last trade" ) @@ -224,7 +227,7 @@ single short-sale transaction""" ) self.assertEqual( - pp2.positions[1].last_sale, + pp2.positions[1].last_sale_price, trades_2[-1].price, "last sale should be price of last trade" ) @@ -285,7 +288,7 @@ cost of sole txn in test" ) self.assertEqual( - ppTotal.positions[1].last_sale, + ppTotal.positions[1].last_sale_price, trades_2[-1].price, "last sale should be price of last trade" ) @@ -367,7 +370,7 @@ trade after cover""" ) self.assertEqual( - pp.positions[1].last_sale, + pp.positions[1].last_sale_price, trades[-1].price, "last sale should be price of last trade" ) @@ -415,10 +418,10 @@ shares in position" pp.calculate_performance() self.assertEqual( - pp.positions[1].last_sale, + pp.positions[1].last_sale_price, trades[-1].price, "should have a last sale of 12, got {val}".format( - val=pp.positions[1].last_sale + val=pp.positions[1].last_sale_price ) ) @@ -456,9 +459,9 @@ shares in position" pp2.calculate_performance() self.assertEqual( - pp2.positions[1].last_sale, + pp2.positions[1].last_sale_price, 10, - "should have a last sale of 10, was {val}".format(val=pp2.positions[1].last_sale) + "should have a last sale of 10, was {val}".format(val=pp2.positions[1].last_sale_price) ) self.assertEqual( @@ -482,7 +485,7 @@ shares in position" pp3.calculate_performance() self.assertEqual( - pp3.positions[1].last_sale, + pp3.positions[1].last_sale_price, 10, "should have a last sale of 10" ) @@ -499,27 +502,58 @@ shares in position" "should be -400 for all trades and transactions in period" ) - - def dtest_daily_performance_calc(self): - hostedAlgo = factories.createAlgo("workingAlgo.py") - btRecord = BackTestRun(duration_unit="Days",duration_count=5,capital_base=25000000) - bt = BackTest(hostedAlgo,btRecord) - start = bt.periodStart - end = bt.periodEnd - #print "{start} to {end}".format(start=start, end=end) + def test_tracker(self): - trades = factories.createTradeHistory(1,[10,11,12,11],[100,100,100,100],start, self.oneday) - #createTransaction(self, sid, amount, price, dt, order_id) - bt.createTransaction(1, 100, 10.0, trades[0].dt + 30*self.onesec, None) - curPeriod = start - bt.positions = {} - dailyPeriods = [] - bt.initialValue = 0.0 - while (bt.mktClose) <= bt.periodEnd: - bt.updatePerformance() - dailyPeriods.append(bt.curPeriod) - bt.nextMarketDay() + trade_count = 100 + sid = 133 + price = [10.1] * trade_count + volume = [100] * trade_count + start_date = datetime.datetime.strptime("01/01/2011","%m/%d/%Y") + start_date = start_date.replace(tzinfo=pytz.utc) + trade_time_increment = datetime.timedelta(days=1) + trade_history = factory.create_trade_history( + sid, + price, + volume, + start_date, + trade_time_increment, + self.trading_environment + ) + + trade_client = TradeSimulationClient(start_date) + start = trade_history[0].dt + end = trade_history[-1].dt + tracker = perf.PerformanceTracker( + start, + end, + 1000.0, + self.trading_environment + ) + + for event in trade_history: + #create a transaction for all but + #one trade, to simulate None transaction + if(event.dt != start): + txn = zp.namedict({ + 'sid' : event.sid, + 'amount' : -25, + 'dt' : event.dt, + 'price' : 10.0, + 'commission' : 0.50 + }) + else: + txn = None + event[zp.TRANSFORM_TYPE.TRANSACTION] = txn + trade_client.queue_event(event) - self.assertEqual(dailyPeriods[0].pnl,0,"the first day's performance should be zero") - self.assertEqual(dailyPeriods[1].pnl,100,"the second day's pnl should be 100 but was {pnl}".format(pnl=dailyPeriods[1].pnl)) + df = trade_client.get_frame() + tracker.update(df) + + #we skip one trade, to test case of None transaction + txn_count = len(trade_history) - 1 + self.assertEqual(tracker.txn_count, txn_count) + + cumulative_pos = tracker.cumulative_performance.positions[sid] + expected_size = txn_count * -25 + self.assertEqual(cumulative_pos.amount, expected_size) \ No newline at end of file diff --git a/zipline/test/test_risk.py b/zipline/test/test_risk.py index bdcf575f..0d293b5e 100644 --- a/zipline/test/test_risk.py +++ b/zipline/test/test_risk.py @@ -26,7 +26,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2006, month=1, day=1, tzinfo=pytz.utc) self.algo_returns_06 = factory.create_returns_from_list(RETURNS, start_date, self.trading_calendar) end_date = datetime.datetime(year=2006, month=12, day=31, tzinfo=pytz.utc) - self.metrics_06 = risk.RiskReport(self.algo_returns_06, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + self.metrics_06 = risk.RiskReport(self.algo_returns_06, self.trading_calendar) def tearDown(self): return @@ -48,7 +48,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2006, month=1, day=1) end_date = datetime.datetime(year=2006, month=12, day=31) returns = factory.create_returns_from_range(start_date, end_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) self.assertEqual([round(x.benchmark_period_returns, 4) for x in metrics.month_periods], [0.0255,0.0005,0.0111,0.0122,-0.0309,0.0001,0.0051,0.0213,0.0246,0.0315,0.0165,0.0126]) self.assertEqual([round(x.benchmark_period_returns, 4) for x in metrics.three_month_periods], @@ -61,7 +61,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2006, month=1, day=1) end_date = datetime.datetime(year=2006, month=12, day=31) returns = factory.create_returns_from_range(start_date, end_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) self.assertEqual([x.trading_days for x in metrics.year_periods],[251]) self.assertEqual([x.trading_days for x in metrics.month_periods],[20,19,23,19,22,22,20,23,20,22,21,20]) @@ -69,7 +69,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2006, month=1, day=1) end_date = datetime.datetime(year=2006, month=12, day=31) returns = factory.create_returns_from_range(start_date, end_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.month_periods], [0.031,0.026,0.024,0.025,0.037,0.047,0.039,0.022,0.023,0.021,0.025,0.019]) self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.three_month_periods], @@ -131,7 +131,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2008, month=1, day=1) end_date = datetime.datetime(year=2008, month=12, day=31) returns = factory.create_returns_from_range(start_date, end_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) self.assertEqual([round(x.benchmark_period_returns, 3) for x in metrics.month_periods], [-0.061,-0.035,-0.006,0.048,0.011,-0.086,-0.01,0.012,-0.091,-0.169,-0.075,0.008]) self.assertEqual([round(x.benchmark_period_returns, 3) for x in metrics.three_month_periods], @@ -144,7 +144,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2008, month=1, day=1) end_date = datetime.datetime(year=2008, month=12, day=31) returns = factory.create_returns_from_range(start_date, end_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) self.assertEqual([x.trading_days for x in metrics.year_periods],[253]) self.assertEqual([x.trading_days for x in metrics.month_periods],[21,20,20,22,21,21,22,21,21,23,19,22]) @@ -152,7 +152,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2008, month=1, day=1) end_date = datetime.datetime(year=2008, month=12, day=31) returns = factory.create_returns_from_range(start_date, end_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.month_periods], [0.07,0.058,0.082,0.054,0.041,0.057,0.068,0.06,0.157,0.244,0.195,0.145]) self.assertEqual([round(x.benchmark_volatility, 3) for x in metrics.three_month_periods], @@ -166,7 +166,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=2006, month=1, day=1) end_date = datetime.datetime(year=2006, month=12, day=31) returns = factory.create_returns_from_range(start_date, end_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) self.assertEqual([round(x.treasury_period_return, 4) for x in metrics.month_periods], [0.0037,0.0034,0.0039,0.0038,0.0040,0.0037,0.0043,0.0043,0.0038,0.0044,0.0043,0.0041]) self.assertEqual([round(x.treasury_period_return, 4) for x in metrics.three_month_periods], @@ -183,7 +183,7 @@ class Risk(unittest.TestCase): start_date = datetime.datetime(year=1991, month=1, day=1) returns = factory.create_returns(365 * 5 + 2, start_date, self.trading_calendar) #1992 and 1996 were leap years returns = returns[:-10] #truncate the returns series to end mid-month - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) total_months = 60 self.check_metrics(metrics, total_months, start_date) @@ -194,7 +194,7 @@ class Risk(unittest.TestCase): #because we may catch the leap of the last year, and i think this func is [start,end) ld = calendar.leapdays(start_date.year, start_date.year + years + 1) returns = factory.create_returns(365 * years + ld, start_date, self.trading_calendar) - metrics = risk.RiskReport(returns, self.benchmark_returns, self.treasury_curves, self.trading_calendar) + metrics = risk.RiskReport(returns, self.trading_calendar) total_months = years * 12 self.check_metrics(metrics, total_months, start_date) @@ -202,10 +202,7 @@ class Risk(unittest.TestCase): self.assert_range_length(metrics.month_periods, total_months, 1, start_date) self.assert_range_length(metrics.three_month_periods, total_months, 3, start_date) self.assert_range_length(metrics.six_month_periods, total_months, 6, start_date) - self.assert_range_length(metrics.year_periods, total_months, 12, start_date) - self.assert_range_length(metrics.three_year_periods, total_months, 36, start_date) - self.assert_range_length(metrics.five_year_periods, total_months, 60, start_date) - + self.assert_range_length(metrics.year_periods, total_months, 12, start_date) def assert_last_day(self, period_end): #30 days has september, april, june and november if(period_end.month in [9,4,6,11]):