From a48db838c1a0fb3e7c71037c08b97401b57fc5bb Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 8 May 2012 13:07:55 -0400 Subject: [PATCH 1/4] refactored transforms to be a package. added vwap, moving average, returns. basic tests for calculations, need to add a test to run the calculations in a zipline. --- zipline/finance/transforms/__init__.py | 0 zipline/finance/transforms/moving_average.py | 92 +++++++++++++++++++ zipline/finance/transforms/returns.py | 44 +++++++++ .../{transforms.py => transforms/vwap.py} | 72 +++++++++------ zipline/test/test_transforms.py | 61 ++++++++++++ 5 files changed, 243 insertions(+), 26 deletions(-) create mode 100644 zipline/finance/transforms/__init__.py create mode 100644 zipline/finance/transforms/moving_average.py create mode 100644 zipline/finance/transforms/returns.py rename zipline/finance/{transforms.py => transforms/vwap.py} (53%) create mode 100644 zipline/test/test_transforms.py diff --git a/zipline/finance/transforms/__init__.py b/zipline/finance/transforms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/zipline/finance/transforms/moving_average.py b/zipline/finance/transforms/moving_average.py new file mode 100644 index 00000000..ee1eed9e --- /dev/null +++ b/zipline/finance/transforms/moving_average.py @@ -0,0 +1,92 @@ +import pandas +from datetime import timedelta +from collections import defaultdict + +from zipline.messaging import BaseTransform + +class MovingAverageTransform(BaseTransform): + + def init(self, daycount=3): + self.daycount = daycount + self.by_sid = defaultdict(MovingAverage) + + def transform(self, event): + cur = self.by_sid(event.sid) + cur.update(event) + self.state['value'] = cur.vwap + return self.state + +class MovingAverage(object): + + def __init__(self, daycount): + self.window = EventWindow(daycount) + self.total = 0.0 + self.average = 0.0 + + def update(self, event): + self.window.update(event) + + self.total += event.price + + for dropped in self.window.dropped_ticks: + self.total -= dropped.price + + if len(self.window.ticks) > 0: + self.average = self.total / len(self.window.ticks) + else: + self.average = 0.0 + +class EventWindow(object): + def __init__(self, daycount): + self.ticks = [] + self.dropped_ticks = [] + self.delta = timedelta(days=daycount) + + def update(self, event): + # add new event + self.ticks.append(event) + # determine which events are expired + last_date = event['dt'] + first_date = last_date - self.delta + + self.dropped_ticks = [] + for tick in self.ticks: + if tick['dt'] <= first_date: + self.dropped_ticks.append(tick) + + # remove the expired events + slice_index = len(self.dropped_ticks) + self.ticks = self.ticks[slice_index:] + +# ------------------------------ +# Experimental +# ------------------------------ + +class EventHistory(object): + + def __init__(self, daycount): + self.ticks = [] + self.dropped_ticks = [] + self.frame = pandas.DataFrame() + self.delta = timedelta(days=daycount) + + def update(self, event): + self.ticks.append(event.__dict__) + self.last_date = event['dt'] + self.first_date = self.last_date - self.delta + + # determine which events are expired + self.dropped_ticks = [] + for tick in self.ticks: + if tick['dt'] < self.first_date: + self.dropped_ticks.append(tick) + + # remove the expired events + slice_index = len(self.dropped_ticks) + self.ticks = self.ticks[slice_index:] + self.frame = pandas.DataFrame( + self.ticks + ) + self.frame.index = self.frame['dt'] + + diff --git a/zipline/finance/transforms/returns.py b/zipline/finance/transforms/returns.py new file mode 100644 index 00000000..3c258bee --- /dev/null +++ b/zipline/finance/transforms/returns.py @@ -0,0 +1,44 @@ +import pandas +from datetime import timedelta +from collections import defaultdict + +from zipline.messaging import BaseTransform + +class WindowTransform(BaseTransform): + + def init(self, daycount=3): + self.daycount = daycount + self.by_sid = defaultdict(DailyReturns) + + def transform(self, event): + cur = self.by_sid(event.sid) + cur.update(event) + self.state['value'] = cur.vwap + return self.state + +class ReturnsFromPriorClose(object): + """ + Calculates a security's returns since the previous close, using the + current price. + """ + + def __init__(self): + self.last_close = None + self.last_event = None + self.returns = 0.0 + + def update(self, event): + next_close = None + if self.last_close: + change = event.price - self.last_close.price + self.returns = change / self.last_close.price + + if self.last_event: + if self.last_event.dt.day != event.dt.day: + # the current event is from the day after + # the last event. Therefore the last event was + # the last close + self.last_close = self.last_event + + # the current event is now the last_event + self.last_event = event \ No newline at end of file diff --git a/zipline/finance/transforms.py b/zipline/finance/transforms/vwap.py similarity index 53% rename from zipline/finance/transforms.py rename to zipline/finance/transforms/vwap.py index cdfcbdc8..4bf18b4c 100644 --- a/zipline/finance/transforms.py +++ b/zipline/finance/transforms/vwap.py @@ -1,8 +1,9 @@ +import pandas from datetime import timedelta -from itertools import ifilter from collections import defaultdict from zipline.messaging import BaseTransform +from zipline.finance.transforms.moving_average import EventWindow, EventHistory class VWAPTransform(BaseTransform): @@ -15,53 +16,72 @@ class VWAPTransform(BaseTransform): cur.update(event) self.state['value'] = cur.vwap return self.state - + class DailyVWAP: """A class that tracks the volume weighted average price based on tick updates.""" def __init__(self, daycount=3): - self.ticks = [] - self.dropped_ticks = [] + self.window = EventWindow(daycount) self.flux = 0.0 self.volume = 0 - self.lastTick = None self.vwap = 0.0 self.delta = timedelta(days=daycount) - + def update(self, event): - - self.ticks.append(event) + + # update the event window + self.window.update(event) + + # add the current event's flux and volume to the tracker flux, volume = self.calculate_flux([event]) self.flux += flux self.volume += volume - - self.last_date = event['dt'] - self.first_date = self.last_date - self.delta - #use a list comprehension to filter the ticks to those within - #desired day range. The dt properties are full datetime objects - #and provide overloads for arithmetic operations. - self.dropped_ticks = [] - for tick in self.ticks: - if tick['dt'] < self.first_date: - self.dropped_ticks.append(tick) - - slice_index = len(self.dropped_ticks) - self.ticks = self.ticks[slice_index:] - dropped_flux, dropped_volume = self.calculate_flux(self.dropped_ticks) - + # subract the expired events flux and volume from the tracker + dropped = self.window.dropped_ticks + dropped_flux, dropped_volume = self.calculate_flux(dropped) + self.flux -= dropped_flux self.volume -= dropped_volume - + if(self.volume != 0): self.vwap = self.flux / self.volume else: self.vwap = None - + def calculate_flux(self, ticks): flux = 0.0 volume = 0 for tick in ticks: flux += tick['volume'] * tick['price'] volume += tick['volume'] - return flux, volume \ No newline at end of file + return flux, volume + + +# ------------------------------ +# Experimental +# ------------------------------ + +class DailyVWAP_df(object): + + def __init__(self, daycount=3): + self.history = EventHistory(daycount) + self.vwap = None + + def update(self, event): + self.history.update(event) + frame = self.history.frame + + window = len(frame) + value = pandas.rolling_sum( + frame['price'] * frame['volume'], + window + ) + volume = pandas.rolling_sum( + frame['volume'], + window + ) + + vwap = value / volume + self.vwap = vwap[-1] + diff --git a/zipline/test/test_transforms.py b/zipline/test/test_transforms.py new file mode 100644 index 00000000..dd7031c9 --- /dev/null +++ b/zipline/test/test_transforms.py @@ -0,0 +1,61 @@ +from datetime import timedelta + +from unittest2 import TestCase +import zipline.test.factory as factory +from zipline.finance.transforms.vwap import DailyVWAP, DailyVWAP_df +from zipline.finance.transforms.returns import ReturnsFromPriorClose +from zipline.finance.transforms.moving_average import MovingAverage + + +class FinanceTestCase(TestCase): + def setUp(self): + self.trading_environment = factory.create_trading_environment() + def test_vwap(self): + + trade_history = factory.create_trade_history( + 133, + [10.0, 10.0, 10.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + self.trading_environment + ) + + vwap = DailyVWAP(daycount=2) + for trade in trade_history: + vwap.update(trade) + + self.assertEqual(vwap.vwap, 10.75) + + + def test_returns(self): + trade_history = factory.create_trade_history( + 133, + [10.0, 10.0, 10.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + self.trading_environment + ) + + returns = ReturnsFromPriorClose() + for trade in trade_history: + returns.update(trade) + + + self.assertEqual(returns.returns, .1) + + + def test_moving_average(self): + trade_history = factory.create_trade_history( + 133, + [10.0, 10.0, 10.0, 11.0], + [100, 100, 100, 300], + timedelta(days=1), + self.trading_environment + ) + + ma = MovingAverage(daycount=2) + for trade in trade_history: + ma.update(trade) + + + self.assertEqual(ma.average, 10.5) \ No newline at end of file From 602f70d0fe1bd69eddea6f7684d3ea6cf508b05d Mon Sep 17 00:00:00 2001 From: fawce Date: Tue, 8 May 2012 17:12:41 -0400 Subject: [PATCH 2/4] flattened the finance package. draft test for vwap in a complete zipline. --- .../moving_average.py => movingaverage.py} | 6 ++- zipline/finance/{transforms => }/returns.py | 0 zipline/finance/transforms/__init__.py | 0 zipline/finance/{transforms => }/vwap.py | 2 +- zipline/test/test_transforms.py | 46 ++++++++++++++++--- 5 files changed, 46 insertions(+), 8 deletions(-) rename zipline/finance/{transforms/moving_average.py => movingaverage.py} (92%) rename zipline/finance/{transforms => }/returns.py (100%) delete mode 100644 zipline/finance/transforms/__init__.py rename zipline/finance/{transforms => }/vwap.py (96%) diff --git a/zipline/finance/transforms/moving_average.py b/zipline/finance/movingaverage.py similarity index 92% rename from zipline/finance/transforms/moving_average.py rename to zipline/finance/movingaverage.py index ee1eed9e..3041014e 100644 --- a/zipline/finance/transforms/moving_average.py +++ b/zipline/finance/movingaverage.py @@ -13,7 +13,7 @@ class MovingAverageTransform(BaseTransform): def transform(self, event): cur = self.by_sid(event.sid) cur.update(event) - self.state['value'] = cur.vwap + self.state['value'] = cur.average return self.state class MovingAverage(object): @@ -37,6 +37,10 @@ class MovingAverage(object): self.average = 0.0 class EventWindow(object): + """ + Tracks a window of the event history. Use an instance to track the events + inside your window to efficiently calculate rolling statistics. + """ def __init__(self, daycount): self.ticks = [] self.dropped_ticks = [] diff --git a/zipline/finance/transforms/returns.py b/zipline/finance/returns.py similarity index 100% rename from zipline/finance/transforms/returns.py rename to zipline/finance/returns.py diff --git a/zipline/finance/transforms/__init__.py b/zipline/finance/transforms/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/zipline/finance/transforms/vwap.py b/zipline/finance/vwap.py similarity index 96% rename from zipline/finance/transforms/vwap.py rename to zipline/finance/vwap.py index 4bf18b4c..a3dfc99c 100644 --- a/zipline/finance/transforms/vwap.py +++ b/zipline/finance/vwap.py @@ -3,7 +3,7 @@ from datetime import timedelta from collections import defaultdict from zipline.messaging import BaseTransform -from zipline.finance.transforms.moving_average import EventWindow, EventHistory +from zipline.finance.movingaverage import EventWindow, EventHistory class VWAPTransform(BaseTransform): diff --git a/zipline/test/test_transforms.py b/zipline/test/test_transforms.py index dd7031c9..121803ff 100644 --- a/zipline/test/test_transforms.py +++ b/zipline/test/test_transforms.py @@ -1,15 +1,46 @@ from datetime import timedelta - +from collections import defaultdict from unittest2 import TestCase + import zipline.test.factory as factory -from zipline.finance.transforms.vwap import DailyVWAP, DailyVWAP_df -from zipline.finance.transforms.returns import ReturnsFromPriorClose -from zipline.finance.transforms.moving_average import MovingAverage +import zipline.util as qutil +from zipline.finance.vwap import DailyVWAP, VWAPTransform, DailyVWAP_df +from zipline.finance.returns import ReturnsFromPriorClose +from zipline.finance.movingaverage import MovingAverage +from zipline.lines import SimulatedTrading +from zipline.simulator import AddressAllocator, Simulator -class FinanceTestCase(TestCase): +allocator = AddressAllocator(1000) + +class ZiplineWithTransformsTestCase(TestCase): + leased_sockets = defaultdict(list) + + def setUp(self): + qutil.configure_logging() + self.trading_environment = factory.create_trading_environment() + self.zipline_test_config = { + 'allocator':allocator, + 'sid':133 + } + + def test_vwap_tnfm(self): + zipline = SimulatedTrading.create_test_zipline( + **self.zipline_test_config + ) + + vwap = VWAPTransform("vwap_10", daycount=10) + zipline.add_transform(vwap) + + zipline.simulate(blocking=True) + + self.assertTrue(zipline.sim.ready()) + self.assertFalse(zipline.sim.exception) + +class FinanceTransformsTestCase(TestCase): def setUp(self): self.trading_environment = factory.create_trading_environment() + def test_vwap(self): trade_history = factory.create_trade_history( @@ -58,4 +89,7 @@ class FinanceTestCase(TestCase): ma.update(trade) - self.assertEqual(ma.average, 10.5) \ No newline at end of file + self.assertEqual(ma.average, 10.5) + + + \ No newline at end of file From 81de47774fad60f56640e772a96a887ef0ffe250 Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 10 May 2012 13:51:02 -0400 Subject: [PATCH 3/4] fixed allocation issue --- zipline/finance/movingaverage.py | 3 +++ zipline/finance/vwap.py | 7 +++++-- zipline/lines.py | 2 ++ zipline/test/test_transforms.py | 2 ++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/zipline/finance/movingaverage.py b/zipline/finance/movingaverage.py index 3041014e..7c87d92a 100644 --- a/zipline/finance/movingaverage.py +++ b/zipline/finance/movingaverage.py @@ -15,6 +15,9 @@ class MovingAverageTransform(BaseTransform): cur.update(event) self.state['value'] = cur.average return self.state + + def create_vwap(self): + return DailyVWAP(self.daycount) class MovingAverage(object): diff --git a/zipline/finance/vwap.py b/zipline/finance/vwap.py index a3dfc99c..1120e946 100644 --- a/zipline/finance/vwap.py +++ b/zipline/finance/vwap.py @@ -9,18 +9,21 @@ class VWAPTransform(BaseTransform): def init(self, daycount=3): self.daycount = daycount - self.by_sid = defaultdict(DailyVWAP) + self.by_sid = defaultdict(self.create_vwap) def transform(self, event): cur = self.by_sid(event.sid) cur.update(event) self.state['value'] = cur.vwap return self.state + + def create_vwap(self): + return DailyVWAP(self.daycount) class DailyVWAP: """A class that tracks the volume weighted average price based on tick updates.""" - def __init__(self, daycount=3): + def __init__(self, daycount): self.window = EventWindow(daycount) self.flux = 0.0 self.volume = 0 diff --git a/zipline/lines.py b/zipline/lines.py index 26d01f67..932d9e2f 100644 --- a/zipline/lines.py +++ b/zipline/lines.py @@ -143,6 +143,8 @@ class SimulatedTrading(object): sockets[7], logging = qutil.LOGGER ) + + self.con.cancel_socket = self.allocator.lease(1)[0] # TODO: Not freeform self.con.manage( diff --git a/zipline/test/test_transforms.py b/zipline/test/test_transforms.py index 121803ff..e2fb3666 100644 --- a/zipline/test/test_transforms.py +++ b/zipline/test/test_transforms.py @@ -17,6 +17,8 @@ class ZiplineWithTransformsTestCase(TestCase): leased_sockets = defaultdict(list) def setUp(self): + # skip ahead 100 spots + allocator.lease(100) qutil.configure_logging() self.trading_environment = factory.create_trading_environment() self.zipline_test_config = { From add3f10b2e6e06a12a90adaf2af6096b087f32a6 Mon Sep 17 00:00:00 2001 From: fawce Date: Thu, 10 May 2012 15:13:00 -0400 Subject: [PATCH 4/4] removed expiremental pandas code --- zipline/finance/movingaverage.py | 32 -------------------------------- zipline/finance/vwap.py | 31 +------------------------------ zipline/test/test_transforms.py | 2 +- 3 files changed, 2 insertions(+), 63 deletions(-) diff --git a/zipline/finance/movingaverage.py b/zipline/finance/movingaverage.py index 7c87d92a..db495d9d 100644 --- a/zipline/finance/movingaverage.py +++ b/zipline/finance/movingaverage.py @@ -1,4 +1,3 @@ -import pandas from datetime import timedelta from collections import defaultdict @@ -64,36 +63,5 @@ class EventWindow(object): # remove the expired events slice_index = len(self.dropped_ticks) self.ticks = self.ticks[slice_index:] - -# ------------------------------ -# Experimental -# ------------------------------ - -class EventHistory(object): - - def __init__(self, daycount): - self.ticks = [] - self.dropped_ticks = [] - self.frame = pandas.DataFrame() - self.delta = timedelta(days=daycount) - - def update(self, event): - self.ticks.append(event.__dict__) - self.last_date = event['dt'] - self.first_date = self.last_date - self.delta - - # determine which events are expired - self.dropped_ticks = [] - for tick in self.ticks: - if tick['dt'] < self.first_date: - self.dropped_ticks.append(tick) - - # remove the expired events - slice_index = len(self.dropped_ticks) - self.ticks = self.ticks[slice_index:] - self.frame = pandas.DataFrame( - self.ticks - ) - self.frame.index = self.frame['dt'] diff --git a/zipline/finance/vwap.py b/zipline/finance/vwap.py index 1120e946..f409ce4d 100644 --- a/zipline/finance/vwap.py +++ b/zipline/finance/vwap.py @@ -3,7 +3,7 @@ from datetime import timedelta from collections import defaultdict from zipline.messaging import BaseTransform -from zipline.finance.movingaverage import EventWindow, EventHistory +from zipline.finance.movingaverage import EventWindow class VWAPTransform(BaseTransform): @@ -59,32 +59,3 @@ class DailyVWAP: flux += tick['volume'] * tick['price'] volume += tick['volume'] return flux, volume - - -# ------------------------------ -# Experimental -# ------------------------------ - -class DailyVWAP_df(object): - - def __init__(self, daycount=3): - self.history = EventHistory(daycount) - self.vwap = None - - def update(self, event): - self.history.update(event) - frame = self.history.frame - - window = len(frame) - value = pandas.rolling_sum( - frame['price'] * frame['volume'], - window - ) - volume = pandas.rolling_sum( - frame['volume'], - window - ) - - vwap = value / volume - self.vwap = vwap[-1] - diff --git a/zipline/test/test_transforms.py b/zipline/test/test_transforms.py index e2fb3666..6a2bf204 100644 --- a/zipline/test/test_transforms.py +++ b/zipline/test/test_transforms.py @@ -4,7 +4,7 @@ from unittest2 import TestCase import zipline.test.factory as factory import zipline.util as qutil -from zipline.finance.vwap import DailyVWAP, VWAPTransform, DailyVWAP_df +from zipline.finance.vwap import DailyVWAP, VWAPTransform from zipline.finance.returns import ReturnsFromPriorClose from zipline.finance.movingaverage import MovingAverage from zipline.lines import SimulatedTrading