From 5f86ee72ee6cdce59a19b8f4c9af7e55446ed1f7 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 14:49:01 -0400 Subject: [PATCH 1/9] MAINT: Move search day distance function to module level. --- zipline/finance/risk.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index 9e158e85..04c40b1d 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -173,6 +173,18 @@ def alpha(algorithm_period_return, treasury_period_return, (treasury_period_return + beta * (benchmark_period_returns - treasury_period_return)) +########################### +# End Risk Metric Section # +########################### + + +def search_day_distance(end_date, dt): + tdd = trading.environment.trading_day_distance(dt, end_date) + if tdd is None: + return None + assert tdd >= 0 + return tdd + class RiskMetricsBase(object): def __init__(self, start_date, end_date, returns): @@ -438,7 +450,7 @@ class RiskMetricsBase(object): rate = self.get_treasury_rate(prev_day) if rate is not None: search_day = prev_day - search_dist = self.search_day_distance(prev_day) + search_dist = search_day_distance(self.end_date, prev_day) break if search_day: @@ -464,13 +476,6 @@ that date doesn't exceed treasury history range." ) raise Exception(message) - def search_day_distance(self, dt): - tdd = trading.environment.trading_day_distance(dt, self.end_date) - if tdd is None: - return None - assert tdd >= 0 - return tdd - def get_treasury_rate(self, day): rate = None From b461c0d91c4f0b94b4f0b54b38ce973660cda454 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 14:53:50 -0400 Subject: [PATCH 2/9] MAINT: Move get_treasury_rate to risk module level. --- zipline/finance/risk.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index 04c40b1d..3d828d0c 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -178,6 +178,21 @@ def alpha(algorithm_period_return, treasury_period_return, ########################### +def get_treasury_rate(treasury_curves, treasury_duration, day): + rate = None + + curve = treasury_curves[day] + # 1month note data begins in 8/2001, + # so we can use 3month instead. + idx = TREASURY_DURATIONS.index(treasury_duration) + for duration in TREASURY_DURATIONS[idx:]: + rate = curve[duration] + if rate is not None: + break + + return rate + + def search_day_distance(end_date, dt): tdd = trading.environment.trading_day_distance(dt, end_date) if tdd is None: @@ -435,7 +450,9 @@ class RiskMetricsBase(object): search_day = None if end_day in self.treasury_curves: - rate = self.get_treasury_rate(end_day) + rate = get_treasury_rate(self.treasury_curves, + self.treasury_duration, + end_day) if rate is not None: search_day = end_day @@ -447,7 +464,9 @@ class RiskMetricsBase(object): # Find rightmost value less than or equal to end_day i = bisect.bisect_right(search_days, end_day) for prev_day in search_days[i - 1::-1]: - rate = self.get_treasury_rate(prev_day) + rate = get_treasury_rate(self.treasury_curves, + self.treasury_duration, + prev_day) if rate is not None: search_day = prev_day search_dist = search_day_distance(self.end_date, prev_day) @@ -476,20 +495,6 @@ that date doesn't exceed treasury history range." ) raise Exception(message) - def get_treasury_rate(self, day): - rate = None - - curve = self.treasury_curves[day] - # 1month note data begins in 8/2001, - # so we can use 3month instead. - idx = TREASURY_DURATIONS.index(self.treasury_duration) - for duration in TREASURY_DURATIONS[idx:]: - rate = curve[duration] - if rate is not None: - break - - return rate - class RiskMetricsIterative(RiskMetricsBase): """Iterative version of RiskMetrics. From 8ea52e04211b779e2b7db3112b0a702752b5a7b7 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 14:57:20 -0400 Subject: [PATCH 3/9] MAINT: Factor out start and end date choose_treasury parameters. Preparing for move of method to a module level function. --- zipline/finance/risk.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index 3d828d0c..ff1bb383 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -235,7 +235,10 @@ class RiskMetricsBase(object): self.benchmark_returns) self.algorithm_volatility = self.calculate_volatility( self.algorithm_returns) - self.treasury_period_return = self.choose_treasury() + self.treasury_period_return = self.choose_treasury( + self.start_date, + self.end_date + ) self.sharpe = self.calculate_sharpe() self.sortino = self.calculate_sortino() self.information = self.calculate_information() @@ -423,8 +426,8 @@ class RiskMetricsBase(object): return 1.0 - math.exp(max_drawdown) - def choose_treasury(self): - td = self.end_date - self.start_date + def choose_treasury(self, start_date, end_date): + td = end_date - start_date if td.days <= 31: self.treasury_duration = '1month' elif td.days <= 93: @@ -446,7 +449,7 @@ class RiskMetricsBase(object): else: self.treasury_duration = '30year' - end_day = self.end_date.replace(hour=0, minute=0, second=0) + end_day = end_date.replace(hour=0, minute=0, second=0) search_day = None if end_day in self.treasury_curves: @@ -469,7 +472,7 @@ class RiskMetricsBase(object): prev_day) if rate is not None: search_day = prev_day - search_dist = search_day_distance(self.end_date, prev_day) + search_dist = search_day_distance(end_date, prev_day) break if search_day: @@ -478,7 +481,7 @@ class RiskMetricsBase(object): message = "No rate within 1 trading day of end date = \ {dt} and term = {term}. Using {search_day}. Check that date doesn't exceed \ treasury history range." - message = message.format(dt=self.end_date, + message = message.format(dt=end_date, term=self.treasury_duration, search_day=search_day) log.warn(message) @@ -490,7 +493,7 @@ treasury history range." message = "No rate for end date = {dt} and term = {term}. Check \ that date doesn't exceed treasury history range." message = message.format( - dt=self.end_date, + dt=end_date, term=self.treasury_duration ) raise Exception(message) @@ -566,7 +569,10 @@ algorithm_returns ({algo_count}) in range {start} : {end}" self.calculate_volatility(self.benchmark_returns)) self.algorithm_volatility.append( self.calculate_volatility(self.algorithm_returns)) - self.treasury_period_return = self.choose_treasury() + self.treasury_period_return = self.choose_treasury( + self.start_date, + self.end_date + ) self.excess_returns.append( self.algorithm_period_returns[-1] - self.treasury_period_return) self.beta.append(self.calculate_beta()[0]) From 39038131dbd62222a717240f7176f8ccc9023b5c Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 15:03:20 -0400 Subject: [PATCH 4/9] MAINT: Remove saving of treasury duration. The treasury_duration member in RiskMetrics is never used except for in unit tests. Remove the saving of treasury_duration in preparation for the move of the choose_treasury method out of the RiskMetrics classes. Down the line, if we do restore the sanving of treasury_duration, choose_treasury can return a tuple that includes treasury_duration instead of just returning the rate. --- tests/test_risk_compare_batch_iterative.py | 6 ----- zipline/finance/risk.py | 28 +++++++++++----------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/tests/test_risk_compare_batch_iterative.py b/tests/test_risk_compare_batch_iterative.py index 4de3fab3..7ae27d86 100644 --- a/tests/test_risk_compare_batch_iterative.py +++ b/tests/test_risk_compare_batch_iterative.py @@ -88,12 +88,6 @@ class RiskCompareIterativeToBatch(unittest.TestCase): self.assertEqual( risk_metrics_original.end_date, risk_metrics_refactor.end_date) - self.assertEqual( - risk_metrics_original.treasury_duration, - risk_metrics_refactor.treasury_duration) - self.assertEqual( - risk_metrics_original.treasury_curve, - risk_metrics_refactor.treasury_curve) self.assertEqual( risk_metrics_original.treasury_period_return, risk_metrics_refactor.treasury_period_return) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index ff1bb383..2498f6f4 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -429,32 +429,32 @@ class RiskMetricsBase(object): def choose_treasury(self, start_date, end_date): td = end_date - start_date if td.days <= 31: - self.treasury_duration = '1month' + treasury_duration = '1month' elif td.days <= 93: - self.treasury_duration = '3month' + treasury_duration = '3month' elif td.days <= 186: - self.treasury_duration = '6month' + treasury_duration = '6month' elif td.days <= 366: - self.treasury_duration = '1year' + treasury_duration = '1year' elif td.days <= 365 * 2 + 1: - self.treasury_duration = '2year' + treasury_duration = '2year' elif td.days <= 365 * 3 + 1: - self.treasury_duration = '3year' + treasury_duration = '3year' elif td.days <= 365 * 5 + 2: - self.treasury_duration = '5year' + treasury_duration = '5year' elif td.days <= 365 * 7 + 2: - self.treasury_duration = '7year' + treasury_duration = '7year' elif td.days <= 365 * 10 + 2: - self.treasury_duration = '10year' + treasury_duration = '10year' else: - self.treasury_duration = '30year' + treasury_duration = '30year' end_day = end_date.replace(hour=0, minute=0, second=0) search_day = None if end_day in self.treasury_curves: rate = get_treasury_rate(self.treasury_curves, - self.treasury_duration, + treasury_duration, end_day) if rate is not None: search_day = end_day @@ -468,7 +468,7 @@ class RiskMetricsBase(object): i = bisect.bisect_right(search_days, end_day) for prev_day in search_days[i - 1::-1]: rate = get_treasury_rate(self.treasury_curves, - self.treasury_duration, + treasury_duration, prev_day) if rate is not None: search_day = prev_day @@ -482,7 +482,7 @@ class RiskMetricsBase(object): {dt} and term = {term}. Using {search_day}. Check that date doesn't exceed \ treasury history range." message = message.format(dt=end_date, - term=self.treasury_duration, + term=treasury_duration, search_day=search_day) log.warn(message) @@ -494,7 +494,7 @@ treasury history range." that date doesn't exceed treasury history range." message = message.format( dt=end_date, - term=self.treasury_duration + term=treasury_duration ) raise Exception(message) From 2e603fa9367afbf6f64a0336fd7df299fdf130fe Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 15:07:37 -0400 Subject: [PATCH 5/9] MAINT: Factor out treasury_curve argument from risk choose_benchmark Move the reference to self.treasury_curve to a parameter, on the path of making this method a module level function. --- zipline/finance/risk.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index 2498f6f4..b01f6679 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -236,6 +236,7 @@ class RiskMetricsBase(object): self.algorithm_volatility = self.calculate_volatility( self.algorithm_returns) self.treasury_period_return = self.choose_treasury( + self.treasury_curves, self.start_date, self.end_date ) @@ -426,7 +427,7 @@ class RiskMetricsBase(object): return 1.0 - math.exp(max_drawdown) - def choose_treasury(self, start_date, end_date): + def choose_treasury(self, treasury_curves, start_date, end_date): td = end_date - start_date if td.days <= 31: treasury_duration = '1month' @@ -452,8 +453,8 @@ class RiskMetricsBase(object): end_day = end_date.replace(hour=0, minute=0, second=0) search_day = None - if end_day in self.treasury_curves: - rate = get_treasury_rate(self.treasury_curves, + if end_day in treasury_curves: + rate = get_treasury_rate(treasury_curves, treasury_duration, end_day) if rate is not None: @@ -467,7 +468,7 @@ class RiskMetricsBase(object): # Find rightmost value less than or equal to end_day i = bisect.bisect_right(search_days, end_day) for prev_day in search_days[i - 1::-1]: - rate = get_treasury_rate(self.treasury_curves, + rate = get_treasury_rate(treasury_curves, treasury_duration, prev_day) if rate is not None: @@ -487,7 +488,7 @@ treasury history range." log.warn(message) if search_day: - self.treasury_curve = self.treasury_curves[search_day] + treasury_curves[search_day] return rate * (td.days + 1) / 365 message = "No rate for end date = {dt} and term = {term}. Check \ @@ -570,6 +571,7 @@ algorithm_returns ({algo_count}) in range {start} : {end}" self.algorithm_volatility.append( self.calculate_volatility(self.algorithm_returns)) self.treasury_period_return = self.choose_treasury( + self.treasury_curves, self.start_date, self.end_date ) From 0cc953e00f7d1d5a40801d3474543330d0e49a25 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 15:15:26 -0400 Subject: [PATCH 6/9] MAINT: Move choose_treasury method from risk metrics class to module. --- zipline/finance/risk.py | 149 ++++++++++++++++++++-------------------- 1 file changed, 75 insertions(+), 74 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index b01f6679..a32a14ca 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -201,6 +201,79 @@ def search_day_distance(end_date, dt): return tdd +def choose_treasury(treasury_curves, start_date, end_date): + td = end_date - start_date + if td.days <= 31: + treasury_duration = '1month' + elif td.days <= 93: + treasury_duration = '3month' + elif td.days <= 186: + treasury_duration = '6month' + elif td.days <= 366: + treasury_duration = '1year' + elif td.days <= 365 * 2 + 1: + treasury_duration = '2year' + elif td.days <= 365 * 3 + 1: + treasury_duration = '3year' + elif td.days <= 365 * 5 + 2: + treasury_duration = '5year' + elif td.days <= 365 * 7 + 2: + treasury_duration = '7year' + elif td.days <= 365 * 10 + 2: + treasury_duration = '10year' + else: + treasury_duration = '30year' + + end_day = end_date.replace(hour=0, minute=0, second=0) + search_day = None + + if end_day in treasury_curves: + rate = get_treasury_rate(treasury_curves, + treasury_duration, + end_day) + if rate is not None: + search_day = end_day + + if not search_day: + # in case end date is not a trading day or there is no treasury + # data, search for the previous day with an interest rate. + search_days = treasury_curves.keys() + + # Find rightmost value less than or equal to end_day + i = bisect.bisect_right(search_days, end_day) + for prev_day in search_days[i - 1::-1]: + rate = get_treasury_rate(treasury_curves, + treasury_duration, + prev_day) + if rate is not None: + search_day = prev_day + search_dist = search_day_distance(end_date, prev_day) + break + + if search_day: + if (search_dist is None or search_dist > 1) and \ + search_days[0] <= end_day <= search_days[-1]: + message = "No rate within 1 trading day of end date = \ +{dt} and term = {term}. Using {search_day}. Check that date doesn't exceed \ +treasury history range." + message = message.format(dt=end_date, + term=treasury_duration, + search_day=search_day) + log.warn(message) + + if search_day: + treasury_curves[search_day] + return rate * (td.days + 1) / 365 + + message = "No rate for end date = {dt} and term = {term}. Check \ +that date doesn't exceed treasury history range." + message = message.format( + dt=end_date, + term=treasury_duration + ) + raise Exception(message) + + class RiskMetricsBase(object): def __init__(self, start_date, end_date, returns): @@ -235,7 +308,7 @@ class RiskMetricsBase(object): self.benchmark_returns) self.algorithm_volatility = self.calculate_volatility( self.algorithm_returns) - self.treasury_period_return = self.choose_treasury( + self.treasury_period_return = choose_treasury( self.treasury_curves, self.start_date, self.end_date @@ -427,78 +500,6 @@ class RiskMetricsBase(object): return 1.0 - math.exp(max_drawdown) - def choose_treasury(self, treasury_curves, start_date, end_date): - td = end_date - start_date - if td.days <= 31: - treasury_duration = '1month' - elif td.days <= 93: - treasury_duration = '3month' - elif td.days <= 186: - treasury_duration = '6month' - elif td.days <= 366: - treasury_duration = '1year' - elif td.days <= 365 * 2 + 1: - treasury_duration = '2year' - elif td.days <= 365 * 3 + 1: - treasury_duration = '3year' - elif td.days <= 365 * 5 + 2: - treasury_duration = '5year' - elif td.days <= 365 * 7 + 2: - treasury_duration = '7year' - elif td.days <= 365 * 10 + 2: - treasury_duration = '10year' - else: - treasury_duration = '30year' - - end_day = end_date.replace(hour=0, minute=0, second=0) - search_day = None - - if end_day in treasury_curves: - rate = get_treasury_rate(treasury_curves, - treasury_duration, - end_day) - if rate is not None: - search_day = end_day - - if not search_day: - # in case end date is not a trading day or there is no treasury - # data, search for the previous day with an interest rate. - search_days = self.treasury_curves.keys() - - # Find rightmost value less than or equal to end_day - i = bisect.bisect_right(search_days, end_day) - for prev_day in search_days[i - 1::-1]: - rate = get_treasury_rate(treasury_curves, - treasury_duration, - prev_day) - if rate is not None: - search_day = prev_day - search_dist = search_day_distance(end_date, prev_day) - break - - if search_day: - if (search_dist is None or search_dist > 1) and \ - search_days[0] <= end_day <= search_days[-1]: - message = "No rate within 1 trading day of end date = \ -{dt} and term = {term}. Using {search_day}. Check that date doesn't exceed \ -treasury history range." - message = message.format(dt=end_date, - term=treasury_duration, - search_day=search_day) - log.warn(message) - - if search_day: - treasury_curves[search_day] - return rate * (td.days + 1) / 365 - - message = "No rate for end date = {dt} and term = {term}. Check \ -that date doesn't exceed treasury history range." - message = message.format( - dt=end_date, - term=treasury_duration - ) - raise Exception(message) - class RiskMetricsIterative(RiskMetricsBase): """Iterative version of RiskMetrics. @@ -570,7 +571,7 @@ algorithm_returns ({algo_count}) in range {start} : {end}" self.calculate_volatility(self.benchmark_returns)) self.algorithm_volatility.append( self.calculate_volatility(self.algorithm_returns)) - self.treasury_period_return = self.choose_treasury( + self.treasury_period_return = choose_treasury( self.treasury_curves, self.start_date, self.end_date From cf60eeb46b2b79aa48a054b6dac48f3256342073 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 15:36:01 -0400 Subject: [PATCH 7/9] MAINT: Refactors out selection of treasury duration its own function. Instead of having the duration selection logic in choose_treasury, break out duration specific logig into another function. --- zipline/finance/risk.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index a32a14ca..4f91abd2 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -201,7 +201,7 @@ def search_day_distance(end_date, dt): return tdd -def choose_treasury(treasury_curves, start_date, end_date): +def select_treasury_duration(start_date, end_date): td = end_date - start_date if td.days <= 31: treasury_duration = '1month' @@ -224,6 +224,11 @@ def choose_treasury(treasury_curves, start_date, end_date): else: treasury_duration = '30year' + return treasury_duration + + +def choose_treasury(treasury_curves, start_date, end_date): + treasury_duration = select_treasury_duration(start_date, end_date) end_day = end_date.replace(hour=0, minute=0, second=0) search_day = None @@ -262,7 +267,7 @@ treasury history range." log.warn(message) if search_day: - treasury_curves[search_day] + td = end_date - start_date return rate * (td.days + 1) / 365 message = "No rate for end date = {dt} and term = {term}. Check \ From bb3e9727dc75339e8d117a5b80f826db98136fe8 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 15:47:14 -0400 Subject: [PATCH 8/9] MAINT: Mask treasury_curves used by risk metrics to period range. So that calculations that leverage the range of the treasury_curves, like `pd.Series.searchsorted` will not overshoot the 'end' of the range we are calculating risk metrics. --- zipline/finance/risk.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index 4f91abd2..29a54c6d 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -282,7 +282,11 @@ that date doesn't exceed treasury history range." class RiskMetricsBase(object): def __init__(self, start_date, end_date, returns): - self.treasury_curves = trading.environment.treasury_curves + treasury_curves = trading.environment.treasury_curves + mask = ((treasury_curves.index >= start_date) & + (treasury_curves.index <= end_date)) + + self.treasury_curves = treasury_curves[mask] self.start_date = start_date self.end_date = end_date From a66d6866f5a11ebd08df990fe5367db0e4fb6ca4 Mon Sep 17 00:00:00 2001 From: Eddie Hebert Date: Wed, 3 Apr 2013 15:55:04 -0400 Subject: [PATCH 9/9] MAINT: Use searchsorted instead of bisect on treasury curves. Remove use of .keys() and creation of a new list of the curve Series's values. --- zipline/finance/risk.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zipline/finance/risk.py b/zipline/finance/risk.py index 29a54c6d..7059ffd6 100644 --- a/zipline/finance/risk.py +++ b/zipline/finance/risk.py @@ -58,7 +58,6 @@ Risk Report import logbook import datetime import math -import bisect import numpy as np import numpy.linalg as la from dateutil.relativedelta import relativedelta @@ -242,10 +241,10 @@ def choose_treasury(treasury_curves, start_date, end_date): if not search_day: # in case end date is not a trading day or there is no treasury # data, search for the previous day with an interest rate. - search_days = treasury_curves.keys() + search_days = treasury_curves.index # Find rightmost value less than or equal to end_day - i = bisect.bisect_right(search_days, end_day) + i = search_days.searchsorted(end_day) for prev_day in search_days[i - 1::-1]: rate = get_treasury_rate(treasury_curves, treasury_duration,