diff --git a/tests/risk/__init__.py b/tests/risk/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/risk/answer_key.py b/tests/risk/answer_key.py new file mode 100644 index 00000000..7f3931ba --- /dev/null +++ b/tests/risk/answer_key.py @@ -0,0 +1,163 @@ +# +# Copyright 2013 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import os + +import numpy as np +import xlrd +import requests + + +def col_letter_to_index(col_letter): + # Only supports single letter, + # but answer key doesn't need multi-letter, yet. + return ord(col_letter) - 65 + +DIR = os.path.dirname(os.path.realpath(__file__)) + +ANSWER_KEY_CHECKSUMS_PATH = os.path.join(DIR, 'risk-answer-key-checksums') +ANSWER_KEY_CHECKSUMS = open(ANSWER_KEY_CHECKSUMS_PATH, 'r').read().splitlines() + +ANSWER_KEY_PATH = os.path.join(DIR, 'risk-answer-key.xls') + +ANSWER_KEY_EXISTS = os.path.exists(ANSWER_KEY_PATH) + +ANSWER_KEY_DL_TEMPLATE = """ +https://s3.amazonaws.com/zipline-test-data/risk/{md5}+/risk-answer-key.xls +""".strip() + + +def ensure_latest_answer_key(): + """ + Get the latest answer key from a publically available location. + + Logic for determining what and when to download is as such: + + - If there is no local xls file, then get the lastest answer key, + as defined by the last row in the checksum file. + - If there is a local xls file: + -- If the xls's checksum is in the checksum file: + --- If the xls's checksum does not match the latest, then grab the + the latest checksum and replace the local checksum file. + --- If the xls's checksum matches the latest, then skip download, and + use the local xls as a cached copy. + -- If the xls's checksum is not in the checksum file, then leave the + local file alone, assuming that the local xls's md5 is not in the list due + to local modifications during development. + + It is possible that md5's could collide, if that is ever case, we should + then find an alternative naming scheme. + + The xls answer sheet is not kept in SCM, because its size is on the order + of 20MB, and every edit would increase the repo size. + + xlsl and ods have smaller outputs and could be more friendly to SCM, but: + - not using xlsl, because currently the xlsl that is generated by + LibreOffice is not readable by the xldr module. + - not using ods, because of the lack of a module as facile as xldr for + extracting the data from the ods format. + """ + + answer_key_dl_checksum = None + + local_answer_key_exists = os.path.exists(ANSWER_KEY_PATH) + if local_answer_key_exists: + with open(ANSWER_KEY_PATH, 'r') as f: + md5 = hashlib.md5() + while True: + buf = f.read(1024) + if not buf: + break + md5.update(buf) + local_hash = md5.hexdigest() + + if local_hash in ANSWER_KEY_CHECKSUMS: + # Assume previously downloaded version. + # Check for latest. + if local_hash != ANSWER_KEY_CHECKSUMS[-1]: + # More recent checksum, download + answer_key_dl_checksum = ANSWER_KEY_CHECKSUMS[-1] + else: + # Assume local copy that is being developed on + answer_key_dl_checksum = None + + if answer_key_dl_checksum: + res = requests.get( + ANSWER_KEY_DL_TEMPLATE.format(md5=answer_key_dl_checksum)) + with open(ANSWER_KEY_PATH, 'w') as f: + f.write(res.content) + +# Get latest answer key on load. +ensure_latest_answer_key() + + +class DataIndex(object): + """ + Coordinates for the spreadsheet, using the values as seen in the notebook. + The python-excel libraries use 0 index, while the spreadsheet in a GUI + uses a 1 index. + """ + def __init__(self, sheet_name, col, row_start, row_end): + self.sheet_name = sheet_name + self.col = col + self.row_start = row_start + self.row_end = row_end + + @property + def col_index(self): + return col_letter_to_index(self.col) + + @property + def row_start_index(self): + return self.row_start - 1 + + @property + def row_end_index(self): + return self.row_end - 1 + + +class AnswerKey(object): + + RETURNS = DataIndex('Sim', 'D', 4, 255) + + # Below matches the inconsistent capitalization in spreadsheet + BENCHMARK_PERIOD_RETURNS = { + 'Monthly': DataIndex('s_p', 'P', 8, 19), + '3-Month': DataIndex('s_p', 'Q', 10, 19), + '6-month': DataIndex('s_p', 'R', 13, 19), + 'year': DataIndex('s_p', 'S', 19, 19), + } + + BENCHMARK_PERIOD_VOLATILITY = { + 'Monthly': DataIndex('s_p', 'T', 8, 19), + '3-Month': DataIndex('s_p', 'U', 10, 19), + '6-month': DataIndex('s_p', 'V', 13, 19), + 'year': DataIndex('s_p', 'W', 19, 19), + } + + def __init__(self): + self.workbook = xlrd.open_workbook(ANSWER_KEY_PATH) + + self.sheets = {} + self.sheets['Sim'] = self.workbook.sheet_by_name('Sim') + self.sheets['s_p'] = self.workbook.sheet_by_name('s_p') + + def get_values(self, data_index, decimal=4): + return [np.round(x, decimal) for x in + self.sheets[data_index.sheet_name].col_values( + data_index.col_index, + data_index.row_start_index, + data_index.row_end_index + 1)] diff --git a/tests/risk/risk-answer-key-checksums b/tests/risk/risk-answer-key-checksums new file mode 100644 index 00000000..f1ec4cc6 --- /dev/null +++ b/tests/risk/risk-answer-key-checksums @@ -0,0 +1 @@ +3ac0773c4be4e9e5bacd9c6fa0e03e15 diff --git a/tests/test_risk.py b/tests/risk/test_risk.py similarity index 84% rename from tests/test_risk.py rename to tests/risk/test_risk.py index 52edef7d..8d816259 100644 --- a/tests/test_risk.py +++ b/tests/risk/test_risk.py @@ -16,12 +16,19 @@ import unittest import datetime import calendar +import numpy as np import pytz import zipline.finance.risk as risk from zipline.utils import factory from zipline.finance.trading import SimulationParameters +from . answer_key import AnswerKey + +ANSWER_KEY = AnswerKey() + +RETURNS = ANSWER_KEY.get_values(AnswerKey.RETURNS) + class TestRisk(unittest.TestCase): @@ -93,44 +100,26 @@ class TestRisk(unittest.TestCase): def test_benchmark_returns_06(self): returns = factory.create_returns_from_range(self.sim_params) metrics = risk.RiskReport(returns, self.sim_params) + answer_key_month_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_RETURNS['Monthly']) self.assertEqual([round(x.benchmark_period_returns, 4) for x in metrics.month_periods], - [0.0255, - 0.0005, - 0.0111, - 0.0122, - -0.0309, - 0.0001, - 0.0051, - 0.0213, - 0.0246, - 0.0315, - 0.0165, - 0.0126]) + answer_key_month_periods) + answer_key_three_month_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_RETURNS['3-Month']) self.assertEqual([round(x.benchmark_period_returns, 4) for x in metrics.three_month_periods], - [0.0373, - 0.0239, - -0.0083, - -0.0191, - -0.0259, - 0.0266, - 0.0517, - 0.0793, - 0.0743, - 0.0617]) + answer_key_three_month_periods) + answer_key_six_month_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_RETURNS['6-month']) self.assertEqual([round(x.benchmark_period_returns, 4) for x in metrics.six_month_periods], - [0.0176, - -0.0027, - 0.0181, - 0.0316, - 0.0514, - 0.1028, - 0.1166]) + answer_key_six_month_periods) + answer_key_year_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_RETURNS['year']) self.assertEqual([round(x.benchmark_period_returns, 4) for x in metrics.year_periods], - [0.1362]) + answer_key_year_periods) def test_trading_days_06(self): returns = factory.create_returns_from_range(self.sim_params) @@ -143,47 +132,33 @@ class TestRisk(unittest.TestCase): def test_benchmark_volatility_06(self): returns = factory.create_returns_from_range(self.sim_params) metrics = risk.RiskReport(returns, self.sim_params) - self.assertEqual([round(x.benchmark_volatility, 3) + answer_key_month_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_VOLATILITY['Monthly'], + decimal=3) + self.assertEqual([np.round(x.benchmark_volatility, 3) for x in metrics.month_periods], - [0.031, - 0.026, - 0.024, - 0.025, - 0.037, - 0.047, - 0.039, - 0.022, - 0.023, - 0.021, - 0.025, - 0.019]) + answer_key_month_periods) - self.assertEqual([round(x.benchmark_volatility, 3) + answer_key_three_month_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_VOLATILITY['3-Month'], + decimal=3) + self.assertEqual([np.round(x.benchmark_volatility, 3) for x in metrics.three_month_periods], - [0.047, - 0.042, - 0.050, - 0.064, - 0.070, - 0.064, - 0.049, - 0.037, - 0.039, - 0.037]) + answer_key_three_month_periods) - self.assertEqual([round(x.benchmark_volatility, 3) + answer_key_six_month_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_VOLATILITY['6-month'], + decimal=3) + self.assertEqual([np.round(x.benchmark_volatility, 3) for x in metrics.six_month_periods], - [0.079, - 0.082, - 0.081, - 0.081, - 0.080, - 0.074, - 0.061]) + answer_key_six_month_periods) - self.assertEqual([round(x.benchmark_volatility, 3) + answer_key_year_periods = ANSWER_KEY.get_values( + AnswerKey.BENCHMARK_PERIOD_VOLATILITY['year'], + decimal=3) + self.assertEqual([np.round(x.benchmark_volatility, 3) for x in metrics.year_periods], - [0.100]) + answer_key_year_periods) def test_algorithm_returns_06(self): self.assertEqual([round(x.algorithm_period_returns, 3) @@ -834,256 +809,3 @@ class TestRisk(unittest.TestCase): ) self.assert_month(start_date.month, col[-1].end_date.month) self.assert_last_day(col[-1].end_date) - -RETURNS = [ - 0.0093, - -0.0193, - 0.0351, - 0.0396, - 0.0338, - -0.0211, - 0.0389, - 0.0326, - -0.0137, - -0.0411, - -0.0032, - 0.0149, - 0.0133, - 0.0348, - 0.042, - -0.0455, - 0.0262, - -0.0461, - 0.0021, - -0.0273, - -0.0429, - 0.0427, - -0.0104, - 0.0346, - -0.0311, - 0.0003, - 0.0211, - 0.0248, - -0.0215, - 0.004, - 0.0267, - 0.0029, - -0.0369, - 0.0057, - 0.0298, - -0.0179, - -0.0361, - -0.0401, - -0.0123, - -0.005, - 0.0203, - -0.041, - 0.0011, - 0.0118, - 0.0103, - -0.0184, - -0.0437, - 0.0411, - -0.0242, - -0.0054, - -0.0039, - -0.0273, - -0.0075, - 0.0064, - -0.0376, - 0.0424, - 0.0399, - 0.019, - 0.0236, - -0.0284, - -0.0341, - 0.0266, - 0.05, - 0.0069, - -0.0442, - -0.016, - 0.0173, - 0.0348, - -0.0404, - -0.0068, - -0.0376, - 0.0356, - 0.0043, - -0.0481, - -0.0134, - 0.0257, - 0.0442, - 0.0234, - 0.0394, - 0.0376, - -0.0147, - -0.0098, - 0.0474, - -0.0102, - 0.0138, - 0.0286, - 0.0347, - 0.0279, - -0.0067, - 0.0462, - -0.0432, - 0.0247, - 0.0174, - -0.0305, - -0.0317, - -0.0068, - 0.0264, - -0.0257, - -0.0328, - 0.0092, - 0.0288, - -0.002, - 0.0288, - 0.028, - -0.0093, - 0.0178, - -0.0365, - -0.0086, - -0.0133, - -0.0309, - 0.0473, - -0.0149, - 0.0378, - -0.0316, - -0.0292, - -0.0453, - -0.0451, - 0.0093, - 0.0397, - -0.0361, - -0.0168, - -0.0494, - -0.0143, - -0.0405, - -0.0349, - 0.0069, - 0.0378, - -0.0233, - -0.0492, - 0.018, - -0.0386, - 0.0339, - 0.0119, - 0.0454, - 0.0118, - -0.011, - -0.0254, - 0.0266, - -0.0366, - -0.0211, - 0.0399, - 0.0307, - 0.035, - -0.0402, - 0.0304, - -0.0031, - 0.0256, - 0.0134, - -0.0019, - -0.0235, - -0.0058, - -0.0117, - 0.0051, - -0.0451, - -0.0466, - -0.0124, - 0.0283, - -0.0499, - 0.0318, - -0.0028, - 0.0203, - 0.005, - 0.0085, - 0.0048, - 0.0277, - 0.0159, - -0.0149, - 0.035, - 0.0404, - -0.01, - 0.0377, - 0.0302, - 0.0046, - -0.0328, - -0.0469, - 0.0071, - -0.0382, - -0.0214, - 0.0429, - 0.0145, - -0.0279, - -0.0172, - 0.0423, - 0.041, - -0.0183, - 0.0137, - -0.0412, - -0.0348, - 0.0302, - 0.0248, - 0.0051, - -0.0298, - -0.0103, - -0.0333, - -0.0399, - 0.0485, - -0.0166, - 0.0384, - 0.0259, - -0.0163, - 0.0357, - 0.0308, - -0.0386, - 0.0481, - -0.0446, - -0.0282, - -0.0037, - 0.0202, - 0.0216, - 0.0113, - 0.0194, - 0.0392, - 0.0016, - 0.0268, - -0.0155, - -0.027, - 0.02, - 0.0216, - -0.0009, - 0.022, - 0.0, - 0.041, - 0.0133, - -0.0382, - 0.0495, - -0.0221, - -0.0329, - -0.0033, - -0.0089, - -0.0129, - -0.0252, - 0.048, - -0.0307, - -0.0357, - 0.0033, - -0.0412, - -0.0407, - 0.0455, - 0.0159, - -0.0051, - -0.0274, - -0.0213, - 0.0361, - 0.0051, - -0.0378, - 0.0084, - 0.0066, - -0.0103, - -0.0037, - 0.0478, - -0.0278] diff --git a/tests/test_risk_compare_batch_iterative.py b/tests/risk/test_risk_compare_batch_iterative.py similarity index 100% rename from tests/test_risk_compare_batch_iterative.py rename to tests/risk/test_risk_compare_batch_iterative.py