Merge reading of risk unit test expected results from a spreadsheet.

To help tentpole the risk results with an alternative implementation. Also, the spreadsheet provided is what the original answers were based from, reading from said spreadsheet should help prevent drift.
2026-08-03 12:40:47 +08:00 · 2013-07-18 14:31:21 -04:00
parent 940ddd22d3 e5e38a437b
commit 9a629d8781
5 changed files with 203 additions and 317 deletions
@@ -0,0 +1,163 @@
+#
+# Copyright 2013 Quantopian, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+import os
+
+import numpy as np
+import xlrd
+import requests
+
+
+def col_letter_to_index(col_letter):
+    # Only supports single letter,
+    # but answer key doesn't need multi-letter, yet.
+    return ord(col_letter) - 65
+
+DIR = os.path.dirname(os.path.realpath(__file__))
+
+ANSWER_KEY_CHECKSUMS_PATH = os.path.join(DIR, 'risk-answer-key-checksums')
+ANSWER_KEY_CHECKSUMS = open(ANSWER_KEY_CHECKSUMS_PATH, 'r').read().splitlines()
+
+ANSWER_KEY_PATH = os.path.join(DIR, 'risk-answer-key.xls')
+
+ANSWER_KEY_EXISTS = os.path.exists(ANSWER_KEY_PATH)
+
+ANSWER_KEY_DL_TEMPLATE = """
+https://s3.amazonaws.com/zipline-test-data/risk/{md5}+/risk-answer-key.xls
+""".strip()
+
+
+def ensure_latest_answer_key():
+    """
+    Get the latest answer key from a publically available location.
+
+    Logic for determining what and when to download is as such:
+
+    - If there is no local xls file, then get the lastest answer key,
+    as defined by the last row in the checksum file.
+    - If there is a local xls file:
+    -- If the xls's checksum is in the checksum file:
+    --- If the xls's checksum does not match the latest, then grab the
+    the latest checksum and replace the local checksum file.
+    --- If the xls's checksum matches the latest, then skip download, and
+    use the local xls as a cached copy.
+    -- If the xls's checksum is not in the checksum file, then leave the
+    local file alone, assuming that the local xls's md5 is not in the list due
+    to local modifications during development.
+
+    It is possible that md5's could collide, if that is ever case, we should
+    then find an alternative naming scheme.
+
+    The xls answer sheet is not kept in SCM, because its size is on the order
+    of 20MB, and every edit would increase the repo size.
+
+    xlsl and ods have smaller outputs and could be more friendly to SCM, but:
+    - not using xlsl, because currently the xlsl that is generated by
+    LibreOffice is not readable by the xldr module.
+    - not using ods, because of the lack of a module as facile as xldr for
+    extracting the data from the ods format.
+    """
+
+    answer_key_dl_checksum = None
+
+    local_answer_key_exists = os.path.exists(ANSWER_KEY_PATH)
+    if local_answer_key_exists:
+        with open(ANSWER_KEY_PATH, 'r') as f:
+            md5 = hashlib.md5()
+            while True:
+                buf = f.read(1024)
+                if not buf:
+                    break
+                md5.update(buf)
+        local_hash = md5.hexdigest()
+
+        if local_hash in ANSWER_KEY_CHECKSUMS:
+            # Assume previously downloaded version.
+            # Check for latest.
+            if local_hash != ANSWER_KEY_CHECKSUMS[-1]:
+                # More recent checksum, download
+                answer_key_dl_checksum = ANSWER_KEY_CHECKSUMS[-1]
+            else:
+                # Assume local copy that is being developed on
+                answer_key_dl_checksum = None
+
+    if answer_key_dl_checksum:
+        res = requests.get(
+            ANSWER_KEY_DL_TEMPLATE.format(md5=answer_key_dl_checksum))
+        with open(ANSWER_KEY_PATH, 'w') as f:
+            f.write(res.content)
+
+# Get latest answer key on load.
+ensure_latest_answer_key()
+
+
+class DataIndex(object):
+    """
+    Coordinates for the spreadsheet, using the values as seen in the notebook.
+    The python-excel libraries use 0 index, while the spreadsheet in a GUI
+    uses a 1 index.
+    """
+    def __init__(self, sheet_name, col, row_start, row_end):
+        self.sheet_name = sheet_name
+        self.col = col
+        self.row_start = row_start
+        self.row_end = row_end
+
+    @property
+    def col_index(self):
+        return col_letter_to_index(self.col)
+
+    @property
+    def row_start_index(self):
+        return self.row_start - 1
+
+    @property
+    def row_end_index(self):
+        return self.row_end - 1
+
+
+class AnswerKey(object):
+
+    RETURNS = DataIndex('Sim', 'D', 4, 255)
+
+    # Below matches the inconsistent capitalization in spreadsheet
+    BENCHMARK_PERIOD_RETURNS = {
+        'Monthly': DataIndex('s_p', 'P', 8, 19),
+        '3-Month': DataIndex('s_p', 'Q', 10, 19),
+        '6-month': DataIndex('s_p', 'R', 13, 19),
+        'year': DataIndex('s_p', 'S', 19, 19),
+    }
+
+    BENCHMARK_PERIOD_VOLATILITY = {
+        'Monthly': DataIndex('s_p', 'T', 8, 19),
+        '3-Month': DataIndex('s_p', 'U', 10, 19),
+        '6-month': DataIndex('s_p', 'V', 13, 19),
+        'year': DataIndex('s_p', 'W', 19, 19),
+    }
+
+    def __init__(self):
+        self.workbook = xlrd.open_workbook(ANSWER_KEY_PATH)
+
+        self.sheets = {}
+        self.sheets['Sim'] = self.workbook.sheet_by_name('Sim')
+        self.sheets['s_p'] = self.workbook.sheet_by_name('s_p')
+
+    def get_values(self, data_index, decimal=4):
+        return [np.round(x, decimal) for x in
+                self.sheets[data_index.sheet_name].col_values(
+                    data_index.col_index,
+                    data_index.row_start_index,
+                    data_index.row_end_index + 1)]
@@ -0,0 +1 @@
+3ac0773c4be4e9e5bacd9c6fa0e03e15
@@ -16,12 +16,19 @@
 import unittest
 import datetime
 import calendar
+import numpy as np
 import pytz
 import zipline.finance.risk as risk
 from zipline.utils import factory

 from zipline.finance.trading import SimulationParameters

+from . answer_key import AnswerKey
+
+ANSWER_KEY = AnswerKey()
+
+RETURNS = ANSWER_KEY.get_values(AnswerKey.RETURNS)
+

 class TestRisk(unittest.TestCase):

@@ -93,44 +100,26 @@ class TestRisk(unittest.TestCase):
    def test_benchmark_returns_06(self):
        returns = factory.create_returns_from_range(self.sim_params)
        metrics = risk.RiskReport(returns, self.sim_params)
+        answer_key_month_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_RETURNS['Monthly'])
        self.assertEqual([round(x.benchmark_period_returns, 4)
                          for x in metrics.month_periods],
-                         [0.0255,
-                          0.0005,
-                          0.0111,
-                          0.0122,
-                          -0.0309,
-                          0.0001,
-                          0.0051,
-                          0.0213,
-                          0.0246,
-                          0.0315,
-                          0.0165,
-                          0.0126])
+                         answer_key_month_periods)
+        answer_key_three_month_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_RETURNS['3-Month'])
        self.assertEqual([round(x.benchmark_period_returns, 4)
                          for x in metrics.three_month_periods],
-                         [0.0373,
-                          0.0239,
-                          -0.0083,
-                          -0.0191,
-                          -0.0259,
-                          0.0266,
-                          0.0517,
-                          0.0793,
-                          0.0743,
-                          0.0617])
+                         answer_key_three_month_periods)
+        answer_key_six_month_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_RETURNS['6-month'])
        self.assertEqual([round(x.benchmark_period_returns, 4)
                          for x in metrics.six_month_periods],
-                         [0.0176,
-                          -0.0027,
-                          0.0181,
-                          0.0316,
-                          0.0514,
-                          0.1028,
-                          0.1166])
+                         answer_key_six_month_periods)
+        answer_key_year_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_RETURNS['year'])
        self.assertEqual([round(x.benchmark_period_returns, 4)
                          for x in metrics.year_periods],
-                         [0.1362])
+                         answer_key_year_periods)

    def test_trading_days_06(self):
        returns = factory.create_returns_from_range(self.sim_params)
@@ -143,47 +132,33 @@ class TestRisk(unittest.TestCase):
    def test_benchmark_volatility_06(self):
        returns = factory.create_returns_from_range(self.sim_params)
        metrics = risk.RiskReport(returns, self.sim_params)
-        self.assertEqual([round(x.benchmark_volatility, 3)
+        answer_key_month_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_VOLATILITY['Monthly'],
+            decimal=3)
+        self.assertEqual([np.round(x.benchmark_volatility, 3)
                          for x in metrics.month_periods],
-                         [0.031,
-                          0.026,
-                          0.024,
-                          0.025,
-                          0.037,
-                          0.047,
-                          0.039,
-                          0.022,
-                          0.023,
-                          0.021,
-                          0.025,
-                          0.019])
+                         answer_key_month_periods)

-        self.assertEqual([round(x.benchmark_volatility, 3)
+        answer_key_three_month_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_VOLATILITY['3-Month'],
+            decimal=3)
+        self.assertEqual([np.round(x.benchmark_volatility, 3)
                          for x in metrics.three_month_periods],
-                         [0.047,
-                          0.042,
-                          0.050,
-                          0.064,
-                          0.070,
-                          0.064,
-                          0.049,
-                          0.037,
-                          0.039,
-                          0.037])
+                         answer_key_three_month_periods)

-        self.assertEqual([round(x.benchmark_volatility, 3)
+        answer_key_six_month_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_VOLATILITY['6-month'],
+            decimal=3)
+        self.assertEqual([np.round(x.benchmark_volatility, 3)
                          for x in metrics.six_month_periods],
-                         [0.079,
-                          0.082,
-                          0.081,
-                          0.081,
-                          0.080,
-                          0.074,
-                          0.061])
+                         answer_key_six_month_periods)

-        self.assertEqual([round(x.benchmark_volatility, 3)
+        answer_key_year_periods = ANSWER_KEY.get_values(
+            AnswerKey.BENCHMARK_PERIOD_VOLATILITY['year'],
+            decimal=3)
+        self.assertEqual([np.round(x.benchmark_volatility, 3)
                          for x in metrics.year_periods],
-                         [0.100])
+                         answer_key_year_periods)

    def test_algorithm_returns_06(self):
        self.assertEqual([round(x.algorithm_period_returns, 3)
@@ -834,256 +809,3 @@ class TestRisk(unittest.TestCase):
            )
            self.assert_month(start_date.month, col[-1].end_date.month)
            self.assert_last_day(col[-1].end_date)
-
-RETURNS = [
-    0.0093,
-    -0.0193,
-    0.0351,
-    0.0396,
-    0.0338,
-    -0.0211,
-    0.0389,
-    0.0326,
-    -0.0137,
-    -0.0411,
-    -0.0032,
-    0.0149,
-    0.0133,
-    0.0348,
-    0.042,
-    -0.0455,
-    0.0262,
-    -0.0461,
-    0.0021,
-    -0.0273,
-    -0.0429,
-    0.0427,
-    -0.0104,
-    0.0346,
-    -0.0311,
-    0.0003,
-    0.0211,
-    0.0248,
-    -0.0215,
-    0.004,
-    0.0267,
-    0.0029,
-    -0.0369,
-    0.0057,
-    0.0298,
-    -0.0179,
-    -0.0361,
-    -0.0401,
-    -0.0123,
-    -0.005,
-    0.0203,
-    -0.041,
-    0.0011,
-    0.0118,
-    0.0103,
-    -0.0184,
-    -0.0437,
-    0.0411,
-    -0.0242,
-    -0.0054,
-    -0.0039,
-    -0.0273,
-    -0.0075,
-    0.0064,
-    -0.0376,
-    0.0424,
-    0.0399,
-    0.019,
-    0.0236,
-    -0.0284,
-    -0.0341,
-    0.0266,
-    0.05,
-    0.0069,
-    -0.0442,
-    -0.016,
-    0.0173,
-    0.0348,
-    -0.0404,
-    -0.0068,
-    -0.0376,
-    0.0356,
-    0.0043,
-    -0.0481,
-    -0.0134,
-    0.0257,
-    0.0442,
-    0.0234,
-    0.0394,
-    0.0376,
-    -0.0147,
-    -0.0098,
-    0.0474,
-    -0.0102,
-    0.0138,
-    0.0286,
-    0.0347,
-    0.0279,
-    -0.0067,
-    0.0462,
-    -0.0432,
-    0.0247,
-    0.0174,
-    -0.0305,
-    -0.0317,
-    -0.0068,
-    0.0264,
-    -0.0257,
-    -0.0328,
-    0.0092,
-    0.0288,
-    -0.002,
-    0.0288,
-    0.028,
-    -0.0093,
-    0.0178,
-    -0.0365,
-    -0.0086,
-    -0.0133,
-    -0.0309,
-    0.0473,
-    -0.0149,
-    0.0378,
-    -0.0316,
-    -0.0292,
-    -0.0453,
-    -0.0451,
-    0.0093,
-    0.0397,
-    -0.0361,
-    -0.0168,
-    -0.0494,
-    -0.0143,
-    -0.0405,
-    -0.0349,
-    0.0069,
-    0.0378,
-    -0.0233,
-    -0.0492,
-    0.018,
-    -0.0386,
-    0.0339,
-    0.0119,
-    0.0454,
-    0.0118,
-    -0.011,
-    -0.0254,
-    0.0266,
-    -0.0366,
-    -0.0211,
-    0.0399,
-    0.0307,
-    0.035,
-    -0.0402,
-    0.0304,
-    -0.0031,
-    0.0256,
-    0.0134,
-    -0.0019,
-    -0.0235,
-    -0.0058,
-    -0.0117,
-    0.0051,
-    -0.0451,
-    -0.0466,
-    -0.0124,
-    0.0283,
-    -0.0499,
-    0.0318,
-    -0.0028,
-    0.0203,
-    0.005,
-    0.0085,
-    0.0048,
-    0.0277,
-    0.0159,
-    -0.0149,
-    0.035,
-    0.0404,
-    -0.01,
-    0.0377,
-    0.0302,
-    0.0046,
-    -0.0328,
-    -0.0469,
-    0.0071,
-    -0.0382,
-    -0.0214,
-    0.0429,
-    0.0145,
-    -0.0279,
-    -0.0172,
-    0.0423,
-    0.041,
-    -0.0183,
-    0.0137,
-    -0.0412,
-    -0.0348,
-    0.0302,
-    0.0248,
-    0.0051,
-    -0.0298,
-    -0.0103,
-    -0.0333,
-    -0.0399,
-    0.0485,
-    -0.0166,
-    0.0384,
-    0.0259,
-    -0.0163,
-    0.0357,
-    0.0308,
-    -0.0386,
-    0.0481,
-    -0.0446,
-    -0.0282,
-    -0.0037,
-    0.0202,
-    0.0216,
-    0.0113,
-    0.0194,
-    0.0392,
-    0.0016,
-    0.0268,
-    -0.0155,
-    -0.027,
-    0.02,
-    0.0216,
-    -0.0009,
-    0.022,
-    0.0,
-    0.041,
-    0.0133,
-    -0.0382,
-    0.0495,
-    -0.0221,
-    -0.0329,
-    -0.0033,
-    -0.0089,
-    -0.0129,
-    -0.0252,
-    0.048,
-    -0.0307,
-    -0.0357,
-    0.0033,
-    -0.0412,
-    -0.0407,
-    0.0455,
-    0.0159,
-    -0.0051,
-    -0.0274,
-    -0.0213,
-    0.0361,
-    0.0051,
-    -0.0378,
-    0.0084,
-    0.0066,
-    -0.0103,
-    -0.0037,
-    0.0478,
-    -0.0278]