mirror of
https://github.com/wassname/catalyst.git
synced 2026-08-01 12:20:21 +08:00
BUG: Fix error during benchmark update over empty period.
On ranges with missing data from Yahoo, e.g.: On 2013-04-2 the date range of April 2013-03-29 failed because of the first day in the range being Good Friday, and the API not yet updating for the Monday after. Handle the 404 that is found by raising and warning that no benchmark data was found, but continuing on.
This commit is contained in:
+27
-15
@@ -12,7 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import collections
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
@@ -30,6 +30,10 @@ from loader_utils import (
|
||||
)
|
||||
from zipline.protocol import DailyReturn
|
||||
|
||||
|
||||
class BenchmarkDataNotFoundError(Exception):
|
||||
pass
|
||||
|
||||
_BENCHMARK_MAPPING = {
|
||||
# Need to add 'symbol'
|
||||
'volume': (int, 'Volume'),
|
||||
@@ -52,27 +56,35 @@ def get_raw_benchmark_data(start_date, end_date, symbol):
|
||||
|
||||
# create benchmark files
|
||||
# ^GSPC 19500103
|
||||
params = {
|
||||
's': symbol,
|
||||
# end_date month, zero indexed
|
||||
'd': end_date.month - 1,
|
||||
# end_date day str(int(todate[6:8])) #day
|
||||
'e': end_date.day,
|
||||
# end_date year str(int(todate[0:4]))
|
||||
'f': end_date.year,
|
||||
# daily frequency
|
||||
'g': 'd',
|
||||
params = collections.OrderedDict((
|
||||
('s', symbol),
|
||||
# start_date month, zero indexed
|
||||
'a': start_date.month - 1,
|
||||
('a', start_date.month - 1),
|
||||
# start_date day
|
||||
'b': start_date.day,
|
||||
('b', start_date.day),
|
||||
# start_date year
|
||||
'c': start_date.year
|
||||
}
|
||||
('c', start_date.year),
|
||||
# end_date month, zero indexed
|
||||
('d', end_date.month - 1),
|
||||
# end_date day str(int(todate[6:8])) #day
|
||||
('e', end_date.day),
|
||||
# end_date year str(int(todate[0:4]))
|
||||
('f', end_date.year),
|
||||
# daily frequency
|
||||
('g', 'd'),
|
||||
))
|
||||
|
||||
res = requests.get('http://ichart.yahoo.com/table.csv',
|
||||
params=params)
|
||||
|
||||
if not res.ok:
|
||||
raise BenchmarkDataNotFoundError("""
|
||||
No benchmark data found for date range.
|
||||
start_date={start_date}, end_date={end_date}, url={url}""".strip().
|
||||
format(start_date=start_date,
|
||||
end_date=end_date,
|
||||
url=res.url))
|
||||
|
||||
return csv.DictReader(StringIO(res.content))
|
||||
|
||||
|
||||
|
||||
+17
-9
@@ -20,7 +20,10 @@ import msgpack
|
||||
from collections import OrderedDict
|
||||
from datetime import timedelta
|
||||
|
||||
import logbook
|
||||
|
||||
from treasuries import get_treasury_data
|
||||
import benchmarks
|
||||
from benchmarks import get_benchmark_returns
|
||||
|
||||
from zipline.protocol import DailyReturn
|
||||
@@ -28,6 +31,7 @@ from zipline.utils.date_utils import tuple_to_date
|
||||
from zipline.utils.tradingcalendar import trading_days
|
||||
from operator import attrgetter
|
||||
|
||||
logger = logbook.Logger('Loader')
|
||||
|
||||
# TODO: Make this path customizable.
|
||||
DATA_PATH = os.path.join(
|
||||
@@ -128,16 +132,20 @@ def update_benchmarks(symbol, last_date):
|
||||
for packed_date, returns in bm_list:
|
||||
benchmark_data.append((packed_date, returns))
|
||||
|
||||
start = last_date + timedelta(days=1)
|
||||
for daily_return in get_benchmark_returns(symbol, start_date=start):
|
||||
date_as_tuple = daily_return.date.timetuple()[0:6] + \
|
||||
(daily_return.date.microsecond,)
|
||||
# Not ideal but massaging data into expected format
|
||||
benchmark = (date_as_tuple, daily_return.returns)
|
||||
benchmark_data.append(benchmark)
|
||||
try:
|
||||
start = last_date + timedelta(days=1)
|
||||
for daily_return in get_benchmark_returns(symbol, start_date=start):
|
||||
date_as_tuple = daily_return.date.timetuple()[0:6] + \
|
||||
(daily_return.date.microsecond,)
|
||||
# Not ideal but massaging data into expected format
|
||||
benchmark = (date_as_tuple, daily_return.returns)
|
||||
benchmark_data.append(benchmark)
|
||||
|
||||
with get_datafile(get_benchmark_filename(symbol), mode='wb') as bmark_fp:
|
||||
bmark_fp.write(msgpack.dumps(benchmark_data))
|
||||
with get_datafile(
|
||||
get_benchmark_filename(symbol), mode='wb') as bmark_fp:
|
||||
bmark_fp.write(msgpack.dumps(benchmark_data))
|
||||
except benchmarks.BenchmarkDataNotFoundError as exc:
|
||||
logger.warn(exc)
|
||||
|
||||
|
||||
def get_benchmark_filename(symbol):
|
||||
|
||||
Reference in New Issue
Block a user