diff --git a/zipline/data/minute_bars.py b/zipline/data/minute_bars.py index e35a73c9..df1b4328 100644 --- a/zipline/data/minute_bars.py +++ b/zipline/data/minute_bars.py @@ -109,6 +109,7 @@ class BcolzMinuteBarMetadata(object): The factor by which the pricing data is multiplied so that the float data can be stored as an integer. """ + FORMAT_VERSION = 1 METADATA_FILENAME = 'metadata.json' @@ -122,6 +123,13 @@ class BcolzMinuteBarMetadata(object): with open(path) as fp: raw_data = json.load(fp) + try: + version = raw_data['minutes_per_day'] + except KeyError: + # Version was first written with version 1, assume 0, + # if version does not match. + version = 0 + first_trading_day = pd.Timestamp( raw_data['first_trading_day'], tz='UTC') market_opens = pd.to_datetime(raw_data['market_opens'], @@ -131,19 +139,34 @@ class BcolzMinuteBarMetadata(object): unit='m', utc=True) ohlc_ratio = raw_data['ohlc_ratio'] - return cls(first_trading_day, - market_opens, - market_closes, - ohlc_ratio) - def __init__(self, first_trading_day, - market_opens, - market_closes, - ohlc_ratio): + if version == 0: + # version 0 always assumed US equities. + minutes_per_day = US_EQUITIES_MINUTES_PER_DAY + else: + minutes_per_day = raw_data['minutes_per_day'] + + return cls( + first_trading_day, + market_opens, + market_closes, + ohlc_ratio, + minutes_per_day, + ) + + def __init__( + self, + first_trading_day, + market_opens, + market_closes, + ohlc_ratio, + minutes_per_day, + ): self.first_trading_day = first_trading_day self.market_opens = market_opens self.market_closes = market_closes self.ohlc_ratio = ohlc_ratio + self.minutes_per_day = minutes_per_day def write(self, rootdir): """ @@ -161,6 +184,7 @@ class BcolzMinuteBarMetadata(object): float data can be stored as an integer. """ metadata = { + 'version': self.FORMAT_VERSION, 'first_trading_day': str(self.first_trading_day.date()), 'market_opens': self.market_opens.values. astype('datetime64[m]'). @@ -169,6 +193,7 @@ class BcolzMinuteBarMetadata(object): astype('datetime64[m]'). astype(np.int64).tolist(), 'ohlc_ratio': self.ohlc_ratio, + 'minutes_per_day': self.minutes_per_day } with open(self.metadata_path(rootdir), 'w+') as fp: json.dump(metadata, fp) @@ -300,6 +325,7 @@ class BcolzMinuteBarWriter(object): self._market_opens, self._market_closes, self._ohlc_ratio, + self._minutes_per_day, ) metadata.write(self._rootdir) @@ -669,6 +695,8 @@ class BcolzMinuteBarReader(object): self._ohlc_inverse = 1.0 / metadata.ohlc_ratio + self._minutes_per_day = metadata.minutes_per_day + self._carrays = { field: LRUCache(maxsize=sid_cache_size) for field in self.FIELDS @@ -704,7 +732,7 @@ class BcolzMinuteBarReader(object): market_closes = self._market_closes.values.astype('datetime64[m]') minutes_per_day = (market_closes - market_opens).astype(np.int64) early_indices = np.where( - minutes_per_day != US_EQUITIES_MINUTES_PER_DAY - 1)[0] + minutes_per_day != self._minutes_per_day - 1)[0] early_opens = self._market_opens[early_indices] early_closes = self._market_closes[early_indices] minutes = [(market_open, early_close) @@ -736,7 +764,7 @@ class BcolzMinuteBarReader(object): end_pos = ( self._find_position_of_minute(market_open) + - US_EQUITIES_MINUTES_PER_DAY + self._minutes_per_day - 1 ) @@ -855,14 +883,14 @@ class BcolzMinuteBarReader(object): dt_minutes, start_date_minutes, volumes, - US_EQUITIES_MINUTES_PER_DAY + self._minutes_per_day, ) def _pos_to_minute(self, pos): minute_epoch = minute_value( self._market_open_values, pos, - US_EQUITIES_MINUTES_PER_DAY + self._minutes_per_day ) return pd.Timestamp(minute_epoch, tz='UTC', unit="m") @@ -890,7 +918,7 @@ class BcolzMinuteBarReader(object): self._market_open_values, self._market_close_values, minute_dt.value / NANOS_IN_MINUTE, - US_EQUITIES_MINUTES_PER_DAY, + self._minutes_per_day, ) def load_raw_arrays(self, fields, start_dt, end_dt, sids):