diff --git a/catalyst/algorithm.py b/catalyst/algorithm.py index 139a2395..1e548a43 100644 --- a/catalyst/algorithm.py +++ b/catalyst/algorithm.py @@ -431,6 +431,7 @@ class TradingAlgorithm(object): If get_loader is None, constructs an ExplodingPipelineEngine """ + print 'using all_dates for {}'.format(data_frequency) if get_loader is not None: if data_frequency == 'daily': all_dates = self.trading_calendar.all_sessions diff --git a/catalyst/data/_minute_bar_internal.pyx b/catalyst/data/_minute_bar_internal.pyx index 9ebb0841..ddc35258 100644 --- a/catalyst/data/_minute_bar_internal.pyx +++ b/catalyst/data/_minute_bar_internal.pyx @@ -44,7 +44,7 @@ def five_minute_value(ndarray[long_t, ndim=1] market_opens, q = cython.cdiv(pos, five_minutes_per_day) r = cython.cmod(pos, five_minutes_per_day) - return market_opens[q] + r + return market_opens[q] + 5 * r def find_position_of_minute(ndarray[long_t, ndim=1] market_opens, ndarray[long_t, ndim=1] market_closes, @@ -112,10 +112,14 @@ def find_position_of_five_minute(ndarray[long_t, ndim=1] market_opens, market_open = market_opens[market_open_loc] market_close = market_closes[market_open_loc] - if not forward_fill and ((five_minute_val - market_open) >= five_minutes_per_day): + val_open_offset = (five_minute_val - market_open)/5 + close_open_offset = (market_close - market_open)/5 + + if not forward_fill and val_open_offset >= five_minutes_per_day: raise ValueError("Given five minutes is not between an open and a close") - delta = int_min(five_minute_val - market_open, market_close - market_open) + # clamp offset to close index + delta = int_min(val_open_offset, close_open_offset) return (market_open_loc * five_minutes_per_day) + delta diff --git a/catalyst/data/data_portal.py b/catalyst/data/data_portal.py index 69aa166e..79dc1f7c 100644 --- a/catalyst/data/data_portal.py +++ b/catalyst/data/data_portal.py @@ -689,9 +689,11 @@ class DataPortal(object): if pd.isnull(query_dt): # no last traded dt, bail + print 'ffill, no dt {} for, {}'.format(query_dt, column) if column == 'volume': return 0 else: + print 'ffill, no dt, field == nan' return np.nan else: # If not forward filling, we just want dt. @@ -700,14 +702,17 @@ class DataPortal(object): try: result = reader.get_value(asset.sid, query_dt, column) except NoDataOnDate: + print 'no data for {} on date {}'.format(column, query_dt) if column == 'volume': return 0 else: return np.nan if not ffill or (dt == query_dt) or (dt.date() == query_dt.date()): + #print 'already have data' return result + #print 'adjusting..' # the value we found came from a different day, so we have to adjust # the data if there are any adjustments on that day barrier return self.get_adjusted_value( diff --git a/catalyst/data/five_minute_bars.py b/catalyst/data/five_minute_bars.py index 9021dc0a..8f8de210 100644 --- a/catalyst/data/five_minute_bars.py +++ b/catalyst/data/five_minute_bars.py @@ -1151,6 +1151,7 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader): if field != 'volume': value *= self._ohlc_ratio_inverse_for_sid(sid) + #print 'minute pos: {}, {}: {}'.format(minute_pos, field, value) return value def get_last_traded_dt(self, asset, dt): @@ -1161,8 +1162,8 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader): def _find_last_traded_five_minute_position(self, asset, dt): volumes = self._open_minute_file('volume', asset) - start_date_minute = asset.start_date.value / NANOS_IN_FIVE_MINUTE - dt_minute = dt.value / NANOS_IN_FIVE_MINUTE + start_date_minute = asset.start_date.value / NANOS_IN_MINUTE + dt_minute = dt.value / NANOS_IN_MINUTE try: # if we know of a dt before which this asset has no volume, @@ -1227,7 +1228,7 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader): return find_position_of_five_minute( self._market_open_values, self._market_close_values, - minute_dt.value / NANOS_IN_FIVE_MINUTE, + minute_dt.value / NANOS_IN_MINUTE, self._five_minutes_per_day, False, ) @@ -1252,11 +1253,19 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader): (minutes in range, sids) with a dtype of float64, containing the values for the respective field over start and end dt range. """ + print 'start_dt:', start_dt + print 'end_dt:', end_dt + start_idx = self._find_position_of_five_minute(start_dt) end_idx = self._find_position_of_five_minute(end_dt) + print 'start_idx:', start_idx + print 'end_idex:', end_idx + num_minutes = (end_idx - start_idx + 1) + print 'num_minutes:', num_minutes + results = [] indices_to_exclude = self._exclusion_indices_for_range( @@ -1293,6 +1302,8 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader): out[:len(where), i][where] = values[where] results.append(out) + + print 'results:', results return results diff --git a/catalyst/pipeline/loaders/crypto_pricing_loader.py b/catalyst/pipeline/loaders/crypto_pricing_loader.py index a3ec1114..3901370a 100644 --- a/catalyst/pipeline/loaders/crypto_pricing_loader.py +++ b/catalyst/pipeline/loaders/crypto_pricing_loader.py @@ -37,6 +37,7 @@ class CryptoPricingLoader(PipelineLoader): cal = get_calendar('OPEN') + print 'CryptoPricingLoader-{}'.format(data_frequency) if data_frequency == 'daily': reader = bundle.daily_bar_reader all_sessions = cal.all_sessions @@ -57,6 +58,7 @@ class CryptoPricingLoader(PipelineLoader): self.raw_price_loader = reader self._columns = dataset.columns self._all_sessions = all_sessions + self._data_frequency = data_frequency @classmethod def from_files(cls, pricing_path):