Five minute simulation data working

catalyst/algorithm.py

@@ -431,6 +431,7 @@ class TradingAlgorithm(object):
 
         If get_loader is None, constructs an ExplodingPipelineEngine
         """
+        print 'using all_dates for {}'.format(data_frequency)
         if get_loader is not None:
             if data_frequency == 'daily':
                 all_dates = self.trading_calendar.all_sessions

catalyst/data/_minute_bar_internal.pyx

@@ -44,7 +44,7 @@ def five_minute_value(ndarray[long_t, ndim=1] market_opens,
     q = cython.cdiv(pos, five_minutes_per_day)
     r = cython.cmod(pos, five_minutes_per_day)
 
-    return market_opens[q] + r
+    return market_opens[q] + 5 * r
 
 def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
                             ndarray[long_t, ndim=1] market_closes,
@@ -112,10 +112,14 @@ def find_position_of_five_minute(ndarray[long_t, ndim=1] market_opens,
     market_open = market_opens[market_open_loc]
     market_close = market_closes[market_open_loc]
 
-    if not forward_fill and ((five_minute_val - market_open) >= five_minutes_per_day):
+    val_open_offset = (five_minute_val - market_open)/5
+    close_open_offset = (market_close - market_open)/5
+
+    if not forward_fill and val_open_offset >= five_minutes_per_day:
         raise ValueError("Given five minutes is not between an open and a close")
 
-    delta = int_min(five_minute_val - market_open, market_close - market_open)
+    # clamp offset to close index
+    delta = int_min(val_open_offset, close_open_offset)
 
     return (market_open_loc * five_minutes_per_day) + delta
 

catalyst/data/data_portal.py

@@ -689,9 +689,11 @@ class DataPortal(object):
 
             if pd.isnull(query_dt):
                 # no last traded dt, bail
+                print 'ffill, no dt {} for, {}'.format(query_dt, column)
                 if column == 'volume':
                     return 0
                 else:
+                    print 'ffill, no dt, field == nan'
                     return np.nan
         else:
             # If not forward filling, we just want dt.
@@ -700,14 +702,17 @@ class DataPortal(object):
         try:
             result = reader.get_value(asset.sid, query_dt, column)
         except NoDataOnDate:
+            print 'no data for {} on date {}'.format(column, query_dt)
             if column == 'volume':
                 return 0
             else:
                 return np.nan
 
         if not ffill or (dt == query_dt) or (dt.date() == query_dt.date()):
+            #print 'already have data'
             return result
 
+        #print 'adjusting..'
         # the value we found came from a different day, so we have to adjust
         # the data if there are any adjustments on that day barrier
         return self.get_adjusted_value(

catalyst/data/five_minute_bars.py

@@ -1151,6 +1151,7 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader):
 
         if field != 'volume':
             value *= self._ohlc_ratio_inverse_for_sid(sid)
+        #print 'minute pos: {}, {}: {}'.format(minute_pos, field, value)
         return value
 
     def get_last_traded_dt(self, asset, dt):
@@ -1161,8 +1162,8 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader):
 
     def _find_last_traded_five_minute_position(self, asset, dt):
         volumes = self._open_minute_file('volume', asset)
-        start_date_minute = asset.start_date.value / NANOS_IN_FIVE_MINUTE
-        dt_minute = dt.value / NANOS_IN_FIVE_MINUTE
+        start_date_minute = asset.start_date.value / NANOS_IN_MINUTE
+        dt_minute = dt.value / NANOS_IN_MINUTE
 
         try:
             # if we know of a dt before which this asset has no volume,
@@ -1227,7 +1228,7 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader):
         return find_position_of_five_minute(
             self._market_open_values,
             self._market_close_values,
-            minute_dt.value / NANOS_IN_FIVE_MINUTE,
+            minute_dt.value / NANOS_IN_MINUTE,
             self._five_minutes_per_day,
             False,
         )
@@ -1252,11 +1253,19 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader):
             (minutes in range, sids) with a dtype of float64, containing the
             values for the respective field over start and end dt range.
         """
+        print 'start_dt:', start_dt
+        print 'end_dt:', end_dt
+
         start_idx = self._find_position_of_five_minute(start_dt)
         end_idx = self._find_position_of_five_minute(end_dt)
 
+        print 'start_idx:', start_idx
+        print 'end_idex:', end_idx
+
         num_minutes = (end_idx - start_idx + 1)
 
+        print 'num_minutes:', num_minutes
+
         results = []
 
         indices_to_exclude = self._exclusion_indices_for_range(
@@ -1293,6 +1302,8 @@ class BcolzFiveMinuteBarReader(FiveMinuteBarReader):
                     out[:len(where), i][where] = values[where]
 
             results.append(out)
+
+        print 'results:', results
         return results
 
 

catalyst/pipeline/loaders/crypto_pricing_loader.py

@@ -37,6 +37,7 @@ class CryptoPricingLoader(PipelineLoader):
 
         cal = get_calendar('OPEN')
 
+        print 'CryptoPricingLoader-{}'.format(data_frequency)
         if data_frequency == 'daily':
             reader = bundle.daily_bar_reader
             all_sessions = cal.all_sessions
@@ -57,6 +58,7 @@ class CryptoPricingLoader(PipelineLoader):
         self.raw_price_loader = reader
         self._columns = dataset.columns
         self._all_sessions = all_sessions
+        self._data_frequency = data_frequency
 
     @classmethod
     def from_files(cls, pricing_path):