WIP: Fixes 1/1000 price issue in history, and works with full coins. Requires matching-version 'catalyst ingest'

catalyst/data/_equities.pyx

@@ -217,7 +217,7 @@ cpdef _read_bcolz_data(ctable_t table,
 
         if column_name in ['open', 'high', 'low', 'close']:
             where_nan = (outbuf == 0)
-            outbuf_as_float = outbuf.astype(float64) * .000001
+            outbuf_as_float = outbuf.astype(float64) * .000000001
             outbuf_as_float[where_nan] = NAN
             results.append(outbuf_as_float)
         elif column_name != 'volume':

catalyst/data/bundles/poloniex.py

@@ -105,7 +105,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
         # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
         # on disk, which we compensate here to get the right pricing amounts
         # ref: data/us_equity_pricing.py
-        scale = 1000
+        scale = 1
         raw.loc[:, 'open'] /= scale
         raw.loc[:, 'high'] /= scale
         raw.loc[:, 'low'] /= scale

catalyst/data/history_loader.py

@@ -38,7 +38,7 @@ from catalyst.utils.numpy_utils import float64_dtype
 from catalyst.utils.pandas_utils import find_in_sorted_index
 
 # Default number of decimal places used for rounding asset prices.
-DEFAULT_ASSET_PRICE_DECIMALS = 3
+DEFAULT_ASSET_PRICE_DECIMALS = 9
 
 
 class HistoryCompatibleUSEquityAdjustmentReader(object):

catalyst/data/us_equity_pricing.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from __future__ import division     # Python2 req to have division of ints yield float
+
 from errno import ENOENT
 from functools import partial
 from os import remove
@@ -80,7 +83,6 @@ from catalyst.utils.cli import (
 from ._equities import _compute_row_slices, _read_bcolz_data
 from ._adjustments import load_adjustments_from_sqlite
 
-
 logger = logbook.Logger('UsEquityPricing')
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
@@ -116,6 +118,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
 UINT32_MAX = iinfo(uint32).max
 UINT64_MAX = iinfo(uint64).max
 
+PRICE_ADJUSTMENT_FACTOR = 1000000000    # Provides 9 decimals resolution. Also affects _equities.pyx L220
+
 
 def check_uint32_safe(value, colname):
     if value >= UINT32_MAX:
@@ -433,7 +437,7 @@ class BcolzDailyBarWriter(object):
             return raw_data
 
         winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
-        processed = (raw_data[list(OHLC)] * 1000000).astype('uint64')
+        processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         dates = raw_data.index.values.astype('datetime64[s]')
         check_uint32_safe(dates.max().view(np.int64), 'day')
         processed['day'] = dates.astype('uint32')
@@ -519,7 +523,6 @@ class BcolzDailyBarReader(SessionBarReader):
         # Need to test keeping the entire array in memory for the course of a
         # process first.
         self._spot_cols = {}
-        self.PRICE_ADJUSTMENT_FACTOR = 0.001
         self._read_all_threshold = read_all_threshold
 
     @lazyval
@@ -763,7 +766,7 @@ class BcolzDailyBarReader(SessionBarReader):
             if price == 0:
                 return nan
             else:
-                return price * 0.001
+                return price / PRICE_ADJUSTMENT_FACTOR
         else:
             return price