diff --git a/catalyst/data/_equities.pyx b/catalyst/data/_equities.pyx index 81f9a66e..563fa56a 100644 --- a/catalyst/data/_equities.pyx +++ b/catalyst/data/_equities.pyx @@ -217,7 +217,7 @@ cpdef _read_bcolz_data(ctable_t table, if column_name in ['open', 'high', 'low', 'close']: where_nan = (outbuf == 0) - outbuf_as_float = outbuf.astype(float64) * .000001 + outbuf_as_float = outbuf.astype(float64) * .000000001 outbuf_as_float[where_nan] = NAN results.append(outbuf_as_float) elif column_name != 'volume': diff --git a/catalyst/data/bundles/poloniex.py b/catalyst/data/bundles/poloniex.py index 4dcdf7bc..eb4fc735 100644 --- a/catalyst/data/bundles/poloniex.py +++ b/catalyst/data/bundles/poloniex.py @@ -105,7 +105,7 @@ class PoloniexBundle(BaseCryptoPricingBundle): # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored # on disk, which we compensate here to get the right pricing amounts # ref: data/us_equity_pricing.py - scale = 1000 + scale = 1 raw.loc[:, 'open'] /= scale raw.loc[:, 'high'] /= scale raw.loc[:, 'low'] /= scale diff --git a/catalyst/data/history_loader.py b/catalyst/data/history_loader.py index 8bc00563..e33f5f6e 100644 --- a/catalyst/data/history_loader.py +++ b/catalyst/data/history_loader.py @@ -38,7 +38,7 @@ from catalyst.utils.numpy_utils import float64_dtype from catalyst.utils.pandas_utils import find_in_sorted_index # Default number of decimal places used for rounding asset prices. -DEFAULT_ASSET_PRICE_DECIMALS = 3 +DEFAULT_ASSET_PRICE_DECIMALS = 9 class HistoryCompatibleUSEquityAdjustmentReader(object): diff --git a/catalyst/data/us_equity_pricing.py b/catalyst/data/us_equity_pricing.py index 03d01a4e..901a0e60 100644 --- a/catalyst/data/us_equity_pricing.py +++ b/catalyst/data/us_equity_pricing.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import division # Python2 req to have division of ints yield float + from errno import ENOENT from functools import partial from os import remove @@ -80,7 +83,6 @@ from catalyst.utils.cli import ( from ._equities import _compute_row_slices, _read_bcolz_data from ._adjustments import load_adjustments_from_sqlite - logger = logbook.Logger('UsEquityPricing') OHLC = frozenset(['open', 'high', 'low', 'close']) @@ -116,6 +118,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = { UINT32_MAX = iinfo(uint32).max UINT64_MAX = iinfo(uint64).max +PRICE_ADJUSTMENT_FACTOR = 1000000000 # Provides 9 decimals resolution. Also affects _equities.pyx L220 + def check_uint32_safe(value, colname): if value >= UINT32_MAX: @@ -433,7 +437,7 @@ class BcolzDailyBarWriter(object): return raw_data winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC) - processed = (raw_data[list(OHLC)] * 1000000).astype('uint64') + processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64') dates = raw_data.index.values.astype('datetime64[s]') check_uint32_safe(dates.max().view(np.int64), 'day') processed['day'] = dates.astype('uint32') @@ -519,7 +523,6 @@ class BcolzDailyBarReader(SessionBarReader): # Need to test keeping the entire array in memory for the course of a # process first. self._spot_cols = {} - self.PRICE_ADJUSTMENT_FACTOR = 0.001 self._read_all_threshold = read_all_threshold @lazyval @@ -763,7 +766,7 @@ class BcolzDailyBarReader(SessionBarReader): if price == 0: return nan else: - return price * 0.001 + return price / PRICE_ADJUSTMENT_FACTOR else: return price