Merge branch 'poloniex-exchange' into concurrent-exchanges

catalyst/data/_equities.pyx

@@ -217,7 +217,7 @@ cpdef _read_bcolz_data(ctable_t table,
 
         if column_name in ['open', 'high', 'low', 'close']:
             where_nan = (outbuf == 0)
-            outbuf_as_float = outbuf.astype(float64) * .000001
+            outbuf_as_float = outbuf.astype(float64) * .000000001
             outbuf_as_float[where_nan] = NAN
             results.append(outbuf_as_float)
         elif column_name != 'volume':

catalyst/data/bundles/base.py

@@ -491,7 +491,7 @@ class BaseBundle(object):
                     data_frequency,
                 )
                 raw_data.index = pd.to_datetime(raw_data.index, utc=True)
-                raw_data.index = raw_data.index.tz_localize('UTC')
+                #raw_data.index = raw_data.index.tz_localize('UTC')
 
                 # Filter incoming data to fit start and end sessions.
                 raw_data = raw_data[

catalyst/data/bundles/base_pricing.py

@@ -24,6 +24,7 @@ class BasePricingBundle(BaseBundle):
             ('start_date', 'datetime64[ns]'),
             ('end_date', 'datetime64[ns]'),
             ('ac_date', 'datetime64[ns]'),
+            ('min_trade_size', 'float'),
         ]
 
     @lazyval

catalyst/data/bundles/poloniex.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import sys
+
 from datetime import datetime
 
 import pandas as pd
@@ -23,6 +25,8 @@ from catalyst.data.bundles.core import register_bundle
 from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
 from catalyst.utils.memoize import lazyval
 
+from catalyst.curate.poloniex import PoloniexCurator
+
 class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def name(self):
@@ -36,7 +40,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
     def frequencies(self):
         return set((
             'daily',
-            #'5-minute',
+            'minute',
         ))
 
     @lazyval
@@ -75,12 +79,14 @@ class PoloniexBundle(BaseCryptoPricingBundle):
         start_date = sym_data.index[0]
         end_date = sym_data.index[-1]
         ac_date = end_date + pd.Timedelta(days=1)
+        min_trade_size = 0.00000001
 
         return (
             sym_md.symbol,
             start_date,
             end_date,
             ac_date,
+            min_trade_size,
         )
 
     def fetch_raw_symbol_frame(self,
@@ -90,24 +96,30 @@ class PoloniexBundle(BaseCryptoPricingBundle):
                                start_date,
                                end_date,
                                frequency):
+
         # TODO: replace this with direct exchange call
         # The end date and frequency should be used to calculate the number of bars
-        raw = pd.read_json(
-            self._format_data_url(
-                api_key,
-                symbol,
-                start_date,
-                end_date,
-                frequency,
-            ),
-            orient='records',
-        )
-        raw.set_index('date', inplace=True)
+        if(frequency == 'minute'):
+            pc = PoloniexCurator()
+            raw = pc.onemin_to_dataframe(symbol, start_date, end_date)
+
+        else:
+            raw = pd.read_json(
+                self._format_data_url(
+                    api_key,
+                    symbol,
+                    start_date,
+                    end_date,
+                    frequency,
+                ),
+                orient='records',
+            )
+            raw.set_index('date', inplace=True)
 
         # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
         # on disk, which we compensate here to get the right pricing amounts
         # ref: data/us_equity_pricing.py
-        scale = 1000
+        scale = 1
         raw.loc[:, 'open'] /= scale
         raw.loc[:, 'high'] /= scale
         raw.loc[:, 'low'] /= scale
@@ -169,4 +181,9 @@ register_bundle(PoloniexBundle, ['USDT_BTC',])
 For a production environment make sure to use (to bundle all pairs):
 register_bundle(PoloniexBundle)
 '''
-register_bundle(PoloniexBundle, create_writers=False)
+
+if 'ingest' in sys.argv and '-c' in sys.argv:
+    register_bundle(PoloniexBundle)
+else:
+    register_bundle(PoloniexBundle, create_writers=False)
+

catalyst/data/history_loader.py

@@ -38,7 +38,7 @@ from catalyst.utils.numpy_utils import float64_dtype
 from catalyst.utils.pandas_utils import find_in_sorted_index
 
 # Default number of decimal places used for rounding asset prices.
-DEFAULT_ASSET_PRICE_DECIMALS = 3
+DEFAULT_ASSET_PRICE_DECIMALS = 9
 
 
 class HistoryCompatibleUSEquityAdjustmentReader(object):

catalyst/data/resample.py

@@ -156,7 +156,10 @@ class DailyHistoryAggregator(object):
             cache = self._caches[field] = (session, market_open, {})
 
         _, market_open, entries = cache
-        market_open = market_open.tz_localize('UTC')
+        try:
+            market_open = market_open.tz_localize('UTC')
+        except TypeError:
+            market_open = market_open.tz_convert('UTC')
         if dt != market_open:
             prev_dt = dt_value - self._one_min
         else:

catalyst/data/us_equity_pricing.py

@@ -11,6 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
+from __future__ import division     # Python2 req to have division of ints yield float
+
 from errno import ENOENT
 from functools import partial
 from os import remove
@@ -80,7 +83,6 @@ from catalyst.utils.cli import (
 from ._equities import _compute_row_slices, _read_bcolz_data
 from ._adjustments import load_adjustments_from_sqlite
 
-
 logger = logbook.Logger('UsEquityPricing')
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
@@ -116,6 +118,8 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
 UINT32_MAX = iinfo(uint32).max
 UINT64_MAX = iinfo(uint64).max
 
+PRICE_ADJUSTMENT_FACTOR = 1000000000    # Provides 9 decimals resolution. Also affects _equities.pyx L220
+
 
 def check_uint32_safe(value, colname):
     if value >= UINT32_MAX:
@@ -433,7 +437,7 @@ class BcolzDailyBarWriter(object):
             return raw_data
 
         winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
-        processed = (raw_data[list(OHLC)] * 1000000).astype('uint64')
+        processed = (raw_data[list(OHLC)] * PRICE_ADJUSTMENT_FACTOR).astype('uint64')
         dates = raw_data.index.values.astype('datetime64[s]')
         check_uint32_safe(dates.max().view(np.int64), 'day')
         processed['day'] = dates.astype('uint32')
@@ -519,7 +523,6 @@ class BcolzDailyBarReader(SessionBarReader):
         # Need to test keeping the entire array in memory for the course of a
         # process first.
         self._spot_cols = {}
-        self.PRICE_ADJUSTMENT_FACTOR = 0.001
         self._read_all_threshold = read_all_threshold
 
     @lazyval
@@ -763,7 +766,7 @@ class BcolzDailyBarReader(SessionBarReader):
             if price == 0:
                 return nan
             else:
-                return price * 0.001
+                return price / PRICE_ADJUSTMENT_FACTOR
         else:
             return price