change pandas dependancy

BLD: Dockerfile[-dev]: s/quantopian/enigmampc/g

Dockerfile

@@ -1,11 +1,11 @@
 #
 # Dockerfile for an image with the currently checked out version of catalyst installed. To build:
 #
-#    docker build -t quantopian/catalyst .
+#    docker build -t enigmampc/catalyst .
 #
 # To run the container:
 #
-#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalyst -it quantopian/catalyst
+#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalyst -it enigmampc/catalyst
 #
 # To access Jupyter when running docker locally (you may need to add NAT rules):
 #

Dockerfile-dev

@@ -1,15 +1,15 @@
 #
 # Dockerfile for an image with the currently checked out version of catalyst installed. To build:
 #
-#    docker build -t quantopian/catalystdev -f Dockerfile-dev .
+#    docker build -t enigmampc/catalystdev -f Dockerfile-dev .
 #
-# Note: the dev build requires a quantopian/catalyst image, which you can build as follows:
+# Note: the dev build requires a enigmampc/catalyst image, which you can build as follows:
 #
-#    docker build -t quantopian/catalyst -f Dockerfile .
+#    docker build -t enigmampc/catalyst -f Dockerfile .
 #
 # To run the container:
 #
-#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalystdev -it quantopian/catalystdev
+#    docker run -v /path/to/your/notebooks:/projects -v ~/.catalyst:/root/.catalyst -p 8888:8888/tcp --name catalystdev -it enigmampc/catalystdev
 #
 # To access Jupyter when running docker locally (you may need to add NAT rules):
 #
@@ -25,7 +25,7 @@
 #
 #    docker exec -it catalystdev catalyst run -f /projects/my_algo.py --start 2015-1-1 --end 2016-1-1 /projects/result.pickle
 #
-FROM quantopian/catalyst
+FROM enigmampc/catalyst
 
 WORKDIR /catalyst
 

catalyst/_protocol.pyx

@@ -16,7 +16,6 @@ import warnings
 from contextlib import contextmanager
 from functools import wraps
 
-from pandas.tslib import normalize_date
 import pandas as pd
 import numpy as np
 
@@ -564,7 +563,7 @@ cdef class BarData:
             })
 
     cdef bool _is_stale_for_asset(self, asset, dt, adjusted_dt, data_portal):
-        session_label = normalize_date(dt) # FIXME
+        session_label = dt.normalize_date() # FIXME
 
         if not asset.is_alive_for_session(session_label):
             return False

catalyst/algorithm.py

@@ -21,7 +21,6 @@ import logbook
 import pytz
 import pandas as pd
 from contextlib2 import ExitStack
-from pandas.tseries.tools import normalize_date
 import numpy as np
 
 from itertools import chain, repeat
@@ -1345,7 +1344,7 @@ class TradingAlgorithm(object):
         # Make sure the asset exists, and that there is a last price for it.
         # FIXME: we should use BarData's can_trade logic here, but I haven't
         # yet found a good way to do that.
-        normalized_date = normalize_date(self.datetime)
+        normalized_date = self.datetime.normalize()
 
         if normalized_date < asset.start_date:
             raise CannotOrderDelistedAsset(
@@ -1392,7 +1391,7 @@ class TradingAlgorithm(object):
             )
 
         if asset.auto_close_date:
-            day = normalize_date(self.get_datetime())
+            day = self.get_datetime().normalize()
 
             if day > min(asset.end_date, asset.auto_close_date):
                 # If we are after the asset's end date or auto close date, warn
@@ -2475,7 +2474,7 @@ class TradingAlgorithm(object):
         """
         Internal implementation of `pipeline_output`.
         """
-        today = normalize_date(self.get_datetime())
+        today = self.get_datetime().normalize()
         data = NO_DATA = object()
         try:
             data = self._pipeline_cache.unwrap(today)

catalyst/constants.py

@@ -24,8 +24,7 @@ AUTO_INGEST = False
 
 AUTH_SERVER = 'https://data.enigma.co'
 
-# TODO: switch to mainnet
-ETH_REMOTE_NODE = 'https://rinkeby.infura.io/'
+ETH_REMOTE_NODE = 'https://mainnet.infura.io'
 
 MARKETPLACE_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
                        'catalyst/master/catalyst/marketplace/' \
@@ -35,7 +34,6 @@ MARKETPLACE_CONTRACT_ABI = 'https://raw.githubusercontent.com/enigmampc/' \
                            'catalyst/master/catalyst/marketplace/' \
                            'contract_marketplace_abi.json'
 
-# TODO: switch to mainnet
 ENIGMA_CONTRACT = 'https://raw.githubusercontent.com/enigmampc/' \
                   'catalyst/master/catalyst/marketplace/' \
                   'contract_enigma_address.txt'

catalyst/data/data_portal.py

@@ -20,7 +20,6 @@ import numpy as np
 from numpy import float64, int64, nan
 import pandas as pd
 from pandas import isnull
-from pandas.tslib import normalize_date
 from six import iteritems
 from six.moves import reduce
 
@@ -439,7 +438,7 @@ class DataPortal(object):
                     (isinstance(asset, (Asset, ContinuousFuture))))
 
     def _get_fetcher_value(self, asset, field, dt):
-        day = normalize_date(dt)
+        day = dt.normalize()
 
         try:
             return \
@@ -1130,7 +1129,7 @@ class DataPortal(object):
         if self._asset_start_dates[sid] > dt:
             raise NoTradeDataAvailableTooEarly(
                 sid=sid,
-                dt=normalize_date(dt),
+                dt=dt.normalize(),
                 start_dt=start_date
             )
 
@@ -1138,7 +1137,7 @@ class DataPortal(object):
         if self._asset_end_dates[sid] < dt:
             raise NoTradeDataAvailableTooLate(
                 sid=sid,
-                dt=normalize_date(dt),
+                dt=dt.normalize(),
                 end_dt=end_date
             )
 
@@ -1262,7 +1261,7 @@ class DataPortal(object):
         if self._extra_source_df is None:
             return []
 
-        day = normalize_date(dt)
+        day = dt.normalize()
 
         if day in self._extra_source_df.index:
             assets = self._extra_source_df.loc[day]['sid']

catalyst/data/history_loader.py

@@ -21,7 +21,6 @@ from abc import (
 from numpy import concatenate
 from lru import LRU
 from pandas import isnull
-from pandas.tslib import normalize_date
 from toolz import sliding_window
 
 from six import with_metaclass
@@ -93,8 +92,8 @@ class HistoryCompatibleUSEquityAdjustmentReader(object):
             The adjustments as a dict of loc -> Float64Multiply
         """
         sid = int(asset)
-        start = normalize_date(dts[0])
-        end = normalize_date(dts[-1])
+        start = dts[0].normalize()
+        end = dts[-1].normalize()
         adjs = {}
         if field != 'volume':
             mergers = self._adjustments_reader.get_adjustments_for_sid(

catalyst/data/us_equity_pricing.py

@@ -49,7 +49,6 @@ from pandas import (
     to_datetime,
     Timestamp,
 )
-from pandas.tslib import iNaT
 from six import (
     iteritems,
     string_types,
@@ -422,7 +421,7 @@ class BcolzDailyBarWriter(object):
         )
 
         full_table.attrs['first_trading_day'] = (
-            earliest_date if earliest_date is not None else iNaT
+            earliest_date if earliest_date is not None else NaT
         )
 
         full_table.attrs['first_row'] = first_row

catalyst/exchange/exchange_data_portal.py

@@ -1,5 +1,4 @@
 import abc
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -301,34 +300,20 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         )
         adj_bar_count = candle_size * bar_count
 
-        if data_frequency == "minute":
-            # for minute frequency always request data until the
-            # current minute (do not include the current minute)
-            last_dt_for_series = end_dt - datetime.timedelta(minutes=1)
-
-            # read the minute bundles for daily frequency to
-            # support last partial candle
-            # TODO: optimize this by applying this logic only for the last day
-            if adj_data_frequency == 'daily':
-                adj_data_frequency = 'minute'
-                adj_bar_count = adj_bar_count * 1440
-
-        else:  # data_frequency == "daily":
-            last_dt_for_series = end_dt
+        if data_frequency == 'minute' and adj_data_frequency == 'daily':
+            end_dt = end_dt.floor('1D')
 
         series = bundle.get_history_window_series_and_load(
             assets=assets,
-            end_dt=last_dt_for_series,
+            end_dt=end_dt,
             bar_count=adj_bar_count,
             field=field,
             data_frequency=adj_data_frequency,
             algo_end_dt=self._last_available_session,
         )
 
-        start_dt = get_start_dt(last_dt_for_series, adj_bar_count,
-                                adj_data_frequency, False)
+        start_dt = get_start_dt(end_dt, adj_bar_count, adj_data_frequency)
         df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
-
         return df
 
     def get_exchange_spot_value(self,

catalyst/finance/performance/tracker.py

@@ -62,7 +62,6 @@ from __future__ import division
 import logbook
 
 import pandas as pd
-from pandas.tseries.tools import normalize_date
 
 from catalyst.finance.performance.period import PerformancePeriod
 from catalyst.errors import NoFurtherDataError
@@ -344,7 +343,7 @@ class PerformanceTracker(object):
         """
         self.position_tracker.sync_last_sale_prices(dt, False, data_portal)
         self.update_performance()
-        todays_date = normalize_date(dt)
+        todays_date = dt.normalize()
         account = self.get_account(False)
 
         bench_returns = self.all_benchmark_returns.loc[todays_date:dt]

catalyst/finance/risk/cumulative.py

@@ -18,7 +18,6 @@ import logbook
 import numpy as np
 
 import pandas as pd
-from pandas.tseries.tools import normalize_date
 
 from six import iteritems
 
@@ -80,7 +79,7 @@ class RiskMetricsCumulative(object):
         # on the first day.
         self.day_before_start = self.start_session - self.sessions.freq
 
-        last_day = normalize_date(sim_params.end_session)
+        last_day = sim_params.end_session.normalize()
         if last_day not in self.sessions:
             last_day = pd.tseries.index.DatetimeIndex(
                 [last_day]

catalyst/finance/trading.py

@@ -16,7 +16,6 @@ from functools import partial
 
 import logbook
 import pandas as pd
-from pandas.tslib import normalize_date
 from six import string_types
 from sqlalchemy import create_engine
 
@@ -164,8 +163,8 @@ class SimulationParameters(object):
         # chop off any minutes or hours on the given start and end dates,
         # as we only support session labels here (and we represent session
         # labels as midnight UTC).
-        self._start_session = normalize_date(start_session)
-        self._end_session = normalize_date(end_session)
+        self._start_session = start_session.normalize()
+        self._end_session = end_session.normalize()
         self._capital_base = capital_base
 
         self._emission_rate = emission_rate

catalyst/gens/tradesimulation.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 from contextlib2 import ExitStack
 from logbook import Logger, Processor
-from pandas.tslib import normalize_date
 from catalyst.protocol import BarData
 from catalyst.utils.api_support import ZiplineAPI
 from six import viewkeys
@@ -229,7 +228,7 @@ class AlgorithmSimulator(object):
                 elif action == SESSION_END:
                     # End of the session.
                     if emission_rate == 'daily':
-                        handle_benchmark(normalize_date(dt))
+                        handle_benchmark(dt).normalize()
                     execute_order_cancellation_policy()
 
                     yield self._get_daily_message(dt, algo, algo.perf_tracker)

catalyst/utils/calendars/trading_calendar.py

@@ -640,12 +640,9 @@ class TradingCalendar(with_metaclass(ABCMeta)):
         """
         sched = self.schedule
 
-        # `market_open` and `market_close` should be timezone aware, but pandas
-        # 0.16.1 does not appear to support this:
-        # http://pandas.pydata.org/pandas-docs/stable/whatsnew.html#datetime-with-tz  # noqa
         return (
-            sched.at[session_label, 'market_open'].tz_localize('UTC'),
-            sched.at[session_label, 'market_close'].tz_localize('UTC'),
+            sched.at[session_label, 'market_open'],
+            sched.at[session_label, 'market_close'],
         )
 
     def session_open(self, session_label):

catalyst/utils/factory.py

@@ -117,9 +117,9 @@ def create_dividend(sid, payment, declared_date, ex_date, pay_date):
         'net_amount': payment,
         'payment_sid': None,
         'ratio': None,
-        'declared_date': pd.tslib.normalize_date(declared_date),
-        'ex_date': pd.tslib.normalize_date(ex_date),
-        'pay_date': pd.tslib.normalize_date(pay_date),
+        'declared_date': pd.tslib.declared_date.normalize(),
+        'ex_date': pd.tslib.ex_date.normalize(),
+        'pay_date': pd.tslib.pay_date.normalize(),
         'type': DATASOURCE_TYPE.DIVIDEND,
         'source_id': 'MockDividendSource'
     })
@@ -134,9 +134,9 @@ def create_stock_dividend(sid, payment_sid, ratio, declared_date,
         'ratio': ratio,
         'net_amount': None,
         'gross_amount': None,
-        'dt': pd.tslib.normalize_date(declared_date),
-        'ex_date': pd.tslib.normalize_date(ex_date),
-        'pay_date': pd.tslib.normalize_date(pay_date),
+        'dt': pd.tslib.declared_date.normalize(),
+        'ex_date': pd.tslib.ex_date.normalize(),
+        'pay_date': pd.tslib.pay_date.normalize(),
         'type': DATASOURCE_TYPE.DIVIDEND,
         'source_id': 'MockDividendSource'
     })

catalyst/utils/run_algo.py

@@ -263,8 +263,8 @@ def _run(handle_data,
         # We still need to support bundles for other misc data, but we
         # can handle this later.
 
-        if start != pd.tslib.normalize_date(start) or \
-                        end != pd.tslib.normalize_date(end):
+        if start != pd.tslib.start.normalize() or \
+                        end != pd.tslib.end.normalize():
             # todo: add to Sim_Params the option to start & end at specific times 
             log.warn(
                 "Catalyst currently starts and ends on the start and "

docs/source/releases.rst

@@ -8,7 +8,8 @@ Version 0.5.8
 
 Bug Fixes
 ~~~~~~~~~
-- Fix Data Marketplace release on mainnet
+- Fix proper release of Data Marketplace on mainnet.
+
 
 Version 0.5.7
 ^^^^^^^^^^^^^
@@ -22,7 +23,7 @@ Build
 
 Bug Fixes
 ~~~~~~~~~
-- fixes in storing and loading the state :issue:`214`, 
+- Added arguments to the ``reduce`` function in tha Asset class :issue:`214`, 
   :issue:`287`
 
 Version 0.5.6

etc/python2.7-environment.yml

@@ -5,7 +5,6 @@ channels:
 dependencies:
 - certifi=2016.2.28=py27_0
 - mkl=2017.0.3
-- matplotlib=2.1.2=py36_0
 - numpy=1.13.1=py27_0
 - openssl=1.0.2l
 - pip=9.0.1=py27_1
@@ -40,7 +39,7 @@ dependencies:
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.1.0
+  - matplotlib==2.1.2
   - multipledispatch==0.4.9
   - networkx==2.0
   - numexpr==2.6.4

setup.py

@@ -165,7 +165,7 @@ def _filter_requirements(lines_iter, filter_names=None,
 
 REQ_UPPER_BOUNDS = {
     'bcolz': '<1',
-    'pandas': '<0.20',
+    'pandas': '>=0.22',
     'empyrical': '<0.2.2',
 }
 

tests/finance/test_slippage.py

@@ -21,7 +21,6 @@ import datetime
 from math import sqrt
 
 from nose_parameterized import parameterized
-from pandas.tslib import normalize_date
 import numpy as np
 import pandas as pd
 import pytz
@@ -1108,8 +1107,8 @@ class OrdersStopTestCase(WithSimParams,
             )),
         )
         days = pd.date_range(
-            start=normalize_date(self.minutes[0]),
-            end=normalize_date(self.minutes[-1])
+            start=self.minutes[0].normalize(),
+            end=self.minutes[-1].normalize()
         )
         with tmp_bcolz_equity_minute_bar_reader(
                 self.trading_calendar, days, assets) as reader:

tests/pipeline/test_pipeline_algo.py

@@ -27,7 +27,6 @@ from pandas import (
     Series,
     Timestamp,
 )
-from pandas.tseries.tools import normalize_date
 from six import iteritems, itervalues
 
 from catalyst.algorithm import TradingAlgorithm
@@ -530,7 +529,7 @@ class PipelineAlgorithmTestCase(WithBcolzEquityDailyBarReaderFromCSVs,
             attach_pipeline(pipeline, 'test')
 
         def handle_data(context, data):
-            today = normalize_date(get_datetime())
+            today = get_datetime().normalize()
             results = pipeline_output('test')
             expect_over_300 = {
                 AAPL: today < self.AAPL_split_date,

tests/test_fetcher.py

@@ -395,7 +395,6 @@ def handle_data(context, data):
 
             algocode = """
 from pandas import Timestamp
-from pandas.tseries.tools import normalize_date
 from catalyst.api import fetch_csv, record, sid, get_datetime
 
 def initialize(context):
@@ -411,7 +410,7 @@ def initialize(context):
     context.bar_count = 0
 
 def handle_data(context, data):
-    expected = context.expected_sids[normalize_date(get_datetime())]
+    expected = context.expected_sids[get_datetime().normalize()]
     actual = data.fetcher_assets
     for stk in expected:
         if stk not in actual: