Merge branch 'develop' - Release 0.3.1

FIX: bundle start_dt cannot be earlier than asset_start
FIX: prior raise of AuthNotFound, now generates empty auth.json, and raises AuthEmpty when live
FIX: os.path.join to make BUNDLE_NAME_TEMPLATE compatible across OSes

catalyst/examples/buy_btc.py

@@ -0,0 +1,10 @@
+from catalyst.api import order, record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usd')
+
+
+def handle_data(context, data):
+    order(context.asset, 1)
+    record(btc=data.current(context.asset, 'price'))

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,10 +1,10 @@
 import base64
+import datetime
 import hashlib
 import hmac
 import json
 import re
 import time
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -22,10 +22,10 @@ from catalyst.exchange.exchange_errors import (
     InvalidOrderStyle, OrderCancelError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
+from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.protocol import Account
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request

catalyst/exchange/bittrex/bittrex.py

@@ -5,18 +5,17 @@ from catalyst.assets._assets import TradingPair
 from logbook import Logger
 from six.moves import urllib
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
     CreateOrderError
-from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
-
+from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.constants import LOG_LEVEL
+from catalyst.finance.order import Order, ORDER_STATUS
 
 log = Logger('Bittrex', level=LOG_LEVEL)
 

catalyst/exchange/data_portal_exchange.py

@@ -19,17 +19,13 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.bundle_utils import get_start_dt
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
-    PricingDataBeforeTradingError,
+    PricingDataNotLoadedError)
-    PricingDataNotLoadedError, InvalidHistoryFrequencyError,
-    BundleNotFoundError)
-
-from catalyst.constants import LOG_LEVEL
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 

catalyst/exchange/exchange.py

@@ -9,14 +9,14 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_adj_dates
+    get_delta, get_periods
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
-    InvalidHistoryFrequencyError, MismatchingFrequencyError, \
+    InvalidHistoryFrequencyError, PricingDataNotLoadedError
-    BundleNotFoundError, NoDataAvailableOnExchange, PricingDataNotLoadedError
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -24,8 +24,6 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('Exchange', level=LOG_LEVEL)
 
 

catalyst/exchange/exchange_algorithm.py

@@ -26,6 +26,7 @@ from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
     BcolzMinuteBarReader
 from catalyst.errors import OrderInBeforeTradingStart
@@ -51,10 +52,8 @@ from catalyst.utils.api_support import (
     disallowed_in_before_trading_start)
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
     expect_types
-from catalyst.utils.preprocess import preprocess
 from catalyst.utils.math_utils import round_nearest
-
+from catalyst.utils.preprocess import preprocess
-from catalyst.constants import LOG_LEVEL
 
 log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 

catalyst/exchange/exchange_bcolz.py

@@ -3,7 +3,6 @@ import numpy as np
 from catalyst import get_calendar
 from catalyst.data.minute_bars import BcolzMinuteBarReader, \
     BcolzMinuteBarWriter
-from catalyst.exchange.bundle_utils import get_periods, get_periods_range
 
 
 class BcolzExchangeBarWriter(BcolzMinuteBarWriter):

catalyst/exchange/exchange_blotter.py

@@ -1,13 +1,12 @@
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import Transaction
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
 # It seems like we need to accept greater slippage risk in cryptos

catalyst/exchange/exchange_bundle.py

@@ -3,9 +3,10 @@ import shutil
 from datetime import timedelta
 
 import pandas as pd
-from logbook import Logger, INFO
+from logbook import Logger
 
 from catalyst import get_calendar
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
@@ -14,18 +15,16 @@ from catalyst.exchange.bundle_utils import range_in_bundle, \
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
-    InvalidHistoryFrequencyError, PricingDataBeforeTradingError, \
+    InvalidHistoryFrequencyError, TempBundleNotFoundError, \
-    TempBundleNotFoundError, NoDataAvailableOnExchange, \
+    NoDataAvailableOnExchange, \
     PricingDataNotLoadedError
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('exchange_bundle', level=LOG_LEVEL)
 
-BUNDLE_NAME_TEMPLATE = '{root}/{frequency}_bundle'
+BUNDLE_NAME_TEMPLATE = os.path.join('{root}','{frequency}_bundle')
 
 def _cachpath(symbol, type_):
     return '-'.join([symbol, type_])
@@ -172,7 +171,7 @@ class ExchangeBundle:
                 invalid_data_behavior='raise'
             )
         except BcolzMinuteOverlappingData as e:
-            log.warn('chunk already exists: {}'.format(e))
+            log.debug('chunk already exists: {}'.format(e))
         except Exception as e:
             log.warn('error when writing data: {}, trying again'.format(e))
 
@@ -319,6 +318,9 @@ class ExchangeBundle:
             except NoDataAvailableOnExchange:
                 continue
 
+            start_dt = max(start_dt, self.calendar.first_trading_session)
+            start_dt = max(start_dt, asset_start)
+
             # Aligning start / end dates with the daily calendar
             sessions = get_periods_range(start_dt, end_dt, data_frequency) \
                 if data_frequency == 'minute' \

catalyst/exchange/exchange_errors.py

@@ -1,10 +1,13 @@
-import sys, traceback
+import sys
+import traceback
+
 from catalyst.errors import ZiplineError
 
 
 def silent_except_hook(exctype, excvalue, exctraceback):
     if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError,
-                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, ]:
+                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, 
+                   ExchangeAuthEmpty ]:
         fn = traceback.extract_tb(exctraceback)[-1][0]
         ln = traceback.extract_tb(exctraceback)[-1][1]
         print "Error traceback: {1} (line {2})\n" \
@@ -63,6 +66,13 @@ class ExchangeAuthNotFound(ZiplineError):
     ).strip()
 
 
+class ExchangeAuthEmpty(ZiplineError):
+    msg = (
+        'Please enter your API token key and secret for exchange {exchange} '
+        'in the following file: {filename}'
+    ).strip()
+
+
 class ExchangeSymbolsNotFound(ZiplineError):
     msg = (
         'Unable to download or find a local copy of symbols.json for exchange '

catalyst/exchange/exchange_portfolio.py

@@ -1,9 +1,8 @@
 import numpy as np
 from logbook import Logger
 
-from catalyst.protocol import Portfolio, Positions, Position
-
 from catalyst.constants import LOG_LEVEL
+from catalyst.protocol import Portfolio, Positions, Position
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 

catalyst/exchange/exchange_utils.py

@@ -8,7 +8,8 @@ import pandas as pd
 
 from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
     ExchangeSymbolsNotFound
-from catalyst.utils.paths import data_root, ensure_directory, last_modified_time
+from catalyst.utils.paths import data_root, ensure_directory, \
+    last_modified_time
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'
@@ -64,11 +65,10 @@ def get_exchange_auth(exchange_name, environ=None):
             data = json.load(data_file)
             return data
     else:
-        raise ExchangeAuthNotFound(
+        data = dict(name=exchange_name, key='', secret='')
-            exchange=exchange_name,
+        with open(filename, 'w') as f:
-            filename=filename
+            json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':'))
-        )
+            return data
-
 
 def get_algo_folder(algo_name, environ=None):
     if not environ:

catalyst/exchange/live_graph_clock.py

@@ -10,7 +10,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from datetime import timedelta
 
 import pandas as pd
 from catalyst.gens.sim_engine import (
@@ -19,11 +18,10 @@ from catalyst.gens.sim_engine import (
 )
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import \
     MismatchingBaseCurrenciesExchanges
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 

catalyst/exchange/poloniex/poloniex.py

@@ -1,39 +1,32 @@
-import base64
-import hashlib
-import hmac
 import json
-import re
+import json
 import time
 from collections import defaultdict
 
 import numpy as np
 import pandas as pd
 import pytz
-import requests
-# import six
-from six import iteritems
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
+# import six
+from six import iteritems
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
+from catalyst.constants import LOG_LEVEL
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-
 # from websocket import create_connection
 from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError,
+    InvalidOrderStyle, OrphanOrderReverseError)
-    OrphanOrderReverseError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
+    ExchangeStopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
+from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
+from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
-
+from catalyst.protocol import Account
-from catalyst.constants import LOG_LEVEL
 
 log = Logger('Poloniex', level=LOG_LEVEL)
 

catalyst/exchange/simple_clock.py

@@ -16,13 +16,13 @@ from time import sleep
 import pandas as pd
 from catalyst.gens.sim_engine import (
     BAR,
-    SESSION_START,
+    SESSION_START
-    MINUTE_END,
-    SESSION_END
 )
 from logbook import Logger
 
-log = Logger('ExchangeClock')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 
 class SimpleClock(object):

catalyst/utils/run_algo.py

@@ -36,11 +36,11 @@ from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
+    ExchangeRequestError, ExchangeAuthEmpty,
     ExchangeRequestErrorTooManyAttempts,
     BaseCurrencyNotFoundError, ExchangeNotFoundError)
 from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object
+    get_algo_object, get_exchange_folder
 from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
@@ -166,6 +166,12 @@ def _run(handle_data,
 
         # This corresponds to the json file containing api token info
         exchange_auth = get_exchange_auth(exchange_name)
+
+        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
+            raise ExchangeAuthEmpty(
+                    exchange=exchange_name.title(), 
+                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
+
         if exchange_name == 'bitfinex':
             exchanges[exchange_name] = Bitfinex(
                 key=exchange_auth['key'],
@@ -237,8 +243,11 @@ def _run(handle_data,
                 balances = exchange.get_balances()
             except ExchangeRequestError as e:
                 if attempt_index < 20:
-                    log.warn('exchange error when retrieving balances, {} '
+                    log.warn(
-                             'trying again in 5 seconds'.format(e))
+                        'could not retrieve balances on {}: {}'.format(
+                            exchange.name, e
+                        )
+                    )
                     sleep(5)
                     return fetch_capital_base(exchange, attempt_index + 1)
 

docs/source/beginner-tutorial.rst

@@ -429,6 +429,10 @@ and allows us to plot the price of bitcoin. For example, we could easily
 examine now how our portfolio value changed over time compared to the
 bitcoin price.
 
+.. code-block:: python
+
+    %load_ext catalyst
+
 .. code-block:: python
 
     %pylab inline
@@ -484,7 +488,8 @@ a function we use in the ``handle_data()`` section:
 
 .. code-block:: python
 
-    %%catalyst --start 2016-1-1 --end 2017-9-30 -x bitfinex -o dma.pickle
+    %%catalyst --start 2016-4-1 --end 2017-9-30 -x bitfinex
+
     from catalyst.api import order, record, symbol, order_target
 
     def initialize(context):
@@ -492,16 +497,16 @@ a function we use in the ``handle_data()`` section:
        context.asset = symbol('btc_usd')
 
     def handle_data(context, data):
-       # Skip first 300 days to get full windows
+       # Skip first 150 days to get full windows
        context.i += 1
-       if context.i < 300:
+       if context.i < 150:
            return
 
        # Compute averages
        # data.history() has to be called with the same params
        # from above and returns a pandas dataframe.
-       short_mavg = data.history(context.asset, 'price', bar_count=100, frequency="1d").mean()
+       short_mavg = data.history(context.asset, 'price', bar_count=50, frequency="1d").mean()
-       long_mavg = data.history(context.asset, 'price', bar_count=300, frequency="1d").mean()
+       long_mavg = data.history(context.asset, 'price', bar_count=150, frequency="1d").mean()
 
        # Trading logic
        if short_mavg > long_mavg:
@@ -518,7 +523,7 @@ a function we use in the ``handle_data()`` section:
 
     def analyze(context, perf):
        import matplotlib.pyplot as plt
-       fig = plt.figure()
+       fig = plt.figure(figsize=(12,12))
        ax1 = fig.add_subplot(211)
        perf.portfolio_value.plot(ax=ax1)
        ax1.set_ylabel('portfolio value in $')

etc/python2.7-environment.yml

@@ -1,30 +1,24 @@
 name: catalyst
 channels:
-- statiskit
 - defaults
 dependencies:
 - certifi=2016.2.28=py27_0
-- coverage=4.4.1=py27_0
+- libgfortran=3.0.0=1
-- nose=1.3.7=py27_1
+- mkl=2017.0.3=0
+- numpy=1.13.1=py27_0
 - openssl=1.0.2l=0
-- path.py=10.3.1=py27_0
 - pip=9.0.1=py27_1
 - python=2.7.13=0
-- pyyaml=3.12=py27_0
 - readline=6.2=2
-- setuptools=36.4.0=py27_0
+- scipy=0.19.1=np113py27_0
-- six=1.10.0=py27_0
+- setuptools=36.4.0=py27_1
 - sqlite=3.13.0=0
 - tk=8.5.18=0
 - wheel=0.29.0=py27_0
-- yaml=0.1.6=0
 - zlib=1.2.11=0
-- libdev=1.0.0=py27_0
-- python-dev=1.0.0=py27_0
-- python-scons=3.0.0=py27_0
 - pip:
-  - alembic==0.9.5
+  - alembic==0.9.6
-  - backports.shutil-get-terminal-size==1.0.0
+  - backports.functools-lru-cache==1.4
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
@@ -32,36 +26,22 @@ dependencies:
   - contextlib2==0.5.5
   - cycler==0.10.0
   - cyordereddict==1.0.0
-  - cython==0.26.1
+  - cython==0.27.1
   - decorator==4.1.2
   - empyrical==0.2.1
-  - enigma-catalyst>=0.2.dev2
-  - enum34==1.1.6
-  - functools32==3.2.3.post2
   - idna==2.6
   - intervaltree==2.1.0
-  - ipdb==0.10.3
-  - ipdbplugin==1.4.5
-  - ipython==5.5.0
-  - ipython-genutils==0.2.0
   - logbook==1.1.0
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.0.2
+  - matplotlib==2.1.0
   - multipledispatch==0.4.9
-  - networkx==1.11
+  - networkx==2.0
   - numexpr==2.6.4
-  - numpy==1.13.1
   - pandas==0.19.2
   - pandas-datareader==0.5.0
-  - pathlib2==2.3.0
   - patsy==0.4.1
-  - pexpect==4.2.1
-  - pickleshare==0.7.4
-  - prompt-toolkit==1.0.15
-  - ptyprocess==0.5.2
-  - pygments==2.2.0
   - pyparsing==2.2.0
   - python-dateutil==2.6.1
   - python-editor==1.0.3
@@ -69,16 +49,12 @@ dependencies:
   - requests==2.18.4
   - requests-file==1.4.2
   - requests-ftp==0.3.1
-  - scandir==1.5
+  - six==1.11.0
-  - scipy==0.19.1
-  - scons==3.0.0a20170821
-  - simplegeneric==0.8.1
   - sortedcontainers==1.5.7
   - sqlalchemy==1.1.14
   - statsmodels==0.8.0
   - subprocess32==3.2.7
   - tables==3.4.2
   - toolz==0.8.2
-  - traitlets==4.3.2
   - urllib3==1.22
-  - wcwidth==0.1.7
+  - enigma-catalyst>=0.3