Merge branch 'develop' - Release 0.3.1

FIX: bundle start_dt cannot be earlier than asset_start
FIX: prior raise of AuthNotFound, now generates empty auth.json, and raises AuthEmpty when live
FIX: os.path.join to make BUNDLE_NAME_TEMPLATE compatible across OSes

catalyst/examples/buy_btc.py

@@ -0,0 +1,10 @@
+from catalyst.api import order, record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usd')
+
+
+def handle_data(context, data):
+    order(context.asset, 1)
+    record(btc=data.current(context.asset, 'price'))

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,10 +1,10 @@
 import base64
+import datetime
 import hashlib
 import hmac
 import json
 import re
 import time
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -22,10 +22,10 @@ from catalyst.exchange.exchange_errors import (
     InvalidOrderStyle, OrderCancelError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
+from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.protocol import Account
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request

catalyst/exchange/bittrex/bittrex.py

@@ -5,18 +5,17 @@ from catalyst.assets._assets import TradingPair
 from logbook import Logger
 from six.moves import urllib
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
     CreateOrderError
-from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
-
-from catalyst.constants import LOG_LEVEL
+from catalyst.finance.execution import LimitOrder, StopLimitOrder
+from catalyst.finance.order import Order, ORDER_STATUS
 
 log = Logger('Bittrex', level=LOG_LEVEL)
 

catalyst/exchange/data_portal_exchange.py

@@ -19,17 +19,13 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.bundle_utils import get_start_dt
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
-    PricingDataBeforeTradingError,
-    PricingDataNotLoadedError, InvalidHistoryFrequencyError,
-    BundleNotFoundError)
-
-from catalyst.constants import LOG_LEVEL
+    PricingDataNotLoadedError)
 
 log = Logger('DataPortalExchange', level=LOG_LEVEL)
 

catalyst/exchange/exchange.py

@@ -9,14 +9,14 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_adj_dates
+    get_delta, get_periods
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
-    InvalidHistoryFrequencyError, MismatchingFrequencyError, \
-    BundleNotFoundError, NoDataAvailableOnExchange, PricingDataNotLoadedError
+    InvalidHistoryFrequencyError, PricingDataNotLoadedError
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -24,8 +24,6 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('Exchange', level=LOG_LEVEL)
 
 

catalyst/exchange/exchange_algorithm.py

@@ -26,6 +26,7 @@ from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
     BcolzMinuteBarReader
 from catalyst.errors import OrderInBeforeTradingStart
@@ -51,10 +52,8 @@ from catalyst.utils.api_support import (
     disallowed_in_before_trading_start)
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
     expect_types
-from catalyst.utils.preprocess import preprocess
 from catalyst.utils.math_utils import round_nearest
-
-from catalyst.constants import LOG_LEVEL
+from catalyst.utils.preprocess import preprocess
 
 log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 

catalyst/exchange/exchange_bcolz.py

@@ -3,7 +3,6 @@ import numpy as np
 from catalyst import get_calendar
 from catalyst.data.minute_bars import BcolzMinuteBarReader, \
     BcolzMinuteBarWriter
-from catalyst.exchange.bundle_utils import get_periods, get_periods_range
 
 
 class BcolzExchangeBarWriter(BcolzMinuteBarWriter):

catalyst/exchange/exchange_blotter.py

@@ -1,13 +1,12 @@
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import Transaction
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('exchange_blotter', level=LOG_LEVEL)
 
 # It seems like we need to accept greater slippage risk in cryptos

catalyst/exchange/exchange_bundle.py

@@ -3,9 +3,10 @@ import shutil
 from datetime import timedelta
 
 import pandas as pd
-from logbook import Logger, INFO
+from logbook import Logger
 
 from catalyst import get_calendar
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
@@ -14,18 +15,16 @@ from catalyst.exchange.bundle_utils import range_in_bundle, \
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
-    InvalidHistoryFrequencyError, PricingDataBeforeTradingError, \
-    TempBundleNotFoundError, NoDataAvailableOnExchange, \
+    InvalidHistoryFrequencyError, TempBundleNotFoundError, \
+    NoDataAvailableOnExchange, \
     PricingDataNotLoadedError
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('exchange_bundle', level=LOG_LEVEL)
 
-BUNDLE_NAME_TEMPLATE = '{root}/{frequency}_bundle'
+BUNDLE_NAME_TEMPLATE = os.path.join('{root}','{frequency}_bundle')
 
 def _cachpath(symbol, type_):
     return '-'.join([symbol, type_])
@@ -172,7 +171,7 @@ class ExchangeBundle:
                 invalid_data_behavior='raise'
             )
         except BcolzMinuteOverlappingData as e:
-            log.warn('chunk already exists: {}'.format(e))
+            log.debug('chunk already exists: {}'.format(e))
         except Exception as e:
             log.warn('error when writing data: {}, trying again'.format(e))
 
@@ -319,6 +318,9 @@ class ExchangeBundle:
             except NoDataAvailableOnExchange:
                 continue
 
+            start_dt = max(start_dt, self.calendar.first_trading_session)
+            start_dt = max(start_dt, asset_start)
+
             # Aligning start / end dates with the daily calendar
             sessions = get_periods_range(start_dt, end_dt, data_frequency) \
                 if data_frequency == 'minute' \

catalyst/exchange/exchange_errors.py

@@ -1,10 +1,13 @@
-import sys, traceback
+import sys
+import traceback
+
 from catalyst.errors import ZiplineError
 
 
 def silent_except_hook(exctype, excvalue, exctraceback):
     if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError,
-                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, ]:
+                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, 
+                   ExchangeAuthEmpty ]:
         fn = traceback.extract_tb(exctraceback)[-1][0]
         ln = traceback.extract_tb(exctraceback)[-1][1]
         print "Error traceback: {1} (line {2})\n" \
@@ -63,6 +66,13 @@ class ExchangeAuthNotFound(ZiplineError):
     ).strip()
 
 
+class ExchangeAuthEmpty(ZiplineError):
+    msg = (
+        'Please enter your API token key and secret for exchange {exchange} '
+        'in the following file: {filename}'
+    ).strip()
+
+
 class ExchangeSymbolsNotFound(ZiplineError):
     msg = (
         'Unable to download or find a local copy of symbols.json for exchange '

catalyst/exchange/exchange_portfolio.py

@@ -1,9 +1,8 @@
 import numpy as np
 from logbook import Logger
 
-from catalyst.protocol import Portfolio, Positions, Position
-
 from catalyst.constants import LOG_LEVEL
+from catalyst.protocol import Portfolio, Positions, Position
 
 log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 

catalyst/exchange/exchange_utils.py

@@ -8,7 +8,8 @@ import pandas as pd
 
 from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
     ExchangeSymbolsNotFound
-from catalyst.utils.paths import data_root, ensure_directory, last_modified_time
+from catalyst.utils.paths import data_root, ensure_directory, \
+    last_modified_time
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'
@@ -64,11 +65,10 @@ def get_exchange_auth(exchange_name, environ=None):
             data = json.load(data_file)
             return data
     else:
-        raise ExchangeAuthNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
-
+        data = dict(name=exchange_name, key='', secret='')
+        with open(filename, 'w') as f:
+            json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':'))
+            return data
 
 def get_algo_folder(algo_name, environ=None):
     if not environ:

catalyst/exchange/live_graph_clock.py

@@ -10,7 +10,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from datetime import timedelta
 
 import pandas as pd
 from catalyst.gens.sim_engine import (
@@ -19,11 +18,10 @@ from catalyst.gens.sim_engine import (
 )
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import \
     MismatchingBaseCurrenciesExchanges
 
-from catalyst.constants import LOG_LEVEL
-
 log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 

catalyst/exchange/poloniex/poloniex.py

@@ -1,39 +1,32 @@
-import base64
-import hashlib
-import hmac
 import json
-import re
+import json
 import time
 from collections import defaultdict
 
 import numpy as np
 import pandas as pd
 import pytz
-import requests
-# import six
-from six import iteritems
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
+# import six
+from six import iteritems
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-
+from catalyst.constants import LOG_LEVEL
 # from websocket import create_connection
 from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError,
-    OrphanOrderReverseError)
+    InvalidOrderStyle, OrphanOrderReverseError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
+    ExchangeStopLimitOrder
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
+from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
+from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
-
-from catalyst.constants import LOG_LEVEL
+from catalyst.protocol import Account
 
 log = Logger('Poloniex', level=LOG_LEVEL)
 

catalyst/exchange/simple_clock.py

@@ -16,13 +16,13 @@ from time import sleep
 import pandas as pd
 from catalyst.gens.sim_engine import (
     BAR,
-    SESSION_START,
-    MINUTE_END,
-    SESSION_END
+    SESSION_START
 )
 from logbook import Logger
 
-log = Logger('ExchangeClock')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 
 class SimpleClock(object):

catalyst/utils/run_algo.py

@@ -36,11 +36,11 @@ from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
+    ExchangeRequestError, ExchangeAuthEmpty,
     ExchangeRequestErrorTooManyAttempts,
     BaseCurrencyNotFoundError, ExchangeNotFoundError)
 from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object
+    get_algo_object, get_exchange_folder
 from logbook import Logger
 
 from catalyst.constants import LOG_LEVEL
@@ -166,6 +166,12 @@ def _run(handle_data,
 
         # This corresponds to the json file containing api token info
         exchange_auth = get_exchange_auth(exchange_name)
+
+        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
+            raise ExchangeAuthEmpty(
+                    exchange=exchange_name.title(), 
+                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
+
         if exchange_name == 'bitfinex':
             exchanges[exchange_name] = Bitfinex(
                 key=exchange_auth['key'],
@@ -237,8 +243,11 @@ def _run(handle_data,
                 balances = exchange.get_balances()
             except ExchangeRequestError as e:
                 if attempt_index < 20:
-                    log.warn('exchange error when retrieving balances, {} '
-                             'trying again in 5 seconds'.format(e))
+                    log.warn(
+                        'could not retrieve balances on {}: {}'.format(
+                            exchange.name, e
+                        )
+                    )
                     sleep(5)
                     return fetch_capital_base(exchange, attempt_index + 1)