Merge branch 'develop' - Release 0.3.1

FIX: bundle start_dt cannot be earlier than asset_start
FIX: prior raise of AuthNotFound, now generates empty auth.json, and raises AuthEmpty when live
FIX: os.path.join to make BUNDLE_NAME_TEMPLATE compatible across OSes

catalyst/algorithm.py

@@ -138,8 +138,9 @@ from catalyst.gens.sim_engine import MinuteSimulationClock
 from catalyst.sources.benchmark_source import BenchmarkSource
 from catalyst.catalyst_warnings import ZiplineDeprecationWarning
 
+from catalyst.constants import LOG_LEVEL
 
-log = logbook.Logger("ZiplineLog")
+log = logbook.Logger("CatalystLog", level=LOG_LEVEL)
 
 
 class TradingAlgorithm(object):

catalyst/assets/assets.py

@@ -76,7 +76,9 @@ from catalyst.utils.numpy_utils import as_column
 from catalyst.utils.preprocess import preprocess
 from catalyst.utils.sqlite_utils import group_into_chunks, coerce_string_to_eng
 
-log = Logger('assets.py')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('assets.py', level=LOG_LEVEL)
 
 # A set of fields that need to be converted to strings before building an
 # Asset to avoid unicode fields

catalyst/constants.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+import logbook
+
+LOG_LEVEL = logbook.INFO

catalyst/data/_equities.pyx

@@ -215,7 +215,7 @@ cpdef _read_bcolz_data(ctable_t table,
                 else:
                     continue
 
-        if column_name in ['open', 'high', 'low', 'close']:
+        if column_name in ['open', 'high', 'low', 'close', 'volume']:
             where_nan = (outbuf == 0)
             outbuf_as_float = outbuf.astype(float64) * .000000001
             outbuf_as_float[where_nan] = NAN

catalyst/data/bundles/base.py

@@ -30,8 +30,10 @@ from catalyst.utils.cli import (
 )
 from catalyst.utils.memoize import lazyval
 
+from catalyst.constants import LOG_LEVEL
+
 logbook.StderrHandler().push_application()
-log = logbook.Logger(__name__)
+log = logbook.Logger(__name__, level=LOG_LEVEL)
 
 DEFAULT_RETRIES = 5
 

catalyst/data/bundles/quandl.py

@@ -40,7 +40,9 @@ from catalyst.utils.cli import maybe_show_progress
 
 from . import core as bundles
 
-log = Logger(__name__)
+from catalyst.constants import LOG_LEVEL
+
+log = Logger(__name__, level=LOG_LEVEL)
 seconds_per_call = (pd.Timedelta('10 minutes') / 2000).total_seconds()
 
 class QuandlBundle(BaseEquityPricingBundle):

catalyst/data/data_portal.py

@@ -68,7 +68,9 @@ from catalyst.errors import (
     HistoryWindowStartsBeforeData,
 )
 
-log = Logger('DataPortal')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('DataPortal', level=LOG_LEVEL)
 
 BASE_FIELDS = frozenset([
     "open",

catalyst/data/loader.py

@@ -32,7 +32,9 @@ from ..utils.paths import (
     data_root,
 )
 
-logger = logbook.Logger('Loader')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('Loader', level=LOG_LEVEL)
 
 # Mapping from index symbol to appropriate bond data
 INDEX_MAPPING = {

catalyst/data/minute_bars.py

@@ -44,7 +44,9 @@ from catalyst.utils.calendars import get_calendar
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.memoize import lazyval
 
-logger = logbook.Logger('MinuteBars')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('MinuteBars', level=LOG_LEVEL)
 
 US_EQUITIES_MINUTES_PER_DAY = 390
 FUTURES_MINUTES_PER_DAY = 1440

catalyst/data/us_equity_pricing.py

@@ -83,7 +83,9 @@ from catalyst.utils.cli import (
 from ._equities import _compute_row_slices, _read_bcolz_data
 from ._adjustments import load_adjustments_from_sqlite
 
-logger = logbook.Logger('UsEquityPricing')
+from catalyst.constants import LOG_LEVEL
+
+logger = logbook.Logger('UsEquityPricing', level=LOG_LEVEL)
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
 OHLCV = frozenset(['open', 'high', 'low', 'close', 'volume'])

catalyst/examples/buy_btc.py

@@ -0,0 +1,10 @@
+from catalyst.api import order, record, symbol
+
+
+def initialize(context):
+    context.asset = symbol('btc_usd')
+
+
+def handle_data(context, data):
+    order(context.asset, 1)
+    record(btc=data.current(context.asset, 'price'))

catalyst/examples/buy_btc_simple.py

@@ -1,8 +1,8 @@
 from catalyst.api import order, record, symbol
 
 def initialize(context):
-   context.asset = symbol('btc_usd')
+    context.asset = symbol('btc_usd')
 
 def handle_data(context, data):
-   order(asset, 1)
-   record(btc=data.current(context.asset, 'price'))
+    order(context.asset, 1)
+    record(btc = data.current(context.asset, 'price'))

catalyst/exchange/asset_finder_exchange.py

@@ -1,6 +1,8 @@
 from logbook import Logger
 
-log = Logger('AssetFinderExchange')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('AssetFinderExchange', level=LOG_LEVEL)
 
 
 class AssetFinderExchange(object):
@@ -41,9 +43,9 @@ class AssetFinderExchange(object):
         """
         for sid in sids:
             if sid in self._asset_cache:
-                log.info('got asset from cache: {}'.format(sid))
+                log.debug('got asset from cache: {}'.format(sid))
             else:
-                log.info('fetching asset: {}'.format(sid))
+                log.debug('fetching asset: {}'.format(sid))
         return list()
 
     def lookup_symbol(self, symbol, exchange, as_of_date=None, fuzzy=False):

catalyst/exchange/bitfinex/bitfinex.py

@@ -1,10 +1,10 @@
 import base64
+import datetime
 import hashlib
 import hmac
 import json
 import re
 import time
-import datetime
 
 import numpy as np
 import pandas as pd
@@ -22,10 +22,10 @@ from catalyst.exchange.exchange_errors import (
     InvalidOrderStyle, OrderCancelError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
+from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.protocol import Account
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
@@ -33,7 +33,9 @@ requests.adapters.DEFAULT_RETRIES = 20
 
 BITFINEX_URL = 'https://api.bitfinex.com'
 
-log = Logger('Bitfinex')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Bitfinex', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 

catalyst/exchange/bittrex/bittrex.py

@@ -5,18 +5,19 @@ from catalyst.assets._assets import TradingPair
 from logbook import Logger
 from six.moves import urllib
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.bittrex.bittrex_api import Bittrex_api
 from catalyst.exchange.exchange import Exchange
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     ExchangeRequestError, InvalidOrderStyle, OrderNotFound, OrderCancelError, \
     CreateOrderError
-from catalyst.finance.execution import LimitOrder, StopLimitOrder
-from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
+from catalyst.finance.execution import LimitOrder, StopLimitOrder
+from catalyst.finance.order import Order, ORDER_STATUS
 
-log = Logger('Bittrex')
+log = Logger('Bittrex', level=LOG_LEVEL)
 
 URL2 = 'https://bittrex.com/Api/v2.0'
 

catalyst/exchange/data_portal_exchange.py

@@ -19,17 +19,15 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import DataPortal
-from catalyst.exchange.bundle_utils import get_start_dt
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     ExchangeBarDataError,
-    PricingDataBeforeTradingError,
-    PricingDataNotLoadedError, InvalidHistoryFrequencyError,
-    BundleNotFoundError)
+    PricingDataNotLoadedError)
 
-log = Logger('DataPortalExchange')
+log = Logger('DataPortalExchange', level=LOG_LEVEL)
 
 
 class DataPortalExchangeBase(DataPortal):

catalyst/exchange/exchange.py

@@ -9,14 +9,14 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.data_portal import BASE_FIELDS
 from catalyst.exchange.bundle_utils import get_start_dt, \
-    get_delta, get_periods, get_adj_dates
+    get_delta, get_periods
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
     InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \
-    InvalidHistoryFrequencyError, MismatchingFrequencyError, \
-    BundleNotFoundError, NoDataAvailableOnExchange, PricingDataNotLoadedError
+    InvalidHistoryFrequencyError, PricingDataNotLoadedError
 from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
     ExchangeLimitOrder, ExchangeStopOrder
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
@@ -24,7 +24,7 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols
 from catalyst.finance.order import ORDER_STATUS
 from catalyst.finance.transaction import Transaction
 
-log = Logger('Exchange')
+log = Logger('Exchange', level=LOG_LEVEL)
 
 
 class Exchange:

catalyst/exchange/exchange_algorithm.py

@@ -26,6 +26,7 @@ from catalyst.assets._assets import TradingPair
 
 import catalyst.protocol as zp
 from catalyst.algorithm import TradingAlgorithm
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteBarWriter, \
     BcolzMinuteBarReader
 from catalyst.errors import OrderInBeforeTradingStart
@@ -51,10 +52,10 @@ from catalyst.utils.api_support import (
     disallowed_in_before_trading_start)
 from catalyst.utils.input_validation import error_keywords, ensure_upper_case, \
     expect_types
-from catalyst.utils.preprocess import preprocess
 from catalyst.utils.math_utils import round_nearest
+from catalyst.utils.preprocess import preprocess
 
-log = logbook.Logger('exchange_algorithm')
+log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL)
 
 
 class ExchangeAlgorithmExecutor(AlgorithmSimulator):

catalyst/exchange/exchange_bcolz.py

@@ -3,7 +3,6 @@ import numpy as np
 from catalyst import get_calendar
 from catalyst.data.minute_bars import BcolzMinuteBarReader, \
     BcolzMinuteBarWriter
-from catalyst.exchange.bundle_utils import get_periods, get_periods_range
 
 
 class BcolzExchangeBarWriter(BcolzMinuteBarWriter):

catalyst/exchange/exchange_blotter.py

@@ -1,12 +1,13 @@
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.finance.blotter import Blotter
 from catalyst.finance.commission import CommissionModel
 from catalyst.finance.slippage import SlippageModel
 from catalyst.finance.transaction import Transaction
 
-log = Logger('exchange_blotter')
+log = Logger('exchange_blotter', level=LOG_LEVEL)
 
 # It seems like we need to accept greater slippage risk in cryptos
 # Orders won't often close at Equity levels.

catalyst/exchange/exchange_bundle.py

@@ -3,9 +3,10 @@ import shutil
 from datetime import timedelta
 
 import pandas as pd
-from logbook import Logger, INFO
+from logbook import Logger
 
 from catalyst import get_calendar
+from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
 from catalyst.exchange.bundle_utils import range_in_bundle, \
@@ -14,23 +15,20 @@ from catalyst.exchange.bundle_utils import range_in_bundle, \
 from catalyst.exchange.exchange_bcolz import BcolzExchangeBarReader, \
     BcolzExchangeBarWriter
 from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \
-    InvalidHistoryFrequencyError, PricingDataBeforeTradingError, \
-    TempBundleNotFoundError, NoDataAvailableOnExchange, \
+    InvalidHistoryFrequencyError, TempBundleNotFoundError, \
+    NoDataAvailableOnExchange, \
     PricingDataNotLoadedError
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
 from catalyst.utils.paths import ensure_directory
 
+log = Logger('exchange_bundle', level=LOG_LEVEL)
+
+BUNDLE_NAME_TEMPLATE = os.path.join('{root}','{frequency}_bundle')
 
 def _cachpath(symbol, type_):
     return '-'.join([symbol, type_])
 
-
-BUNDLE_NAME_TEMPLATE = '{root}/{frequency}_bundle'
-log = Logger('exchange_bundle')
-log.level = INFO
-
-
 class ExchangeBundle:
     def __init__(self, exchange):
         self.exchange = exchange
@@ -173,7 +171,7 @@ class ExchangeBundle:
                 invalid_data_behavior='raise'
             )
         except BcolzMinuteOverlappingData as e:
-            log.warn('chunk already exists: {}'.format(e))
+            log.debug('chunk already exists: {}'.format(e))
         except Exception as e:
             log.warn('error when writing data: {}, trying again'.format(e))
 
@@ -320,6 +318,9 @@ class ExchangeBundle:
             except NoDataAvailableOnExchange:
                 continue
 
+            start_dt = max(start_dt, self.calendar.first_trading_session)
+            start_dt = max(start_dt, asset_start)
+
             # Aligning start / end dates with the daily calendar
             sessions = get_periods_range(start_dt, end_dt, data_frequency) \
                 if data_frequency == 'minute' \

catalyst/exchange/exchange_errors.py

@@ -1,10 +1,13 @@
-import sys, traceback
+import sys
+import traceback
+
 from catalyst.errors import ZiplineError
 
 
 def silent_except_hook(exctype, excvalue, exctraceback):
     if exctype in [PricingDataBeforeTradingError, PricingDataNotLoadedError,
-                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, ]:
+                   SymbolNotFoundOnExchange, NoDataAvailableOnExchange, 
+                   ExchangeAuthEmpty ]:
         fn = traceback.extract_tb(exctraceback)[-1][0]
         ln = traceback.extract_tb(exctraceback)[-1][1]
         print "Error traceback: {1} (line {2})\n" \
@@ -63,6 +66,13 @@ class ExchangeAuthNotFound(ZiplineError):
     ).strip()
 
 
+class ExchangeAuthEmpty(ZiplineError):
+    msg = (
+        'Please enter your API token key and secret for exchange {exchange} '
+        'in the following file: {filename}'
+    ).strip()
+
+
 class ExchangeSymbolsNotFound(ZiplineError):
     msg = (
         'Unable to download or find a local copy of symbols.json for exchange '

catalyst/exchange/exchange_portfolio.py

@@ -1,9 +1,10 @@
 import numpy as np
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.protocol import Portfolio, Positions, Position
 
-log = Logger('ExchangePortfolio')
+log = Logger('ExchangePortfolio', level=LOG_LEVEL)
 
 
 class ExchangePortfolio(Portfolio):

catalyst/exchange/exchange_utils.py

@@ -8,7 +8,8 @@ import pandas as pd
 
 from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
     ExchangeSymbolsNotFound
-from catalyst.utils.paths import data_root, ensure_directory, last_modified_time
+from catalyst.utils.paths import data_root, ensure_directory, \
+    last_modified_time
 
 SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
               '{exchange}/symbols.json'
@@ -64,11 +65,10 @@ def get_exchange_auth(exchange_name, environ=None):
             data = json.load(data_file)
             return data
     else:
-        raise ExchangeAuthNotFound(
-            exchange=exchange_name,
-            filename=filename
-        )
-
+        data = dict(name=exchange_name, key='', secret='')
+        with open(filename, 'w') as f:
+            json.dump(data, f, sort_keys=False, indent=2, separators=(',', ':'))
+            return data
 
 def get_algo_folder(algo_name, environ=None):
     if not environ:

catalyst/exchange/live_graph_clock.py

@@ -10,7 +10,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from datetime import timedelta
 
 import pandas as pd
 from catalyst.gens.sim_engine import (
@@ -19,11 +18,11 @@ from catalyst.gens.sim_engine import (
 )
 from logbook import Logger
 
+from catalyst.constants import LOG_LEVEL
 from catalyst.exchange.exchange_errors import \
     MismatchingBaseCurrenciesExchanges
 
-
-log = Logger('LiveGraphClock')
+log = Logger('LiveGraphClock', level=LOG_LEVEL)
 
 
 class LiveGraphClock(object):

catalyst/exchange/poloniex/poloniex.py

@@ -1,39 +1,34 @@
-import base64
-import hashlib
-import hmac
 import json
-import re
+import json
 import time
 from collections import defaultdict
 
 import numpy as np
 import pandas as pd
 import pytz
-import requests
-# import six
-from six import iteritems
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
+# import six
+from six import iteritems
 
-from catalyst.exchange.exchange_bundle import ExchangeBundle
-from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
-
+from catalyst.constants import LOG_LEVEL
 # from websocket import create_connection
 from catalyst.exchange.exchange import Exchange
+from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
     ExchangeRequestError,
     InvalidHistoryFrequencyError,
-    InvalidOrderStyle, OrderCancelError,
-    OrphanOrderReverseError)
+    InvalidOrderStyle, OrphanOrderReverseError)
 from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
-    ExchangeStopLimitOrder, ExchangeStopOrder
-from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.protocol import Account
+    ExchangeStopLimitOrder
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
+from catalyst.exchange.poloniex.poloniex_api import Poloniex_api
+from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.finance.transaction import Transaction
+from catalyst.protocol import Account
 
-log = Logger('Poloniex')
+log = Logger('Poloniex', level=LOG_LEVEL)
 
 
 class Poloniex(Exchange):

catalyst/exchange/simple_clock.py

@@ -16,13 +16,13 @@ from time import sleep
 import pandas as pd
 from catalyst.gens.sim_engine import (
     BAR,
-    SESSION_START,
-    MINUTE_END,
-    SESSION_END
+    SESSION_START
 )
 from logbook import Logger
 
-log = Logger('ExchangeClock')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('ExchangeClock', level=LOG_LEVEL)
 
 
 class SimpleClock(object):

catalyst/finance/blotter.py

@@ -34,7 +34,9 @@ from catalyst.finance.commission import (
 from catalyst.finance.cancel_policy import NeverCancel
 from catalyst.utils.input_validation import expect_types
 
-log = Logger('Blotter')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Blotter', level=LOG_LEVEL)
 warning_logger = Logger('AlgoWarning')
 
 

catalyst/finance/controls.py

@@ -24,7 +24,9 @@ from catalyst.errors import (
     TradingControlViolation,
 )
 
-log = logbook.Logger('TradingControl')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('TradingControl', level=LOG_LEVEL)
 
 
 class TradingControl(with_metaclass(abc.ABCMeta)):

catalyst/finance/performance/period.py

@@ -88,7 +88,10 @@ from six import itervalues, iteritems
 
 import catalyst.protocol as zp
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
+
 TRADE_TYPE = zp.DATASOURCE_TYPE.TRADE
 
 

catalyst/finance/performance/position.py

@@ -40,7 +40,9 @@ import logbook
 from catalyst.assets import Future, Asset
 from catalyst.utils.input_validation import expect_types
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class Position(object):

catalyst/finance/performance/position_tracker.py

@@ -32,7 +32,9 @@ from catalyst.assets import (
 )
 from . position import positiondict
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 PositionStats = namedtuple('PositionStats',

catalyst/finance/performance/tracker.py

@@ -70,7 +70,9 @@ import catalyst.finance.risk as risk
 
 from . position_tracker import PositionTracker
 
-log = logbook.Logger('Performance')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Performance', level=LOG_LEVEL)
 
 
 class PerformanceTracker(object):

catalyst/finance/risk/cumulative.py

@@ -38,7 +38,9 @@ from empyrical import (
     sortino_ratio,
 )
 
-log = logbook.Logger('Risk Cumulative')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Cumulative', level=LOG_LEVEL)
 
 
 choose_treasury = functools.partial(choose_treasury, lambda *args: '10year',

catalyst/finance/risk/period.py

@@ -36,7 +36,9 @@ from empyrical import (
     sortino_ratio
 )
 
-log = logbook.Logger('Risk Period')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Period', level=LOG_LEVEL)
 
 choose_treasury = functools.partial(risk.choose_treasury,
                                     risk.select_treasury_duration)

catalyst/finance/risk/report.py

@@ -63,7 +63,9 @@ from dateutil.relativedelta import relativedelta
 
 from . period import RiskMetricsPeriod
 
-log = logbook.Logger('Risk Report')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk Report', level=LOG_LEVEL)
 
 
 class RiskReport(object):

catalyst/finance/risk/risk.py

@@ -61,7 +61,9 @@ Risk Report
 import logbook
 import numpy as np
 
-log = logbook.Logger('Risk')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Risk', level=LOG_LEVEL)
 
 
 TREASURY_DURATIONS = [

catalyst/finance/trading.py

@@ -26,7 +26,9 @@ from catalyst.data.loader import load_market_data
 from catalyst.utils.calendars import get_calendar
 from catalyst.utils.memoize import remember_last
 
-log = logbook.Logger('Trading')
+from catalyst.constants import LOG_LEVEL
+
+log = logbook.Logger('Trading', level=LOG_LEVEL)
 
 
 DEFAULT_CAPITAL_BASE = 1e5

catalyst/gens/tradesimulation.py

@@ -27,7 +27,9 @@ from catalyst.gens.sim_engine import (
     BEFORE_TRADING_START_BAR
 )
 
-log = Logger('Trade Simulation')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('Trade Simulation', level=LOG_LEVEL)
 
 
 class AlgorithmSimulator(object):

catalyst/sources/requests_csv.py

@@ -23,7 +23,9 @@ from catalyst.protocol import (
 )
 from catalyst.assets import Equity
 
-logger = Logger('Requests Source Logger')
+from catalyst.constants import LOG_LEVEL
+
+logger = Logger('Requests Source Logger', level=LOG_LEVEL)
 
 
 def roll_dts_to_midnight(dts, trading_day):

catalyst/utils/run_algo.py

@@ -36,14 +36,16 @@ from catalyst.exchange.data_portal_exchange import DataPortalExchangeLive, \
 from catalyst.exchange.asset_finder_exchange import AssetFinderExchange
 from catalyst.exchange.exchange_portfolio import ExchangePortfolio
 from catalyst.exchange.exchange_errors import (
-    ExchangeRequestError,
+    ExchangeRequestError, ExchangeAuthEmpty,
     ExchangeRequestErrorTooManyAttempts,
     BaseCurrencyNotFoundError, ExchangeNotFoundError)
 from catalyst.exchange.exchange_utils import get_exchange_auth, \
-    get_algo_object
+    get_algo_object, get_exchange_folder
 from logbook import Logger
 
-log = Logger('run_algo')
+from catalyst.constants import LOG_LEVEL
+
+log = Logger('run_algo', level=LOG_LEVEL)
 
 
 class _RunAlgoError(click.ClickException, ValueError):
@@ -164,6 +166,12 @@ def _run(handle_data,
 
         # This corresponds to the json file containing api token info
         exchange_auth = get_exchange_auth(exchange_name)
+
+        if live and (exchange_auth['key'] == '' or exchange_auth['secret'] == ''):
+            raise ExchangeAuthEmpty(
+                    exchange=exchange_name.title(), 
+                    filename=os.path.join(get_exchange_folder(exchange_name, environ), 'auth.json') )
+
         if exchange_name == 'bitfinex':
             exchanges[exchange_name] = Bitfinex(
                 key=exchange_auth['key'],
@@ -235,8 +243,11 @@ def _run(handle_data,
                 balances = exchange.get_balances()
             except ExchangeRequestError as e:
                 if attempt_index < 20:
-                    log.warn('exchange error when retrieving balances, {} '
-                             'trying again in 5 seconds'.format(e))
+                    log.warn(
+                        'could not retrieve balances on {}: {}'.format(
+                            exchange.name, e
+                        )
+                    )
                     sleep(5)
                     return fetch_capital_base(exchange, attempt_index + 1)
 

docs/source/beginner-tutorial.rst

@@ -52,7 +52,7 @@ My first algorithm
 ~~~~~~~~~~~~~~~~~~
 
 Lets take a look at a very simple algorithm from the ``examples``
-directory, ``buy_btc.py``:
+directory: `buy_btc_simple.py <https://github.com/enigmampc/catalyst/blob/master/catalyst/examples/buy_btc_simple.py>`_:
 
 .. code-block:: python
 
@@ -225,16 +225,16 @@ Thus, to execute our algorithm from above and save the results to
 
 .. code-block:: python
 
-    catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2016-9-29 -o buy_simple_btc_out.pickle
+    catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle
 
 
-.. 
-.. parsed-literal
+.. parsed-literal:: 
 
-..    AAPL
-..    [2015-11-04 22:45:32.820166] INFO: Performance: Simulated 3521 trading days out of 3521.
-..    [2015-11-04 22:45:32.820314] INFO: Performance: first open: 2000-01-03 14:31:00+00:00
-..    [2015-11-04 22:45:32.820401] INFO: Performance: last close: 2013-12-31 21:00:00+00:00
+    INFO: run_algo: running algo in backtest mode
+    INFO: exchange_algorithm: initialized trading algorithm in backtest mode
+    INFO: Performance: Simulated 639 trading days out of 639.
+    INFO: Performance: first open: 2016-01-01 00:00:00+00:00
+    INFO: Performance: last close: 2017-09-30 23:59:00+00:00
 
 
 ``run`` first calls the ``initialize()`` function, and then
@@ -255,7 +255,7 @@ slippage model that ``catalyst`` uses).
 
 Let's take a quick look at the performance ``DataFrame``. For this, we
 use ``pandas`` from inside the IPython Notebook and print the first ten
-rows. Note that ``catalyst`` makes heavy usage of 
+rows. and print the first ten rows. Note that ``catalyst`` makes heavy usage of 
 `pandas <http://pandas.pydata.org/>`_, especially for data input and 
 outputting so it's worth spending some time to learn it.
 
@@ -265,17 +265,200 @@ outputting so it's worth spending some time to learn it.
     perf = pd.read_pickle('buy_btc_simple_out.pickle') # read in perf DataFrame
     perf.head()
 
+.. raw:: html
+
+    <div style="max-height:1000px;max-width:1500px;overflow:auto;">
+      <table border="1" class="dataframe">
+        <thead>
+          <tr style="text-align: right;">
+            <th></th>
+            <th>algo_volatility</th>
+            <th>algorithm_period_return</th>
+            <th>alpha</th>
+            <th>benchmark_period_return</th>
+            <th>benchmark_volatility</th>
+            <th>beta</th>
+            <th>btc</th>
+            <th>capital_used</th>
+            <th>ending_cash</th>
+            <th>ending_exposure</th>
+            <th>...</th>
+            <th>short_exposure</th>
+            <th>short_value</th>
+            <th>shorts_count</th>
+            <th>sortino</th>
+            <th>starting_cash</th>
+            <th>starting_exposure</th>
+            <th>starting_value</th>
+            <th>trading_days</th>
+            <th>transactions</th>
+            <th>treasury_period_return</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr>
+            <th>2016-01-01 23:59:00+00:00</th>
+            <td>NaN</td>
+            <td>0.000000e+00</td>
+            <td>NaN</td>
+            <td>-0.010937</td>
+            <td>NaN</td>
+            <td>NaN</td>
+            <td>433.979999</td>
+            <td>0.000000</td>
+            <td>1.000000e+07</td>
+            <td>0.00</td>
+            <td>...</td>
+            <td>0</td>
+            <td>0</td>
+            <td>0</td>
+            <td>NaN</td>
+            <td>1.000000e+07</td>
+            <td>0.00</td>
+            <td>0.00</td>
+            <td>1</td>
+            <td>[]</td>
+            <td>0.0227</td>
+          </tr>
+          <tr>
+            <th>2016-01-02 23:59:00+00:00</th>
+            <td>0.000011</td>
+            <td>-9.536708e-07</td>
+            <td>-0.000170</td>
+            <td>-0.006480</td>
+            <td>0.173338</td>
+            <td>-0.000062</td>
+            <td>432.700000</td>
+            <td>-442.236708</td>
+            <td>9.999558e+06</td>
+            <td>432.70</td>
+            <td>...</td>
+            <td>0</td>
+            <td>0</td>
+            <td>0</td>
+            <td>-11.224972</td>
+            <td>1.000000e+07</td>
+            <td>0.00</td>
+            <td>0.00</td>
+            <td>2</td>
+            <td>[{u'order_id': u'7869f7828fa140328eb40477bb7de...</td>
+            <td>0.0227</td>
+          </tr>
+          <tr>
+            <th>2016-01-03 23:59:00+00:00</th>
+            <td>0.000011</td>
+            <td>-2.328842e-06</td>
+            <td>-0.000176</td>
+            <td>-0.026512</td>
+            <td>0.197857</td>
+            <td>0.000009</td>
+            <td>428.390000</td>
+            <td>-437.831716</td>
+            <td>9.999120e+06</td>
+            <td>856.78</td>
+            <td>...</td>
+            <td>0</td>
+            <td>0</td>
+            <td>0</td>
+            <td>-12.754262</td>
+            <td>9.999558e+06</td>
+            <td>432.70</td>
+            <td>432.70</td>
+            <td>3</td>
+            <td>[{u'order_id': u'be62ff77760c4599abaac43be9cc9...</td>
+            <td>0.0227</td>
+          </tr>
+          <tr>
+            <th>2016-01-04 23:59:00+00:00</th>
+            <td>0.000011</td>
+            <td>-2.380954e-06</td>
+            <td>-0.000139</td>
+            <td>-0.008640</td>
+            <td>0.269790</td>
+            <td>0.000020</td>
+            <td>432.900000</td>
+            <td>-442.441116</td>
+            <td>9.998677e+06</td>
+            <td>1298.70</td>
+            <td>...</td>
+            <td>0</td>
+            <td>0</td>
+            <td>0</td>
+            <td>-11.287205</td>
+            <td>9.999120e+06</td>
+            <td>856.78</td>
+            <td>856.78</td>
+            <td>4</td>
+            <td>[{u'order_id': u'd6dca79513214346a646079213526...</td>
+            <td>0.0224</td>
+          </tr>
+          <tr>
+            <th>2016-01-05 23:59:00+00:00</th>
+            <td>0.000011</td>
+            <td>-3.650729e-06</td>
+            <td>-0.000158</td>
+            <td>-0.021426</td>
+            <td>0.245989</td>
+            <td>0.000024</td>
+            <td>431.840000</td>
+            <td>-441.357754</td>
+            <td>9.998236e+06</td>
+            <td>1727.36</td>
+            <td>...</td>
+            <td>0</td>
+            <td>0</td>
+            <td>0</td>
+            <td>-12.333847</td>
+            <td>9.998677e+06</td>
+            <td>1298.70</td>
+            <td>1298.70</td>
+            <td>5</td>
+            <td>[{u'order_id': u'505275d6646a41f3856b22b16678d...</td>
+            <td>0.0225</td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+
+|
 There is a row for each trading day, starting on the first day of our 
 simulation Jan 1st, 2016. In the columns you can find various
-information about the state of your algorithm. The very first column
+information about the state of your algorithm. The column
 ``btc`` was placed there by the ``record()`` function mentioned earlier
 and allows us to plot the price of bitcoin. For example, we could easily
 examine now how our portfolio value changed over time compared to the
 bitcoin price.
 
-Our algorithm performance as assessed by the
-``portfolio_value`` closely matches that of the bitcoin price. This
-is not surprising as our algorithm only bought bitcoin every chance it got.
+.. code-block:: python
+
+    %load_ext catalyst
+
+.. code-block:: python
+
+    %pylab inline
+    figsize(12, 12)
+    import matplotlib.pyplot as plt
+
+    ax1 = plt.subplot(211)
+    perf.portfolio_value.plot(ax=ax1)
+    ax1.set_ylabel('portfolio value')
+    ax2 = plt.subplot(212, sharex=ax1)
+    perf.btc.plot(ax=ax2)
+    ax2.set_ylabel('bitcoin price')
+
+.. parsed-literal::
+
+    Populating the interactive namespace from numpy and matplotlib
+
+.. parsed-literal::
+
+    <matplotlib.text.Text at 0x10eaeadd0>
+
+.. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/buy_btc_simple_graph.png
+
+Our algorithm performance as assessed by the ``portfolio_value`` closely 
+matches that of the bitcoin price. This is not surprising as our algorithm 
+only bought bitcoin every chance it got.
 
 
 Access to previous prices using ``history``
@@ -305,23 +488,25 @@ a function we use in the ``handle_data()`` section:
 
 .. code-block:: python
 
-    from catalyst.api import order, record, symbol
+    %%catalyst --start 2016-4-1 --end 2017-9-30 -x bitfinex
+
+    from catalyst.api import order, record, symbol, order_target
 
     def initialize(context):
        context.i = 0
        context.asset = symbol('btc_usd')
 
-   def handle_data(context, data):
-       # Skip first 300 days to get full windows
+    def handle_data(context, data):
+       # Skip first 150 days to get full windows
        context.i += 1
-       if context.i < 300:
+       if context.i < 150:
            return
 
        # Compute averages
        # data.history() has to be called with the same params
        # from above and returns a pandas dataframe.
-       short_mavg = data.history(context.asset, 'price', bar_count=100, frequency="1d").mean()
-       long_mavg = data.history(context.asset, 'price', bar_count=300, frequency="1d").mean()
+       short_mavg = data.history(context.asset, 'price', bar_count=50, frequency="1d").mean()
+       long_mavg = data.history(context.asset, 'price', bar_count=150, frequency="1d").mean()
 
        # Trading logic
        if short_mavg > long_mavg:
@@ -336,6 +521,46 @@ a function we use in the ``handle_data()`` section:
               short_mavg=short_mavg,
               long_mavg=long_mavg)
 
+    def analyze(context, perf):
+       import matplotlib.pyplot as plt
+       fig = plt.figure(figsize=(12,12))
+       ax1 = fig.add_subplot(211)
+       perf.portfolio_value.plot(ax=ax1)
+       ax1.set_ylabel('portfolio value in $')
+
+       ax2 = fig.add_subplot(212)
+       perf['btc'].plot(ax=ax2)
+       perf[['short_mavg', 'long_mavg']].plot(ax=ax2)
+
+       perf_trans = perf.ix[[t != [] for t in perf.transactions]]
+       buys = perf_trans.ix[[t[0]['amount'] > 0 for t in perf_trans.transactions]]
+       sells = perf_trans.ix[
+           [t[0]['amount'] < 0 for t in perf_trans.transactions]]
+       ax2.plot(buys.index, perf.short_mavg.ix[buys.index],
+                '^', markersize=10, color='m')
+       ax2.plot(sells.index, perf.short_mavg.ix[sells.index],
+                'v', markersize=10, color='k')
+       ax2.set_ylabel('price in $')
+       plt.legend(loc=0)
+       plt.show()
+
+Here we are explicitly defining an ``analyze()`` function that gets
+automatically called once the backtest is done.
+
+Although it might not be directly apparent, the power of ``history()``
+(pun intended) can not be under-estimated as most algorithms make use of
+prior market developments in one form or another. You could easily
+devise a strategy that trains a classifier with
+`scikit-learn <http://scikit-learn.org/stable/>`__ which tries to
+predict future market movements based on past prices (note, that most of
+the ``scikit-learn`` functions require ``numpy.ndarray``\ s rather than
+``pandas.DataFrame``\ s, so you can simply pass the underlying
+``ndarray`` of a ``DataFrame`` via ``.values``).
+
+We also used the ``order_target()`` function above. This and other
+functions like it can make order management and portfolio rebalancing
+much easier.
+
 
 Conclusions
 ~~~~~~~~~~~

etc/python2.7-environment.yml

@@ -1,30 +1,24 @@
 name: catalyst
 channels:
-- statiskit
 - defaults
 dependencies:
 - certifi=2016.2.28=py27_0
-- coverage=4.4.1=py27_0
-- nose=1.3.7=py27_1
+- libgfortran=3.0.0=1
+- mkl=2017.0.3=0
+- numpy=1.13.1=py27_0
 - openssl=1.0.2l=0
-- path.py=10.3.1=py27_0
 - pip=9.0.1=py27_1
 - python=2.7.13=0
-- pyyaml=3.12=py27_0
 - readline=6.2=2
-- setuptools=36.4.0=py27_0
-- six=1.10.0=py27_0
+- scipy=0.19.1=np113py27_0
+- setuptools=36.4.0=py27_1
 - sqlite=3.13.0=0
 - tk=8.5.18=0
 - wheel=0.29.0=py27_0
-- yaml=0.1.6=0
 - zlib=1.2.11=0
-- libdev=1.0.0=py27_0
-- python-dev=1.0.0=py27_0
-- python-scons=3.0.0=py27_0
 - pip:
-  - alembic==0.9.5
-  - backports.shutil-get-terminal-size==1.0.0
+  - alembic==0.9.6
+  - backports.functools-lru-cache==1.4
   - bcolz==0.12.1
   - bottleneck==1.2.1
   - chardet==3.0.4
@@ -32,36 +26,22 @@ dependencies:
   - contextlib2==0.5.5
   - cycler==0.10.0
   - cyordereddict==1.0.0
-  - cython==0.26.1
+  - cython==0.27.1
   - decorator==4.1.2
   - empyrical==0.2.1
-  - enigma-catalyst>=0.2.dev2
-  - enum34==1.1.6
-  - functools32==3.2.3.post2
   - idna==2.6
   - intervaltree==2.1.0
-  - ipdb==0.10.3
-  - ipdbplugin==1.4.5
-  - ipython==5.5.0
-  - ipython-genutils==0.2.0
   - logbook==1.1.0
   - lru-dict==1.1.6
   - mako==1.0.7
   - markupsafe==1.0
-  - matplotlib==2.0.2
+  - matplotlib==2.1.0
   - multipledispatch==0.4.9
-  - networkx==1.11
+  - networkx==2.0
   - numexpr==2.6.4
-  - numpy==1.13.1
   - pandas==0.19.2
   - pandas-datareader==0.5.0
-  - pathlib2==2.3.0
   - patsy==0.4.1
-  - pexpect==4.2.1
-  - pickleshare==0.7.4
-  - prompt-toolkit==1.0.15
-  - ptyprocess==0.5.2
-  - pygments==2.2.0
   - pyparsing==2.2.0
   - python-dateutil==2.6.1
   - python-editor==1.0.3
@@ -69,16 +49,12 @@ dependencies:
   - requests==2.18.4
   - requests-file==1.4.2
   - requests-ftp==0.3.1
-  - scandir==1.5
-  - scipy==0.19.1
-  - scons==3.0.0a20170821
-  - simplegeneric==0.8.1
+  - six==1.11.0
   - sortedcontainers==1.5.7
   - sqlalchemy==1.1.14
   - statsmodels==0.8.0
   - subprocess32==3.2.7
   - tables==3.4.2
   - toolz==0.8.2
-  - traitlets==4.3.2
   - urllib3==1.22
-  - wcwidth==0.1.7
+  - enigma-catalyst>=0.3

etc/requirements.txt

@@ -1,7 +1,7 @@
 # Incompatible with earlier PIP versions
 pip>=7.1.0
 # bcolz fails to install if this is not in the build_requires.
-setuptools>18.0
+setuptools>36.0
 
 # Logging
 Logbook==0.12.5