BLD: Working on the sample algo for intro videos. Made auto-ingestion configurable.

catalyst/constants.py

@@ -4,4 +4,6 @@ import logbook
 
 LOG_LEVEL = logbook.INFO
 
-DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
+DATE_TIME_FORMAT = '%Y-%m-%d %H:%M'
+
+AUTO_INGEST = False

catalyst/examples/momemtum.py

@@ -3,6 +3,7 @@
 # going to sell.  Hopefully we'll ride the waves.
 
 import pandas as pd
+import talib
 # To run an algorithm in Catalyst, you need two functions: initialize and 
 # handle_data.
 from logbook import Logger
@@ -31,9 +32,16 @@ def initialize(context):
     context.eth_btc = symbol('eth_usdt')
     context.max_amount = 0.01
     context.base_price = None
+    context.current_day = None
+    context.yesterdy = None
 
 
 def handle_data(context, data):
+    today = data.current_dt.floor('1D')
+    if today != context.current_day:
+        context.traded_today = False
+        context.current_day = today
+
     # This handle_data function is where the real work is done.  Our data is
     # minute-level tick data, and each minute is called a frame.  This function
     # runs on each frame of the data.
@@ -44,16 +52,21 @@ def handle_data(context, data):
     bars = data.history(
         context.eth_btc,
         fields=['close', 'volume'],
-        bar_count=3,
-        frequency='1D'
+        bar_count=100,
+        frequency='30T'
+    )
+    # Use TA-Lib to calculate MACD data using calibrated settings
+    macd_raw, signal, macd_hist = talib.MACD(
+        bars['close'].values, fastperiod=30, slowperiod=40, signalperiod=45
     )
-    vwap = stats_utils.vwap(bars)
 
     # We need a variable for the current price of the security to compare to
     # the average.
     current = data.current(context.eth_btc, fields=['close', 'volume'])
     price = current['close']
-    log.info('{}: price: {}, vwap: {}'.format(data.current_dt, price, vwap))
+    log.info(
+        '{}: price: {}, macd: {}'.format(data.current_dt, price, macd_raw[-1])
+    )
 
     # If base_price is not set, we use the current value. This is the
     # price at the first bar which we reference to calculate price_change.
@@ -64,7 +77,8 @@ def handle_data(context, data):
     record(
         price=price,
         volume=current['volume'],
-        vwap=vwap,
+        macd=macd_raw[-1],
+        signal=signal[-1],
         price_change=price_change,
     )
 
@@ -77,23 +91,17 @@ def handle_data(context, data):
     # portfolio object.  The portfolio object tracks your positions, cash,
     # cost basis of specific holdings, and more.  In this line, we calculate
     # how long or short our position is at this minute.   
-    position_amount = context.portfolio.positions[context.eth_btc].amount
+    pos_amount = context.portfolio.positions[context.eth_btc].amount
 
-    # This is the meat of the algorithm, placed in this if statement.  If the
-    # price of the security is .5% less than the 3-day volume weighted average
-    # price AND we haven't reached our maximum short, then we call the order
-    # command and sell 100 shares.  Similarly, if the stock is .5% higher than
-    # the 3-day average AND we haven't reached our maximum long, then we call
-    # the order command and buy 100 shares.
-    if price > vwap * 1.01 and position_amount < context.max_amount:
-        order_target_percent(
-            context.eth_btc, 1, style=LimitOrder(price * 1.02)
-        )
+    if macd_hist[-1] > 0 and data.can_trade(context.eth_btc) \
+            and pos_amount == 0 and not context.traded_today:
+        order_target_percent(context.eth_btc, 0.75)
+        context.traded_today = True
 
-    elif price < vwap * 0.995 and position_amount > 0:
-        order_target_percent(
-            context.eth_btc, 0, style=LimitOrder(price * 0.98)
-        )
+    elif macd_hist[-1] < 0 and data.can_trade(context.eth_btc) \
+            and pos_amount > 0 and context.traded_today:
+        order_target_percent(context.eth_btc, 0)
+        context.traded_today = True
 
 
 def analyze(context=None, results=None):
@@ -153,19 +161,19 @@ def analyze(context=None, results=None):
     ax5.set_ylabel('Percent Change')
 
     ax6 = plt.subplot(615, sharex=ax1)
-    results.loc[:, 'vwap'].plot(ax=ax6)
-    ax6.set_ylabel('VWAP')
+    results.loc[:, 'macd'].plot(ax=ax6)
+    ax6.set_ylabel('MACD')
 
     ax6.plot(
         buys.index,
-        results.loc[buys.index, 'vwap'],
+        results.loc[buys.index, 'macd'],
         '^',
         markersize=10,
         color='g',
     )
     ax6.plot(
         sells.index,
-        results.loc[sells.index, 'vwap'],
+        results.loc[sells.index, 'macd'],
         'v',
         markersize=10,
         color='r',
@@ -182,13 +190,13 @@ def analyze(context=None, results=None):
 # Backtest
 run_algorithm(
     capital_base=1,
-    data_frequency='minute',
+    data_frequency='daily',
     initialize=initialize,
     handle_data=handle_data,
     analyze=analyze,
     exchange_name='poloniex',
     algo_namespace=algo_namespace,
     base_currency='usdt',
-    start=pd.to_datetime('2017-5-15', utc=True),
-    end=pd.to_datetime('2017-5-20', utc=True),
+    start=pd.to_datetime('2016-10-1', utc=True),
+    end=pd.to_datetime('2017-10-31', utc=True),
 )

catalyst/exchange/exchange_bundle.py

@@ -13,7 +13,7 @@ from pytz import UTC
 from six import itervalues
 
 from catalyst import get_calendar
-from catalyst.constants import DATE_TIME_FORMAT
+from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST
 from catalyst.constants import LOG_LEVEL
 from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
     BcolzMinuteBarMetadata
@@ -701,7 +701,46 @@ class ExchangeBundle:
         Series
 
         """
-        try:
+        if AUTO_INGEST:
+            try:
+                series = self.get_history_window_series(
+                    assets=assets,
+                    end_dt=end_dt,
+                    bar_count=bar_count,
+                    field=field,
+                    data_frequency=data_frequency
+                )
+                return pd.DataFrame(series)
+
+            except PricingDataNotLoadedError:
+                start_dt = get_start_dt(end_dt, bar_count, data_frequency)
+                log.info(
+                    'pricing data for {symbol} not found in range '
+                    '{start} to {end}, updating the bundles.'.format(
+                        symbol=[asset.symbol for asset in assets],
+                        start=start_dt,
+                        end=end_dt
+                    )
+                )
+                self.ingest_assets(
+                    assets=assets,
+                    start_dt=start_dt,
+                    end_dt=algo_end_dt,
+                    data_frequency=data_frequency,
+                    show_progress=True,
+                    show_breakdown=True
+                )
+                series = self.get_history_window_series(
+                    assets=assets,
+                    end_dt=end_dt,
+                    bar_count=bar_count,
+                    field=field,
+                    data_frequency=data_frequency,
+                    reset_reader=True
+                )
+                return series
+
+        else:
             series = self.get_history_window_series(
                 assets=assets,
                 end_dt=end_dt,
@@ -711,34 +750,6 @@ class ExchangeBundle:
             )
             return pd.DataFrame(series)
 
-        except PricingDataNotLoadedError:
-            start_dt = get_start_dt(end_dt, bar_count, data_frequency)
-            log.info(
-                'pricing data for {symbol} not found in range '
-                '{start} to {end}, updating the bundles.'.format(
-                    symbol=[asset.symbol for asset in assets],
-                    start=start_dt,
-                    end=end_dt
-                )
-            )
-            self.ingest_assets(
-                assets=assets,
-                start_dt=start_dt,
-                end_dt=algo_end_dt,
-                data_frequency=data_frequency,
-                show_progress=True,
-                show_breakdown=True
-            )
-            series = self.get_history_window_series(
-                assets=assets,
-                end_dt=end_dt,
-                bar_count=bar_count,
-                field=field,
-                data_frequency=data_frequency,
-                reset_reader=True
-            )
-            return series
-
     def get_spot_values(self,
                         assets,
                         field,
@@ -782,7 +793,9 @@ class ExchangeBundle:
                 exchange=self.exchange.name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=dt,
+                end_dt=dt
             )
 
     def get_history_window_series(self,
@@ -810,7 +823,9 @@ class ExchangeBundle:
                 exchange=self.exchange.name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=start_dt,
+                end_dt=end_dt
             )
 
         for asset in assets:
@@ -828,7 +843,9 @@ class ExchangeBundle:
                     exchange=self.exchange.name,
                     symbols=asset.symbol,
                     symbol_list=asset.symbol,
-                    data_frequency=data_frequency
+                    data_frequency=data_frequency,
+                    start_dt=asset_start_dt,
+                    end_dt=asset_end_dt
                 )
 
         series = dict()
@@ -848,7 +865,9 @@ class ExchangeBundle:
                 exchange=self.exchange.name,
                 symbols=symbols,
                 symbol_list=','.join(symbols),
-                data_frequency=data_frequency
+                data_frequency=data_frequency,
+                start_dt=start_dt,
+                end_dt=end_dt
             )
 
         periods = self.get_calendar_periods_range(

catalyst/exchange/exchange_data_portal.py

@@ -6,7 +6,7 @@ import pandas as pd
 from catalyst.assets._assets import TradingPair
 from logbook import Logger
 
-from catalyst.constants import LOG_LEVEL
+from catalyst.constants import LOG_LEVEL, AUTO_INGEST
 from catalyst.data.data_portal import DataPortal
 from catalyst.exchange.exchange_bundle import ExchangeBundle
 from catalyst.exchange.exchange_errors import (
@@ -378,24 +378,28 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
         else:
             dt = dt.floor('1 min')
 
-        try:
-            return bundle.get_spot_values(assets, field, dt, data_frequency)
-
-        except PricingDataNotLoadedError:
-            log.info(
-                'pricing data for {symbol} not found on {dt}'
-                ', updating the bundles.'.format(
-                    symbol=[asset.symbol for asset in assets],
-                    dt=dt
+        if AUTO_INGEST:
+            try:
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency
                 )
-            )
-            bundle.ingest_assets(
-                assets=assets,
-                start_dt=self._first_trading_day,
-                end_dt=self._last_available_session,
-                data_frequency=data_frequency,
-                show_progress=True
-            )
-            return bundle.get_spot_values(
-                assets, field, dt, data_frequency, True
-            )
+            except PricingDataNotLoadedError:
+                log.info(
+                    'pricing data for {symbol} not found on {dt}'
+                    ', updating the bundles.'.format(
+                        symbol=[asset.symbol for asset in assets],
+                        dt=dt
+                    )
+                )
+                bundle.ingest_assets(
+                    assets=assets,
+                    start_dt=self._first_trading_day,
+                    end_dt=self._last_available_session,
+                    data_frequency=data_frequency,
+                    show_progress=True
+                )
+                return bundle.get_spot_values(
+                    assets, field, dt, data_frequency, True
+                )
+        else:
+            return bundle.get_spot_values(assets, field, dt, data_frequency)

catalyst/exchange/exchange_errors.py

@@ -211,12 +211,11 @@ class PricingDataBeforeTradingError(ZiplineError):
 
 
 class PricingDataNotLoadedError(ZiplineError):
-    msg = ('Pricing data {field} for trading pairs {symbols} trading on '
-           'exchange {exchange} since {first_trading_day} is unavailable. '
-           'The bundle data is either out-of-date or has not been loaded yet. '
-           'Please ingest data using the command '
-           '`catalyst ingest-exchange -x {exchange} -f {data_frequency} -i {symbol_list}`. '
-           'See catalyst documentation for details.').strip()
+    msg = ('Missing data for {exchange} {symbols} in date range '
+           '[{start_dt} - {end_dt}]'
+           '\nPlease run: `catalyst ingest-exchange -x {exchange} -f '
+           '{data_frequency} -i {symbol_list}`. See catalyst documentation '
+           'for details.').strip()
 
 
 class ApiCandlesError(ZiplineError):