diff --git a/catalyst/constants.py b/catalyst/constants.py index da6a957f..cde29914 100644 --- a/catalyst/constants.py +++ b/catalyst/constants.py @@ -4,4 +4,6 @@ import logbook LOG_LEVEL = logbook.INFO -DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' \ No newline at end of file +DATE_TIME_FORMAT = '%Y-%m-%d %H:%M' + +AUTO_INGEST = False \ No newline at end of file diff --git a/catalyst/examples/momemtum.py b/catalyst/examples/momemtum.py index dd928fd6..09dece3d 100644 --- a/catalyst/examples/momemtum.py +++ b/catalyst/examples/momemtum.py @@ -3,6 +3,7 @@ # going to sell. Hopefully we'll ride the waves. import pandas as pd +import talib # To run an algorithm in Catalyst, you need two functions: initialize and # handle_data. from logbook import Logger @@ -31,9 +32,16 @@ def initialize(context): context.eth_btc = symbol('eth_usdt') context.max_amount = 0.01 context.base_price = None + context.current_day = None + context.yesterdy = None def handle_data(context, data): + today = data.current_dt.floor('1D') + if today != context.current_day: + context.traded_today = False + context.current_day = today + # This handle_data function is where the real work is done. Our data is # minute-level tick data, and each minute is called a frame. This function # runs on each frame of the data. @@ -44,16 +52,21 @@ def handle_data(context, data): bars = data.history( context.eth_btc, fields=['close', 'volume'], - bar_count=3, - frequency='1D' + bar_count=100, + frequency='30T' + ) + # Use TA-Lib to calculate MACD data using calibrated settings + macd_raw, signal, macd_hist = talib.MACD( + bars['close'].values, fastperiod=30, slowperiod=40, signalperiod=45 ) - vwap = stats_utils.vwap(bars) # We need a variable for the current price of the security to compare to # the average. current = data.current(context.eth_btc, fields=['close', 'volume']) price = current['close'] - log.info('{}: price: {}, vwap: {}'.format(data.current_dt, price, vwap)) + log.info( + '{}: price: {}, macd: {}'.format(data.current_dt, price, macd_raw[-1]) + ) # If base_price is not set, we use the current value. This is the # price at the first bar which we reference to calculate price_change. @@ -64,7 +77,8 @@ def handle_data(context, data): record( price=price, volume=current['volume'], - vwap=vwap, + macd=macd_raw[-1], + signal=signal[-1], price_change=price_change, ) @@ -77,23 +91,17 @@ def handle_data(context, data): # portfolio object. The portfolio object tracks your positions, cash, # cost basis of specific holdings, and more. In this line, we calculate # how long or short our position is at this minute. - position_amount = context.portfolio.positions[context.eth_btc].amount + pos_amount = context.portfolio.positions[context.eth_btc].amount - # This is the meat of the algorithm, placed in this if statement. If the - # price of the security is .5% less than the 3-day volume weighted average - # price AND we haven't reached our maximum short, then we call the order - # command and sell 100 shares. Similarly, if the stock is .5% higher than - # the 3-day average AND we haven't reached our maximum long, then we call - # the order command and buy 100 shares. - if price > vwap * 1.01 and position_amount < context.max_amount: - order_target_percent( - context.eth_btc, 1, style=LimitOrder(price * 1.02) - ) + if macd_hist[-1] > 0 and data.can_trade(context.eth_btc) \ + and pos_amount == 0 and not context.traded_today: + order_target_percent(context.eth_btc, 0.75) + context.traded_today = True - elif price < vwap * 0.995 and position_amount > 0: - order_target_percent( - context.eth_btc, 0, style=LimitOrder(price * 0.98) - ) + elif macd_hist[-1] < 0 and data.can_trade(context.eth_btc) \ + and pos_amount > 0 and context.traded_today: + order_target_percent(context.eth_btc, 0) + context.traded_today = True def analyze(context=None, results=None): @@ -153,19 +161,19 @@ def analyze(context=None, results=None): ax5.set_ylabel('Percent Change') ax6 = plt.subplot(615, sharex=ax1) - results.loc[:, 'vwap'].plot(ax=ax6) - ax6.set_ylabel('VWAP') + results.loc[:, 'macd'].plot(ax=ax6) + ax6.set_ylabel('MACD') ax6.plot( buys.index, - results.loc[buys.index, 'vwap'], + results.loc[buys.index, 'macd'], '^', markersize=10, color='g', ) ax6.plot( sells.index, - results.loc[sells.index, 'vwap'], + results.loc[sells.index, 'macd'], 'v', markersize=10, color='r', @@ -182,13 +190,13 @@ def analyze(context=None, results=None): # Backtest run_algorithm( capital_base=1, - data_frequency='minute', + data_frequency='daily', initialize=initialize, handle_data=handle_data, analyze=analyze, exchange_name='poloniex', algo_namespace=algo_namespace, base_currency='usdt', - start=pd.to_datetime('2017-5-15', utc=True), - end=pd.to_datetime('2017-5-20', utc=True), + start=pd.to_datetime('2016-10-1', utc=True), + end=pd.to_datetime('2017-10-31', utc=True), ) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 2a92799f..ff295575 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -13,7 +13,7 @@ from pytz import UTC from six import itervalues from catalyst import get_calendar -from catalyst.constants import DATE_TIME_FORMAT +from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST from catalyst.constants import LOG_LEVEL from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ BcolzMinuteBarMetadata @@ -701,7 +701,46 @@ class ExchangeBundle: Series """ - try: + if AUTO_INGEST: + try: + series = self.get_history_window_series( + assets=assets, + end_dt=end_dt, + bar_count=bar_count, + field=field, + data_frequency=data_frequency + ) + return pd.DataFrame(series) + + except PricingDataNotLoadedError: + start_dt = get_start_dt(end_dt, bar_count, data_frequency) + log.info( + 'pricing data for {symbol} not found in range ' + '{start} to {end}, updating the bundles.'.format( + symbol=[asset.symbol for asset in assets], + start=start_dt, + end=end_dt + ) + ) + self.ingest_assets( + assets=assets, + start_dt=start_dt, + end_dt=algo_end_dt, + data_frequency=data_frequency, + show_progress=True, + show_breakdown=True + ) + series = self.get_history_window_series( + assets=assets, + end_dt=end_dt, + bar_count=bar_count, + field=field, + data_frequency=data_frequency, + reset_reader=True + ) + return series + + else: series = self.get_history_window_series( assets=assets, end_dt=end_dt, @@ -711,34 +750,6 @@ class ExchangeBundle: ) return pd.DataFrame(series) - except PricingDataNotLoadedError: - start_dt = get_start_dt(end_dt, bar_count, data_frequency) - log.info( - 'pricing data for {symbol} not found in range ' - '{start} to {end}, updating the bundles.'.format( - symbol=[asset.symbol for asset in assets], - start=start_dt, - end=end_dt - ) - ) - self.ingest_assets( - assets=assets, - start_dt=start_dt, - end_dt=algo_end_dt, - data_frequency=data_frequency, - show_progress=True, - show_breakdown=True - ) - series = self.get_history_window_series( - assets=assets, - end_dt=end_dt, - bar_count=bar_count, - field=field, - data_frequency=data_frequency, - reset_reader=True - ) - return series - def get_spot_values(self, assets, field, @@ -782,7 +793,9 @@ class ExchangeBundle: exchange=self.exchange.name, symbols=symbols, symbol_list=','.join(symbols), - data_frequency=data_frequency + data_frequency=data_frequency, + start_dt=dt, + end_dt=dt ) def get_history_window_series(self, @@ -810,7 +823,9 @@ class ExchangeBundle: exchange=self.exchange.name, symbols=symbols, symbol_list=','.join(symbols), - data_frequency=data_frequency + data_frequency=data_frequency, + start_dt=start_dt, + end_dt=end_dt ) for asset in assets: @@ -828,7 +843,9 @@ class ExchangeBundle: exchange=self.exchange.name, symbols=asset.symbol, symbol_list=asset.symbol, - data_frequency=data_frequency + data_frequency=data_frequency, + start_dt=asset_start_dt, + end_dt=asset_end_dt ) series = dict() @@ -848,7 +865,9 @@ class ExchangeBundle: exchange=self.exchange.name, symbols=symbols, symbol_list=','.join(symbols), - data_frequency=data_frequency + data_frequency=data_frequency, + start_dt=start_dt, + end_dt=end_dt ) periods = self.get_calendar_periods_range( diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index a6cf4db7..43feaa9c 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -6,7 +6,7 @@ import pandas as pd from catalyst.assets._assets import TradingPair from logbook import Logger -from catalyst.constants import LOG_LEVEL +from catalyst.constants import LOG_LEVEL, AUTO_INGEST from catalyst.data.data_portal import DataPortal from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import ( @@ -378,24 +378,28 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): else: dt = dt.floor('1 min') - try: - return bundle.get_spot_values(assets, field, dt, data_frequency) - - except PricingDataNotLoadedError: - log.info( - 'pricing data for {symbol} not found on {dt}' - ', updating the bundles.'.format( - symbol=[asset.symbol for asset in assets], - dt=dt + if AUTO_INGEST: + try: + return bundle.get_spot_values( + assets, field, dt, data_frequency ) - ) - bundle.ingest_assets( - assets=assets, - start_dt=self._first_trading_day, - end_dt=self._last_available_session, - data_frequency=data_frequency, - show_progress=True - ) - return bundle.get_spot_values( - assets, field, dt, data_frequency, True - ) + except PricingDataNotLoadedError: + log.info( + 'pricing data for {symbol} not found on {dt}' + ', updating the bundles.'.format( + symbol=[asset.symbol for asset in assets], + dt=dt + ) + ) + bundle.ingest_assets( + assets=assets, + start_dt=self._first_trading_day, + end_dt=self._last_available_session, + data_frequency=data_frequency, + show_progress=True + ) + return bundle.get_spot_values( + assets, field, dt, data_frequency, True + ) + else: + return bundle.get_spot_values(assets, field, dt, data_frequency) diff --git a/catalyst/exchange/exchange_errors.py b/catalyst/exchange/exchange_errors.py index 00e53395..35f320d8 100644 --- a/catalyst/exchange/exchange_errors.py +++ b/catalyst/exchange/exchange_errors.py @@ -211,12 +211,11 @@ class PricingDataBeforeTradingError(ZiplineError): class PricingDataNotLoadedError(ZiplineError): - msg = ('Pricing data {field} for trading pairs {symbols} trading on ' - 'exchange {exchange} since {first_trading_day} is unavailable. ' - 'The bundle data is either out-of-date or has not been loaded yet. ' - 'Please ingest data using the command ' - '`catalyst ingest-exchange -x {exchange} -f {data_frequency} -i {symbol_list}`. ' - 'See catalyst documentation for details.').strip() + msg = ('Missing data for {exchange} {symbols} in date range ' + '[{start_dt} - {end_dt}]' + '\nPlease run: `catalyst ingest-exchange -x {exchange} -f ' + '{data_frequency} -i {symbol_list}`. See catalyst documentation ' + 'for details.').strip() class ApiCandlesError(ZiplineError):