From b64e5929b4090c3445e3fd1af5f088d20568e961 Mon Sep 17 00:00:00 2001 From: fredfortier Date: Fri, 10 Nov 2017 17:39:36 -0500 Subject: [PATCH] BUG: resolve issue #61 by adjusting our perf conventions to match zipline exactly. --- catalyst/examples/momemtum.py | 67 +++++++++++++------------ catalyst/exchange/exchange_algorithm.py | 55 +++++++++----------- 2 files changed, 59 insertions(+), 63 deletions(-) diff --git a/catalyst/examples/momemtum.py b/catalyst/examples/momemtum.py index 09dece3d..b4eabfae 100644 --- a/catalyst/examples/momemtum.py +++ b/catalyst/examples/momemtum.py @@ -1,18 +1,17 @@ # For this example, we're going to write a simple momentum script. When the # stock goes up quickly, we're going to buy; when it goes down quickly, we're # going to sell. Hopefully we'll ride the waves. +from collections import OrderedDict import pandas as pd import talib -# To run an algorithm in Catalyst, you need two functions: initialize and +# To run an algorithm in Catalyst, you need two functions: initialize and # handle_data. from logbook import Logger from catalyst import run_algorithm from catalyst.api import symbol, record, order_target_percent, \ get_open_orders -from catalyst.exchange import stats_utils -from catalyst.finance.execution import LimitOrder # We give a name to the algorithm which Catalyst will use to persist its state. # In this example, Catalyst will create the `.catalyst/data/live_algos` @@ -37,27 +36,30 @@ def initialize(context): def handle_data(context, data): + # This handle_data function is where the real work is done. Our data is + # minute-level tick data, and each minute is called a frame. This function + # runs on each frame of the data. + + # We flag the first period of each day. + # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle + # would only execute once. today = data.current_dt.floor('1D') if today != context.current_day: context.traded_today = False context.current_day = today - # This handle_data function is where the real work is done. Our data is - # minute-level tick data, and each minute is called a frame. This function - # runs on each frame of the data. - - # We're computing the volume-weighted-average-price of the security + # We're computing the volume-weighted-average-price of the security # defined above, in the context.eth_btc variable. For this example, we're # using three bars on the daily chart. - bars = data.history( + prices = data.history( context.eth_btc, - fields=['close', 'volume'], + fields='close', bar_count=100, frequency='30T' ) # Use TA-Lib to calculate MACD data using calibrated settings macd_raw, signal, macd_hist = talib.MACD( - bars['close'].values, fastperiod=30, slowperiod=40, signalperiod=45 + prices.values, fastperiod=30, slowperiod=40, signalperiod=45 ) # We need a variable for the current price of the security to compare to @@ -99,7 +101,7 @@ def handle_data(context, data): context.traded_today = True elif macd_hist[-1] < 0 and data.can_trade(context.eth_btc) \ - and pos_amount > 0 and context.traded_today: + and pos_amount > 0 and not context.traded_today: order_target_percent(context.eth_btc, 0) context.traded_today = True @@ -129,8 +131,8 @@ def analyze(context=None, results=None): # Transaction have an exact timestamp while stVats are daily. # We adjust the time to the end of each period to place them on the graph. - for t in all_trans: - t['dt'] = t['dt'].replace(hour=23, minute=59) + # for t in all_trans: + # t['dt'] = t['dt'].replace(hour=23, minute=59) buys = results.loc[[t['dt'] for t in all_trans if t['amount'] > 0], :] sells = results.loc[[t['dt'] for t in all_trans if t['amount'] < 0], :] @@ -161,23 +163,24 @@ def analyze(context=None, results=None): ax5.set_ylabel('Percent Change') ax6 = plt.subplot(615, sharex=ax1) - results.loc[:, 'macd'].plot(ax=ax6) - ax6.set_ylabel('MACD') + results.loc[:, 'macd'].plot(ax=ax6, label='macd') - ax6.plot( - buys.index, - results.loc[buys.index, 'macd'], - '^', - markersize=10, - color='g', - ) - ax6.plot( - sells.index, - results.loc[sells.index, 'macd'], - 'v', - markersize=10, - color='r', - ) + # ax6.plot( + # buys.index, + # results.loc[buys.index, 'macd'], + # '^', + # markersize=10, + # color='g', + # label='buys' + # ) + # ax6.plot( + # sells.index, + # results.loc[sells.index, 'macd'], + # 'v', + # markersize=10, + # color='r', + # label='sells' + # ) plt.legend(loc=3) @@ -197,6 +200,6 @@ run_algorithm( exchange_name='poloniex', algo_namespace=algo_namespace, base_currency='usdt', - start=pd.to_datetime('2016-10-1', utc=True), - end=pd.to_datetime('2017-10-31', utc=True), + start=pd.to_datetime('2017-6-1', utc=True), + end=pd.to_datetime('2017-8-1', utc=True), ) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index c05e559f..0d71a058 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -16,6 +16,7 @@ import signal import sys from collections import deque from datetime import timedelta +from itertools import chain from os import listdir from os.path import isfile, join from time import sleep @@ -23,12 +24,11 @@ from time import sleep import logbook import pandas as pd from catalyst.assets._assets import TradingPair +from six import itervalues import catalyst.protocol as zp from catalyst.algorithm import TradingAlgorithm from catalyst.constants import LOG_LEVEL -from catalyst.data.minute_bars import BcolzMinuteBarWriter, \ - BcolzMinuteBarReader from catalyst.errors import OrderInBeforeTradingStart from catalyst.exchange.exchange_blotter import ExchangeBlotter from catalyst.exchange.exchange_errors import ( @@ -182,17 +182,19 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): # we want the key to be absent, not just empty # Only include transactions for given dt - stats['transactions'] = dict() + stats['transactions'] = [] for date in period.processed_transactions: if start_dt <= date < end_dt: - stats['transactions'][date] = \ - period.processed_transactions[date] + transactions = period.processed_transactions[date] + for t in transactions: + stats['transactions'].append(t.to_dict()) - stats['orders'] = dict() + stats['orders'] = [] for date in period.orders_by_modified: if start_dt <= date < end_dt: - stats['orders'][date] = \ - period.orders_by_modified[date] + orders = period.orders_by_modified[date] + for order in orders: + stats['orders'].append(orders[order].to_dict()) return stats @@ -201,6 +203,7 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase): def __init__(self, *args, **kwargs): super(ExchangeTradingAlgorithmBacktest, self).__init__(*args, **kwargs) + self.frame_stats = list() self.blotter = ExchangeBlotter( data_frequency=self.data_frequency, # Default to NeverCancel in catalyst @@ -245,6 +248,19 @@ class ExchangeTradingAlgorithmBacktest(ExchangeTradingAlgorithmBase): else: return MarketOrder() + def handle_data(self, data): + super(ExchangeTradingAlgorithmBacktest, self).handle_data(data) + + minute_stats = self.prepare_period_stats( + data.current_dt, data.current_dt + timedelta(minutes=1)) + self.frame_stats.append(minute_stats) + + def analyze(self, perf): + stats = pd.DataFrame(self.frame_stats) + stats.set_index('period_close', inplace=True, drop=False) + + super(ExchangeTradingAlgorithmBacktest, self).analyze(perf) + class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): def __init__(self, *args, **kwargs): @@ -273,34 +289,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.stats_minutes = 5 super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs) - # TODO: fix precision before re-enabling - # self._create_minute_writer() signal.signal(signal.SIGINT, self.signal_handler) log.info('initialized trading algorithm in live mode') - def _create_minute_writer(self): - root = get_exchange_minute_writer_root(self.exchange.name) - filename = os.path.join(root, 'metadata.json') - - if os.path.isfile(filename): - writer = BcolzMinuteBarWriter.open( - root, self.sim_params.end_session) - else: - # TODO: need to be able to write more precise numbers - writer = BcolzMinuteBarWriter( - rootdir=root, - calendar=self.trading_calendar, - minutes_per_day=1440, - start_session=self.sim_params.start_session, - end_session=self.sim_params.end_session, - write_metadata=True - ) - - self.exchange.minute_writer = writer - self.exchange.minute_reader = BcolzMinuteBarReader(root) - def signal_handler(self, signal, frame): """ Handles the keyboard interruption signal.