From 82d318a1c0237216c32ac06c579d1ccebd276f3e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 15:51:55 -0500 Subject: [PATCH 1/5] BUG: fixed issue #216 with bad candle data --- catalyst/exchange/ccxt/ccxt_exchange.py | 20 +- catalyst/support/issue_216.py | 376 ++++++++++++++++++++++++ tests/exchange/test_ccxt.py | 9 +- 3 files changed, 392 insertions(+), 13 deletions(-) create mode 100644 catalyst/support/issue_216.py diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 9cf6c6f4..c5fee495 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -425,15 +425,12 @@ class CCXT(Exchange): 'Please provide either start_dt or end_dt, not both.' ) - elif end_dt is not None: - # Make sure that end_dt really wants data in the past - # if it's close to now, we skip the 'since' parameters to - # lower the probability of error - bars_to_now = pd.date_range( - end_dt, pd.Timestamp.utcnow(), freq=freq - ) - # See: https://github.com/ccxt/ccxt/issues/1360 - if len(bars_to_now) > 1 or self.name in ['poloniex']: + if start_dt is None: + # TODO: determine why binance is failing + if end_dt is None and self.name not in ['binance']: + end_dt = pd.Timestamp.utcnow() + + if end_dt is not None: dt_range = get_periods_range( end_dt=end_dt, periods=bar_count, @@ -441,10 +438,13 @@ class CCXT(Exchange): ) start_dt = dt_range[0] - since = None if start_dt is not None: + # Convert out start date to a UNIX timestamp, then translate to + # milliseconds delta = start_dt - get_epoch() since = int(delta.total_seconds()) * 1000 + else: + since = None candles = dict() for index, asset in enumerate(assets): diff --git a/catalyst/support/issue_216.py b/catalyst/support/issue_216.py new file mode 100644 index 00000000..8a038da0 --- /dev/null +++ b/catalyst/support/issue_216.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# !/usr/bin/env python2 + +import sys +import os +import pandas as pd +import signal +# import talib + +from logbook import Logger + +from catalyst import run_algorithm +from catalyst.api import ( + symbol, + record, + order, + order_target, + order_target_percent, + get_open_orders +) +from catalyst.finance import commission + + +# from base.telegrambot import TelegramBot + + +class GracefulKiller: + # Source: https://stackoverflow.com/a/31464349 + def __init__(self, context): + self.kill_now = False + self.signal = 0 + self.context = context + signal.signal(signal.SIGINT, self.exit_gracefully) + + def exit_gracefully(self, signum, frame): + self.kill_now = True + self.signal = signum + if hasattr(self.context, + 'telegram_bot') and self.context.telegram_bot is not None: + self.context.telegram_bot.updater.stop() + sys.exit(0) + + def exit(self): + return self.kill_now + + +class SimulationParameters: + MODE = 'paper' + CAPITAL_BASE = 1000 + """ + Capital base used on this simulation + """ + + DATA_FREQUECY = 'minute' + + EXCHANGE_NAME = 'bitfinex' + # EXCHANGE_NAME = 'binance' + """ + Exchange used on this simulation + """ + + DATA_DIR = '/home/av/Dropbox/simulations/data' + ALGO_NAMESPACE = os.path.basename(__file__).split('.')[0] + ALGO_NAMESPACE_IMAGE = '{}/{}/{}.png'.format(DATA_DIR, 'images', + ALGO_NAMESPACE) + ALGO_NAMESPACE_RESULTS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, 'tables', + ALGO_NAMESPACE + '_results') + ALGO_NAMESPACE_TRANSACTIONS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, + 'tables', + ALGO_NAMESPACE + '_transactions') + BASE_CURRENCY = 'usd' + # BASE_CURRENCY = 'usdt' + + # SHORT PERIOD + START_DATE = '2017-09-07' + """ + Start date used on this simulation + """ + END_DATE = '2017-12-12' + """ + End date used on this simulation + """ + + SKIP_FIRST_CANDLES = 0 + + # CANDLES_SAMPLE_RATE = 60 + # CANDLES_SAMPLE_RATE = 30 + CANDLES_SAMPLE_RATE = 1 + """ + Candle interval used on this simulation (in minutes) + """ + + # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + # 30 minute interval ohlcv data (the standard data required for candlestick or + # indicators/signals) + # 30T means 30 minutes re-sampling of one minute data. + # CANDLES_FREQUENCY = '60T' + # CANDLES_FREQUENCY = '30T' + CANDLES_FREQUENCY = '1T' + CANDLES_BUFFER_SIZE = 48 + COIN_PAIR = 'btc_usd' + # COIN_PAIR = 'btc_usdt' + """ + Coin pair used on this simulation + """ + + # TRANSACTIONS + COMMISSION_FEE = 0.0030 + BUY_MIN_AMOUNT = 5 # i.e: USD + SELL_MIN_AMOUNT = 0.001 # i.e: USD + BUY_SELL_PERCENTAGE = 1 # 0.50 + BUY_PERCENTAGE = BUY_SELL_PERCENTAGE + SELL_PERCENTAGE = BUY_SELL_PERCENTAGE + + BASE_PRICE = 'close' + """ + Base price used (close / Heiken Ashi) + """ + + +log = None +parameters = None + + +def print_facts(context): + context.log.info(""" +Index: {} +Date: {} +Candle: +O: {} +H: {} +L: {} +C: {} +V: {} +Metrics: +... +Portfolio: +Base price: {} +Base coin (coin2/usd): {} +Amount (coin1/btc): {} +""".format( + # Facts + context.i, + context.curr_minute, + context.candles_open[-1], + context.candles_high[-1], + context.candles_low[-1], + context.candles_close[-1], + context.candles_volume[-1], + # Metrics + # ... + # Portfolio + context.curr_base_price, + context.portfolio.cash, + context.portfolio.positions[context.coin_pair].amount, + )) + + +def print_facts_telegram(context): + price = context.curr_base_price + amount = context.portfolio.positions[context.coin_pair].amount + pnl = context.portfolio.pnl + capital_used = context.portfolio.capital_used + portfolio_value = context.portfolio.portfolio_value + portfolio_returns = context.portfolio.returns + starting_cash = context.portfolio.starting_cash + cash = context.portfolio.cash + + msg = """ +Status... +Price: {} +Starting cash: {} +Cash: {} +Capital used: {} +Amount: {} +Portfolio value: {} +Returns: {} +PnL: {} + """.format( + price, + starting_cash, + cash, + capital_used, + amount, + portfolio_value, + portfolio_returns, + pnl, + ) + if hasattr(context, 'telegram_bot') and context.telegram_bot is not None: + context.telegram_bot.msg(msg) + + +def default_initialize(context): + # FIXME: set_benchmark + # set_benchmark(symbol(context.parameters.COIN_PAIR)) + + context.coin_pair = symbol(context.parameters.COIN_PAIR) + context.base_price = None + context.current_day = None + context.counter = -1 + context.i = 0 + + context.candles_sample_rate = context.parameters.CANDLES_SAMPLE_RATE + context.candles_frequency = context.parameters.CANDLES_FREQUENCY + context.candles_buffer_size = context.parameters.CANDLES_BUFFER_SIZE + context.set_commission( + commission.PerShare(cost=context.parameters.COMMISSION_FEE)) + + +def default_handle_data(context, data): + context.curr_minute = data.current_dt + context.counter += 1 + + if context.candles_sample_rate == 1: + context.i += 1 + elif context.counter % context.candles_sample_rate != 0: + context.i += 1 + return + + if context.i < context.parameters.SKIP_FIRST_CANDLES: + return + + context.candles_open = data.history( + context.coin_pair, + 'open', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_high = data.history( + context.coin_pair, + 'high', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_low = data.history( + context.coin_pair, + 'low', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_close = data.history( + context.coin_pair, + 'price', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_volume = data.history( + context.coin_pair, + 'volume', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + + # FIXME: Here is the error! + # The candles_close frame shows more or less always a value of 94, while + # bitcoin price is very different from that + print(context.candles_close) + + context.base_prices = context.candles_close + cash = context.portfolio.cash + amount = context.portfolio.positions[context.coin_pair].amount + price = data.current(context.coin_pair, 'price') + order_id = None + context.last_base_price = context.base_prices[-2] + context.curr_base_price = context.base_prices[-1] + + # TA calculations + # ... + + # Sanity checks + # assert cash >= 0 + if cash < 0: + import ipdb; + ipdb.set_trace() # BREAKPOINT + + print_facts(context) + print_facts_telegram(context) + + # Order management + net_shares = 0 + if context.counter == 2: + brute_shares = (cash / price) * context.parameters.BUY_PERCENTAGE + share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE + net_shares = brute_shares - share_commission_fee + buy_order_id = order(context.coin_pair, net_shares) + + if context.counter == 3: + brute_shares = amount * context.parameters.SELL_PERCENTAGE + share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE + net_shares = -(brute_shares - share_commission_fee) + sell_order_id = order(context.coin_pair, net_shares) + + # Record + record( + price=price, + foo='bar', + # volume=current['volume'], + # price_change=price_change, + # Metrics + cash=cash, + # buy=context.buy, + # sell=context.sell + ) + + +def default_analyze(context=None, perf=None): + pass + + +def initialize(context): + global log + context.parameters = parameters + context.log = Logger(context.parameters.ALGO_NAMESPACE) + log = context.log + default_initialize(context) + context.killer = GracefulKiller(context) + context.telegram_bot = None + + # TELEGRAM_TOKEN='token' + # context.telegram_bot = TelegramBot() + # context.telegram_bot.initialize(TELEGRAM_TOKEN, context) + + +if __name__ == '__main__': + # Parameters: + parameters = SimulationParameters() + start_date = pd.to_datetime(parameters.START_DATE, utc=True) + end_date = pd.to_datetime(parameters.END_DATE, utc=True) + + if parameters.MODE == 'backtest': + results = run_algorithm( + capital_base=parameters.CAPITAL_BASE, + data_frequency=parameters.DATA_FREQUECY, + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + start=start_date, + end=end_date, + live=False, + live_graph=False + ) + + returns_daily = results + results.to_csv('{}'.format(parameters.ALGO_NAMESPACE_RESULTS_TABLE)) + + # returns_daily = returns_minutely.add(1).groupby(pd.TimeGrouper('24H')).prod().add(-1) + + # FIXME: pyfolio integration + # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results) + # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results[:'2017-01-01']) + # pyfolio.create_full_tear_sheet(*pf_data) + + elif parameters.MODE == 'paper': + results = run_algorithm( + capital_base=parameters.CAPITAL_BASE, + data_frequency=parameters.DATA_FREQUECY, + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + live=True, + simulate_orders=True, + live_graph=False + ) + + elif parameters.MODE == 'live': + results = run_algorithm( + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + live=True, + live_graph=True + ) diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index af455cc4..38323347 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -1,8 +1,7 @@ import pandas as pd from logbook import Logger -from catalyst.testing import ZiplineTestCase -from catalyst.testing.fixtures import WithLogger +from catalyst.exchange.utils.stats_utils import set_print_settings from .base import BaseExchangeTestCase from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange_execution import ExchangeLimitOrder @@ -61,12 +60,16 @@ class TestCCXT(BaseExchangeTestCase): freq='30T', assets=[self.exchange.get_asset('eth_btc')], bar_count=200, - start_dt=pd.to_datetime('2017-09-01', utc=True) + # start_dt=pd.to_datetime('2017-09-01', utc=True), ) for asset in candles: df = pd.DataFrame(candles[asset]) df.set_index('last_traded', drop=True, inplace=True) + + set_print_settings() + print(df.head(10)) + print(df.tail(10)) pass def test_tickers(self): From a820f66bdc3e032675b624b5cb81143a64a2f99c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 16:57:23 -0500 Subject: [PATCH 2/5] BLD: adjusted unit test --- tests/exchange/test_ccxt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 38323347..16fe944f 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -14,7 +14,7 @@ log = Logger('test_ccxt') class TestCCXT(BaseExchangeTestCase): @classmethod def setup(self): - exchange_name = 'binance' + exchange_name = 'bittrex' auth = get_exchange_auth(exchange_name) self.exchange = CCXT( exchange_name=exchange_name, @@ -57,7 +57,7 @@ class TestCCXT(BaseExchangeTestCase): def test_get_candles(self): log.info('retrieving candles') candles = self.exchange.get_candles( - freq='30T', + freq='1T', assets=[self.exchange.get_asset('eth_btc')], bar_count=200, # start_dt=pd.to_datetime('2017-09-01', utc=True), @@ -68,6 +68,7 @@ class TestCCXT(BaseExchangeTestCase): df.set_index('last_traded', drop=True, inplace=True) set_print_settings() + print('got {} candles'.format(len(df))) print(df.head(10)) print(df.tail(10)) pass From 00f232e2d7426b8593295ee3e4e6488a39a4866b Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 17:10:41 -0500 Subject: [PATCH 3/5] BUG: fixed sample algo --- catalyst/examples/dual_moving_average.py | 63 ++++++++++++++---------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index ff5dfc5e..363edba1 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -20,8 +20,8 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages - short_window = 50 - long_window = 200 + short_window = 2 + long_window = 2 # Skip as many bars as long_window to properly compute the average context.i += 1 @@ -32,16 +32,18 @@ def handle_data(context, data): # moving average with the appropriate parameters. We choose to use # minute bars for this simulation -> freq="1m" # Returns a pandas dataframe. - short_mavg = data.history(context.asset, + short_data = data.history(context.asset, 'price', bar_count=short_window, - frequency="1m", - ).mean() - long_mavg = data.history(context.asset, + frequency="1T", + ) + short_mavg = short_data.mean() + long_data = data.history(context.asset, 'price', bar_count=long_window, - frequency="1m", - ).mean() + frequency="1T", + ) + long_mavg = long_data.mean() # Let's keep the price of our asset in a more handy variable price = data.current(context.asset, 'price') @@ -82,7 +84,6 @@ def handle_data(context, data): def analyze(context, perf): - # Get the base_currency that was passed as a parameter to the simulation exchange = list(context.exchanges.values())[0] base_currency = exchange.base_currency.upper() @@ -93,7 +94,7 @@ def analyze(context, perf): ax1.legend_.remove() ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) + ax1.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) # Second chart: Plot asset price, moving averages and buys/sells ax2 = plt.subplot(412, sharex=ax1) @@ -104,9 +105,9 @@ def analyze(context, perf): ax2.set_ylabel('{asset}\n({base})'.format( asset=context.asset.symbol, base=base_currency - )) + )) start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) + ax2.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) transaction_df = extract_transactions(perf) if not transaction_df.empty: @@ -136,28 +137,40 @@ def analyze(context, perf): ax3.legend_.remove() ax3.set_ylabel('Percent Change') start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) + ax3.yaxis.set_ticks(np.arange(start, end, (end - start) / 5)) # Fourth chart: Plot our cash ax4 = plt.subplot(414, sharex=ax1) perf.cash.plot(ax=ax4) ax4.set_ylabel('Cash\n({})'.format(base_currency)) start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) + ax4.yaxis.set_ticks(np.arange(0, end, end / 5)) plt.show() if __name__ == '__main__': run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) + capital_base=1000, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=NAMESPACE, + base_currency='usd', + simulate_orders=True, + live=True, + ) + # run_algorithm( + # capital_base=1000, + # data_frequency='minute', + # initialize=initialize, + # handle_data=handle_data, + # analyze=analyze, + # exchange_name='bitfinex', + # algo_namespace=NAMESPACE, + # base_currency='usd', + # start=pd.to_datetime('2017-9-22', utc=True), + # end=pd.to_datetime('2017-9-23', utc=True), + # ) From e56e1f8e21188df373182a8afcec5cdad97d0cfa Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 17:26:48 -0500 Subject: [PATCH 4/5] BUG: fixed an issue with open orders --- catalyst/examples/buy_low_sell_high.py | 6 +++--- catalyst/exchange/exchange_algorithm.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/catalyst/examples/buy_low_sell_high.py b/catalyst/examples/buy_low_sell_high.py index 7dc95b5b..075e2f71 100644 --- a/catalyst/examples/buy_low_sell_high.py +++ b/catalyst/examples/buy_low_sell_high.py @@ -60,7 +60,7 @@ def _handle_data(context, data): rsi=rsi, ) - orders = get_open_orders(context.asset) + orders = context.blotter.open_orders if orders: log.info('skipping bar until all open orders execute') return @@ -146,11 +146,11 @@ if __name__ == '__main__': live = True if live: run_algorithm( - capital_base=0.001, + capital_base=1000, initialize=initialize, handle_data=handle_data, analyze=analyze, - exchange_name='binance', + exchange_name='bittrex', live=True, algo_namespace=algo_namespace, base_currency='btc', diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index da4aee11..aeff9e4e 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -391,8 +391,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): log.warn("Can't initialize signal handler inside another thread." "Exit should be handled by the user.") - log.info('initialized trading algorithm in live mode') - def interrupt_algorithm(self): self.is_running = False @@ -874,6 +872,13 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): raise NotImplementedError() def _get_open_orders(self, asset=None): + if self.simulate_orders: + raise ValueError( + 'The get_open_orders() method only works in live mode. ' + 'The purpose is to list open orders on the exchange ' + 'regardless who placed them. To list the open orders of ' + 'this algo, use `context.blotter.open_orders`.' + ) if asset: exchange = self.exchanges[asset.exchange] return exchange.get_open_orders(asset) @@ -907,6 +912,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): If an asset is passed then this will return a list of the open orders for this asset. """ + # TODO: should this be a shortcut to the open orders in the blotter? return retry( action=self._get_open_orders, attempts=self.attempts['get_open_orders_attempts'], From 09ad397ee6c711b7531ab24ce642ba8b78159707 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 17:31:09 -0500 Subject: [PATCH 5/5] DOC: adjusted the release notes --- docs/source/releases.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 9289d56a..7c5dca1d 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,6 +2,14 @@ Release Notes ============= +Version 0.5.2 +^^^^^^^^^^^^^ +**Release Date**: 2018-02-08 + +Bug Fixes +~~~~~~~~~ +- Fixed an issue with live candle values :issue:`216` and :issue:`199` + Version 0.5.1 ^^^^^^^^^^^^^ **Release Date**: 2018-02-07