From 82d318a1c0237216c32ac06c579d1ccebd276f3e Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Thu, 8 Feb 2018 15:51:55 -0500 Subject: [PATCH] BUG: fixed issue #216 with bad candle data --- catalyst/exchange/ccxt/ccxt_exchange.py | 20 +- catalyst/support/issue_216.py | 376 ++++++++++++++++++++++++ tests/exchange/test_ccxt.py | 9 +- 3 files changed, 392 insertions(+), 13 deletions(-) create mode 100644 catalyst/support/issue_216.py diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 9cf6c6f4..c5fee495 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -425,15 +425,12 @@ class CCXT(Exchange): 'Please provide either start_dt or end_dt, not both.' ) - elif end_dt is not None: - # Make sure that end_dt really wants data in the past - # if it's close to now, we skip the 'since' parameters to - # lower the probability of error - bars_to_now = pd.date_range( - end_dt, pd.Timestamp.utcnow(), freq=freq - ) - # See: https://github.com/ccxt/ccxt/issues/1360 - if len(bars_to_now) > 1 or self.name in ['poloniex']: + if start_dt is None: + # TODO: determine why binance is failing + if end_dt is None and self.name not in ['binance']: + end_dt = pd.Timestamp.utcnow() + + if end_dt is not None: dt_range = get_periods_range( end_dt=end_dt, periods=bar_count, @@ -441,10 +438,13 @@ class CCXT(Exchange): ) start_dt = dt_range[0] - since = None if start_dt is not None: + # Convert out start date to a UNIX timestamp, then translate to + # milliseconds delta = start_dt - get_epoch() since = int(delta.total_seconds()) * 1000 + else: + since = None candles = dict() for index, asset in enumerate(assets): diff --git a/catalyst/support/issue_216.py b/catalyst/support/issue_216.py new file mode 100644 index 00000000..8a038da0 --- /dev/null +++ b/catalyst/support/issue_216.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# !/usr/bin/env python2 + +import sys +import os +import pandas as pd +import signal +# import talib + +from logbook import Logger + +from catalyst import run_algorithm +from catalyst.api import ( + symbol, + record, + order, + order_target, + order_target_percent, + get_open_orders +) +from catalyst.finance import commission + + +# from base.telegrambot import TelegramBot + + +class GracefulKiller: + # Source: https://stackoverflow.com/a/31464349 + def __init__(self, context): + self.kill_now = False + self.signal = 0 + self.context = context + signal.signal(signal.SIGINT, self.exit_gracefully) + + def exit_gracefully(self, signum, frame): + self.kill_now = True + self.signal = signum + if hasattr(self.context, + 'telegram_bot') and self.context.telegram_bot is not None: + self.context.telegram_bot.updater.stop() + sys.exit(0) + + def exit(self): + return self.kill_now + + +class SimulationParameters: + MODE = 'paper' + CAPITAL_BASE = 1000 + """ + Capital base used on this simulation + """ + + DATA_FREQUECY = 'minute' + + EXCHANGE_NAME = 'bitfinex' + # EXCHANGE_NAME = 'binance' + """ + Exchange used on this simulation + """ + + DATA_DIR = '/home/av/Dropbox/simulations/data' + ALGO_NAMESPACE = os.path.basename(__file__).split('.')[0] + ALGO_NAMESPACE_IMAGE = '{}/{}/{}.png'.format(DATA_DIR, 'images', + ALGO_NAMESPACE) + ALGO_NAMESPACE_RESULTS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, 'tables', + ALGO_NAMESPACE + '_results') + ALGO_NAMESPACE_TRANSACTIONS_TABLE = '{}/{}/{}.csv'.format(DATA_DIR, + 'tables', + ALGO_NAMESPACE + '_transactions') + BASE_CURRENCY = 'usd' + # BASE_CURRENCY = 'usdt' + + # SHORT PERIOD + START_DATE = '2017-09-07' + """ + Start date used on this simulation + """ + END_DATE = '2017-12-12' + """ + End date used on this simulation + """ + + SKIP_FIRST_CANDLES = 0 + + # CANDLES_SAMPLE_RATE = 60 + # CANDLES_SAMPLE_RATE = 30 + CANDLES_SAMPLE_RATE = 1 + """ + Candle interval used on this simulation (in minutes) + """ + + # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + # 30 minute interval ohlcv data (the standard data required for candlestick or + # indicators/signals) + # 30T means 30 minutes re-sampling of one minute data. + # CANDLES_FREQUENCY = '60T' + # CANDLES_FREQUENCY = '30T' + CANDLES_FREQUENCY = '1T' + CANDLES_BUFFER_SIZE = 48 + COIN_PAIR = 'btc_usd' + # COIN_PAIR = 'btc_usdt' + """ + Coin pair used on this simulation + """ + + # TRANSACTIONS + COMMISSION_FEE = 0.0030 + BUY_MIN_AMOUNT = 5 # i.e: USD + SELL_MIN_AMOUNT = 0.001 # i.e: USD + BUY_SELL_PERCENTAGE = 1 # 0.50 + BUY_PERCENTAGE = BUY_SELL_PERCENTAGE + SELL_PERCENTAGE = BUY_SELL_PERCENTAGE + + BASE_PRICE = 'close' + """ + Base price used (close / Heiken Ashi) + """ + + +log = None +parameters = None + + +def print_facts(context): + context.log.info(""" +Index: {} +Date: {} +Candle: +O: {} +H: {} +L: {} +C: {} +V: {} +Metrics: +... +Portfolio: +Base price: {} +Base coin (coin2/usd): {} +Amount (coin1/btc): {} +""".format( + # Facts + context.i, + context.curr_minute, + context.candles_open[-1], + context.candles_high[-1], + context.candles_low[-1], + context.candles_close[-1], + context.candles_volume[-1], + # Metrics + # ... + # Portfolio + context.curr_base_price, + context.portfolio.cash, + context.portfolio.positions[context.coin_pair].amount, + )) + + +def print_facts_telegram(context): + price = context.curr_base_price + amount = context.portfolio.positions[context.coin_pair].amount + pnl = context.portfolio.pnl + capital_used = context.portfolio.capital_used + portfolio_value = context.portfolio.portfolio_value + portfolio_returns = context.portfolio.returns + starting_cash = context.portfolio.starting_cash + cash = context.portfolio.cash + + msg = """ +Status... +Price: {} +Starting cash: {} +Cash: {} +Capital used: {} +Amount: {} +Portfolio value: {} +Returns: {} +PnL: {} + """.format( + price, + starting_cash, + cash, + capital_used, + amount, + portfolio_value, + portfolio_returns, + pnl, + ) + if hasattr(context, 'telegram_bot') and context.telegram_bot is not None: + context.telegram_bot.msg(msg) + + +def default_initialize(context): + # FIXME: set_benchmark + # set_benchmark(symbol(context.parameters.COIN_PAIR)) + + context.coin_pair = symbol(context.parameters.COIN_PAIR) + context.base_price = None + context.current_day = None + context.counter = -1 + context.i = 0 + + context.candles_sample_rate = context.parameters.CANDLES_SAMPLE_RATE + context.candles_frequency = context.parameters.CANDLES_FREQUENCY + context.candles_buffer_size = context.parameters.CANDLES_BUFFER_SIZE + context.set_commission( + commission.PerShare(cost=context.parameters.COMMISSION_FEE)) + + +def default_handle_data(context, data): + context.curr_minute = data.current_dt + context.counter += 1 + + if context.candles_sample_rate == 1: + context.i += 1 + elif context.counter % context.candles_sample_rate != 0: + context.i += 1 + return + + if context.i < context.parameters.SKIP_FIRST_CANDLES: + return + + context.candles_open = data.history( + context.coin_pair, + 'open', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_high = data.history( + context.coin_pair, + 'high', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_low = data.history( + context.coin_pair, + 'low', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_close = data.history( + context.coin_pair, + 'price', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + context.candles_volume = data.history( + context.coin_pair, + 'volume', + bar_count=context.candles_buffer_size, + frequency=context.candles_frequency) + + # FIXME: Here is the error! + # The candles_close frame shows more or less always a value of 94, while + # bitcoin price is very different from that + print(context.candles_close) + + context.base_prices = context.candles_close + cash = context.portfolio.cash + amount = context.portfolio.positions[context.coin_pair].amount + price = data.current(context.coin_pair, 'price') + order_id = None + context.last_base_price = context.base_prices[-2] + context.curr_base_price = context.base_prices[-1] + + # TA calculations + # ... + + # Sanity checks + # assert cash >= 0 + if cash < 0: + import ipdb; + ipdb.set_trace() # BREAKPOINT + + print_facts(context) + print_facts_telegram(context) + + # Order management + net_shares = 0 + if context.counter == 2: + brute_shares = (cash / price) * context.parameters.BUY_PERCENTAGE + share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE + net_shares = brute_shares - share_commission_fee + buy_order_id = order(context.coin_pair, net_shares) + + if context.counter == 3: + brute_shares = amount * context.parameters.SELL_PERCENTAGE + share_commission_fee = brute_shares * context.parameters.COMMISSION_FEE + net_shares = -(brute_shares - share_commission_fee) + sell_order_id = order(context.coin_pair, net_shares) + + # Record + record( + price=price, + foo='bar', + # volume=current['volume'], + # price_change=price_change, + # Metrics + cash=cash, + # buy=context.buy, + # sell=context.sell + ) + + +def default_analyze(context=None, perf=None): + pass + + +def initialize(context): + global log + context.parameters = parameters + context.log = Logger(context.parameters.ALGO_NAMESPACE) + log = context.log + default_initialize(context) + context.killer = GracefulKiller(context) + context.telegram_bot = None + + # TELEGRAM_TOKEN='token' + # context.telegram_bot = TelegramBot() + # context.telegram_bot.initialize(TELEGRAM_TOKEN, context) + + +if __name__ == '__main__': + # Parameters: + parameters = SimulationParameters() + start_date = pd.to_datetime(parameters.START_DATE, utc=True) + end_date = pd.to_datetime(parameters.END_DATE, utc=True) + + if parameters.MODE == 'backtest': + results = run_algorithm( + capital_base=parameters.CAPITAL_BASE, + data_frequency=parameters.DATA_FREQUECY, + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + start=start_date, + end=end_date, + live=False, + live_graph=False + ) + + returns_daily = results + results.to_csv('{}'.format(parameters.ALGO_NAMESPACE_RESULTS_TABLE)) + + # returns_daily = returns_minutely.add(1).groupby(pd.TimeGrouper('24H')).prod().add(-1) + + # FIXME: pyfolio integration + # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results) + # pf_data = pyfolio.utils.extract_rets_pos_txn_from_zipline(results[:'2017-01-01']) + # pyfolio.create_full_tear_sheet(*pf_data) + + elif parameters.MODE == 'paper': + results = run_algorithm( + capital_base=parameters.CAPITAL_BASE, + data_frequency=parameters.DATA_FREQUECY, + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + live=True, + simulate_orders=True, + live_graph=False + ) + + elif parameters.MODE == 'live': + results = run_algorithm( + initialize=initialize, + handle_data=default_handle_data, + analyze=default_analyze, + exchange_name=parameters.EXCHANGE_NAME, + algo_namespace=parameters.ALGO_NAMESPACE, + base_currency=parameters.BASE_CURRENCY, + live=True, + live_graph=True + ) diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index af455cc4..38323347 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -1,8 +1,7 @@ import pandas as pd from logbook import Logger -from catalyst.testing import ZiplineTestCase -from catalyst.testing.fixtures import WithLogger +from catalyst.exchange.utils.stats_utils import set_print_settings from .base import BaseExchangeTestCase from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange_execution import ExchangeLimitOrder @@ -61,12 +60,16 @@ class TestCCXT(BaseExchangeTestCase): freq='30T', assets=[self.exchange.get_asset('eth_btc')], bar_count=200, - start_dt=pd.to_datetime('2017-09-01', utc=True) + # start_dt=pd.to_datetime('2017-09-01', utc=True), ) for asset in candles: df = pd.DataFrame(candles[asset]) df.set_index('last_traded', drop=True, inplace=True) + + set_print_settings() + print(df.head(10)) + print(df.tail(10)) pass def test_tickers(self):