diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 01770413..0d4b3c57 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -24,7 +24,8 @@ from catalyst.exchange.exchange_execution import ExchangeLimitOrder from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ get_exchange_folder, get_catalyst_symbol, \ get_exchange_auth -from exchange.utils.datetime_utils import from_ms_timestamp, get_epoch, \ +from catalyst.exchange.utils.datetime_utils import from_ms_timestamp, \ + get_epoch, \ get_periods_range from catalyst.finance.order import Order, ORDER_STATUS from catalyst.finance.transaction import Transaction @@ -424,27 +425,36 @@ class CCXT(Exchange): ) elif end_dt is not None: - dt_range = get_periods_range( - end_dt=end_dt, - periods=bar_count, - freq=freq, + # Make sure that end_dt really wants data in the past + # if it's close to now, we skip the 'since' parameters to + # lower the probability of error + bars_to_now = pd.date_range( + end_dt, pd.Timestamp.utcnow(), freq=freq ) - # skip the left bound of the range since the open range is - # on the right bound - start_dt = dt_range[1] + if len(bars_to_now) > 1: + dt_range = get_periods_range( + end_dt=end_dt, + periods=bar_count, + freq=freq, + ) + # with some exchanges, skip the left bound of the range + # since the open range is on the right bound + if self.name in ['poloniex']: + start_dt = dt_range[1] + else: + start_dt = dt_range[0] - ms = None + since = None if start_dt is not None: - if end_dt is not None: - delta = start_dt - get_epoch() - ms = int(delta.total_seconds()) * 1000 + delta = start_dt - get_epoch() + since = int(delta.total_seconds()) * 1000 candles = dict() for index, asset in enumerate(assets): ohlcvs = self.api.fetch_ohlcv( symbol=symbols[index], timeframe=timeframe, - since=ms, + since=since, limit=bar_count, params={} ) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index c873962e..4ef2cc7a 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -13,7 +13,8 @@ from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ PricingDataNotLoadedError, \ NoDataAvailableOnExchange, NoValueForField, LastCandleTooEarlyError, \ TickerNotFoundError, NotEnoughCashError -from exchange.utils.datetime_utils import get_delta, get_periods_range, \ +from catalyst.exchange.utils.datetime_utils import get_delta, \ + get_periods_range, \ get_periods, get_start_dt, get_frequency from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ resample_history_df, has_bundle @@ -497,39 +498,37 @@ class Exchange: freq, candle_size, unit, data_frequency = get_frequency( frequency, data_frequency ) - adj_bar_count = candle_size * bar_count - - start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) - # The get_history method supports multiple asset candles = self.get_candles( freq=freq, assets=assets, bar_count=bar_count, - start_dt=start_dt if not is_current else None, end_dt=end_dt if not is_current else None, ) series = dict() for asset in candles: + first_candle = candles[asset][0] asset_series = self.get_series_from_candles( candles=candles[asset], - start_dt=start_dt, + start_dt=first_candle['last_traded'], end_dt=end_dt, data_frequency=frequency, field=field, ) - if end_dt is not None: - delta = get_delta(candle_size, data_frequency) - adj_end_dt = end_dt - delta - last_traded = asset_series.index[-1] - if last_traded < adj_end_dt: - raise LastCandleTooEarlyError( - last_traded=last_traded, - end_dt=adj_end_dt, - exchange=self.name, - ) + # Checking to make sure that the dates match + delta = get_delta(candle_size, data_frequency) + adj_end_dt = end_dt - delta + last_traded = asset_series.index[-1] + + if last_traded < adj_end_dt: + raise LastCandleTooEarlyError( + last_traded=last_traded, + end_dt=adj_end_dt, + exchange=self.name, + ) + series[asset] = asset_series df = pd.DataFrame(series) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 9df3f42e..38aed7c7 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -22,7 +22,7 @@ from catalyst.exchange.exchange_errors import EmptyValuesInBundleError, \ PricingDataNotLoadedError, DataCorruptionError, PricingDataValueError from catalyst.exchange.utils.bundle_utils import range_in_bundle, \ get_bcolz_chunk, get_df_from_arrays, get_assets -from exchange.utils.datetime_utils import get_delta, get_start_dt, \ +from catalyst.exchange.utils.datetime_utils import get_delta, get_start_dt, \ get_period_label, get_month_start_end, get_year_start_end from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ save_exchange_symbols, mixin_market_params, get_catalyst_symbol diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index dd4507f6..511bba69 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -10,7 +10,7 @@ from catalyst.exchange.exchange_errors import ( ExchangeRequestError, PricingDataNotLoadedError) from catalyst.exchange.utils.exchange_utils import resample_history_df, group_assets_by_exchange -from exchange.utils.datetime_utils import get_frequency +from catalyst.exchange.utils.datetime_utils import get_frequency from logbook import Logger from redo import retry diff --git a/catalyst/exchange/utils/datetime_utils.py b/catalyst/exchange/utils/datetime_utils.py index 6e7b9d60..2a8cb886 100644 --- a/catalyst/exchange/utils/datetime_utils.py +++ b/catalyst/exchange/utils/datetime_utils.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, date import pandas as pd import pytz -from exchange.exchange_errors import InvalidHistoryFrequencyError, \ +from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ InvalidHistoryFrequencyAlias diff --git a/tests/exchange/test_ccxt.py b/tests/exchange/test_ccxt.py index 67ba61d2..0ae5b0af 100644 --- a/tests/exchange/test_ccxt.py +++ b/tests/exchange/test_ccxt.py @@ -15,7 +15,7 @@ log = Logger('test_ccxt') class TestCCXT(BaseExchangeTestCase): @classmethod def setup(self): - exchange_name = 'binance' + exchange_name = 'bitfinex' auth = get_exchange_auth(exchange_name) self.exchange = CCXT( exchange_name=exchange_name, @@ -58,7 +58,7 @@ class TestCCXT(BaseExchangeTestCase): def test_get_candles(self): log.info('retrieving candles') candles = self.exchange.get_candles( - freq='5T', + freq='30T', assets=[self.exchange.get_asset('eth_btc')], bar_count=200, start_dt=pd.to_datetime('2017-09-01', utc=True) diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index 21ef60d5..091ebd0e 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -85,6 +85,8 @@ class TestSuiteBundle: df, assets, '{}_{}'.format(freq, source) ) + print('saved {} test results: {}'.format(end_dt, folder)) + assert_frame_equal( right=data['bundle'], left=data['exchange'], @@ -102,7 +104,6 @@ class TestSuiteBundle: with open(os.path.join(folder, 'compare.txt'), 'w+') as handle: handle.write(e.args[0]) - print('saved test results: {}'.format(folder)) pass def test_validate_bundles(self): @@ -116,7 +117,7 @@ class TestSuiteBundle: # population=exchange_population, # features=[bundle], # ) # Type: list[Exchange] - exchanges = [get_exchange('poloniex', skip_init=True)] + exchanges = [get_exchange('bitfinex', skip_init=True)] data_portal = TestSuiteBundle.get_data_portal(exchanges) for exchange in exchanges: