From 5c77cfc68d2dc153067f56faa9205adb5b84c030 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 9 Feb 2018 01:38:22 -0500 Subject: [PATCH] BUG: for issue #219, fixed a resampling issue --- catalyst/exchange/exchange_bundle.py | 41 ++++++++----------- catalyst/exchange/exchange_data_portal.py | 2 - catalyst/exchange/utils/exchange_utils.py | 2 +- .../exchange/test_suites/test_suite_bundle.py | 2 + 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index 38aed7c7..569fa6e5 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -458,7 +458,7 @@ class ExchangeBundle: last_entry = None if start is None or \ - (earliest_trade is not None and earliest_trade > start): + (earliest_trade is not None and earliest_trade > start): start = earliest_trade if last_entry is not None and (end is None or end > last_entry): @@ -600,14 +600,14 @@ class ExchangeBundle: if show_breakdown: for asset in chunks: with maybe_show_progress( - chunks[asset], - show_progress, - label='Ingesting {frequency} price data for ' - '{symbol} on {exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, - symbol=asset.symbol - )) as it: + chunks[asset], + show_progress, + label='Ingesting {frequency} price data for ' + '{symbol} on {exchange}'.format( + exchange=self.exchange_name, + frequency=data_frequency, + symbol=asset.symbol + )) as it: for chunk in it: problems += self.ingest_ctable( asset=chunk['asset'], @@ -625,13 +625,13 @@ class ExchangeBundle: key=lambda chunk: pd.to_datetime(chunk['period']) ) with maybe_show_progress( - all_chunks, - show_progress, - label='Ingesting {frequency} price data on ' - '{exchange}'.format( - exchange=self.exchange_name, - frequency=data_frequency, - )) as it: + all_chunks, + show_progress, + label='Ingesting {frequency} price data on ' + '{exchange}'.format( + exchange=self.exchange_name, + frequency=data_frequency, + )) as it: for chunk in it: problems += self.ingest_ctable( asset=chunk['asset'], @@ -830,7 +830,6 @@ class ExchangeBundle: field, data_frequency, algo_end_dt=None, - trailing_bar_count=None, force_auto_ingest=False ): """ @@ -858,7 +857,6 @@ class ExchangeBundle: bar_count=bar_count, field=field, data_frequency=data_frequency, - trailing_bar_count=trailing_bar_count, ) return pd.DataFrame(series) @@ -887,7 +885,6 @@ class ExchangeBundle: field=field, data_frequency=data_frequency, reset_reader=True, - trailing_bar_count=trailing_bar_count, ) return series @@ -898,7 +895,6 @@ class ExchangeBundle: bar_count=bar_count, field=field, data_frequency=data_frequency, - trailing_bar_count=trailing_bar_count, ) return pd.DataFrame(series) @@ -962,12 +958,7 @@ class ExchangeBundle: bar_count, field, data_frequency, - trailing_bar_count=None, reset_reader=False): - if trailing_bar_count: - delta = get_delta(trailing_bar_count, data_frequency) - end_dt += delta - start_dt = get_start_dt(end_dt, bar_count, data_frequency, False) start_dt, _ = self.get_adj_dates( start_dt, end_dt, assets, data_frequency diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index 511bba69..8f9665dc 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -298,7 +298,6 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): frequency, data_frequency ) adj_bar_count = candle_size * bar_count - trailing_bar_count = candle_size - 1 if data_frequency == 'minute' and adj_data_frequency == 'daily': end_dt = end_dt.floor('1D') @@ -310,7 +309,6 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase): field=field, data_frequency=adj_data_frequency, algo_end_dt=self._last_available_session, - trailing_bar_count=trailing_bar_count, ) df = resample_history_df(pd.DataFrame(series), freq, field) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 3c87b510..132845bb 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -540,7 +540,7 @@ def resample_history_df(df, freq, field): else: raise ValueError('Invalid field.') - resampled_df = df.resample(freq).agg(agg) + resampled_df = df.resample(freq, closed='left', label='left').agg(agg) return resampled_df diff --git a/tests/exchange/test_suites/test_suite_bundle.py b/tests/exchange/test_suites/test_suite_bundle.py index ada0dcfd..33ed7c2f 100644 --- a/tests/exchange/test_suites/test_suite_bundle.py +++ b/tests/exchange/test_suites/test_suite_bundle.py @@ -2,6 +2,7 @@ import random import os import pandas as pd +from datetime import timedelta from logbook import TestHandler from pandas.util.testing import assert_frame_equal @@ -159,6 +160,7 @@ class TestSuiteBundle: if end_dt is None or asset_end_dt < end_dt: end_dt = asset_end_dt + end_dt = end_dt + timedelta(minutes=3) dt_range = pd.date_range( end=end_dt, periods=bar_count, freq=freq )