From a85b6c798ab39c99218d45af3ed9fa649a0cb3c1 Mon Sep 17 00:00:00 2001 From: fredfortier Date: Tue, 7 Nov 2017 13:42:47 -0500 Subject: [PATCH] BUG: fixes related to issue #47 and added a verbose ingestion option. --- catalyst/__main__.py | 10 ++++++++-- catalyst/examples/simple_loop.py | 4 ++-- catalyst/exchange/exchange_bundle.py | 14 +++++++------- tests/exchange/test_bundle.py | 17 ++++++++++++----- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/catalyst/__main__.py b/catalyst/__main__.py index 8b7f001b..638d096e 100644 --- a/catalyst/__main__.py +++ b/catalyst/__main__.py @@ -490,8 +490,13 @@ def live(ctx, default=True, help='Print progress information to the terminal.' ) +@click.option( + '--verbose/--no-verbose`', + default=False, + help='Show a progress indicator for every currency pair.' +) def ingest_exchange(exchange_name, data_frequency, start, end, - include_symbols, exclude_symbols, show_progress): + include_symbols, exclude_symbols, show_progress, verbose): """ Ingest data for the given exchange. """ @@ -509,7 +514,8 @@ def ingest_exchange(exchange_name, data_frequency, start, end, exclude_symbols=exclude_symbols, start=start, end=end, - show_progress=show_progress + show_progress=show_progress, + show_breakdown=verbose ) diff --git a/catalyst/examples/simple_loop.py b/catalyst/examples/simple_loop.py index 276369cc..3927c4d2 100644 --- a/catalyst/examples/simple_loop.py +++ b/catalyst/examples/simple_loop.py @@ -21,7 +21,7 @@ def handle_data(context, data): context.asset, fields='price', bar_count=16, - frequency='5T' + frequency='60T' ) rsi = talib.RSI(prices.values, timeperiod=14)[-1] print('got rsi: {}'.format(rsi)) @@ -32,7 +32,7 @@ def handle_data(context, data): run_algorithm( capital_base=250, start=pd.to_datetime('2016-6-1', utc=True), - end=pd.to_datetime('2016-12-31', utc=True), + end=pd.to_datetime('2017-11-1', utc=True), data_frequency='daily', initialize=initialize, handle_data=handle_data, diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index b5c66d7b..c317812a 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -528,7 +528,7 @@ class ExchangeBundle: return chunks def ingest_assets(self, assets, data_frequency, start_dt=None, end_dt=None, - show_progress=False, asset_chunks=False): + show_progress=False, show_breakdown=False): """ Determine if data is missing from the bundle and attempt to ingest it. @@ -539,7 +539,7 @@ class ExchangeBundle: start_dt: datetime end_dt: datetime show_progress: bool - asset_chunks: bool + show_breakdown: bool """ if start_dt is None: @@ -565,7 +565,7 @@ class ExchangeBundle: # This is the common writer for the entire exchange bundle # we want to give an end_date far in time writer = self.get_writer(start_dt, end_dt, data_frequency) - if asset_chunks: + if show_breakdown: for asset in chunks: with maybe_show_progress( chunks[asset], @@ -612,7 +612,7 @@ class ExchangeBundle: def ingest(self, data_frequency, include_symbols=None, exclude_symbols=None, start=None, end=None, - show_progress=True, environ=os.environ): + show_progress=True, show_breakdown=True, environ=os.environ): """ Inject data based on specified parameters. @@ -631,7 +631,7 @@ class ExchangeBundle: for frequency in data_frequency.split(','): self.ingest_assets(assets, frequency, start, end, - show_progress, True) + show_progress, show_breakdown) def get_history_window_series_and_load(self, assets, @@ -684,7 +684,7 @@ class ExchangeBundle: end_dt=algo_end_dt, data_frequency=data_frequency, show_progress=True, - asset_chunks=True + show_breakdown=True ) series = self.get_history_window_series( assets=assets, @@ -692,7 +692,7 @@ class ExchangeBundle: bar_count=bar_count, field=field, data_frequency=data_frequency, - reset_reader=False + reset_reader=True ) return series diff --git a/tests/exchange/test_bundle.py b/tests/exchange/test_bundle.py index ba055dca..2cf98b08 100644 --- a/tests/exchange/test_bundle.py +++ b/tests/exchange/test_bundle.py @@ -442,15 +442,19 @@ class TestExchangeBundle: data_frequency = 'minute' exchange = get_exchange(exchange_name) - asset = exchange.get_asset('neo_usd') + asset = exchange.get_asset('eth_btc') + start_dt = pd.to_datetime('2016-5-31', utc=True) + end_dt = pd.to_datetime('2016-6-1', utc=True) self._bundle_to_csv( asset=asset, exchange=exchange, data_frequency=data_frequency, filename='{}_{}_{}'.format( exchange_name, data_frequency, asset.symbol - ) + ), + start_dt=start_dt, + end_dt=end_dt ) def bundle_to_csv(self): @@ -478,12 +482,15 @@ class TestExchangeBundle: pass def _bundle_to_csv(self, asset, exchange, data_frequency, filename, - path=None): + path=None, start_dt=None, end_dt=None): bundle = ExchangeBundle(exchange) reader = bundle.get_reader(data_frequency, path=path) - start_dt = reader.first_trading_day - end_dt = reader.last_available_dt + if start_dt is None: + start_dt = reader.first_trading_day + + if end_dt is None: + end_dt = reader.last_available_dt if data_frequency == 'daily': end_dt = end_dt - pd.Timedelta(hours=23, minutes=59)