From 215de33c35c79241de37c80d77fd299ba9ba9a14 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 5 Jan 2018 02:46:11 -0500 Subject: [PATCH 01/14] BUG: Fixed issue with updating positions after restoring the state of an algo --- catalyst/exchange/exchange_algorithm.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index df30b368..9365d11d 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -271,9 +271,9 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): # Merging latest recorded variables stats.update(self.recorded_vars) - stats['positions'] = cum.position_tracker.get_positions_list() - period = tracker.todays_performance + stats['positions'] = period.position_tracker.get_positions_list() + # we want the key to be absent, not just empty # Only include transactions for given dt stats['transactions'] = [] @@ -495,6 +495,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): period.starting_cash = perf.ending_cash period.starting_exposure = perf.ending_exposure period.starting_value = perf.ending_value + # This does not seem to get updated correctly period.position_tracker = perf.position_tracker self.trading_client = ExchangeAlgorithmExecutor( From 13de3e69efdde22e1001b01019d07bfbe07453ba Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Fri, 5 Jan 2018 22:22:04 -0500 Subject: [PATCH 02/14] BLD: refined CCXT error handlers --- catalyst/exchange/ccxt/ccxt_exchange.py | 65 ++++++++++++++++++------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index f1107ade..ed3b2b24 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -6,12 +6,8 @@ from collections import defaultdict import ccxt import pandas as pd import six -from catalyst.assets._assets import TradingPair -from ccxt import ExchangeNotAvailable, InvalidOrder -from logbook import Logger -from six import string_types - from catalyst.algorithm import MarketOrder +from catalyst.assets._assets import TradingPair from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -23,6 +19,10 @@ from catalyst.exchange.utils.exchange_utils import mixin_market_params, \ from_ms_timestamp, get_epoch, get_exchange_folder, get_catalyst_symbol, \ get_exchange_auth from catalyst.finance.order import Order, ORDER_STATUS +from ccxt import InvalidOrder, NetworkError, \ + ExchangeError +from logbook import Logger +from six import string_types log = Logger('CCXT', level=LOG_LEVEL) @@ -105,7 +105,12 @@ class CCXT(Exchange): with open(filename, 'w+') as f: json.dump(self.markets, f, indent=4) - except ExchangeNotAvailable as e: + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch markets {}: {}'.format( + self.name, e + ) + ) raise ExchangeRequestError(error=e) self.load_assets() @@ -408,6 +413,7 @@ class CCXT(Exchange): def _fetch_symbol_map(self, is_local): try: return self.fetch_symbol_map(is_local) + except ExchangeSymbolsNotFound: return None @@ -559,8 +565,12 @@ class CCXT(Exchange): for key in balances: balances_lower[key.lower()] = balances[key] - except Exception as e: - log.debug('error retrieving balances: {}', e) + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch balance {}: {}'.format( + self.name, e + ) + ) raise ExchangeRequestError(error=e) return balances_lower @@ -703,14 +713,18 @@ class CCXT(Exchange): amount=adj_amount, price=price ) - except ExchangeNotAvailable as e: - log.debug('unable to create order: {}'.format(e)) - raise ExchangeRequestError(error=e) - except InvalidOrder as e: log.warn('the exchange rejected the order: {}'.format(e)) raise CreateOrderError(exchange=self.name, error=e) + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to create order {} / {}: {}'.format( + self.name, symbol, e + ) + ) + raise ExchangeRequestError(error=e) + if 'info' not in result: raise ValueError('cannot use order without info attribute') @@ -735,7 +749,12 @@ class CCXT(Exchange): limit=None, params=dict() ) - except Exception as e: + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch open orders {} / {}: {}'.format( + self.name, asset.symbol, e + ) + ) raise ExchangeRequestError(error=e) orders = [] @@ -758,7 +777,12 @@ class CCXT(Exchange): order_status = self.api.fetch_order(id=order_id, symbol=symbol) order, executed_price = self._create_order(order_status) - except Exception as e: + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to fetch order {} / {}: {}'.format( + self.name, order_id, e + ) + ) raise ExchangeRequestError(error=e) return order, executed_price @@ -777,7 +801,12 @@ class CCXT(Exchange): if asset_or_symbol is not None else None self.api.cancel_order(id=order_id, symbol=symbol) - except Exception as e: + except (ExchangeError, NetworkError) as e: + log.warn( + 'unable to cancel order {} / {}: {}'.format( + self.name, order_id, e + ) + ) raise ExchangeRequestError(error=e) def tickers(self, assets): @@ -828,10 +857,10 @@ class CCXT(Exchange): tickers[asset] = ticker - except ExchangeNotAvailable as e: + except (ExchangeError, NetworkError) as e: log.warn( - 'unable to fetch ticker: {} {}'.format( - self.name, asset.symbol + 'unable to fetch ticker {} / {}: {}'.format( + self.name, asset.symbol, e ) ) raise ExchangeRequestError(error=e) From 5d9708901d7b964919f02f7fd7939d19b5ed7a86 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 19:08:47 -0500 Subject: [PATCH 03/14] BUG: fixed issue #111 related to positions update after restoring algo state --- catalyst/examples/mean_reversion_simple.py | 6 +- catalyst/exchange/exchange_algorithm.py | 99 ++++++++++++++-------- 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 14f7ec99..e46b8df6 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -37,8 +37,8 @@ def initialize(context): context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 55 - context.RSI_OVERBOUGHT = 65 + context.RSI_OVERSOLD = 35 + context.RSI_OVERBOUGHT = 50 context.CANDLE_SIZE = '5T' context.start_time = time.time() @@ -248,7 +248,7 @@ if __name__ == '__main__': if live: run_algorithm( - capital_base=0.03, + capital_base=0.025, initialize=initialize, handle_data=handle_data, analyze=analyze, diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 9365d11d..8e369f8e 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -391,16 +391,20 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): 'before exiting the algorithm.') algo_folder = get_algo_folder(self.algo_namespace) - folder = join(algo_folder, 'daily_perf') + folder = join(algo_folder, 'daily_performance') files = [f for f in listdir(folder) if isfile(join(folder, f))] daily_perf_list = [] for item in files: filename = join(folder, item) + with open(filename, 'rb') as handle: - daily_perf_list.append(pickle.load(handle)) + perf_period = pickle.load(handle) + perf_period_dict = perf_period.to_dict() + daily_perf_list.append(perf_period_dict) stats = pd.DataFrame(daily_perf_list) + stats.set_index('period_close', drop=False, inplace=True) self.analyze(stats) @@ -460,44 +464,62 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): return self._clock - def get_generator(self): - if self.trading_client is not None: - return self.trading_client.transform() + def _init_trading_client(self): + """ + This replaces Ziplines `_create_generator` method. The main difference + is that we are restoring performance tracker objects if available. + This allows us to stop/start algos without loosing their state. - perf = None + """ if self.perf_tracker is None: + # Note from the Zipline dev: + # HACK: When running with the `run` method, we set perf_tracker to + # None so that it will be overwritten here. tracker = self.perf_tracker = PerformanceTracker( sim_params=self.sim_params, trading_calendar=self.trading_calendar, env=self.trading_environment, ) - # Set the dt initially to the period start by forcing it to change. self.on_dt_changed(self.sim_params.start_session) + new_position_tracker = tracker.position_tracker + tracker.position_tracker = None + # Unpacking the perf_tracker and positions if available - perf = get_algo_object( + cum_perf = get_algo_object( algo_name=self.algo_namespace, key='cumulative_performance', ) + if cum_perf is not None: + tracker.cumulative_performance = cum_perf + # Ensure single common position tracker + tracker.position_tracker = cum_perf.position_tracker + + today = pd.Timestamp.utcnow().floor('1D') + todays_perf = get_algo_object( + algo_name=self.algo_namespace, + key=today.strftime('%Y-%m-%d'), + rel_path='daily_performance', + ) + if todays_perf is not None: + # Ensure single common position tracker + if tracker.position_tracker is not None: + todays_perf.position_tracker = tracker.position_tracker + else: + tracker.position_tracker = todays_perf.position_tracker + + tracker.todays_performance = todays_perf + + if tracker.position_tracker is None: + # Use a new position_tracker if not is found in the state + tracker.position_tracker = new_position_tracker if not self.initialized: + # Calls the initialize function of the algorithm self.initialize(*self.initialize_args, **self.initialize_kwargs) self.initialized = True - # Call the simulation trading algorithm for side-effects: - # it creates the perf tracker - # TradingAlgorithm._create_generator(self, self.sim_params) - if perf is not None: - tracker.cumulative_performance = perf - - period = self.perf_tracker.todays_performance - period.starting_cash = perf.ending_cash - period.starting_exposure = perf.ending_exposure - period.starting_value = perf.ending_value - # This does not seem to get updated correctly - period.position_tracker = perf.position_tracker - self.trading_client = ExchangeAlgorithmExecutor( algo=self, sim_params=self.sim_params, @@ -507,6 +529,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): restrictions=self.restrictions, universe_func=self._calculate_universe, ) + + def get_generator(self): + if self.trading_client is None: + self._init_trading_client() + return self.trading_client.transform() def updated_portfolio(self): @@ -677,11 +704,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.frame_stats = list() self.performance_needs_update = False - new_orders = self.perf_tracker.todays_performance.orders_by_id.keys() - if new_orders != self._last_orders: + orders = self.perf_tracker.todays_performance.orders_by_id.keys() + if orders != self._last_orders: self.performance_needs_update = True - self._last_orders = copy.deepcopy(new_orders) + # Saving current orders to detect changes in the next frame + self._last_orders = copy.deepcopy(orders) if self.performance_needs_update: self.perf_tracker.update_performance() @@ -698,7 +726,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.portfolio_needs_update = False log.info( - 'got totals from exchanges, cash: {} positions: {}'.format( + 'portfolio balances, cash: {}, positions: {}'.format( cash, positions_value ) ) @@ -710,18 +738,29 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # every bar no matter if the algorithm places an order or not. self.validate_account_controls() + self._save_algo_state(data) + self.current_day = data.current_dt.floor('1D') + + def _save_algo_state(self, data): + today = data.current_dt.floor('1D') try: self._save_stats_csv(self._process_stats(data)) except Exception as e: log.warn('unable to calculate performance: {}'.format(e)) + log.debug('saving cumulative performance object') save_algo_object( algo_name=self.algo_namespace, key='cumulative_performance', obj=self.perf_tracker.cumulative_performance, ) - - self.current_day = data.current_dt.floor('1D') + log.debug('saving todays performance object') + save_algo_object( + algo_name=self.algo_namespace, + key=today.strftime('%Y-%m-%d'), + obj=self.perf_tracker.todays_performance, + rel_path='daily_performance' + ) def _process_stats(self, data): today = data.current_dt.floor('1D') @@ -764,12 +803,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): start_dt=today, end_dt=data.current_dt ) - save_algo_object( - algo_name=self.algo_namespace, - key=today.strftime('%Y-%m-%d'), - obj=daily_stats, - rel_path='daily_perf' - ) return recorded_cols From ba0208910f59de83c7adbc214888a02e16e25bef Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 19:37:53 -0500 Subject: [PATCH 04/14] BUG: Fixed issue #142 by removing unecessary capital_base check since we are now validating positions and cash against the exchange --- catalyst/exchange/exchange_algorithm.py | 4 -- catalyst/exchange/utils/exchange_utils.py | 14 ++++++ catalyst/utils/run_algo.py | 61 +---------------------- 3 files changed, 15 insertions(+), 64 deletions(-) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 8e369f8e..efd10553 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -551,10 +551,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): positions, returning the available cash, and raising error if the data goes out of sync. - Parameters - ---------- - attempt_index: int - Returns ------- float diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index ab16f71d..ebc678d2 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -733,3 +733,17 @@ def get_candles_df(candles, field, freq, bar_count, end_dt, df.dropna(inplace=True) return df + + +def fetch_capital_base(exchange): + """ + Fetch the base currency amount required to bootstrap + the algorithm against the exchange. + + The algorithm cannot continue without this value. + + :param exchange: the targeted exchange + :param attempt_index: + :return capital_base: the amount of base currency available for + trading + """ diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 015f91e4..af67c29f 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -15,6 +15,7 @@ from catalyst.data.data_portal import DataPortal from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \ TradingPairPricing from catalyst.exchange.utils.factory import get_exchange +from redo import retry try: from pygments import highlight @@ -199,66 +200,6 @@ def _run(handle_data, first_trading_day=pd.to_datetime('today', utc=True) ) - def fetch_capital_base(exchange, attempt_index=0): - """ - Fetch the base currency amount required to bootstrap - the algorithm against the exchange. - - The algorithm cannot continue without this value. - - :param exchange: the targeted exchange - :param attempt_index: - :return capital_base: the amount of base currency available for - trading - """ - try: - log.debug('retrieving capital base in {} to bootstrap ' - 'exchange {}'.format(base_currency, exchange_name)) - balances = exchange.get_balances() - except ExchangeRequestError as e: - if attempt_index < 20: - log.warn( - 'could not retrieve balances on {}: {}'.format( - exchange.name, e - ) - ) - sleep(5) - return fetch_capital_base(exchange, attempt_index + 1) - - else: - raise ExchangeRequestErrorTooManyAttempts( - attempts=attempt_index, - error=e - ) - - if base_currency in balances: - base_currency_available = balances[base_currency]['free'] - log.info( - 'base currency available in the account: {} {}'.format( - base_currency_available, base_currency - ) - ) - - return base_currency_available - else: - raise BaseCurrencyNotFoundError( - base_currency=base_currency, - exchange=exchange_name - ) - - if not simulate_orders: - for exchange_name in exchanges: - exchange = exchanges[exchange_name] - balance = fetch_capital_base(exchange) - - if balance < capital_base: - raise NotEnoughCapitalError( - exchange=exchange_name, - base_currency=base_currency, - balance=balance, - capital_base=capital_base, - ) - sim_params = create_simulation_parameters( start=start, end=end, From d88710501f765e5d2abb1ea353c91306ffebe1c3 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 19:50:58 -0500 Subject: [PATCH 05/14] DOC: updated release notes in preparation for version 0.4.4. --- docs/source/releases.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index e095d0b4..40594aa0 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -2,9 +2,18 @@ Release Notes ============= +Version 0.4.4 +^^^^^^^^^^^^^ +**Release Date**: 2018-01-06 + +Bug Fixes +~~~~~~~~~ +- Removed redundant capital_base validation (:issue:`142`) +- Fixed portfolio update issue with restored state (:issue:`111) + Version 0.4.3 ^^^^^^^^^^^^^ -**Release Date**: 2017-01-05 +**Release Date**: 2018-01-05 Bug Fixes ~~~~~~~~~ @@ -13,7 +22,7 @@ Bug Fixes Version 0.4.2 ^^^^^^^^^^^^^ -**Release Date**: 2017-01-03 +**Release Date**: 2018-01-03 Bug Fixes ~~~~~~~~~ From 9c33ee123c6e5dba0c782edca9abafd0cee6fb7b Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 19:51:41 -0500 Subject: [PATCH 06/14] BLD: Housekeeping --- catalyst/examples/mean_reversion_simple.py | 4 ++-- catalyst/exchange/exchange.py | 3 +-- catalyst/exchange/exchange_algorithm.py | 5 ++--- catalyst/exchange/exchange_asset_finder.py | 3 +-- catalyst/exchange/exchange_blotter.py | 5 ++--- catalyst/exchange/exchange_bundle.py | 9 ++++----- catalyst/exchange/exchange_data_portal.py | 5 ++--- catalyst/exchange/exchange_portfolio.py | 3 +-- catalyst/exchange/exchange_pricing_loader.py | 11 +++++------ catalyst/exchange/live_graph_clock.py | 5 ++--- catalyst/exchange/simple_clock.py | 3 +-- catalyst/exchange/utils/bundle_utils.py | 1 - catalyst/exchange/utils/exchange_utils.py | 19 ++----------------- catalyst/exchange/utils/factory.py | 3 +-- .../exchange/utils/serialization_utils.py | 3 +-- catalyst/exchange/utils/stats_utils.py | 1 - catalyst/exchange/utils/test_utils.py | 1 - 17 files changed, 27 insertions(+), 57 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index e46b8df6..1ba812a2 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -37,8 +37,8 @@ def initialize(context): context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 35 - context.RSI_OVERBOUGHT = 50 + context.RSI_OVERSOLD = 45 + context.RSI_OVERBOUGHT = 55 context.CANDLE_SIZE = '5T' context.start_time = time.time() diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index b6b24d20..75f4ec3c 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -5,8 +5,6 @@ from time import sleep import numpy as np import pandas as pd -from logbook import Logger - from catalyst.constants import LOG_LEVEL from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -19,6 +17,7 @@ from catalyst.exchange.utils.bundle_utils import get_start_dt, \ get_delta, get_periods, get_periods_range from catalyst.exchange.utils.exchange_utils import get_exchange_symbols, \ get_frequency, resample_history_df, has_bundle +from logbook import Logger log = Logger('Exchange', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index efd10553..87bff941 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -18,11 +18,9 @@ from datetime import timedelta from os import listdir from os.path import isfile, join +import catalyst.protocol as zp import logbook import pandas as pd -from redo import retry - -import catalyst.protocol as zp from catalyst.algorithm import TradingAlgorithm from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange_blotter import ExchangeBlotter @@ -49,6 +47,7 @@ from catalyst.utils.api_support import api_method from catalyst.utils.input_validation import error_keywords, ensure_upper_case from catalyst.utils.math_utils import round_nearest from catalyst.utils.preprocess import preprocess +from redo import retry log = logbook.Logger('exchange_algorithm', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_asset_finder.py b/catalyst/exchange/exchange_asset_finder.py index 0a887986..5b41cc7e 100644 --- a/catalyst/exchange/exchange_asset_finder.py +++ b/catalyst/exchange/exchange_asset_finder.py @@ -1,8 +1,7 @@ import pandas as pd -from logbook import Logger - from catalyst.constants import LOG_LEVEL from catalyst.exchange.utils.factory import find_exchanges +from logbook import Logger log = Logger('ExchangeAssetFinder', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index 41a673e1..cb48f939 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -1,8 +1,5 @@ import pandas as pd from catalyst.assets._assets import TradingPair -from logbook import Logger -from redo import retry - from catalyst.constants import LOG_LEVEL from catalyst.exchange.exchange_errors import ExchangeRequestError from catalyst.finance.blotter import Blotter @@ -11,6 +8,8 @@ from catalyst.finance.order import ORDER_STATUS from catalyst.finance.slippage import SlippageModel from catalyst.finance.transaction import create_transaction, Transaction from catalyst.utils.input_validation import expect_types +from logbook import Logger +from redo import retry log = Logger('exchange_blotter', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index c3d032d2..5ceb73db 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -8,12 +8,8 @@ from operator import is_not import numpy as np import pandas as pd import pytz -from catalyst.assets._assets import TradingPair -from logbook import Logger -from pytz import UTC -from six import itervalues - from catalyst import get_calendar +from catalyst.assets._assets import TradingPair from catalyst.constants import DATE_TIME_FORMAT, AUTO_INGEST from catalyst.constants import LOG_LEVEL from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \ @@ -32,6 +28,9 @@ from catalyst.exchange.utils.exchange_utils import get_exchange_folder, \ save_exchange_symbols, mixin_market_params, get_catalyst_symbol from catalyst.utils.cli import maybe_show_progress from catalyst.utils.paths import ensure_directory +from logbook import Logger +from pytz import UTC +from six import itervalues log = Logger('exchange_bundle', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_data_portal.py b/catalyst/exchange/exchange_data_portal.py index a37d1025..6f57b7e7 100644 --- a/catalyst/exchange/exchange_data_portal.py +++ b/catalyst/exchange/exchange_data_portal.py @@ -3,9 +3,6 @@ import abc import numpy as np import pandas as pd from catalyst.assets._assets import TradingPair -from logbook import Logger -from redo import retry - from catalyst.constants import LOG_LEVEL, AUTO_INGEST from catalyst.data.data_portal import DataPortal from catalyst.exchange.exchange_bundle import ExchangeBundle @@ -14,6 +11,8 @@ from catalyst.exchange.exchange_errors import ( PricingDataNotLoadedError) from catalyst.exchange.utils.exchange_utils import get_frequency, \ resample_history_df, group_assets_by_exchange +from logbook import Logger +from redo import retry log = Logger('DataPortalExchange', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_portfolio.py b/catalyst/exchange/exchange_portfolio.py index 71ed9a35..894fea0f 100644 --- a/catalyst/exchange/exchange_portfolio.py +++ b/catalyst/exchange/exchange_portfolio.py @@ -1,8 +1,7 @@ import numpy as np -from logbook import Logger - from catalyst.constants import LOG_LEVEL from catalyst.protocol import Portfolio, Positions, Position +from logbook import Logger log = Logger('ExchangePortfolio', level=LOG_LEVEL) diff --git a/catalyst/exchange/exchange_pricing_loader.py b/catalyst/exchange/exchange_pricing_loader.py index 38663962..3c59f467 100644 --- a/catalyst/exchange/exchange_pricing_loader.py +++ b/catalyst/exchange/exchange_pricing_loader.py @@ -11,12 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from logbook import Logger -from numpy import ( - iinfo, - uint32, -) - from catalyst.constants import LOG_LEVEL from catalyst.data.us_equity_pricing import BcolzDailyBarReader from catalyst.errors import NoFurtherDataError @@ -26,6 +20,11 @@ from catalyst.pipeline.data import DataSet, Column from catalyst.pipeline.loaders.base import PipelineLoader from catalyst.utils.calendars import get_calendar from catalyst.utils.numpy_utils import float64_dtype +from logbook import Logger +from numpy import ( + iinfo, + uint32, +) UINT32_MAX = iinfo(uint32).max diff --git a/catalyst/exchange/live_graph_clock.py b/catalyst/exchange/live_graph_clock.py index 40ad9c24..8a04bef0 100644 --- a/catalyst/exchange/live_graph_clock.py +++ b/catalyst/exchange/live_graph_clock.py @@ -1,13 +1,12 @@ import pandas as pd +from catalyst.constants import LOG_LEVEL +from catalyst.exchange.utils.stats_utils import prepare_stats from catalyst.gens.sim_engine import ( BAR, SESSION_START ) from logbook import Logger -from catalyst.constants import LOG_LEVEL -from catalyst.exchange.utils.stats_utils import prepare_stats - log = Logger('LiveGraphClock', level=LOG_LEVEL) diff --git a/catalyst/exchange/simple_clock.py b/catalyst/exchange/simple_clock.py index cede9429..14f52bc9 100644 --- a/catalyst/exchange/simple_clock.py +++ b/catalyst/exchange/simple_clock.py @@ -14,14 +14,13 @@ from time import sleep import pandas as pd +from catalyst.constants import LOG_LEVEL from catalyst.gens.sim_engine import ( BAR, SESSION_START ) from logbook import Logger -from catalyst.constants import LOG_LEVEL - log = Logger('ExchangeClock', level=LOG_LEVEL) diff --git a/catalyst/exchange/utils/bundle_utils.py b/catalyst/exchange/utils/bundle_utils.py index 6107bf79..e9207511 100644 --- a/catalyst/exchange/utils/bundle_utils.py +++ b/catalyst/exchange/utils/bundle_utils.py @@ -6,7 +6,6 @@ from datetime import timedelta, datetime, date import numpy as np import pandas as pd import pytz - from catalyst.data.bundles.core import download_without_progress from catalyst.exchange.utils.exchange_utils import get_exchange_bundles_folder diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index ebc678d2..f6669c4b 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -8,9 +8,6 @@ from datetime import date, datetime import pandas as pd from catalyst.assets._assets import TradingPair -from six import string_types -from six.moves.urllib import request - from catalyst.constants import DATE_FORMAT, SYMBOLS_URL from catalyst.exchange.exchange_errors import ExchangeSymbolsNotFound, \ InvalidHistoryFrequencyError, InvalidHistoryFrequencyAlias @@ -18,6 +15,8 @@ from catalyst.exchange.utils.serialization_utils import ExchangeJSONEncoder, \ ExchangeJSONDecoder from catalyst.utils.paths import data_root, ensure_directory, \ last_modified_time +from six import string_types +from six.moves.urllib import request def get_sid(symbol): @@ -733,17 +732,3 @@ def get_candles_df(candles, field, freq, bar_count, end_dt, df.dropna(inplace=True) return df - - -def fetch_capital_base(exchange): - """ - Fetch the base currency amount required to bootstrap - the algorithm against the exchange. - - The algorithm cannot continue without this value. - - :param exchange: the targeted exchange - :param attempt_index: - :return capital_base: the amount of base currency available for - trading - """ diff --git a/catalyst/exchange/utils/factory.py b/catalyst/exchange/utils/factory.py index 17499442..5650c42b 100644 --- a/catalyst/exchange/utils/factory.py +++ b/catalyst/exchange/utils/factory.py @@ -1,13 +1,12 @@ import os -from logbook import Logger - from catalyst.constants import LOG_LEVEL from catalyst.exchange.ccxt.ccxt_exchange import CCXT from catalyst.exchange.exchange import Exchange from catalyst.exchange.exchange_errors import ExchangeAuthEmpty from catalyst.exchange.utils.exchange_utils import get_exchange_auth, \ get_exchange_folder, is_blacklist +from logbook import Logger log = Logger('factory', level=LOG_LEVEL) exchange_cache = dict() diff --git a/catalyst/exchange/utils/serialization_utils.py b/catalyst/exchange/utils/serialization_utils.py index 4b098a02..62ab74d5 100644 --- a/catalyst/exchange/utils/serialization_utils.py +++ b/catalyst/exchange/utils/serialization_utils.py @@ -3,9 +3,8 @@ import re from json import JSONEncoder import pandas as pd -from six import string_types - from catalyst.constants import DATE_TIME_FORMAT +from six import string_types class ExchangeJSONEncoder(json.JSONEncoder): diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py index dd2d4899..0957e4a6 100644 --- a/catalyst/exchange/utils/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -8,7 +8,6 @@ import time import numpy as np import pandas as pd from catalyst.assets._assets import TradingPair - from catalyst.exchange.utils.exchange_utils import get_algo_folder from catalyst.utils.paths import data_root, ensure_directory diff --git a/catalyst/exchange/utils/test_utils.py b/catalyst/exchange/utils/test_utils.py index caae1e23..ac5021be 100644 --- a/catalyst/exchange/utils/test_utils.py +++ b/catalyst/exchange/utils/test_utils.py @@ -3,7 +3,6 @@ import random import tempfile from catalyst.assets._assets import TradingPair - from catalyst.exchange.utils.exchange_utils import get_exchange_folder from catalyst.exchange.utils.factory import find_exchanges from catalyst.utils.paths import ensure_directory From 30448a65c51148628d8596ed09b8363da4a82fb5 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sat, 6 Jan 2018 19:58:11 -0500 Subject: [PATCH 07/14] BLD: Housekeeping --- catalyst/utils/run_algo.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index af67c29f..405ebdd5 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -4,18 +4,15 @@ import sys import warnings from datetime import timedelta from runpy import run_path -from time import sleep import click import pandas as pd -from logbook import Logger - from catalyst.data.bundles import load from catalyst.data.data_portal import DataPortal from catalyst.exchange.exchange_pricing_loader import ExchangePricingLoader, \ TradingPairPricing from catalyst.exchange.utils.factory import get_exchange -from redo import retry +from logbook import Logger try: from pygments import highlight @@ -41,9 +38,6 @@ from catalyst.exchange.exchange_algorithm import ( from catalyst.exchange.exchange_data_portal import DataPortalExchangeLive, \ DataPortalExchangeBacktest from catalyst.exchange.exchange_asset_finder import ExchangeAssetFinder -from catalyst.exchange.exchange_errors import ( - ExchangeRequestError, ExchangeRequestErrorTooManyAttempts, - BaseCurrencyNotFoundError, NotEnoughCapitalError) from catalyst.constants import LOG_LEVEL From 33f94b3ef935272efcbd2e7bd1459cfc903c005f Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Sun, 7 Jan 2018 02:32:12 -0500 Subject: [PATCH 08/14] BLD: for issue #144, skipped cash verification when there are open orders. --- catalyst/examples/mean_reversion_simple.py | 4 ++-- catalyst/exchange/exchange_algorithm.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 1ba812a2..f5b89d73 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -37,8 +37,8 @@ def initialize(context): context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 45 - context.RSI_OVERBOUGHT = 55 + context.RSI_OVERSOLD = 50 + context.RSI_OVERBOUGHT = 60 context.CANDLE_SIZE = '5T' context.start_time = time.time() diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 87bff941..1f4063f4 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -582,10 +582,18 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if base_currency is None: base_currency = exchange.base_currency + # Don't check the cash if there are open orders. This could + # results in false positives. + orders = [] + for asset in self.blotter.open_orders: + asset_orders = self.blotter.open_orders[asset] + orders += asset_orders + + required_cash = self.portfolio.cash if not orders else None cash, positions_value = exchange.sync_positions( positions=exchange_positions, check_balances=check_balances, - cash=self.portfolio.cash, + cash=required_cash, ) total_cash += cash total_positions_value += positions_value From 55d1fee82d02dca9a4e5c1701e37edeaa37d149c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 8 Jan 2018 17:01:58 -0500 Subject: [PATCH 09/14] DOC: added an alpha warning message in response to issue #146 --- catalyst/utils/run_algo.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 405ebdd5..7aee2ff8 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -4,6 +4,7 @@ import sys import warnings from datetime import timedelta from runpy import run_path +from time import sleep import click import pandas as pd @@ -139,6 +140,14 @@ def _run(handle_data, else: click.echo(algotext) + log.warn( + 'Catalyst is currently in ALPHA. It is going through rapid ' + 'development and it is subject to errors. Please use carefully. ' + 'We encourage your to report any issue on GitHub: ' + 'https://github.com/enigmampc/catalyst/issues' + ) + sleep(3) + mode = 'paper-trading' if simulate_orders else 'live-trading' \ if live else 'backtest' log.info('running algo in {mode} mode'.format(mode=mode)) From 141ee65c913395405b0f142df67ca90f776ad57c Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 01:08:57 -0500 Subject: [PATCH 10/14] BLD: working on unit tests. --- catalyst/data/loader.py | 1 + catalyst/examples/mean_reversion_simple.py | 10 +- .../mean_reversion_simple_custom_fees.py | 288 ++++++++++++++++++ catalyst/exchange/exchange_algorithm.py | 3 +- catalyst/exchange/exchange_blotter.py | 10 +- catalyst/exchange/utils/stats_utils.py | 35 ++- catalyst/utils/run_algo.py | 2 +- tests/exchange/test_suites/test_suite_algo.py | 72 +++++ 8 files changed, 404 insertions(+), 17 deletions(-) create mode 100644 catalyst/examples/mean_reversion_simple_custom_fees.py create mode 100644 tests/exchange/test_suites/test_suite_algo.py diff --git a/catalyst/data/loader.py b/catalyst/data/loader.py index bfe6c701..cdaa26a0 100644 --- a/catalyst/data/loader.py +++ b/catalyst/data/loader.py @@ -146,6 +146,7 @@ def load_crypto_market_data(trading_day=None, trading_days=None, exchange = get_exchange( exchange_name='poloniex', base_currency='usdt' ) + exchange.init() benchmark_asset = exchange.get_asset(bm_symbol) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index f5b89d73..7bf60ae9 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -37,14 +37,14 @@ def initialize(context): context.base_price = None context.current_day = None - context.RSI_OVERSOLD = 50 + context.RSI_OVERSOLD = 55 context.RSI_OVERBOUGHT = 60 context.CANDLE_SIZE = '5T' context.start_time = time.time() - # context.set_commission(maker=0.1, taker=0.2) - context.set_slippage(spread=0.0001) + # context.set_commission(maker=0.001, taker=0.002) + # context.set_slippage(spread=0.001) def handle_data(context, data): @@ -248,7 +248,7 @@ if __name__ == '__main__': if live: run_algorithm( - capital_base=0.025, + capital_base=0.1, initialize=initialize, handle_data=handle_data, analyze=analyze, @@ -280,7 +280,7 @@ if __name__ == '__main__': analyze=analyze, exchange_name='bitfinex', algo_namespace=NAMESPACE, - base_currency='eth', + base_currency='btc', start=pd.to_datetime('2017-10-01', utc=True), end=pd.to_datetime('2017-11-10', utc=True), output=out diff --git a/catalyst/examples/mean_reversion_simple_custom_fees.py b/catalyst/examples/mean_reversion_simple_custom_fees.py new file mode 100644 index 00000000..fc44c93e --- /dev/null +++ b/catalyst/examples/mean_reversion_simple_custom_fees.py @@ -0,0 +1,288 @@ +# For this example, we're going to write a simple momentum script. When the +# stock goes up quickly, we're going to buy; when it goes down quickly, we're +# going to sell. Hopefully we'll ride the waves. +import os +import tempfile +import time + +import numpy as np +import pandas as pd +import talib +from logbook import Logger + +from catalyst import run_algorithm +from catalyst.api import symbol, record, order_target_percent, get_open_orders +from catalyst.exchange.utils.stats_utils import extract_transactions +# We give a name to the algorithm which Catalyst will use to persist its state. +# In this example, Catalyst will create the `.catalyst/data/live_algos` +# directory. If we stop and start the algorithm, Catalyst will resume its +# state using the files included in the folder. +from catalyst.utils.paths import ensure_directory + +NAMESPACE = 'mean_reversion_simple' +log = Logger(NAMESPACE) + + +# To run an algorithm in Catalyst, you need two functions: initialize and +# handle_data. + +def initialize(context): + # This initialize function sets any data or variables that you'll use in + # your algorithm. For instance, you'll want to define the trading pair (or + # trading pairs) you want to backtest. You'll also want to define any + # parameters or values you're going to use. + + # In our example, we're looking at Neo in Ether. + context.market = symbol('eth_btc') + context.base_price = None + context.current_day = None + + context.RSI_OVERSOLD = 50 + context.RSI_OVERBOUGHT = 60 + context.CANDLE_SIZE = '5T' + + context.start_time = time.time() + + context.set_commission(maker=0.001, taker=0.002) + # context.set_slippage(spread=0.001) + + +def handle_data(context, data): + # This handle_data function is where the real work is done. Our data is + # minute-level tick data, and each minute is called a frame. This function + # runs on each frame of the data. + + # We flag the first period of each day. + # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle + # would only execute once. This method works with minute and daily + # frequencies. + today = data.current_dt.floor('1D') + if today != context.current_day: + context.traded_today = False + context.current_day = today + + # We're computing the volume-weighted-average-price of the security + # defined above, in the context.market variable. For this example, we're + # using three bars on the 15 min bars. + + # The frequency attribute determine the bar size. We use this convention + # for the frequency alias: + # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases + prices = data.history( + context.market, + fields='close', + bar_count=50, + frequency=context.CANDLE_SIZE + ) + + # Ta-lib calculates various technical indicator based on price and + # volume arrays. + + # In this example, we are comp + rsi = talib.RSI(prices.values, timeperiod=14) + + # We need a variable for the current price of the security to compare to + # the average. Since we are requesting two fields, data.current() + # returns a DataFrame with + current = data.current(context.market, fields=['close', 'volume']) + price = current['close'] + + # If base_price is not set, we use the current value. This is the + # price at the first bar which we reference to calculate price_change. + if context.base_price is None: + context.base_price = price + + price_change = (price - context.base_price) / context.base_price + cash = context.portfolio.cash + + # Now that we've collected all current data for this frame, we use + # the record() method to save it. This data will be available as + # a parameter of the analyze() function for further analysis. + + record( + volume=current['volume'], + price=price, + price_change=price_change, + rsi=rsi[-1], + cash=cash + ) + # We are trying to avoid over-trading by limiting our trades to + # one per day. + if context.traded_today: + return + + # TODO: retest with open orders + # Since we are using limit orders, some orders may not execute immediately + # we wait until all orders are executed before considering more trades. + orders = get_open_orders(context.market) + if len(orders) > 0: + log.info('exiting because orders are open: {}'.format(orders)) + return + + # Exit if we cannot trade + if not data.can_trade(context.market): + return + + # Another powerful built-in feature of the Catalyst backtester is the + # portfolio object. The portfolio object tracks your positions, cash, + # cost basis of specific holdings, and more. In this line, we calculate + # how long or short our position is at this minute. + pos_amount = context.portfolio.positions[context.market].amount + + if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: + log.info( + '{}: buying - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + # Set a style for limit orders, + limit_price = price * 1.005 + order_target_percent( + context.market, 1, limit_price=limit_price + ) + context.traded_today = True + + elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0: + log.info( + '{}: selling - price: {}, rsi: {}'.format( + data.current_dt, price, rsi[-1] + ) + ) + limit_price = price * 0.995 + order_target_percent( + context.market, 0, limit_price=limit_price + ) + context.traded_today = True + + +def analyze(context=None, perf=None): + end = time.time() + log.info('elapsed time: {}'.format(end - context.start_time)) + + import matplotlib.pyplot as plt + # The base currency of the algo exchange + base_currency = context.exchanges.values()[0].base_currency.upper() + + # Plot the portfolio value over time. + ax1 = plt.subplot(611) + perf.loc[:, 'portfolio_value'].plot(ax=ax1) + ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency)) + + # Plot the price increase or decrease over time. + ax2 = plt.subplot(612, sharex=ax1) + perf.loc[:, 'price'].plot(ax=ax2, label='Price') + + ax2.set_ylabel('{asset}\n({base})'.format( + asset=context.market.symbol, base=base_currency + )) + + transaction_df = extract_transactions(perf) + if not transaction_df.empty: + buy_df = transaction_df[transaction_df['amount'] > 0] + sell_df = transaction_df[transaction_df['amount'] < 0] + ax2.scatter( + buy_df.index.to_pydatetime(), + perf.loc[buy_df.index.floor('1 min'), 'price'], + marker='^', + s=100, + c='green', + label='' + ) + ax2.scatter( + sell_df.index.to_pydatetime(), + perf.loc[sell_df.index.floor('1 min'), 'price'], + marker='v', + s=100, + c='red', + label='' + ) + + ax4 = plt.subplot(613, sharex=ax1) + perf.loc[:, 'cash'].plot( + ax=ax4, label='Base Currency ({})'.format(base_currency) + ) + ax4.set_ylabel('Cash\n({})'.format(base_currency)) + + perf['algorithm'] = perf.loc[:, 'algorithm_period_return'] + + ax5 = plt.subplot(614, sharex=ax1) + perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5) + ax5.set_ylabel('Percent\nChange') + + ax6 = plt.subplot(615, sharex=ax1) + perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI') + ax6.set_ylabel('RSI') + ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod') + ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod') + + if not transaction_df.empty: + ax6.scatter( + buy_df.index.to_pydatetime(), + perf.loc[buy_df.index.floor('1 min'), 'rsi'], + marker='^', + s=100, + c='green', + label='' + ) + ax6.scatter( + sell_df.index.to_pydatetime(), + perf.loc[sell_df.index.floor('1 min'), 'rsi'], + marker='v', + s=100, + c='red', + label='' + ) + plt.legend(loc=3) + start, end = ax6.get_ylim() + ax6.yaxis.set_ticks(np.arange(0, end, end / 5)) + + # Show the plot. + plt.gcf().set_size_inches(18, 8) + plt.show() + pass + + +if __name__ == '__main__': + # The execution mode: backtest or live + live = False + + if live: + run_algorithm( + capital_base=0.025, + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + live=True, + algo_namespace=NAMESPACE, + base_currency='btc', + live_graph=False, + simulate_orders=False, + stats_output=None, + ) + + else: + folder = os.path.join( + tempfile.gettempdir(), 'catalyst', NAMESPACE + ) + ensure_directory(folder) + + timestr = time.strftime('%Y%m%d-%H%M%S') + out = os.path.join(folder, '{}.p'.format(timestr)) + # catalyst run -f catalyst/examples/mean_reversion_simple.py \ + # -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion \ + # --data-frequency minute --capital-base 10000 + run_algorithm( + capital_base=0.1, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=NAMESPACE, + base_currency='eth', + start=pd.to_datetime('2017-10-01', utc=True), + end=pd.to_datetime('2017-11-10', utc=True), + output=out + ) + log.info('saved perf stats: {}'.format(out)) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 1f4063f4..a05bb7a7 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -587,7 +587,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): orders = [] for asset in self.blotter.open_orders: asset_orders = self.blotter.open_orders[asset] - orders += asset_orders + if asset_orders: + orders += asset_orders required_cash = self.portfolio.cash if not orders else None cash, positions_value = exchange.sync_positions( diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index cb48f939..26b74457 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -10,6 +10,7 @@ from catalyst.finance.transaction import create_transaction, Transaction from catalyst.utils.input_validation import expect_types from logbook import Logger from redo import retry +from six import iteritems log = Logger('exchange_blotter', level=LOG_LEVEL) @@ -41,6 +42,11 @@ class TradingPairFeeSchedule(CommissionModel): ) ) + def get_maker_taker(self, asset): + maker = self.maker if self.maker is not None else asset.maker + taker = self.taker if self.taker is not None else asset.taker + return maker, taker + def calculate(self, order, transaction): """ Calculate the final fee based on the order parameters. @@ -54,8 +60,7 @@ class TradingPairFeeSchedule(CommissionModel): cost = abs(transaction.amount) * transaction.price asset = order.asset - maker = self.maker if self.maker is not None else asset.maker - taker = self.taker if self.taker is not None else asset.taker + maker, taker = self.get_maker_taker(asset) multiplier = taker if order.limit is not None: @@ -250,6 +255,7 @@ class ExchangeBlotter(Blotter): for order, txn in self.check_open_orders(): order.dt = txn.dt + # TODO: is the commission already on the order object? transactions.append(txn) if not order.open: diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py index 0957e4a6..82894862 100644 --- a/catalyst/exchange/utils/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -10,6 +10,7 @@ import pandas as pd from catalyst.assets._assets import TradingPair from catalyst.exchange.utils.exchange_utils import get_algo_folder from catalyst.utils.paths import data_root, ensure_directory +from operator import itemgetter s3_conn = [] mailgun = [] @@ -260,7 +261,14 @@ def prepare_stats(stats, recorded_cols=list()): return df, columns -def get_pretty_stats(stats, recorded_cols=None, num_rows=10): +def set_print_settings(): + pd.set_option('display.expand_frame_repr', False) + pd.set_option('precision', 8) + pd.set_option('display.width', 1000) + pd.set_option('display.max_colwidth', 1000) + + +def get_pretty_stats(stats, recorded_cols=None, num_rows=10, show_tail=True): """ Format and print the last few rows of a statistics DataFrame. See the pyfolio project for the data structure. @@ -280,17 +288,17 @@ def get_pretty_stats(stats, recorded_cols=None, num_rows=10): """ if isinstance(stats, pd.DataFrame): stats = stats.T.to_dict().values() + stats.sort(key=itemgetter('period_close')) + + if len(stats) > num_rows: + display_stats = stats[-num_rows:] if show_tail else stats[0:num_rows] + else: + display_stats = stats - display_stats = stats[-num_rows:] if len(stats) > num_rows else stats df, columns = prepare_stats( display_stats, recorded_cols=recorded_cols ) - - pd.set_option('display.expand_frame_repr', False) - pd.set_option('precision', 8) - pd.set_option('display.width', 1000) - pd.set_option('display.max_colwidth', 1000) - + set_print_settings() return df.to_string(columns=columns) @@ -438,6 +446,17 @@ def df_to_string(df): return df.to_string() +def extract_orders(perf): + order_list = perf.orders.values + all_orders = [t for sublist in order_list for t in sublist] + all_orders.sort(key=lambda o: o['dt']) + + orders = pd.DataFrame(all_orders) + if not orders.empty: + orders.set_index('dt', inplace=True, drop=True) + return orders + + def extract_transactions(perf): """ Compute indexes for buy and sell transactions diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index 7aee2ff8..d03488ea 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -143,7 +143,7 @@ def _run(handle_data, log.warn( 'Catalyst is currently in ALPHA. It is going through rapid ' 'development and it is subject to errors. Please use carefully. ' - 'We encourage your to report any issue on GitHub: ' + 'We encourage you to report any issue on GitHub: ' 'https://github.com/enigmampc/catalyst/issues' ) sleep(3) diff --git a/tests/exchange/test_suites/test_suite_algo.py b/tests/exchange/test_suites/test_suite_algo.py new file mode 100644 index 00000000..58152635 --- /dev/null +++ b/tests/exchange/test_suites/test_suite_algo.py @@ -0,0 +1,72 @@ +import importlib +from os.path import join, isfile + +import pandas as pd +import os + +from catalyst import run_algorithm +from catalyst.exchange.utils.stats_utils import get_pretty_stats, \ + extract_transactions, set_print_settings, extract_orders +from catalyst.testing.fixtures import WithLogger, ZiplineTestCase +from logbook import TestHandler, WARNING +from pathtools.path import listdir + +filter_algos = [ + 'mean_reversion_simple_custom_fees.py', +] + + +class TestSuiteAlgo(WithLogger, ZiplineTestCase): + @staticmethod + def analyze(context, perf): + set_print_settings() + + transaction_df = extract_transactions(perf) + print('the transactions:\n{}'.format(transaction_df)) + + orders_df = extract_orders(perf) + print('the orders:\n{}'.format(orders_df)) + + stats = get_pretty_stats(perf, show_tail=False, num_rows=5) + print('the stats:\n{}'.format(stats)) + pass + + def test_run_examples(self): + folder = join('..', '..', '..', 'catalyst', 'examples') + files = [f for f in listdir(folder) if isfile(join(folder, f))] + + algo_list = [] + for filename in files: + name = os.path.basename(filename) + if filter_algos and name not in filter_algos: + continue + + module_name = 'catalyst.examples.{}'.format( + name.replace('.py', '') + ) + algo_list.append(module_name) + + for module_name in algo_list: + algo = importlib.import_module(module_name) + namespace = module_name.replace('.', '_') + + log_catcher = TestHandler() + with log_catcher: + run_algorithm( + capital_base=0.1, + data_frequency='minute', + initialize=algo.initialize, + handle_data=algo.handle_data, + analyze=TestSuiteAlgo.analyze, + exchange_name='bitfinex', + algo_namespace='test_{}'.format(namespace), + base_currency='eth', + start=pd.to_datetime('2017-10-01', utc=True), + end=pd.to_datetime('2017-10-02', utc=True), + # output=out + ) + warnings = [record for record in log_catcher.records if + record.level == WARNING] + self.assertEqual(0, len(warnings)) + + pass From ac15413af8732225c8013fedfc05c568cb9e9170 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 01:09:23 -0500 Subject: [PATCH 11/14] DOC: updated release notes for version 0.4.4 --- docs/source/releases.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/releases.rst b/docs/source/releases.rst index 40594aa0..5bcfb7d6 100644 --- a/docs/source/releases.rst +++ b/docs/source/releases.rst @@ -4,12 +4,13 @@ Release Notes Version 0.4.4 ^^^^^^^^^^^^^ -**Release Date**: 2018-01-06 +**Release Date**: 2018-01-09 Bug Fixes ~~~~~~~~~ - Removed redundant capital_base validation (:issue:`142`) -- Fixed portfolio update issue with restored state (:issue:`111) +- Fixed portfolio update issue with restored state (:issue:`111`) +- Skipping cash validation where there are open orders (:issue:`144`) Version 0.4.3 ^^^^^^^^^^^^^ From a49cb558212e08041fd5d7abb2ae7d00f26aa31b Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 11:32:44 -0500 Subject: [PATCH 12/14] BUG: fixed issue #147 related to python 3 compatibility --- catalyst/exchange/exchange_algorithm.py | 6 +++--- catalyst/exchange/exchange_blotter.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index a05bb7a7..782b7ec8 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -129,7 +129,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): @api_method def set_commission(self, maker=None, taker=None): - key = self.blotter.commission_models.keys()[0] + key = list(self.blotter.commission_models.keys())[0] if maker is not None: self.blotter.commission_models[key].maker = maker @@ -138,7 +138,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): @api_method def set_slippage(self, spread=None): - key = self.blotter.slippage_models.keys()[0] + key = list(self.blotter.slippage_models.keys())[0] if spread is not None: self.blotter.slippage_models[key].spread = spread @@ -708,7 +708,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.frame_stats = list() self.performance_needs_update = False - orders = self.perf_tracker.todays_performance.orders_by_id.keys() + orders = list(self.perf_tracker.todays_performance.orders_by_id.keys()) if orders != self._last_orders: self.performance_needs_update = True diff --git a/catalyst/exchange/exchange_blotter.py b/catalyst/exchange/exchange_blotter.py index 26b74457..d638e4bd 100644 --- a/catalyst/exchange/exchange_blotter.py +++ b/catalyst/exchange/exchange_blotter.py @@ -10,7 +10,6 @@ from catalyst.finance.transaction import create_transaction, Transaction from catalyst.utils.input_validation import expect_types from logbook import Logger from redo import retry -from six import iteritems log = Logger('exchange_blotter', level=LOG_LEVEL) From db37c9c6a71b98627327341a471c77a8002edd99 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Tue, 9 Jan 2018 16:57:57 -0500 Subject: [PATCH 13/14] BLD: updated sample algo --- catalyst/examples/mean_reversion_simple.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/catalyst/examples/mean_reversion_simple.py b/catalyst/examples/mean_reversion_simple.py index 7bf60ae9..4178e0f8 100644 --- a/catalyst/examples/mean_reversion_simple.py +++ b/catalyst/examples/mean_reversion_simple.py @@ -43,8 +43,8 @@ def initialize(context): context.start_time = time.time() - # context.set_commission(maker=0.001, taker=0.002) - # context.set_slippage(spread=0.001) + context.set_commission(maker=0.001, taker=0.002) + context.set_slippage(spread=0.001) def handle_data(context, data): @@ -244,7 +244,7 @@ def analyze(context=None, perf=None): if __name__ == '__main__': # The execution mode: backtest or live - live = True + live = False if live: run_algorithm( From b60b50e99af29dcdfdf3d71c9ebaaa1a06a50019 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Wed, 10 Jan 2018 13:31:54 -0500 Subject: [PATCH 14/14] BUG: fixed python3 issue in run_algo --- catalyst/utils/run_algo.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index d03488ea..3d83426c 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -148,8 +148,14 @@ def _run(handle_data, ) sleep(3) - mode = 'paper-trading' if simulate_orders else 'live-trading' \ - if live else 'backtest' + if live: + if simulate_orders: + mode = 'paper-trading' + else: + mode = 'live-trading' + else: + mode = 'backtest' + log.info('running algo in {mode} mode'.format(mode=mode)) exchange_name = exchange