diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 98966a82..b9c319f1 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -375,13 +375,23 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if error: log.warning(error) - self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats') + # in order to save paper & live files separately + self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live' - self.custom_signals_stats = \ - get_algo_df(self.algo_namespace, 'custom_signals_stats') + self.pnl_stats = get_algo_df( + self.algo_namespace, + 'pnl_stats_{}'.format(self.mode_name), + ) - self.exposure_stats = \ - get_algo_df(self.algo_namespace, 'exposure_stats') + self.custom_signals_stats = get_algo_df( + self.algo_namespace, + 'custom_signals_stats_{}'.format(self.mode_name) + ) + + self.exposure_stats = get_algo_df( + self.algo_namespace, + 'exposure_stats_{}'.format(self.mode_name) + ) self.is_running = True @@ -515,7 +525,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): """ self.state = get_algo_object( algo_name=self.algo_namespace, - key='context.state', + key='context.state_{}'.format(self.mode_name), ) if self.state is None: self.state = {} @@ -538,7 +548,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # Unpacking the perf_tracker and positions if available cum_perf = get_algo_object( algo_name=self.algo_namespace, - key='cumulative_performance', + key='cumulative_performance_{}'.format(self.mode_name), ) if cum_perf is not None: tracker.cumulative_performance = cum_perf @@ -549,7 +559,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): todays_perf = get_algo_object( algo_name=self.algo_namespace, key=today.strftime('%Y-%m-%d'), - rel_path='daily_performance', + rel_path='daily_performance_{}'.format(self.mode_name), ) if todays_perf is not None: # Ensure single common position tracker @@ -686,7 +696,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): ) self.pnl_stats = pd.concat([self.pnl_stats, df]) - save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats) + save_algo_df( + self.algo_namespace, + 'pnl_stats_{}'.format(self.mode_name), + self.pnl_stats, + ) def add_custom_signals_stats(self, period_stats): """ @@ -707,8 +721,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): ) self.custom_signals_stats = pd.concat([self.custom_signals_stats, df]) - save_algo_df(self.algo_namespace, 'custom_signals_stats', - self.custom_signals_stats) + save_algo_df( + self.algo_namespace, + 'custom_signals_stats_{}'.format(self.mode_name), + self.custom_signals_stats, + ) def add_exposure_stats(self, period_stats): """ @@ -735,7 +752,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.exposure_stats = pd.concat([self.exposure_stats, df]) save_algo_df( - self.algo_namespace, 'exposure_stats', self.exposure_stats + self.algo_namespace, + 'exposure_stats_{}'.format(self.mode_name), + self.exposure_stats ) def nullify_frame_stats(self, now): @@ -759,6 +778,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): obj=self.frame_stats, rel_path='frame_stats' ) + error = remove_old_files( algo_name=self.algo_namespace, today=now, @@ -843,7 +863,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): log.debug('saving cumulative performance object') save_algo_object( algo_name=self.algo_namespace, - key='cumulative_performance', + key='cumulative_performance_{}'.format(self.mode_name), obj=self.perf_tracker.cumulative_performance, ) log.debug('saving todays performance object') @@ -851,12 +871,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): algo_name=self.algo_namespace, key=today.strftime('%Y-%m-%d'), obj=self.perf_tracker.todays_performance, - rel_path='daily_performance' + rel_path='daily_performance_{}'.format(self.mode_name) ) log.debug('saving context.state object') save_algo_object( algo_name=self.algo_namespace, - key='context.state', + key='context.state_{}'.format(self.mode_name), obj=self.state) def _process_stats(self, data): @@ -912,6 +932,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): csv_bytes = stats_to_algo_folder( stats=self.frame_stats, algo_namespace=self.algo_namespace, + folder_name='stats_{}'.format(self.mode_name), recorded_cols=recorded_cols, ) except Exception as e: diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 4f82356c..5c40a26d 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -420,7 +420,7 @@ def clear_frame_stats_directory(algo_name): return error -def remove_old_files(algo_name, today, rel_path): +def remove_old_files(algo_name, today, rel_path, environ=None): """ remove old files from a directory to avoid overloading the disk @@ -430,27 +430,30 @@ def remove_old_files(algo_name, today, rel_path): algo_name: str today: Timestamp rel_path: str + environ: Returns ------- error: str """ + error = None - algo_folder = get_algo_folder(algo_name) + algo_folder = get_algo_folder(algo_name, environ) folder = os.path.join(algo_folder, rel_path) + ensure_directory(folder) # run on all files in the folder for f in os.listdir(folder): - creation_unix = os.path.getctime(f) - creation_time = pd.to_datetime(creation_unix, unit='s', ) + try: + creation_unix = os.path.getctime(os.path.join(folder, f)) + creation_time = pd.to_datetime(creation_unix, unit='s', ) - # if the file is older than 30 days erase it - if today - pd.DateOffset(30) > creation_time: - try: - os.unlink(f) - except OSError: - error = 'unable to erase files in {}'.format(folder) + # if the file is older than 30 days erase it + if today - pd.DateOffset(30) > creation_time: + os.unlink(f) + except OSError: + error = 'unable to erase files in {}'.format(folder) return error diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py index 6e2aab0b..3db79d3b 100644 --- a/catalyst/exchange/utils/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -396,7 +396,8 @@ def email_error(algo_name, dt, e, environ=None): )}) -def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): +def stats_to_algo_folder(stats, algo_namespace, + folder_name, recorded_cols=None): """ Saves the performance stats to the algo local folder. @@ -404,6 +405,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): ---------- stats: list[Object] algo_namespace: str + folder_name: str recorded_cols: list[str] Returns @@ -416,7 +418,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): timestr = time.strftime('%Y%m%d') folder = get_algo_folder(algo_namespace) - stats_folder = os.path.join(folder, 'stats') + stats_folder = os.path.join(folder, folder_name) ensure_directory(stats_folder) filename = os.path.join(stats_folder, '{}.csv'.format(timestr))