diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index 363edba1..00eb01af 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -21,7 +21,7 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages short_window = 2 - long_window = 2 + long_window = 3 # Skip as many bars as long_window to properly compute the average context.i += 1 diff --git a/catalyst/examples/marketplace/github-research.py b/catalyst/examples/marketplace/github-research.py new file mode 100644 index 00000000..b2443597 --- /dev/null +++ b/catalyst/examples/marketplace/github-research.py @@ -0,0 +1,70 @@ +import pandas as pd +import matplotlib.pyplot as plt + +from catalyst import run_algorithm +from catalyst.api import symbol, get_dataset + +START = '2017-01-01' +END = '2017-12-31' + + +def initialize(context): + pass + + +def handle_data(context, data): + context.github = get_dataset('github') + context.github.sort_index(level=0, inplace=True) + + context.zec = data.history(symbol('zec_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + context.xmr = data.history(symbol('xmr_usdt'), + ['price', ], + bar_count=365, + frequency="1d") + + +def analyze(context=None, results=None): + ax1 = plt.subplot(211) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'ZEC']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax1, color='blue') + ax1.legend(loc=2) + ax1.set_title('Zcash') + ax2 = ax1.twinx() + context.zec['price'].loc[START:END].plot(ax=ax2, color='green') + ax2.legend(loc=1) + + ax3 = plt.subplot(212) + idx = pd.IndexSlice + df = context.github.loc[START:END].loc[ + idx[:, [b'XMR']], ['commits']].reset_index( + level='symbol', drop=True) + df.plot(ax=ax3, color='blue') + ax3.legend(loc=2) + ax3.set_title('Monero') + ax4 = ax3.twinx() + context.xmr['price'].loc[START:END].plot(ax=ax4, color='green') + ax4.legend(loc=1) + + plt.show() + + +if __name__ == '__main__': + run_algorithm( + capital_base=1000, + data_frequency='daily', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='poloniex', + algo_namespace='algo-github', + base_currency='usdt', + live=False, + start=pd.to_datetime(END, utc=True), + end=pd.to_datetime(END, utc=True), + ) diff --git a/catalyst/examples/mean_reversion_by_marketcap.py b/catalyst/examples/marketplace/mean_reversion_by_marketcap.py similarity index 100% rename from catalyst/examples/mean_reversion_by_marketcap.py rename to catalyst/examples/marketplace/mean_reversion_by_marketcap.py diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index aeff9e4e..5c9f8771 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -16,7 +16,7 @@ import signal import sys from datetime import timedelta from os import listdir -from os.path import isfile, join +from os.path import isfile, join, exists import catalyst.protocol as zp import logbook @@ -36,9 +36,11 @@ from catalyst.exchange.utils.exchange_utils import ( get_algo_folder, get_algo_df, save_algo_df, + clear_frame_stats_directory, + remove_old_files, group_assets_by_exchange, ) -from catalyst.exchange.utils.stats_utils import get_pretty_stats, stats_to_s3, \ - stats_to_algo_folder +from catalyst.exchange.utils.stats_utils import \ + get_pretty_stats, stats_to_s3, stats_to_algo_folder from catalyst.finance.execution import MarketOrder from catalyst.finance.performance import PerformanceTracker from catalyst.finance.performance.period import calc_period_stats @@ -67,8 +69,8 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): self.current_day = None - if self.simulate_orders is None \ - and self.sim_params.arena == 'backtest': + if self.simulate_orders is None and \ + self.sim_params.arena == 'backtest': self.simulate_orders = True # Operations with retry features @@ -118,7 +120,7 @@ class ExchangeTradingAlgorithmBase(TradingAlgorithm): # be in-line with CXXT and many exchanges. We'll consider # adding more order types in the future. if not isinstance(style, ExchangeLimitOrder) or \ - not isinstance(style, MarketOrder): + not isinstance(style, MarketOrder): raise OrderTypeNotSupported( order_type=style.__class__.__name__ ) @@ -368,6 +370,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self._clock = None self.frame_stats = list() + # erase the frame_stats folder to avoid overloading the disk + error = clear_frame_stats_directory(self.algo_namespace) + if error: + log.warning(error) + self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats') self.custom_signals_stats = \ @@ -392,6 +399,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): "Exit should be handled by the user.") def interrupt_algorithm(self): + """ + + when algorithm comes to an end this function is called. + extracts the stats and calls analyze. + after finishing, it exits the run. + + Parameters + ---------- + + Returns + ------- + + """ self.is_running = False if self._analyze is None: @@ -401,21 +421,31 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): log.info('Exiting the algorithm. Calling `analyze()` ' 'before exiting the algorithm.') + # add the last day stats which is not saved in the directory + current_stats = pd.DataFrame(self.frame_stats) + current_stats.set_index('period_close', drop=False, inplace=True) + + # get the location of the directory algo_folder = get_algo_folder(self.algo_namespace) - folder = join(algo_folder, 'daily_performance') - files = [f for f in listdir(folder) if isfile(join(folder, f))] + folder = join(algo_folder, 'frame_stats') - daily_perf_list = [] - for item in files: - filename = join(folder, item) + if exists(folder): + files = [f for f in listdir(folder) if isfile(join(folder, f))] - with open(filename, 'rb') as handle: - perf_period = pickle.load(handle) - perf_period_dict = perf_period.to_dict() - daily_perf_list.append(perf_period_dict) + period_stats_list = [] + for item in files: + filename = join(folder, item) - stats = pd.DataFrame(daily_perf_list) - stats.set_index('period_close', drop=False, inplace=True) + with open(filename, 'rb') as handle: + perf_period = pickle.load(handle) + period_stats_list.extend(perf_period) + + stats = pd.DataFrame(period_stats_list) + stats.set_index('period_close', drop=False, inplace=True) + + stats = pd.concat([stats, current_stats]) + else: + stats = current_stats self.analyze(stats) @@ -709,6 +739,37 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.algo_namespace, 'exposure_stats', self.exposure_stats ) + def nullify_frame_stats(self, now): + """ + + Save all period_stats to local directory + erase old files from the folder and nullify + self.frame_stats + + Parameters + ---------- + now: Timestamp + + Returns + ------- + + """ + save_algo_object( + algo_name=self.algo_namespace, + key=now.floor('1D').strftime('%Y-%m-%d'), + obj=self.frame_stats, + rel_path='frame_stats' + ) + error = remove_old_files( + algo_name=self.algo_namespace, + today=now, + rel_path='frame_stats' + ) + if error: + log.warning(error) + + self.frame_stats = list() + def handle_data(self, data): """ Wrapper around the handle_data method of each algo. @@ -728,7 +789,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # Resetting the frame stats every day to minimize memory footprint today = data.current_dt.floor('1D') if self.current_day is not None and today > self.current_day: - self.frame_stats = list() + self.nullify_frame_stats(now=data.current_dt) self.performance_needs_update = False orders = list(self.perf_tracker.todays_performance.orders_by_id.keys()) @@ -808,6 +869,8 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # Saving the last hour in memory self.frame_stats.append(frame_stats) + # creating and saving the pnl_stats into the local + # directory self.add_pnl_stats(frame_stats) if self.recorded_vars: self.add_custom_signals_stats(frame_stats) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 3f8d6121..fb24f1c8 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -130,7 +130,7 @@ def get_exchange_symbols(exchange_name, is_local=False, environ=None): filename)).days > 1): try: download_exchange_symbols(exchange_name, environ) - except Exception as e: + except Exception: pass if os.path.isfile(filename): @@ -273,6 +273,7 @@ def get_algo_object(algo_name, key, environ=None, rel_path=None, how='pickle'): key: str environ: rel_path: str + how: str Returns ------- @@ -316,6 +317,7 @@ def save_algo_object(algo_name, key, obj, environ=None, rel_path=None, obj: Object environ: rel_path: str + how: str """ folder = get_algo_folder(algo_name, environ) @@ -392,6 +394,67 @@ def save_algo_df(algo_name, key, df, environ=None, rel_path=None): df.to_csv(handle, encoding='UTF_8') +def clear_frame_stats_directory(algo_name): + """ + remove the outdated directory + to avoid overloading the disk + + Parameters + ---------- + algo_name: str + + Returns + ------- + error: str + + """ + error = None + algo_folder = get_algo_folder(algo_name) + folder = os.path.join(algo_folder, 'frame_stats') + if os.path.exists(folder): + try: + shutil.rmtree(folder) + except OSError: + error = 'unable to remove {}, the analyze ' \ + 'data will be inconsistent'.format(folder) + return error + + +def remove_old_files(algo_name, today, rel_path): + """ + remove old files from a directory + to avoid overloading the disk + + Parameters + ---------- + algo_name: str + today: Timestamp + rel_path: str + + Returns + ------- + error: str + + """ + error = None + algo_folder = get_algo_folder(algo_name) + folder = os.path.join(algo_folder, rel_path) + + # run on all files in the folder + for f in os.listdir(folder): + creation_unix = os.path.getctime(f) + creation_time = pd.to_datetime(creation_unix, unit='s', ) + + # if the file is older than 30 days erase it + if today - pd.DateOffset(30) > creation_time: + try: + os.unlink(f) + except OSError: + error = 'unable to erase files in {}'.format(folder) + + return error + + def get_exchange_minute_writer_root(exchange_name, environ=None): """ The minute writer folder for the exchange. @@ -575,8 +638,9 @@ def mixin_market_params(exchange_name, params, market): params['maker'] = 0.001 params['taker'] = 0.002 - elif 'maker' in market and 'taker' in market \ - and market['maker'] is not None and market['taker'] is not None: + elif 'maker' in market and 'taker' in market and \ + market['maker'] is not None and market['taker'] is not None: + params['maker'] = market['maker'] params['taker'] = market['taker'] diff --git a/catalyst/marketplace/utils/auth_utils.py b/catalyst/marketplace/utils/auth_utils.py index 4979b6f3..ab3c668d 100644 --- a/catalyst/marketplace/utils/auth_utils.py +++ b/catalyst/marketplace/utils/auth_utils.py @@ -47,11 +47,11 @@ def get_key_secret(pubAddr, wallet='mew'): if wallet == 'mew': print('\nObtaining a key/secret pair to streamline all future ' 'requests with the authentication server.\n' - 'Visit https://www.myetherwallet.com/signmsg.html and sign the' + 'Visit https://www.myetherwallet.com/signmsg.html and sign the ' 'following message:\n{}'.format(nonce)) signature = input('Copy and Paste the "sig" field from ' 'the signature here (without the double quotes, ' - 'only the HEX value:\n') + 'only the HEX value):\n') else: raise MarketplaceWalletNotSupported(wallet=wallet) diff --git a/docs/source/install.rst b/docs/source/install.rst index 59279683..5c24159f 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -454,12 +454,22 @@ about matplotlib backends, please refer to the Windows Requirements -------------------- -In Windows, you will first need to install the `Microsoft Visual C++ Compiler -for Python 2.7 -`_. This -package contains the compiler and the set of system headers necessary for -producing binary wheels for Python 2.7 packages. If it's not already in your -system, download it and install it before proceeding to the next step. +In Windows, you will first need to install the Microsoft Visual C++ Compiler, +which is different depending on the version of Python that you plan to use: + +* Python 3.5, 3.6: `Visual C++ 2015 Build Tools + `_, + which installs Visual C++ version 14.0. **This is the recommended version** + +* Python 2.7: `Microsoft Visual C++ Compiler for Python 2.7 + `_, which + installs version Visual C++ version 9.0 + +This package contains the compiler and the set of system headers necessary for +producing binary wheels for Python packages. If it's not already in your +system, download it and install it before proceeding to the next step. If you +need additional help, or are looking for other versions of Visual C++ for +Windows (only advanced users), follow `this link `_. Once you have the above compiler installed, the easiest and best supported way to install Catalyst in Windows is to use :ref:`Conda `. If you didn't