From b2e5b5f73de041040a8e60dc569e25f7b25ff7c2 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 24 Aug 2017 12:55:33 -0600 Subject: [PATCH 1/5] Dropbox -> AWS switch for Poloniex bundle --- catalyst/data/bundles/poloniex.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/catalyst/data/bundles/poloniex.py b/catalyst/data/bundles/poloniex.py index 5d20b636..5e1ddd36 100644 --- a/catalyst/data/bundles/poloniex.py +++ b/catalyst/data/bundles/poloniex.py @@ -42,8 +42,7 @@ class PoloniexBundle(BaseCryptoPricingBundle): @lazyval def tar_url(self): return ( - 'https://www.dropbox.com/s/9naqffawnq8o4r2/' - 'poloniex-bundle.tar?dl=1' + 'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz' ) @lazyval From 24fd0fa6f8ac0d2e08eb81c35cb0b96db39d4812 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 25 Aug 2017 09:29:25 -0600 Subject: [PATCH 2/5] Fix issue #28: buy_and_hodl.py example --- catalyst/examples/buy_and_hodl.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/catalyst/examples/buy_and_hodl.py b/catalyst/examples/buy_and_hodl.py index 57b7f19e..b3f411f4 100644 --- a/catalyst/examples/buy_and_hodl.py +++ b/catalyst/examples/buy_and_hodl.py @@ -23,7 +23,6 @@ from catalyst.api import ( get_open_orders, ) - def initialize(context): context.ASSET_NAME = 'USDT_BTC' context.TARGET_HODL_RATIO = 0.8 @@ -42,8 +41,6 @@ def initialize(context): def handle_data(context, data): context.i += 1 - print 'i:', context.i - starting_cash = context.portfolio.starting_cash target_hodl_value = context.TARGET_HODL_RATIO * starting_cash reserve_value = context.RESERVE_RATIO * starting_cash @@ -73,6 +70,7 @@ def handle_data(context, data): record( price=price, + volume=data[context.asset].volume, cash=cash, starting_cash=context.portfolio.starting_cash, leverage=context.account.leverage, @@ -80,12 +78,13 @@ def handle_data(context, data): def analyze(context=None, results=None): import matplotlib.pyplot as plt + # Plot the portfolio and asset data. - ax1 = plt.subplot(511) + ax1 = plt.subplot(611) results[['portfolio_value']].plot(ax=ax1) ax1.set_ylabel('Portfolio Value (USD)') - ax2 = plt.subplot(512, sharex=ax1) + ax2 = plt.subplot(612, sharex=ax1) ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME)) (context.TICK_SIZE * results[['price']]).plot(ax=ax2) @@ -101,11 +100,11 @@ def analyze(context=None, results=None): color='g', ) - ax3 = plt.subplot(513, sharex=ax1) + ax3 = plt.subplot(613, sharex=ax1) results[['leverage', 'alpha', 'beta']].plot(ax=ax3) ax3.set_ylabel('Leverage ') - ax4 = plt.subplot(514, sharex=ax1) + ax4 = plt.subplot(614, sharex=ax1) results[['starting_cash', 'cash']].plot(ax=ax4) ax4.set_ylabel('Cash (USD)') @@ -119,7 +118,7 @@ def analyze(context=None, results=None): 'benchmark_period_return', ]] - ax5 = plt.subplot(515, sharex=ax1) + ax5 = plt.subplot(615, sharex=ax1) results[[ 'treasury', 'algorithm', @@ -127,8 +126,12 @@ def analyze(context=None, results=None): ]].plot(ax=ax5) ax5.set_ylabel('Percent Change') + ax6 = plt.subplot(616, sharex=ax1) + results[['volume']].plot(ax=ax6) + ax6.set_ylabel('Volume (mCoins/5min)') + plt.legend(loc=3) # Show the plot. plt.gcf().set_size_inches(18, 8) - plt.show() + plt.show() \ No newline at end of file From 3a44a3cc1fd7bd29898b69f60ffb998081fc189f Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 25 Aug 2017 12:44:43 -0600 Subject: [PATCH 3/5] Date fix for treasury_data, starts 1990-01-02 --- catalyst/data/loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalyst/data/loader.py b/catalyst/data/loader.py index b42b424a..15ed6e28 100644 --- a/catalyst/data/loader.py +++ b/catalyst/data/loader.py @@ -136,7 +136,7 @@ def load_crypto_market_data(trading_day=None, ) # Override first_date for treasury data since we have it for many more years # and is independent of crypto data - first_date_treasury = pd.Timestamp('1990-01-01', tz='UTC') + first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC') tc = ensure_treasury_data( bm_symbol, first_date_treasury, From 16f9ab3ba52ed4b6cf1c4c96e3593443ee81a1d6 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 28 Aug 2017 12:20:54 -0600 Subject: [PATCH 4/5] Retrieve cryptobenchmark from bundle, instead of Polo --- catalyst/data/loader.py | 95 +++++++++++++++++++++++++------------- catalyst/utils/run_algo.py | 2 +- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/catalyst/data/loader.py b/catalyst/data/loader.py index 15ed6e28..a9dd8167 100644 --- a/catalyst/data/loader.py +++ b/catalyst/data/loader.py @@ -96,16 +96,15 @@ def has_data_for_dates(series_or_df, first_date, last_date): first, last = dts[[0, -1]].tz_localize(None) return (first <= first_date.tz_localize(None)) and (last >= last_date.tz_localize(None)) -def load_crypto_market_data(trading_day=None, - trading_days=None, - bm_symbol='USDT_BTC', - environ=None): +def load_crypto_market_data(trading_day=None, trading_days=None, bm_symbol='USDT_BTC', + bundle=None, bundle_data=None, environ=None): + if trading_day is None: trading_day = get_calendar('OPEN').trading_day if trading_days is None: trading_days = get_calendar('OPEN').all_sessions - first_date = trading_days[0] + first_date = trading_days[1] now = pd.Timestamp.utcnow() # We expect to have benchmark and treasury data that's current up until @@ -122,8 +121,15 @@ def load_crypto_market_data(trading_day=None, # We'll attempt to download new data if the latest entry in our cache is # before this date. - last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2] - + if(bundle_data): + # If we are using the bundle to retrieve the cryptobenchmark, find the last + # date for which there is trading data in the bundle + asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None) + ix = bundle_data.daily_bar_reader._last_rows[asset.sid] + last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s') + else: + last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2] + br = ensure_crypto_benchmark_data( bm_symbol, first_date, @@ -132,6 +138,8 @@ def load_crypto_market_data(trading_day=None, # We need the trading_day to figure out the close prior to the first # date so that we can compute returns for the first date. trading_day, + bundle, + bundle_data, environ, ) # Override first_date for treasury data since we have it for many more years @@ -240,6 +248,8 @@ def ensure_crypto_benchmark_data(symbol, last_date, now, trading_day, + bundle, + bundle_data, environ=None): filename = get_benchmark_filename(symbol) @@ -248,7 +258,7 @@ def ensure_crypto_benchmark_data(symbol, ('Loading benchmark data for {symbol!r} ' 'from {first_date} to {last_date}'), symbol=symbol, - first_date=first_date - trading_day, + first_date=first_date, last_date=last_date ) @@ -261,34 +271,57 @@ def ensure_crypto_benchmark_data(symbol, environ, ) - if data is not None: return data # If no cached data was found or it was missing any dates then download the # necessary data. - logger.info( - ('Downloading benchmark data for {symbol!r} ' - 'from {first_date} to {last_date}'), - symbol=symbol, - first_date=first_date - trading_day, - last_date=last_date - ) - # Load benchmark symbol from Poloniex API - try: - bundle = PoloniexBundle() - bench_raw = bundle._fetch_symbol_frame( - None, - symbol, - get_calendar(bundle.calendar_name), - first_date - trading_day, - last_date, - 'daily', - ) - except (OSError, IOError, HTTPError): - logger.exception('Failed to fetch new crypto benchmark returns') - raise + if(bundle == 'poloniex'): + ''' + If we're using the Poloniex bundle, we'll get the benchmark from the bundle + instead of downloading it from Poloniex every time we need it. + Poloniex has a captcha for API queries originating from outside the US that + prevents users abroad from getting Catalyst to work + ''' + logger.info( + ('Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'), + symbol=symbol, first_date=first_date, last_date=last_date) + + asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,as_of_date=None) + fields = ['day', 'close'] + raw = bundle_data.daily_bar_reader.load_raw_arrays( + columns=fields, + start_date=first_date - trading_day, + end_date=last_date, + assets=[asset,]) + bench_raw = pd.concat([pd.DataFrame(raw[0], columns=['date']),pd.DataFrame(raw[1], columns=['close'])], axis=1) + bench_raw['date'] = pd.to_datetime(bench_raw['date'],unit='s') + bench_raw.set_index('date', inplace=True) + bench_raw.sort_index(inplace=True) + bench_raw = bench_raw[pd.to_datetime(first_date - trading_day):pd.to_datetime(last_date)] + + else: + # This is how it used to be: downloading the benchmark everytime. + # Leaving this code here to be repurposed in the future for other bundles. + logger.info( + ('Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'), + symbol=symbol, first_date=first_date, last_date=last_date) + + # Load benchmark symbol from Poloniex API + try: + bundle = PoloniexBundle() + bench_raw = bundle._fetch_symbol_frame( + None, + symbol, + get_calendar(bundle.calendar_name), + first_date - trading_day, + last_date, + 'daily', + ) + except (OSError, IOError, HTTPError): + logger.exception('Failed to fetch new crypto benchmark returns') + raise # select close column and compute percent change between days daily_close = bench_raw[['close']] @@ -518,7 +551,7 @@ def _load_cached_data(filename, first_date, last_date, now, resource_name, ) logger.info( - "Cache at {path} does not have data from {start} to {end}.\n", + "Cache at {path} does not have data from {start} to {end}.", start=first_date, end=last_date, path=path, diff --git a/catalyst/utils/run_algo.py b/catalyst/utils/run_algo.py index d63c314a..228d05a3 100644 --- a/catalyst/utils/run_algo.py +++ b/catalyst/utils/run_algo.py @@ -149,7 +149,7 @@ def _run(handle_data, open_calendar = get_calendar('OPEN') env = TradingEnvironment( - load=partial(load_crypto_market_data, environ=environ), + load=partial(load_crypto_market_data, bundle=b, bundle_data=bundle_data, environ=environ), bm_symbol='USDT_BTC', trading_calendar=open_calendar, asset_db_path=connstr, From 01473e51468c3a43fe6a308aedb32755183d6b18 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Mon, 28 Aug 2017 13:15:31 -0600 Subject: [PATCH 5/5] Fixes 1000 scaling price issue --- catalyst/data/bundles/poloniex.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/catalyst/data/bundles/poloniex.py b/catalyst/data/bundles/poloniex.py index 5e1ddd36..4dcdf7bc 100644 --- a/catalyst/data/bundles/poloniex.py +++ b/catalyst/data/bundles/poloniex.py @@ -102,7 +102,10 @@ class PoloniexBundle(BaseCryptoPricingBundle): ) raw.set_index('date', inplace=True) - scale = 1 + # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored + # on disk, which we compensate here to get the right pricing amounts + # ref: data/us_equity_pricing.py + scale = 1000 raw.loc[:, 'open'] /= scale raw.loc[:, 'high'] /= scale raw.loc[:, 'low'] /= scale