Merge branch 'develop'

catalyst/data/bundles/poloniex.py

@@ -42,8 +42,7 @@ class PoloniexBundle(BaseCryptoPricingBundle):
     @lazyval
     def tar_url(self):
         return (
-            'https://www.dropbox.com/s/9naqffawnq8o4r2/'
-            'poloniex-bundle.tar?dl=1'
+            'https://s3.amazonaws.com/enigmaco/catalyst-bundles/poloniex/poloniex-bundle.tar.gz'
         )
 
     @lazyval
@@ -103,7 +102,10 @@ class PoloniexBundle(BaseCryptoPricingBundle):
         )
         raw.set_index('date', inplace=True)
 
-        scale = 1
+        # BcolzDailyBarReader introduces a 1/1000 factor in the way pricing is stored
+        # on disk, which we compensate here to get the right pricing amounts
+        # ref: data/us_equity_pricing.py
+        scale = 1000
         raw.loc[:, 'open'] /= scale
         raw.loc[:, 'high'] /= scale
         raw.loc[:, 'low'] /= scale

catalyst/data/loader.py

@@ -96,16 +96,15 @@ def has_data_for_dates(series_or_df, first_date, last_date):
     first, last = dts[[0, -1]].tz_localize(None)
     return (first <= first_date.tz_localize(None)) and (last >= last_date.tz_localize(None))
 
-def load_crypto_market_data(trading_day=None,
-                            trading_days=None,
-                            bm_symbol='USDT_BTC',
-                            environ=None):
+def load_crypto_market_data(trading_day=None, trading_days=None, bm_symbol='USDT_BTC',
+                            bundle=None, bundle_data=None, environ=None):
+
     if trading_day is None:
         trading_day = get_calendar('OPEN').trading_day
     if trading_days is None:
         trading_days = get_calendar('OPEN').all_sessions
 
-    first_date = trading_days[0]
+    first_date = trading_days[1]
     now = pd.Timestamp.utcnow()
 
     # We expect to have benchmark and treasury data that's current up until
@@ -122,8 +121,15 @@ def load_crypto_market_data(trading_day=None,
 
     # We'll attempt to download new data if the latest entry in our cache is
     # before this date.
-    last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
-
+    if(bundle_data):
+        # If we are using the bundle to retrieve the cryptobenchmark, find the last
+        # date for which there is trading data in the bundle
+        asset = bundle_data.asset_finder.lookup_symbol(symbol=bm_symbol,as_of_date=None)
+        ix = bundle_data.daily_bar_reader._last_rows[asset.sid]
+        last_date = pd.to_datetime(bundle_data.daily_bar_reader._spot_col('day')[ix],unit='s')
+    else:
+        last_date = trading_days[trading_days.get_loc(now, method='ffill') - 2]
+    
     br = ensure_crypto_benchmark_data(
         bm_symbol,
         first_date,
@@ -132,11 +138,13 @@ def load_crypto_market_data(trading_day=None,
         # We need the trading_day to figure out the close prior to the first
         # date so that we can compute returns for the first date.
         trading_day,
+        bundle,
+        bundle_data,
         environ,
     )
     # Override first_date for treasury data since we have it for many more years
     # and is independent of crypto data
-    first_date_treasury = pd.Timestamp('1990-01-01', tz='UTC') 
+    first_date_treasury = pd.Timestamp('1990-01-02', tz='UTC') 
     tc = ensure_treasury_data(
         bm_symbol,
         first_date_treasury,
@@ -240,6 +248,8 @@ def ensure_crypto_benchmark_data(symbol,
                                  last_date,
                                  now,
                                  trading_day,
+                                 bundle,
+                                 bundle_data,
                                  environ=None):
 
     filename = get_benchmark_filename(symbol)
@@ -248,7 +258,7 @@ def ensure_crypto_benchmark_data(symbol,
         ('Loading benchmark data for {symbol!r} '
             'from {first_date} to {last_date}'),
         symbol=symbol,
-        first_date=first_date - trading_day,
+        first_date=first_date,
         last_date=last_date
     )
 
@@ -261,34 +271,57 @@ def ensure_crypto_benchmark_data(symbol,
         environ,
     )
 
-
     if data is not None:
         return data
 
     # If no cached data was found or it was missing any dates then download the
     # necessary data.
-    logger.info(
-        ('Downloading benchmark data for {symbol!r} '
-            'from {first_date} to {last_date}'),
-        symbol=symbol,
-        first_date=first_date - trading_day,
-        last_date=last_date
-    )
 
-    # Load benchmark symbol from Poloniex API
-    try:
-        bundle = PoloniexBundle()
-        bench_raw = bundle._fetch_symbol_frame(
-            None,
-            symbol,
-            get_calendar(bundle.calendar_name),
-            first_date - trading_day,
-            last_date,
-            'daily',
-        )
-    except (OSError, IOError, HTTPError):
-        logger.exception('Failed to fetch new crypto benchmark returns')
-        raise
+    if(bundle == 'poloniex'):
+        '''
+        If we're using the Poloniex bundle, we'll get the benchmark from the bundle
+        instead of downloading it from Poloniex every time we need it.
+        Poloniex has a captcha for API queries originating from outside the US that 
+        prevents users abroad from getting Catalyst to work
+        '''
+        logger.info(
+            ('Retrieving benchmark data from bundle for {symbol!r} from {first_date} to {last_date}'),
+            symbol=symbol, first_date=first_date, last_date=last_date)
+
+        asset = bundle_data.asset_finder.lookup_symbol(symbol=symbol,as_of_date=None)
+        fields = ['day', 'close']
+        raw = bundle_data.daily_bar_reader.load_raw_arrays(
+            columns=fields,
+            start_date=first_date - trading_day,
+            end_date=last_date,
+            assets=[asset,])
+        bench_raw = pd.concat([pd.DataFrame(raw[0], columns=['date']),pd.DataFrame(raw[1], columns=['close'])], axis=1)
+        bench_raw['date'] = pd.to_datetime(bench_raw['date'],unit='s')
+        bench_raw.set_index('date', inplace=True)
+        bench_raw.sort_index(inplace=True)
+        bench_raw = bench_raw[pd.to_datetime(first_date - trading_day):pd.to_datetime(last_date)]
+
+    else:
+        # This is how it used to be: downloading the benchmark everytime. 
+        # Leaving this code here to be repurposed in the future for other bundles.
+        logger.info(
+            ('Downloading benchmark data for {symbol!r} from {first_date} to {last_date}'),
+            symbol=symbol, first_date=first_date, last_date=last_date)
+
+        # Load benchmark symbol from Poloniex API
+        try:
+            bundle = PoloniexBundle()
+            bench_raw = bundle._fetch_symbol_frame(
+                None,
+                symbol,
+                get_calendar(bundle.calendar_name),
+                first_date - trading_day,
+                last_date,
+                'daily',
+            )
+        except (OSError, IOError, HTTPError):
+            logger.exception('Failed to fetch new crypto benchmark returns')
+            raise
 
     # select close column and compute percent change between days
     daily_close = bench_raw[['close']]
@@ -518,7 +551,7 @@ def _load_cached_data(filename, first_date, last_date, now, resource_name,
             )
 
     logger.info(
-        "Cache at {path} does not have data from {start} to {end}.\n",
+        "Cache at {path} does not have data from {start} to {end}.",
         start=first_date,
         end=last_date,
         path=path,

catalyst/examples/buy_and_hodl.py

@@ -23,7 +23,6 @@ from catalyst.api import (
     get_open_orders,
 )
 
-
 def initialize(context):
     context.ASSET_NAME = 'USDT_BTC'
     context.TARGET_HODL_RATIO = 0.8
@@ -42,8 +41,6 @@ def initialize(context):
 def handle_data(context, data):
     context.i += 1
 
-    print 'i:', context.i
-
     starting_cash = context.portfolio.starting_cash
     target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
     reserve_value = context.RESERVE_RATIO * starting_cash
@@ -73,6 +70,7 @@ def handle_data(context, data):
 
     record(
         price=price,
+        volume=data[context.asset].volume,
         cash=cash,
         starting_cash=context.portfolio.starting_cash,
         leverage=context.account.leverage,
@@ -80,12 +78,13 @@ def handle_data(context, data):
 
 def analyze(context=None, results=None):
     import matplotlib.pyplot as plt
+
     # Plot the portfolio and asset data.
-    ax1 = plt.subplot(511)
+    ax1 = plt.subplot(611)
     results[['portfolio_value']].plot(ax=ax1)
     ax1.set_ylabel('Portfolio Value (USD)')
 
-    ax2 = plt.subplot(512, sharex=ax1)
+    ax2 = plt.subplot(612, sharex=ax1)
     ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
     (context.TICK_SIZE * results[['price']]).plot(ax=ax2)
 
@@ -101,11 +100,11 @@ def analyze(context=None, results=None):
         color='g',
     )
 
-    ax3 = plt.subplot(513, sharex=ax1)
+    ax3 = plt.subplot(613, sharex=ax1)
     results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
     ax3.set_ylabel('Leverage ')
 
-    ax4 = plt.subplot(514, sharex=ax1)
+    ax4 = plt.subplot(614, sharex=ax1)
     results[['starting_cash', 'cash']].plot(ax=ax4)
     ax4.set_ylabel('Cash (USD)')
 
@@ -119,7 +118,7 @@ def analyze(context=None, results=None):
         'benchmark_period_return',
     ]]
 
-    ax5 = plt.subplot(515, sharex=ax1)
+    ax5 = plt.subplot(615, sharex=ax1)
     results[[
         'treasury',
         'algorithm',
@@ -127,8 +126,12 @@ def analyze(context=None, results=None):
     ]].plot(ax=ax5)
     ax5.set_ylabel('Percent Change')
 
+    ax6 = plt.subplot(616, sharex=ax1)
+    results[['volume']].plot(ax=ax6)
+    ax6.set_ylabel('Volume (mCoins/5min)')
+
     plt.legend(loc=3)
 
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
-    plt.show()
+    plt.show()

catalyst/utils/run_algo.py

@@ -149,7 +149,7 @@ def _run(handle_data,
             open_calendar = get_calendar('OPEN')
 
             env = TradingEnvironment(
-                load=partial(load_crypto_market_data, environ=environ),
+                load=partial(load_crypto_market_data, bundle=b, bundle_data=bundle_data, environ=environ),
                 bm_symbol='USDT_BTC',
                 trading_calendar=open_calendar,
                 asset_db_path=connstr,