Merge pull request #4 from enigmampc/polo-curate-fixes

Fixes formatting errors in catalyst/curate/poloniex.py

README.rst

@@ -12,7 +12,7 @@ Please visit `<enigma.co>`_ to learn about Catalyst, or refer to the
 
 Catalyst builds on top of the well-established `Zipline <https://github.com/quantopian/zipline>`_ project.
 We did our best to minimize structural changes to the general API to maximize compatibility with existing trading algorithms, developer knowledge, and tutorials.
-For now, please refer to the `Zipline API Docs <https://zipline.io>`_ as a general reference and bring any other questions you have to our #dev channel on `Slack <https://join.slack.com/enigmacatalyst/shared_invite/MTkzMjQ0MTg1NTczLTE0OTY3MjE3MDEtZGZmMTI5YzI3ZA>`_.
+For now, please refer to the `Zipline API Docs <http://zipline.io>`_ as a general reference and bring any other questions you have to our #dev channel on `Slack <https://join.slack.com/enigmacatalyst/shared_invite/MTkzMjQ0MTg1NTczLTE0OTY3MjE3MDEtZGZmMTI5YzI3ZA>`_.
 
 Our primary contributions include the:
 
@@ -134,15 +134,13 @@ the algorithm and plot the resulting performance using matplotlib.
 
 You can find other examples in the ``catalyst/examples`` directory.
 
-Supported Assets
-----------------
+Limitations
+-----------
 
-Currently the poloniex bundle comes prepopulated with data for all 90 registered trading pairs.
-However, due to limitations in how portfolios are currently modeled, we recommend sticking to ``USDT_*`` trading pairs.
-USDT is an independent currency listed on Poloniex whose price is pegged to the US dollar.
-Currently, this list includes: ``USDT_BTC``, ``USDT_DASH``, ``USDT_ETC``, ``USDT_ETH``, ``USDT_LTC``, ``USDT_NXT``, ``USDT_REP``, ``USDT_STR``, ``USDT_XMR``, ``USDT_XRP``, and ``USDT_ZEC``.
-We plan to add support for basing your portfolio in arbitrary currencies and provide native support for modeling ForEx trades in the near future!
+This project is currently in a pre-alpha state and has some limitations we'd like to address:
 
+- *Minimum Denomination:* The smallest tradable unit in Catalyst is equal to 1/1000th of a full coin. We plan to enable more granular increments, but have capped it at 1/1000th for the time being.
+- *Supported Assets:* Currently the poloniex bundle comes prepopulated with data for all 90 registered trading pairs. However, due to limitations in how portfolios are currently modeled, we recommend sticking to ``USDT_*`` trading pairs. USDT is an independent currency listed on Poloniex whose price is pegged to the US dollar. Currently, this list includes: ``USDT_BTC``, ``USDT_DASH``, ``USDT_ETC``, ``USDT_ETH``, ``USDT_LTC``, ``USDT_NXT``, ``USDT_REP``, ``USDT_STR``, ``USDT_XMR``, ``USDT_XRP``, and ``USDT_ZEC``. We plan to add support for basing your portfolio in arbitrary currencies and provide native support for modeling ForEx trades in the near future!
 
 Virtual Environments
 ====================

catalyst/curate/poloniex.py

@@ -6,7 +6,7 @@ import time
 import requests
 import logbook
 
-DT_START        = time.mktime(datetime(2010, 01, 01, 0, 0).timetuple())
+DT_START        = time.mktime(datetime(2010, 1, 1, 0, 0).timetuple())
 CSV_OUT_FOLDER  = '/var/tmp/catalyst/data/poloniex/'
 CONN_RETRIES    = 2
 
@@ -68,8 +68,8 @@ class PoloniexCurator(object):
         return DT_START
 
     def get_data(self, currencyPair, start, end=9999999999, period=300):
-    	url = self._api_path + 'command=returnChartData&currencyPair=' + currencyPair + '&start=' + str(start) + '&end=' + str(end) + '&period=' + str(period)
-        
+        url = self._api_path + 'command=returnChartData&currencyPair=' + currencyPair + '&start=' + str(start) + '&end=' + str(end) + '&period=' + str(period)
+
         try:
             response = requests.get(url)
         except Exception as e:
@@ -77,13 +77,13 @@ class PoloniexCurator(object):
             log.exception(e)
             return None
 
-    	return response.json()
+        return response.json()
 
     '''
     Pulls latest data for a single pair
     '''
     def append_data_single_pair(self, currencyPair, repeat=0):
-    	log.debug('Getting data for %s' % currencyPair)
+        log.debug('Getting data for %s' % currencyPair)
         csv_fn = CSV_OUT_FOLDER + 'crypto_prices-' + currencyPair + '.csv'
         start  = self._get_start_date(csv_fn)
         # Only fetch data if more than 5min have passed since last fetch
@@ -115,10 +115,10 @@ class PoloniexCurator(object):
     Pulls latest data for all currency pairs
     '''
     def append_data(self):
-    	for currencyPair in self.currency_pairs:
-    		self.append_data_single_pair(currencyPair)
-        # Rate limit is 6 calls per second, sleep 1sec/6 to be safe
-    		time.sleep(0.17)
+        for currencyPair in self.currency_pairs:
+            self.append_data_single_pair(currencyPair)
+            # Rate limit is 6 calls per second, sleep 1sec/6 to be safe
+            time.sleep(0.17)
 
     '''
     Returns a data frame for all pairs, or for the requests currency pair.

catalyst/data/_equities.pyx

@@ -21,6 +21,7 @@ from numpy import (
     float64,
     intp,
     uint32,
+    uint64,
     zeros,
 )
 from numpy cimport (
@@ -28,6 +29,7 @@ from numpy cimport (
     intp_t,
     ndarray,
     uint32_t,
+    uint64_t,
     uint8_t,
 )
 from numpy.math cimport NAN
@@ -167,8 +169,8 @@ cpdef _read_bcolz_data(ctable_t table,
         int nassets
         str column_name
         carray_t carray
-        ndarray[dtype=uint32_t, ndim=1] raw_data
-        ndarray[dtype=uint32_t, ndim=2] outbuf
+        ndarray[dtype=uint64_t, ndim=1] raw_data
+        ndarray[dtype=uint64_t, ndim=2] outbuf
         ndarray[dtype=uint8_t, ndim=2, cast=True] where_nan
         ndarray[dtype=float64_t, ndim=2] outbuf_as_float
         intp_t asset
@@ -185,7 +187,7 @@ cpdef _read_bcolz_data(ctable_t table,
         raise ValueError("Incompatible index arrays.")
 
     for column_name in columns:
-        outbuf = zeros(shape=shape, dtype=uint32)
+        outbuf = zeros(shape=shape, dtype=uint64)
         if read_all:
             raw_data = table[column_name][:]
 
@@ -213,11 +215,13 @@ cpdef _read_bcolz_data(ctable_t table,
                 else:
                     continue
 
-        if column_name in {'open', 'high', 'low', 'close'}:
+        if column_name in ['open', 'high', 'low', 'close']:
             where_nan = (outbuf == 0)
-            outbuf_as_float = outbuf.astype(float64) * .001
+            outbuf_as_float = outbuf.astype(float64) * .000001
             outbuf_as_float[where_nan] = NAN
             results.append(outbuf_as_float)
+        elif column_name != 'volume':
+            results.append(outbuf.astype(uint32))
         else:
             results.append(outbuf)
     return results

catalyst/data/loader.py

@@ -20,6 +20,7 @@ import pandas as pd
 import numpy as np
 from pandas_datareader.data import DataReader
 import datetime
+import time
 import pytz
 from six import iteritems
 from six.moves.urllib_error import HTTPError

catalyst/data/us_equity_pricing.py

@@ -34,6 +34,7 @@ from numpy import (
     issubdtype,
     nan,
     uint32,
+    uint64,
 )
 from pandas import (
     DataFrame,
@@ -80,6 +81,7 @@ from ._adjustments import load_adjustments_from_sqlite
 logger = logbook.Logger('UsEquityPricing')
 
 OHLC = frozenset(['open', 'high', 'low', 'close'])
+OHLCV = frozenset(['open', 'high', 'low', 'close', 'volume'])
 US_EQUITY_PRICING_BCOLZ_COLUMNS = (
     'open', 'high', 'low', 'close', 'volume', 'day', 'id'
 )
@@ -109,6 +111,7 @@ SQLITE_STOCK_DIVIDEND_PAYOUT_COLUMN_DTYPES = {
     'ratio': float,
 }
 UINT32_MAX = iinfo(uint32).max
+UINT64_MAX = iinfo(uint64).max
 
 
 def check_uint32_safe(value, colname):
@@ -119,25 +122,25 @@ def check_uint32_safe(value, colname):
 
 
 @expect_element(invalid_data_behavior={'warn', 'raise', 'ignore'})
-def winsorise_uint32(df, invalid_data_behavior, column, *columns):
-    """Drops any record where a value would not fit into a uint32.
+def winsorise_uint64(df, invalid_data_behavior, column, *columns):
+    """Drops any record where a value would not fit into a uint64.
 
     Parameters
     ----------
     df : pd.DataFrame
         The dataframe to winsorise.
     invalid_data_behavior : {'warn', 'raise', 'ignore'}
-        What to do when data is outside the bounds of a uint32.
+        What to do when data is outside the bounds of a uint64.
     *columns : iterable[str]
         The names of the columns to check.
 
     Returns
     -------
     truncated : pd.DataFrame
-        ``df`` with values that do not fit into a uint32 zeroed out.
+        ``df`` with values that do not fit into a uint64 zeroed out.
     """
     columns = list((column,) + columns)
-    mask = df[columns] > UINT32_MAX
+    mask = df[columns] > UINT64_MAX
 
     if invalid_data_behavior != 'ignore':
         mask |= df[columns].isnull()
@@ -150,14 +153,14 @@ def winsorise_uint32(df, invalid_data_behavior, column, *columns):
     if mv.any():
         if invalid_data_behavior == 'raise':
             raise ValueError(
-                '%d values out of bounds for uint32: %r' % (
+                '%d values out of bounds for uint64: %r' % (
                     mv.sum(), df[mask.any(axis=1)],
                 ),
             )
         if invalid_data_behavior == 'warn':
             warnings.warn(
                 'Ignoring %d values because they are out of bounds for'
-                ' uint32: %r' % (
+                ' uint64: %r' % (
                     mv.sum(), df[mask.any(axis=1)],
                 ),
                 stacklevel=3,  # one extra frame for `expect_element`
@@ -239,7 +242,7 @@ class BcolzDailyBarWriter(object):
             Whether or not to show a progress bar while writing.
         invalid_data_behavior : {'warn', 'raise', 'ignore'}, optional
             What to do when data is encountered that is outside the range of
-            a uint32.
+            a uint64.
 
         Returns
         -------
@@ -274,7 +277,7 @@ class BcolzDailyBarWriter(object):
             Whether or not to show a progress bar while writing.
         invalid_data_behavior : {'warn', 'raise', 'ignore'}
             What to do when data is encountered that is outside the range of
-            a uint32.
+            a uint64.
         """
         read = partial(
             read_csv,
@@ -302,7 +305,9 @@ class BcolzDailyBarWriter(object):
 
         # Maps column name -> output carray.
         columns = {
-            k: carray(array([], dtype=uint32))
+            k: carray(array([], dtype=uint64))
+               if k in OHLCV
+               else carray(array([], dtype=uint32))
             for k in US_EQUITY_PRICING_BCOLZ_COLUMNS
         }
 
@@ -417,12 +422,12 @@ class BcolzDailyBarWriter(object):
             # we already have a ctable so do nothing
             return raw_data
 
-        winsorise_uint32(raw_data, invalid_data_behavior, 'volume', *OHLC)
-        processed = (raw_data[list(OHLC)] * 1000).astype('uint32')
+        winsorise_uint64(raw_data, invalid_data_behavior, 'volume', *OHLC)
+        processed = (raw_data[list(OHLC)] * 1000000).astype('uint64')
         dates = raw_data.index.values.astype('datetime64[s]')
         check_uint32_safe(dates.max().view(np.int64), 'day')
         processed['day'] = dates.astype('uint32')
-        processed['volume'] = raw_data.volume.astype('uint32')
+        processed['volume'] = raw_data.volume.astype('uint64')
         return ctable.fromdataframe(processed)
 
 

catalyst/examples/buy_and_hodl.py

@@ -23,19 +23,24 @@ from catalyst.api import (
     get_open_orders,
 )
 
-ASSET = 'USDT_BTC'
-
-TARGET_HODL_RATIO = 0.8
-RESERVE_RATIO = 1.0 - TARGET_HODL_RATIO
 
 def initialize(context):
+    context.ASSET_NAME = 'USDT_ETH'
+    context.TARGET_HODL_RATIO = 0.8
+    context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO
+
+    # For all trading pairs in the poloniex bundle, the default denomination
+    # currently supported by Catalyst is 1/1000th of a full coin. Use this
+    # constant to scale the price of up to that of a full coin if desired.
+    context.TICK_SIZE = 1000.0
+
     context.is_buying = True
-    context.asset = symbol(ASSET)
+    context.asset = symbol(context.ASSET_NAME)
 
 def handle_data(context, data):
-    cash = context.portfolio.cash
-    target_hodl_value = TARGET_HODL_RATIO * context.portfolio.starting_cash
-    reserve_value = RESERVE_RATIO * context.portfolio.starting_cash
+    starting_cash = context.portfolio.starting_cash
+    target_hodl_value = context.TARGET_HODL_RATIO * starting_cash
+    reserve_value = context.RESERVE_RATIO * starting_cash
 
     # Cancel any outstanding orders
     orders = get_open_orders(context.asset) or []
@@ -43,6 +48,7 @@ def handle_data(context, data):
         cancel_order(order)
     
     # Stop buying after passing the reserve threshold
+    cash = context.portfolio.cash
     if cash <= reserve_value:
         context.is_buying = False
 
@@ -74,8 +80,8 @@ def analyze(context=None, results=None):
     ax1.set_ylabel('Portfolio Value (USD)')
 
     ax2 = plt.subplot(512, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=ASSET))
-    results[['price']].plot(ax=ax2)
+    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
+    (context.TICK_SIZE * results[['price']]).plot(ax=ax2)
 
     trans = results.ix[[t != [] for t in results.transactions]]
     buys = trans.ix[
@@ -83,7 +89,7 @@ def analyze(context=None, results=None):
     ]
     ax2.plot(
         buys.index,
-        results.price[buys.index],
+        context.TICK_SIZE * results.price[buys.index],
         '^',
         markersize=10,
         color='g',
@@ -120,14 +126,3 @@ def analyze(context=None, results=None):
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
     plt.show()
-
-
-def _test_args():
-    """Extra arguments to use when catalyst's automated tests run this example.
-    """
-    import pandas as pd
-
-    return {
-        'start': pd.Timestamp('2008', tz='utc'),
-        'end': pd.Timestamp('2013', tz='utc'),
-    }

catalyst/examples/dual_vwap.py

@@ -31,19 +31,24 @@ from catalyst.pipeline import Pipeline
 from catalyst.pipeline.data import CryptoPricing
 from catalyst.pipeline.factors.crypto import VWAP
 
-ASSET = 'USDT_BTC'
-
-TARGET_INVESTMENT_RATIO = 0.8
-SHORT_WINDOW = 30
-LONG_WINDOW = 100
 
 def initialize(context):
+    context.ASSET_NAME = 'USDT_BTC'
+    context.TARGET_INVESTMENT_RATIO = 0.8
+    context.SHORT_WINDOW = 30
+    context.LONG_WINDOW = 100
+
+    # For all trading pairs in the poloniex bundle, the default denomination
+    # currently supported by Catalyst is 1/1000th of a full coin. Use this
+    # constant to scale the price of up to that of a full coin if desired.
+    context.TICK_SIZE = 1000.0
+
     context.i = 0
-    context.asset = symbol(ASSET)
+    context.asset = symbol(context.ASSET_NAME)
 
     set_max_leverage(1.0)
 
-    attach_pipeline(make_pipeline(), 'vwap_pipeline')
+    attach_pipeline(make_pipeline(context), 'vwap_pipeline')
 
     schedule_function(
         rebalance,
@@ -54,12 +59,13 @@ def initialize(context):
 def before_trading_start(context, data):
     context.pipeline_data = pipeline_output('vwap_pipeline')
 
-def make_pipeline():
+def make_pipeline(context):
     return Pipeline(
         columns={
             'price': CryptoPricing.open.latest,
-            'short_mavg': VWAP(window_length=SHORT_WINDOW),
-            'long_mavg': VWAP(window_length=LONG_WINDOW),
+            'volume': CryptoPricing.volume.latest,
+            'short_mavg': VWAP(window_length=context.SHORT_WINDOW),
+            'long_mavg': VWAP(window_length=context.LONG_WINDOW),
         }
     )
 
@@ -67,7 +73,7 @@ def rebalance(context, data):
     context.i += 1
 
     # skip first LONG_WINDOW bars to fill windows
-    if context.i < LONG_WINDOW:
+    if context.i < context.LONG_WINDOW:
         return
 
     # get pipeline data for asset of interest
@@ -78,6 +84,7 @@ def rebalance(context, data):
     short_mavg = pipeline_data.short_mavg
     long_mavg = pipeline_data.long_mavg
     price = pipeline_data.price
+    volume = pipeline_data.volume
 
     # check that order has not already been placed
     open_orders = get_open_orders()
@@ -86,9 +93,15 @@ def rebalance(context, data):
         if data.can_trade(context.asset):
             # adjust portfolio based on comparison of long and short vwap
             if short_mavg > long_mavg:
-                order_target_percent(context.asset, TARGET_INVESTMENT_RATIO)
+                order_target_percent(
+                    context.asset,
+                    context.TARGET_INVESTMENT_RATIO,
+                )
             elif short_mavg < long_mavg:
-                order_target_percent(context.asset, 0.0)
+                order_target_percent(
+                    context.asset,
+                    0.0,
+                )
 
     record(
         price=price,
@@ -96,23 +109,22 @@ def rebalance(context, data):
         leverage=context.account.leverage,
         short_mavg=short_mavg,
         long_mavg=long_mavg,
+        volume=volume,
     )
     
 
 
-# Note: this function can be removed if running
-# this algorithm on quantopian.com
 def analyze(context=None, results=None):
     import matplotlib.pyplot as plt
 
     # Plot the portfolio and asset data.
-    ax1 = plt.subplot(511)
+    ax1 = plt.subplot(611)
     results[['portfolio_value']].plot(ax=ax1)
     ax1.set_ylabel('Portfolio value (USD)')
 
-    ax2 = plt.subplot(512, sharex=ax1)
-    ax2.set_ylabel('{asset} (USD)'.format(asset=ASSET))
-    results[['price', 'short_mavg', 'long_mavg']].plot(ax=ax2)
+    ax2 = plt.subplot(612, sharex=ax1)
+    ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME))
+    (context.TICK_SIZE*results[['price', 'short_mavg', 'long_mavg']]).plot(ax=ax2)
 
     trans = results.ix[[t != [] for t in results.transactions]]
     amounts = [t[0]['amount'] for t in trans.transactions]
@@ -126,24 +138,24 @@ def analyze(context=None, results=None):
 
     ax2.plot(
         buys.index,
-        results.price[buys.index],
+        context.TICK_SIZE * results.price[buys.index],
         '^',
         markersize=10,
         color='g',
     )
     ax2.plot(
         sells.index,
-        results.price[sells.index],
+        context.TICK_SIZE * results.price[sells.index],
         'v',
         markersize=10,
         color='r',
     )
 
-    ax3 = plt.subplot(513, sharex=ax1)
+    ax3 = plt.subplot(613, sharex=ax1)
     results[['leverage', 'alpha', 'beta']].plot(ax=ax3)
     ax3.set_ylabel('Leverage (USD)')
 
-    ax4 = plt.subplot(514, sharex=ax1)
+    ax4 = plt.subplot(614, sharex=ax1)
     results[['cash']].plot(ax=ax4)
     ax4.set_ylabel('Cash (USD)')
 
@@ -157,7 +169,7 @@ def analyze(context=None, results=None):
         'benchmark_period_return',
     ]]
 
-    ax5 = plt.subplot(515, sharex=ax1)
+    ax5 = plt.subplot(615, sharex=ax1)
     results[[
         'treasury',
         'algorithm',
@@ -165,19 +177,12 @@ def analyze(context=None, results=None):
     ]].plot(ax=ax5)
     ax5.set_ylabel('Percent Change')
 
+    ax6 = plt.subplot(616, sharex=ax1)
+    results[['volume']].plot(ax=ax6)
+    ax6.set_ylabel('Volume (mBTC/day)')
+
     plt.legend(loc=3)
 
     # Show the plot.
     plt.gcf().set_size_inches(18, 8)
     plt.show()
-
-
-def _test_args():
-    """Extra arguments to use when catalyst's automated tests run this example.
-    """
-    import pandas as pd
-
-    return {
-        'start': pd.Timestamp('2014-01-01', tz='utc'),
-        'end': pd.Timestamp('2014-11-01', tz='utc'),
-    }