diff --git a/.gitattributes b/.gitattributes index d044edfa..e7871abc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ zipline/_version.py export-subst *.ipynb binary +catalyst/_version.py export-subst diff --git a/catalyst/__init__.py b/catalyst/__init__.py index b4f1dcd4..508137df 100644 --- a/catalyst/__init__.py +++ b/catalyst/__init__.py @@ -80,3 +80,7 @@ __all__ = [ 'run_algorithm', 'utils', ] + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions diff --git a/catalyst/data/bundles/poloniex.py b/catalyst/data/bundles/poloniex.py index 500c7cf9..a50d266b 100644 --- a/catalyst/data/bundles/poloniex.py +++ b/catalyst/data/bundles/poloniex.py @@ -115,12 +115,13 @@ def poloniex_cryptoassets(symbols, start=None, end=None): day_data.volume.sum(), # sum of all volumes ) - # scale to allow trading 100-ths of a coin - daily_bars.loc[:, 'open'] /= 100.0 - daily_bars.loc[:, 'high'] /= 100.0 - daily_bars.loc[:, 'low'] /= 100.0 - daily_bars.loc[:, 'close'] /= 100.0 - daily_bars.loc[:, 'volume'] *= 100.0 + # scale to allow trading 10-ths of a coin + scale = 10.0 + daily_bars.loc[:, 'open'] /= scale + daily_bars.loc[:, 'high'] /= scale + daily_bars.loc[:, 'low'] /= scale + daily_bars.loc[:, 'close'] /= scale + daily_bars.loc[:, 'volume'] *= scale return daily_bars @@ -171,8 +172,10 @@ def poloniex_cryptoassets(symbols, start=None, end=None): yield sid, daily_bars sid += 1 - - daily_bar_writer.write(_pricing_iter()) + daily_bar_writer.write( + _pricing_iter(), + assets=metadata.symbol.index, + ) symbol_map = pd.Series(metadata.symbol.index, metadata.symbol) diff --git a/catalyst/data/bundles/quandl.py b/catalyst/data/bundles/quandl.py index d9bcbdc2..6194c70e 100644 --- a/catalyst/data/bundles/quandl.py +++ b/catalyst/data/bundles/quandl.py @@ -5,6 +5,7 @@ from io import BytesIO from itertools import count import tarfile from time import time, sleep +from datetime import datetime from click import progressbar from logbook import Logger @@ -36,10 +37,17 @@ def _fetch_raw_metadata(api_key, cache, retries, environ): try: raw = pd.read_csv( format_metadata_url(api_key, page_number), + date_parser=pd.tseries.tools.to_datetime, parse_dates=[ 'oldest_available_date', 'newest_available_date', ], + dtypes={ + 'dataset_code': 'int', + 'name': 'str', + 'oldest_available_date': 'str', + 'newest_available_date': 'str', + }, usecols=[ 'dataset_code', 'name', @@ -126,6 +134,10 @@ def fetch_symbol_metadata_frame(api_key, # we need to escape the paren because it is actually splitting on a regex data.asset_name = data.asset_name.str.split(r' \(', 1).str.get(0) data['exchange'] = 'QUANDL' + + data['start_date'] = data['start_date'].astype(datetime) + data['end_date'] = data['end_date'].astype(datetime) + data['auto_close_date'] = data['end_date'] + pd.Timedelta(days=1) return data @@ -313,6 +325,7 @@ def quandl_bundle(environ, dividends, environ.get('QUANDL_DOWNLOAD_ATTEMPTS', 5), ), + assets=metadata.index, show_progress=show_progress, ) adjustment_writer.write( diff --git a/catalyst/data/loader.py b/catalyst/data/loader.py index 18cb445b..15acc567 100644 --- a/catalyst/data/loader.py +++ b/catalyst/data/loader.py @@ -300,12 +300,13 @@ def ensure_crypto_benchmark_data(symbol, first_date, last_date, now, day_data.volume.sum(), # sum of all volumes ) - # scale to allow trading 100-ths of a coin - daily_bars.loc[:, 'open'] /= 100.0 - daily_bars.loc[:, 'high'] /= 100.0 - daily_bars.loc[:, 'low'] /= 100.0 - daily_bars.loc[:, 'close'] /= 100.0 - daily_bars.loc[:, 'volume'] *= 100.0 + # scale to allow trading 10-ths of a coin + scale = 10.0 + daily_bars.loc[:, 'open'] /= scale + daily_bars.loc[:, 'high'] /= scale + daily_bars.loc[:, 'low'] /= scale + daily_bars.loc[:, 'close'] /= scale + daily_bars.loc[:, 'volume'] *= scale return daily_bars diff --git a/catalyst/examples/buy_and_hold.py b/catalyst/examples/buy_and_hold.py index 21eb2224..ab2f9abc 100644 --- a/catalyst/examples/buy_and_hold.py +++ b/catalyst/examples/buy_and_hold.py @@ -22,11 +22,11 @@ from catalyst.api import ( record, ) -TARGET_INVESTMENT_RATIO = 0.5 +TARGET_INVESTMENT_RATIO = 0.8 def initialize(context): context.has_ordered = False - context.asset = symbol('USDT_BTC') + context.asset = symbol('USDT_ETH') def handle_data(context, data): @@ -39,7 +39,7 @@ def handle_data(context, data): context.has_ordered = True record( - USDT_BTC=data[context.asset].price, + USDT_ETH=data[context.asset].price, cash=context.portfolio.cash, leverage=context.account.leverage, ) @@ -52,8 +52,8 @@ def analyze(context=None, results=None): ax1.set_ylabel('Portfolio value (USD)') ax2 = plt.subplot(512, sharex=ax1) - ax2.set_ylabel('USDT_BTC (USD)') - results[['USDT_BTC']].plot(ax=ax2) + ax2.set_ylabel('USDT_ETH (USD)') + results[['USDT_ETH']].plot(ax=ax2) trans = results.ix[[t != [] for t in results.transactions]] buys = trans.ix[ @@ -64,13 +64,13 @@ def analyze(context=None, results=None): ] print 'buys:', buys.head() ax2.plot( - buys.index, results.USDT_BTC[buys.index], + buys.index, results.USDT_ETH[buys.index], '^', markersize=10, color='m', ) ax2.plot( - sells.index, results.USDT_BTC[sells.index], + sells.index, results.USDT_ETH[sells.index], 'v', markersize=10, color='k', diff --git a/catalyst/examples/dual_moving_average_btc.py b/catalyst/examples/dual_moving_average_btc.py index 66a645fb..2b3d4688 100644 --- a/catalyst/examples/dual_moving_average_btc.py +++ b/catalyst/examples/dual_moving_average_btc.py @@ -41,7 +41,7 @@ from catalyst.pipeline.factors.crypto import ( from catalyst.finance.commission import PerDollar from catalyst.finance.slippage import VolumeShareSlippage -TARGET_INVESTMENT_RATIO = 0.1 +TARGET_INVESTMENT_RATIO = 0.8 SHORT_WINDOW = 30 LONG_WINDOW = 100 @@ -70,7 +70,7 @@ def make_pipeline(): 'price': CryptoPricing.close.latest, 'short_mavg': VWAP(window_length=SHORT_WINDOW), 'long_mavg': VWAP(window_length=LONG_WINDOW), - }, + } ) @@ -128,6 +128,7 @@ def rebalance(context, data): # this algorithm on quantopian.com def analyze(context=None, results=None): import matplotlib.pyplot as plt + # Plot the portfolio and asset data. ax1 = plt.subplot(511) results[['portfolio_value']].plot(ax=ax1) diff --git a/catalyst/examples/test.py b/catalyst/examples/test.py new file mode 100644 index 00000000..04d49c31 --- /dev/null +++ b/catalyst/examples/test.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python +# +# Copyright 2014 Quantopian, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +from catalyst.api import ( + order_target_percent, + record, + symbol, + get_open_orders, + set_commission, + set_slippage, + set_max_leverage, + schedule_function, + date_rules, + time_rules, + attach_pipeline, + pipeline_output, +) + +from catalyst.pipeline import Pipeline +from catalyst.pipeline.data import CryptoPricing +from catalyst.pipeline.factors.crypto import ( + VWAP, + SimpleMovingAverage, + MACDSignal, +) + +from catalyst.finance.commission import PerDollar +from catalyst.finance.slippage import VolumeShareSlippage + +TARGET_INVESTMENT_RATIO = 0.2 +SHORT_WINDOW = 30 +LONG_WINDOW = 100 + +def initialize(context): + context.asset = symbol('USDT_BTC') + context.i = 0 + context.macd_cur = None + context.macd_last = None + + set_commission(PerDollar(cost=0.001)) + set_slippage(VolumeShareSlippage()) + set_max_leverage(1.0) + + attach_pipeline(make_pipeline(), 'my_pipeline') + + schedule_function( + rebalance, + date_rules.every_day(), + ) + + +def before_trading_start(context, data): + context.pipeline_data = pipeline_output('my_pipeline') + +def make_pipeline(): + return Pipeline( + columns={ + 'price': CryptoPricing.close.latest, + 'short_mavg': VWAP(window_length=SHORT_WINDOW), + 'long_mavg': VWAP(window_length=LONG_WINDOW), + 'macd': MACDSignal( + fast_period=24, + slow_period=52, + signal_period=18, + ), + }, + ) + + +def handle_data(context, data): + pass + +def rebalance(context, data): + context.i += 1 + + # get pipeline data for asset of interest + pipeline_data = context.pipeline_data + pipeline_data = pipeline_data[pipeline_data.index == context.asset].iloc[0] + + context.macd_last = context.macd_cur + context.macd_cur = pipeline_data.macd + + # skip first LONG_WINDOW bars to fill windows + if context.i < 2: + return + + # retrieve long and short moving averages from pipeline + short_mavg = pipeline_data.short_mavg + long_mavg = pipeline_data.long_mavg + price = pipeline_data.price + + # check that order has not already been placed + open_orders = get_open_orders() + if context.asset not in open_orders: + # check that the asset of interest can currently be traded + if data.can_trade(context.asset): + if context.macd_cur < (0.98 * context.macd_last): + order_target_percent( + context.asset, + TARGET_INVESTMENT_RATIO, + #limit_price=(2 * price), + #stop_price=(0.5 * price), + ) + elif context.macd_cur > (1.02 * context.macd_last): + order_target_percent( + context.asset, + 0.0, + #limit_price=(2 * price), + #stop_price=(0.5 * price), + ) + + """ + # adjust portfolio based on moving averages + if short_mavg > long_mavg: + order_target_percent( + context.asset, + TARGET_INVESTMENT_RATIO, + #limit_price=(2 * price), + #stop_price=(0.5 * price), + ) + elif short_mavg < long_mavg: + order_target_percent( + context.asset, + 0.0, + #limit_price=(2 * price), + #stop_price=(0.5 * price), + ) + """ + + record( + USDT_BTC=price, + cash=context.portfolio.cash, + leverage=context.account.leverage, + short_mavg=short_mavg, + long_mavg=long_mavg, + ) + + + +# Note: this function can be removed if running +# this algorithm on quantopian.com +def analyze(context=None, results=None): + import matplotlib.pyplot as plt + # Plot the portfolio and asset data. + ax1 = plt.subplot(511) + results[['portfolio_value']].plot(ax=ax1) + ax1.set_ylabel('Portfolio value (USD)') + + ax2 = plt.subplot(512, sharex=ax1) + ax2.set_ylabel('USDT_BTC (USD)') + results[['USDT_BTC', 'short_mavg', 'long_mavg']].plot(ax=ax2) + + trans = results.ix[[t != [] for t in results.transactions]] + amounts = [t[0]['amount'] for t in trans.transactions] + print 'amounts:\n', amounts + buys = trans.ix[ + [t[0]['amount'] > 0 for t in trans.transactions] + ] + sells = trans.ix[ + [t[0]['amount'] < 0 for t in trans.transactions] + ] + print 'buys:', buys.head() + ax2.plot( + buys.index, results.USDT_BTC[buys.index], + '^', + markersize=10, + color='m', + ) + ax2.plot( + sells.index, results.USDT_BTC[sells.index], + 'v', + markersize=10, + color='k', + ) + + ax3 = plt.subplot(513, sharex=ax1) + results[['leverage', 'alpha', 'beta']].plot(ax=ax3) + ax3.set_ylabel('Leverage (USD)') + + ax4 = plt.subplot(514, sharex=ax1) + results[['cash']].plot(ax=ax4) + ax4.set_ylabel('Cash (USD)') + + results[[ + 'treasury', + 'algorithm', + 'benchmark', + ]] = results[[ + 'treasury_period_return', + 'algorithm_period_return', + 'benchmark_period_return', + ]] + + ax5 = plt.subplot(515, sharex=ax1) + results[[ + 'treasury', + 'algorithm', + 'benchmark', + ]].plot(ax=ax5) + ax5.set_ylabel('Dollars (USD)') + + plt.legend(loc=3) + + # Show the plot. + plt.gcf().set_size_inches(18, 8) + plt.show() + + +def _test_args(): + """Extra arguments to use when catalyst's automated tests run this example. + """ + import pandas as pd + + return { + 'start': pd.Timestamp('2014-01-01', tz='utc'), + 'end': pd.Timestamp('2014-11-01', tz='utc'), + } diff --git a/catalyst/utils/calendars/trading_calendar.py b/catalyst/utils/calendars/trading_calendar.py index cccb5f28..29a16f97 100644 --- a/catalyst/utils/calendars/trading_calendar.py +++ b/catalyst/utils/calendars/trading_calendar.py @@ -664,13 +664,13 @@ class TradingCalendar(with_metaclass(ABCMeta)): return self.schedule.loc[ start_session_label:end_session_label, 'market_open', - ].dt.tz_localize('UTC') + ].dt.tz_convert('UTC') def session_closes_in_range(self, start_session_label, end_session_label): return self.schedule.loc[ start_session_label:end_session_label, 'market_close', - ].dt.tz_localize('UTC') + ].dt.tz_convert('UTC') @property def all_sessions(self): diff --git a/conda/empyrical/meta.yaml b/conda/empyrical/meta.yaml index 045d11ab..708ce85b 100644 --- a/conda/empyrical/meta.yaml +++ b/conda/empyrical/meta.yaml @@ -1,6 +1,6 @@ package: name: empyrical - version: "0.2.2" + version: "0.2.1" source: fn: empyrical-0.2.2.tar.gz diff --git a/etc/requirements.txt b/etc/requirements.txt index 17832707..7ebfef2c 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -17,7 +17,7 @@ requests-file==1.4.1 # scipy and pandas are required for statsmodels, # statsmodels in turn is required for some pandas packages scipy==0.17.1 -pandas==0.18.1 +pandas==0.19.2 pandas-datareader==0.2.1 # Needed for parts of pandas.stats patsy==0.4.0 @@ -74,6 +74,6 @@ intervaltree==2.1.0 lru-dict==1.1.4 # For financial risk calculations -empyrical==0.2.2 +empyrical==0.2.1 tables==3.3.0 diff --git a/setup.py b/setup.py index 591d068f..34c27d06 100644 --- a/setup.py +++ b/setup.py @@ -162,7 +162,8 @@ def _filter_requirements(lines_iter, filter_names=None, REQ_UPPER_BOUNDS = { 'bcolz': '<1', - 'pandas': '<0.19', + 'pandas': '<0.20', + 'empyrical': '<0.2.2', } @@ -278,7 +279,7 @@ conditional_arguments = { } setup( - name='catalyst', + name='catalyst-hf', url='https://enigma.co', version=versioneer.get_version(), cmdclass=LazyBuildExtCommandClass(versioneer.get_cmdclass()), diff --git a/tests/calendars/test_cfe_calendar.py b/tests/calendars/test_cfe_calendar.py index 596a8675..63cdcc47 100644 --- a/tests/calendars/test_cfe_calendar.py +++ b/tests/calendars/test_cfe_calendar.py @@ -35,9 +35,8 @@ class CFECalendarTestCase(ExchangeCalendarTestBase, TestCase): self.assertTrue(dt in self.calendar.early_closes) market_close = self.calendar.schedule.loc[dt].market_close - market_close = market_close.tz_localize("UTC").tz_convert( - self.calendar.tz - ) + market_close = market_close.tz_convert(self.calendar.tz) + self.assertEqual(12, market_close.hour) self.assertEqual(15, market_close.minute) diff --git a/tests/calendars/test_cme_calendar.py b/tests/calendars/test_cme_calendar.py index 4018959a..e6add56e 100644 --- a/tests/calendars/test_cme_calendar.py +++ b/tests/calendars/test_cme_calendar.py @@ -35,7 +35,5 @@ class CMECalendarTestCase(ExchangeCalendarTestBase, TestCase): market_close = self.calendar.schedule.loc[dt].market_close self.assertEqual( 12, - market_close.tz_localize('UTC').tz_convert( - self.calendar.tz - ).hour + market_close.tz_convert(self.calendar.tz).hour, ) diff --git a/tests/calendars/test_ice_calendar.py b/tests/calendars/test_ice_calendar.py index 45c89e86..21cd8a69 100644 --- a/tests/calendars/test_ice_calendar.py +++ b/tests/calendars/test_ice_calendar.py @@ -48,7 +48,5 @@ class ICECalendarTestCase(ExchangeCalendarTestBase, TestCase): market_close = self.calendar.schedule.loc[dt].market_close self.assertEqual( 13, # all ICE early closes are 1 pm local - market_close.tz_localize("UTC").tz_convert( - self.calendar.tz - ).hour + market_close.tz_convert(self.calendar.tz).hour, ) diff --git a/tests/calendars/test_trading_calendar.py b/tests/calendars/test_trading_calendar.py index 0c34885b..6f4edfde 100644 --- a/tests/calendars/test_trading_calendar.py +++ b/tests/calendars/test_trading_calendar.py @@ -713,7 +713,7 @@ class ExchangeCalendarTestBase(object): the_open = self.calendar.schedule.loc[next_day].market_open - localized_open = the_open.tz_localize("UTC").tz_convert( + localized_open = the_open.tz_convert( self.calendar.tz )