From 20f8a75f4a8906641bbb1b198d05cc15148fff02 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Wed, 21 Feb 2018 20:42:39 +0200 Subject: [PATCH 01/12] BUG: for issue #237, update positions before checking balances --- catalyst/exchange/exchange.py | 11 ++++++++--- catalyst/exchange/exchange_algorithm.py | 21 ++++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index c1b64f7c..3b57256b 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -656,16 +656,21 @@ class Exchange: return df - def _check_low_balance(self, currency, balances, amount): + def _check_low_balance(self, currency, balances, amount, open_orders=None): free = balances[currency]['free'] if currency in balances else 0.0 + if open_orders: + # TODO: make sure that this works + free += sum([order.amount for order in open_orders]) + if free < amount: return free, True else: return free, False - def sync_positions(self, positions, cash=None, check_balances=False): + def sync_positions(self, positions, open_orders=None, cash=None, + check_balances=False): """ Update the portfolio cash and position balances based on the latest ticker prices. @@ -694,7 +699,7 @@ class Exchange: balances=balances, amount=cash, ) - if is_lower: + if is_lower and not open_orders: raise NotEnoughCashError( currency=self.base_currency, exchange=self.name, diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 5c9f8771..c827af4a 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -631,23 +631,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if base_currency is None: base_currency = exchange.base_currency - # Don't check the cash if there are open orders. This could - # results in false positives. orders = [] for asset in self.blotter.open_orders: asset_orders = self.blotter.open_orders[asset] if asset_orders: orders += asset_orders - required_cash = self.portfolio.cash if not orders else None - cash, positions_value = exchange.sync_positions( - positions=exchange_positions, - check_balances=check_balances, - cash=required_cash, - ) - total_cash += cash - total_positions_value += positions_value - # Applying modifications to the original positions for position in exchange_positions: tracker.update_position( @@ -657,6 +646,16 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): last_sale_price=position.last_sale_price, ) + required_cash = self.portfolio.cash if not orders else None + cash, positions_value = exchange.sync_positions( + positions=exchange_positions, + open_orders=orders, + check_balances=check_balances, + cash=required_cash, + ) + total_cash += cash + total_positions_value += positions_value + if not check_balances: total_cash = self.portfolio.cash From 4337abd60a4bfa2f8335dba2c8c3bb9173f38cda Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Wed, 21 Feb 2018 15:51:28 -0700 Subject: [PATCH 02/12] DOC: linking example_algo to their sources --- docs/source/beginner-tutorial.rst | 158 +----- docs/source/example-algos.rst | 898 +----------------------------- 2 files changed, 19 insertions(+), 1037 deletions(-) diff --git a/docs/source/beginner-tutorial.rst b/docs/source/beginner-tutorial.rst index 41afe3c0..291dd205 100644 --- a/docs/source/beginner-tutorial.rst +++ b/docs/source/beginner-tutorial.rst @@ -580,162 +580,8 @@ which you can skim through for now. A copy of this algorithm is available in the ``examples`` directory: `dual_moving_average.py `_. -.. code-block:: python - - import numpy as np - import pandas as pd - from logbook import Logger - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order, record, symbol, order_target_percent, - get_open_orders) - from catalyst.exchange.utils.stats_utils import extract_transactions - - NAMESPACE = 'dual_moving_average' - log = Logger(NAMESPACE) - - def initialize(context): - context.i = 0 - context.asset = symbol('ltc_usd') - context.base_price = None - - - def handle_data(context, data): - # define the windows for the moving averages - short_window = 50 - long_window = 200 - - # Skip as many bars as long_window to properly compute the average - context.i += 1 - if context.i < long_window: - return - - # Compute moving averages calling data.history() for each - # moving average with the appropriate parameters. We choose to use - # minute bars for this simulation -> freq="1m" - # Returns a pandas dataframe. - short_mavg = data.history(context.asset, 'price', - bar_count=short_window, frequency="1m").mean() - long_mavg = data.history(context.asset, 'price', - bar_count=long_window, frequency="1m").mean() - - # Let's keep the price of our asset in a more handy variable - price = data.current(context.asset, 'price') - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - price_change = (price - context.base_price) / context.base_price - - # Save values for later inspection - record(price=price, - cash=context.portfolio.cash, - price_change=price_change, - short_mavg=short_mavg, - long_mavg=long_mavg) - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.asset): - return - - # We check what's our position on our portfolio and trade accordingly - pos_amount = context.portfolio.positions[context.asset].amount - - # Trading logic - if short_mavg > long_mavg and pos_amount == 0: - # we buy 100% of our portfolio for this asset - order_target_percent(context.asset, 1) - elif short_mavg < long_mavg and pos_amount > 0: - # we sell all our positions for this asset - order_target_percent(context.asset, 0) - - - def analyze(context, perf): - - # Get the base_currency that was passed as a parameter to the simulation - exchange = list(context.exchanges.values())[0] - base_currency = exchange.base_currency.upper() - - # First chart: Plot portfolio value using base_currency - ax1 = plt.subplot(411) - perf.loc[:, ['portfolio_value']].plot(ax=ax1) - ax1.legend_.remove() - ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) - start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Second chart: Plot asset price, moving averages and buys/sells - ax2 = plt.subplot(412, sharex=ax1) - perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price') - ax2.legend_.remove() - ax2.set_ylabel('{asset}\n({base})'.format( - asset = context.asset.symbol, - base = base_currency - )) - start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index, 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index, 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - # Third chart: Compare percentage change between our portfolio - # and the price of the asset - ax3 = plt.subplot(413, sharex=ax1) - perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3) - ax3.legend_.remove() - ax3.set_ylabel('Percent Change') - start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Fourth chart: Plot our cash - ax4 = plt.subplot(414, sharex=ax1) - perf.cash.plot(ax=ax4) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) - - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/dual_moving_average.py + :language: python In order to run the code above, you have to ingest the needed data first: diff --git a/docs/source/example-algos.rst b/docs/source/example-algos.rst index 0136b899..ec5b74a0 100644 --- a/docs/source/example-algos.rst +++ b/docs/source/example-algos.rst @@ -52,35 +52,8 @@ Buy BTC Simple Algorithm Source code: `examples/buy_btc_simple.py `_ -.. code-block:: python - - ''' - Run this example, by executing the following from your terminal: - catalyst ingest-exchange -x bitfinex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x bitfinex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle - - If you want to run this code using another exchange, make sure that - the asset is available on that exchange. For example, if you were to run - it for exchange Poloniex, you would need to edit the following line: - - context.asset = symbol('btc_usdt') # note 'usdt' instead of 'usd' - - and specify exchange poloniex as follows: - catalyst ingest-exchange -x poloniex -f daily -i btc_usdt - catalyst run -f buy_btc_simple.py -x poloniex --start 2016-1-1 --end 2017-9-30 -o buy_btc_simple_out.pickle - - To see which assets are available on each exchange, visit: - https://www.enigma.co/catalyst/status - ''' - - from catalyst.api import order, record, symbol - - def initialize(context): - context.asset = symbol('btc_usd') - - def handle_data(context, data): - order(context.asset, 1) - record(btc = data.current(context.asset, 'price')) +.. literalinclude:: ../../catalyst/examples/buy_btc_simple.py + :language: python This simple algorithm does not produce any output nor displays any chart. @@ -90,8 +63,6 @@ This simple algorithm does not produce any output nor displays any chart. Buy and Hodl Algorithm ~~~~~~~~~~~~~~~~~~~~~~ -Source code: `examples/buy_and_hodl.py `_ - First ingest the historical pricing data needed to run this algorithm: .. code-block:: bash @@ -119,157 +90,10 @@ that 2015-3-1 is the earliest date that Catalyst supports (if you choose an earlier date, you'll get an error), and the most recent date you can choose is one day prior to the current date. +Source code: `examples/buy_and_hodl.py `_ -.. code-block:: python - - #!/usr/bin/env python - # - # Copyright 2017 Enigma MPC, Inc. - # Copyright 2015 Quantopian, Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - import pandas as pd - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order_target_value, symbol, record, - cancel_order, get_open_orders, ) - - - def initialize(context): - context.ASSET_NAME = 'btc_usd' - context.TARGET_HODL_RATIO = 0.8 - context.RESERVE_RATIO = 1.0 - context.TARGET_HODL_RATIO - - context.is_buying = True - context.asset = symbol(context.ASSET_NAME) - - context.i = 0 - - - def handle_data(context, data): - context.i += 1 - - starting_cash = context.portfolio.starting_cash - target_hodl_value = context.TARGET_HODL_RATIO * starting_cash - reserve_value = context.RESERVE_RATIO * starting_cash - - # Cancel any outstanding orders - orders = get_open_orders(context.asset) or [] - for order in orders: - cancel_order(order) - - # Stop buying after passing the reserve threshold - cash = context.portfolio.cash - if cash <= reserve_value: - context.is_buying = False - - # Retrieve current asset price from pricing data - price = data.current(context.asset, 'price') - - # Check if still buying and could (approximately) afford another purchase - if context.is_buying and cash > price: - print('buying') - # Place order to make position in asset equal to target_hodl_value - order_target_value( - context.asset, - target_hodl_value, - limit_price=price * 1.1, - ) - - record( - price=price, - volume=data.current(context.asset, 'volume'), - cash=cash, - starting_cash=context.portfolio.starting_cash, - leverage=context.account.leverage, - ) - - - def analyze(context=None, results=None): - - # Plot the portfolio and asset data. - ax1 = plt.subplot(611) - results[['portfolio_value']].plot(ax=ax1) - ax1.set_ylabel('Portfolio Value (USD)') - - ax2 = plt.subplot(612, sharex=ax1) - ax2.set_ylabel('{asset} (USD)'.format(asset=context.ASSET_NAME)) - results[['price']].plot(ax=ax2) - - trans = results.ix[[t != [] for t in results.transactions]] - buys = trans.ix[ - [t[0]['amount'] > 0 for t in trans.transactions] - ] - ax2.scatter( - buys.index.to_pydatetime(), - results.price[buys.index], - marker='^', - s=100, - c='g', - label='' - ) - - ax3 = plt.subplot(613, sharex=ax1) - results[['leverage', 'alpha', 'beta']].plot(ax=ax3) - ax3.set_ylabel('Leverage ') - - ax4 = plt.subplot(614, sharex=ax1) - results[['starting_cash', 'cash']].plot(ax=ax4) - ax4.set_ylabel('Cash (USD)') - - results[[ - 'treasury', - 'algorithm', - 'benchmark', - ]] = results[[ - 'treasury_period_return', - 'algorithm_period_return', - 'benchmark_period_return', - ]] - - ax5 = plt.subplot(615, sharex=ax1) - results[[ - 'treasury', - 'algorithm', - 'benchmark', - ]].plot(ax=ax5) - ax5.set_ylabel('Percent Change') - - ax6 = plt.subplot(616, sharex=ax1) - results[['volume']].plot(ax=ax6) - ax6.set_ylabel('Volume (mCoins/5min)') - - plt.legend(loc=3) - - # Show the plot. - plt.gcf().set_size_inches(18, 8) - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=10000, - data_frequency='daily', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace='buy_and_hodl', - base_currency='usd', - start=pd.to_datetime('2015-03-01', utc=True), - end=pd.to_datetime('2017-10-31', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/buy_and_hodl.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_buy_and_hodl.png @@ -278,166 +102,13 @@ one day prior to the current date. Dual Moving Average Crossover ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Source Code: `examples/dual_moving_average.py `_ - This strategy is covered in detail in the last part of `this tutorial `_. -.. code-block:: python +Source Code: `examples/dual_moving_average.py `_ - import numpy as np - import pandas as pd - from logbook import Logger - import matplotlib.pyplot as plt - - from catalyst import run_algorithm - from catalyst.api import (order, record, symbol, order_target_percent, - get_open_orders) - from catalyst.exchange.stats_utils import extract_transactions - - NAMESPACE = 'dual_moving_average' - log = Logger(NAMESPACE) - - def initialize(context): - context.i = 0 - context.asset = symbol('ltc_usd') - context.base_price = None - - - def handle_data(context, data): - # define the windows for the moving averages - short_window = 50 - long_window = 200 - - # Skip as many bars as long_window to properly compute the average - context.i += 1 - if context.i < long_window: - return - - # Compute moving averages calling data.history() for each - # moving average with the appropriate parameters. We choose to use - # minute bars for this simulation -> freq="1m" - # Returns a pandas dataframe. - short_mavg = data.history(context.asset, 'price', - bar_count=short_window, frequency="1m").mean() - long_mavg = data.history(context.asset, 'price', - bar_count=long_window, frequency="1m").mean() - - # Let's keep the price of our asset in a more handy variable - price = data.current(context.asset, 'price') - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - price_change = (price - context.base_price) / context.base_price - - # Save values for later inspection - record(price=price, - cash=context.portfolio.cash, - price_change=price_change, - short_mavg=short_mavg, - long_mavg=long_mavg) - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.asset): - return - - # We check what's our position on our portfolio and trade accordingly - pos_amount = context.portfolio.positions[context.asset].amount - - # Trading logic - if short_mavg > long_mavg and pos_amount == 0: - # we buy 100% of our portfolio for this asset - order_target_percent(context.asset, 1) - elif short_mavg < long_mavg and pos_amount > 0: - # we sell all our positions for this asset - order_target_percent(context.asset, 0) - - - def analyze(context, perf): - - # Get the base_currency that was passed as a parameter to the simulation - base_currency = context.exchanges.values()[0].base_currency.upper() - - # First chart: Plot portfolio value using base_currency - ax1 = plt.subplot(411) - perf.loc[:, ['portfolio_value']].plot(ax=ax1) - ax1.legend_.remove() - ax1.set_ylabel('Portfolio Value\n({})'.format(base_currency)) - start, end = ax1.get_ylim() - ax1.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Second chart: Plot asset price, moving averages and buys/sells - ax2 = plt.subplot(412, sharex=ax1) - perf.loc[:, ['price','short_mavg','long_mavg']].plot(ax=ax2, label='Price') - ax2.legend_.remove() - ax2.set_ylabel('{asset}\n({base})'.format( - asset = context.asset.symbol, - base = base_currency - )) - start, end = ax2.get_ylim() - ax2.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index, 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index, 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - # Third chart: Compare percentage change between our portfolio - # and the price of the asset - ax3 = plt.subplot(413, sharex=ax1) - perf.loc[:, ['algorithm_period_return', 'price_change']].plot(ax=ax3) - ax3.legend_.remove() - ax3.set_ylabel('Percent Change') - start, end = ax3.get_ylim() - ax3.yaxis.set_ticks(np.arange(start, end, (end-start)/5)) - - # Fourth chart: Plot our cash - ax4 = plt.subplot(414, sharex=ax1) - perf.cash.plot(ax=ax4) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - start, end = ax4.get_ylim() - ax4.yaxis.set_ticks(np.arange(0, end, end/5)) - - plt.show() - - - if __name__ == '__main__': - run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-9-22', utc=True), - end=pd.to_datetime('2017-9-23', utc=True), - ) +.. literalinclude:: ../../catalyst/examples/dual_moving_average.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/tutorial_dual_moving_average.png @@ -447,8 +118,6 @@ This strategy is covered in detail in the last part of Mean Reversion Algorithm ~~~~~~~~~~~~~~~~~~~~~~~~ -Source code: `examples/mean_reversion_simple.py `_ - This algorithm is based on a simple momentum strategy. When the cryptoasset goes up quickly, we're going to buy; when it goes down quickly, we're going to sell. Hopefully, we'll ride the waves. @@ -469,284 +138,10 @@ lines 218-245, so in order to run the algorithm we just type: python mean_reversion_simple.py -.. code-block:: python +Source code: `examples/mean_reversion_simple.py `_ - import os - import tempfile - import time - - import numpy as np - import pandas as pd - import talib - from logbook import Logger - - from catalyst import run_algorithm - from catalyst.api import symbol, record, order_target_percent, get_open_orders - from catalyst.exchange.stats_utils import extract_transactions - # We give a name to the algorithm which Catalyst will use to persist its state. - # In this example, Catalyst will create the `.catalyst/data/live_algos` - # directory. If we stop and start the algorithm, Catalyst will resume its - # state using the files included in the folder. - from catalyst.utils.paths import ensure_directory - - NAMESPACE = 'mean_reversion_simple' - log = Logger(NAMESPACE) - - - # To run an algorithm in Catalyst, you need two functions: initialize and - # handle_data. - - def initialize(context): - # This initialize function sets any data or variables that you'll use in - # your algorithm. For instance, you'll want to define the trading pair (or - # trading pairs) you want to backtest. You'll also want to define any - # parameters or values you're going to use. - - # In our example, we're looking at Neo in USD. - context.neo_eth = symbol('neo_usd') - context.base_price = None - context.current_day = None - - context.RSI_OVERSOLD = 30 - context.RSI_OVERBOUGHT = 80 - context.CANDLE_SIZE = '15T' - - context.start_time = time.time() - - - def handle_data(context, data): - # This handle_data function is where the real work is done. Our data is - # minute-level tick data, and each minute is called a frame. This function - # runs on each frame of the data. - - # We flag the first period of each day. - # Since cryptocurrencies trade 24/7 the `before_trading_starts` handle - # would only execute once. This method works with minute and daily - # frequencies. - today = data.current_dt.floor('1D') - if today != context.current_day: - context.traded_today = False - context.current_day = today - - # We're computing the volume-weighted-average-price of the security - # defined above, in the context.neo_eth variable. For this example, we're - # using three bars on the 15 min bars. - - # The frequency attribute determine the bar size. We use this convention - # for the frequency alias: - # http://pandas.pydata.org/pandas-docs/stable/timeseries.html#offset-aliases - prices = data.history( - context.neo_eth, - fields='close', - bar_count=50, - frequency=context.CANDLE_SIZE - ) - - # Ta-lib calculates various technical indicator based on price and - # volume arrays. - - # In this example, we are comp - rsi = talib.RSI(prices.values, timeperiod=14) - - # We need a variable for the current price of the security to compare to - # the average. Since we are requesting two fields, data.current() - # returns a DataFrame with - current = data.current(context.neo_eth, fields=['close', 'volume']) - price = current['close'] - - # If base_price is not set, we use the current value. This is the - # price at the first bar which we reference to calculate price_change. - if context.base_price is None: - context.base_price = price - - price_change = (price - context.base_price) / context.base_price - cash = context.portfolio.cash - - # Now that we've collected all current data for this frame, we use - # the record() method to save it. This data will be available as - # a parameter of the analyze() function for further analysis. - record( - price=price, - volume=current['volume'], - price_change=price_change, - rsi=rsi[-1], - cash=cash - ) - - # We are trying to avoid over-trading by limiting our trades to - # one per day. - if context.traded_today: - return - - # Since we are using limit orders, some orders may not execute immediately - # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.neo_eth) - if len(orders) > 0: - return - - # Exit if we cannot trade - if not data.can_trade(context.neo_eth): - return - - # Another powerful built-in feature of the Catalyst backtester is the - # portfolio object. The portfolio object tracks your positions, cash, - # cost basis of specific holdings, and more. In this line, we calculate - # how long or short our position is at this minute. - pos_amount = context.portfolio.positions[context.neo_eth].amount - - if rsi[-1] <= context.RSI_OVERSOLD and pos_amount == 0: - log.info( - '{}: buying - price: {}, rsi: {}'.format( - data.current_dt, price, rsi[-1] - ) - ) - # Set a style for limit orders, - limit_price = price * 1.005 - order_target_percent( - context.neo_eth, 1, limit_price=limit_price - ) - context.traded_today = True - - elif rsi[-1] >= context.RSI_OVERBOUGHT and pos_amount > 0: - log.info( - '{}: selling - price: {}, rsi: {}'.format( - data.current_dt, price, rsi[-1] - ) - ) - limit_price = price * 0.995 - order_target_percent( - context.neo_eth, 0, limit_price=limit_price - ) - context.traded_today = True - - - def analyze(context=None, perf=None): - end = time.time() - log.info('elapsed time: {}'.format(end - context.start_time)) - - import matplotlib.pyplot as plt - # The base currency of the algo exchange - base_currency = context.exchanges.values()[0].base_currency.upper() - - # Plot the portfolio value over time. - ax1 = plt.subplot(611) - perf.loc[:, 'portfolio_value'].plot(ax=ax1) - ax1.set_ylabel('Portfolio\nValue\n({})'.format(base_currency)) - - # Plot the price increase or decrease over time. - ax2 = plt.subplot(612, sharex=ax1) - perf.loc[:, 'price'].plot(ax=ax2, label='Price') - - ax2.set_ylabel('{asset}\n({base})'.format( - asset=context.neo_eth.symbol, base=base_currency - )) - - transaction_df = extract_transactions(perf) - if not transaction_df.empty: - buy_df = transaction_df[transaction_df['amount'] > 0] - sell_df = transaction_df[transaction_df['amount'] < 0] - ax2.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index.floor('1 min'), 'price'], - marker='^', - s=100, - c='green', - label='' - ) - ax2.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index.floor('1 min'), 'price'], - marker='v', - s=100, - c='red', - label='' - ) - - ax4 = plt.subplot(613, sharex=ax1) - perf.loc[:, 'cash'].plot( - ax=ax4, label='Base Currency ({})'.format(base_currency) - ) - ax4.set_ylabel('Cash\n({})'.format(base_currency)) - - perf['algorithm'] = perf.loc[:, 'algorithm_period_return'] - - ax5 = plt.subplot(614, sharex=ax1) - perf.loc[:, ['algorithm', 'price_change']].plot(ax=ax5) - ax5.set_ylabel('Percent\nChange') - - ax6 = plt.subplot(615, sharex=ax1) - perf.loc[:, 'rsi'].plot(ax=ax6, label='RSI') - ax6.set_ylabel('RSI') - ax6.axhline(context.RSI_OVERBOUGHT, color='darkgoldenrod') - ax6.axhline(context.RSI_OVERSOLD, color='darkgoldenrod') - - if not transaction_df.empty: - ax6.scatter( - buy_df.index.to_pydatetime(), - perf.loc[buy_df.index.floor('1 min'), 'rsi'], - marker='^', - s=100, - c='green', - label='' - ) - ax6.scatter( - sell_df.index.to_pydatetime(), - perf.loc[sell_df.index.floor('1 min'), 'rsi'], - marker='v', - s=100, - c='red', - label='' - ) - plt.legend(loc=3) - start, end = ax6.get_ylim() - ax6.yaxis.set_ticks(np.arange(0, end, end/5)) - - # Show the plot. - plt.gcf().set_size_inches(18, 8) - plt.show() - pass - - - if __name__ == '__main__': - # The execution mode: backtest or live - MODE = 'backtest' - - if MODE == 'backtest': - folder = os.path.join( - tempfile.gettempdir(), 'catalyst', NAMESPACE - ) - ensure_directory(folder) - - timestr = time.strftime('%Y%m%d-%H%M%S') - out = os.path.join(folder, '{}.p'.format(timestr)) - # catalyst run -f catalyst/examples/mean_reversion_simple.py -x bitfinex -s 2017-10-1 -e 2017-11-10 -c usdt -n mean-reversion --data-frequency minute --capital-base 10000 - run_algorithm( - capital_base=10000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - start=pd.to_datetime('2017-10-01', utc=True), - end=pd.to_datetime('2017-11-10', utc=True), - output=out - ) - log.info('saved perf stats: {}'.format(out)) - - elif MODE == 'live': - run_algorithm( - capital_base=0.5, - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bittrex', - live=True, - algo_namespace=NAMESPACE, - base_currency='usd', - live_graph=False - ) +.. literalinclude:: ../../catalyst/examples/mean_reversion_simple.py + :language: python .. image:: https://s3.amazonaws.com/enigmaco-docs/github.io/example_mean_reversion_simple.png @@ -763,8 +158,6 @@ strategy. Simple Universe ~~~~~~~~~~~~~~~ -Source code: `examples/simple_universe.py `_ - This example aims to provide an easy way for users to learn how to collect data from any given exchange and select a subset of the available currency pairs for trading. You simply need to specify the exchange and @@ -791,142 +184,10 @@ of the file: catalyst ingest-exchange -x bitfinex -f minute -.. code-block:: bash - - python simple_universe.py - -Credits: This code was originally submitted by `Abner Ayala-Acevedo -`_. Thank you! - -.. code-block:: python - - from datetime import timedelta - - import numpy as np - import pandas as pd - - from catalyst import run_algorithm - from catalyst.exchange.utils.exchange_utils import get_exchange_symbols - from catalyst.api import (symbols, ) - - - def initialize(context): - context.i = -1 # minute counter - context.exchange = context.exchanges.values()[0].name.lower() - context.base_currency = context.exchanges.values()[0].base_currency.lower() - - - def handle_data(context, data): - context.i += 1 - lookback_days = 7 # 7 days - - # current date & time in each iteration formatted into a string - now = data.current_dt - date, time = now.strftime('%Y-%m-%d %H:%M:%S').split(' ') - lookback_date = now - timedelta(days=lookback_days) - # keep only the date as a string, discard the time - lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0] - - one_day_in_minutes = 1440 # 60 * 24 assumes data_frequency='minute' - # update universe everyday at midnight - if not context.i % one_day_in_minutes: - context.universe = universe(context, lookback_date, date) - - # get data every 30 minutes - minutes = 30 - # get lookback_days of history data: that is 'lookback' number of bins - lookback = one_day_in_minutes / minutes * lookback_days - if not context.i % minutes and context.universe: - # we iterate for every pair in the current universe - for coin in context.coins: - pair = str(coin.symbol) - - # Get 30 minute interval OHLCV data. This is the standard data - # required for candlestick or indicators/signals. Return Pandas - # DataFrames. 30T means 30-minute re-sampling of one minute data. - # Adjust it to your desired time interval as needed. - opened = fill(data.history(coin, 'open', - bar_count=lookback, frequency='30T')).values - high = fill(data.history(coin, 'high', - bar_count=lookback, frequency='30T')).values - low = fill(data.history(coin, 'low', - bar_count=lookback, frequency='30T')).values - close = fill(data.history(coin, 'price', - bar_count=lookback, frequency='30T')).values - volume = fill(data.history(coin, 'volume', - bar_count=lookback, frequency='30T')).values - - # close[-1] is the last value in the set, which is the equivalent - # to current price (as in the most recent value) - # displays the minute price for each pair every 30 minutes - print('{now}: {pair} -\tO:{o},\tH:{h},\tL:{c},\tC{c},\tV:{v}'.format( - now=now, - pair=pair, - o=opened[-1], - h=high[-1], - l=low[-1], - c=close[-1], - v=volume[-1], - )) - - # ------------------------------------------------------------- - # --------------- Insert Your Strategy Here ------------------- - # ------------------------------------------------------------- - - - def analyze(context=None, results=None): - pass - - - # Get the universe for a given exchange and a given base_currency market - # Example: Poloniex BTC Market - def universe(context, lookback_date, current_date): - # get all the pairs for the given exchange - json_symbols = get_exchange_symbols(context.exchange) - # convert into a DataFrame for easier processing - df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str) - df['base_currency'] = df.apply(lambda row: row.symbol.split('_')[1],axis=1) - df['market_currency'] = df.apply(lambda row: row.symbol.split('_')[0],axis=1) - - # Filter all the pairs to get only the ones for a given base_currency - df = df[df['base_currency'] == context.base_currency] - - # Filter all the pairs to ensure that pair existed in the current date range - df = df[df.start_date < lookback_date] - df = df[df.end_daily >= current_date] - context.coins = symbols(*df.symbol) # convert all the pairs to symbols - - return df.symbol.tolist() - - - # Replace all NA, NAN or infinite values with its nearest value - def fill(series): - if isinstance(series, pd.Series): - return series.replace([np.inf, -np.inf], np.nan).ffill().bfill() - elif isinstance(series, np.ndarray): - return pd.Series(series).replace( - [np.inf, -np.inf], np.nan - ).ffill().bfill().values - else: - return series - - - if __name__ == '__main__': - start_date = pd.to_datetime('2017-11-10', utc=True) - end_date = pd.to_datetime('2017-11-13', utc=True) - - performance = run_algorithm(start=start_date, end=end_date, - capital_base=100.0, # amount of base_currency - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - data_frequency='minute', - base_currency='btc', - live=False, - live_graph=False, - algo_namespace='simple_universe') +Source code: `examples/simple_universe.py `_ +.. literalinclude:: ../../catalyst/examples/simple_universe.py + :language: python .. _portfolio_optimization: @@ -940,135 +201,10 @@ use 180 days of historical data and rebalance every 30 days. This code was used in writting the following article: `Markowitz Portfolio Optimization for Cryptocurrencies `_. -.. code-block:: python +Source code: `examples/simple_universe.py `_ - ''' - You can run this code using the Python interpreter: - - $ python portfolio_optimization.py - ''' - - from __future__ import division - import os - import pytz - import numpy as np - import pandas as pd - from scipy.optimize import minimize - import matplotlib.pyplot as plt - from datetime import datetime - - from catalyst.api import record, symbol, symbols, order_target_percent - from catalyst.utils.run_algo import run_algorithm - - np.set_printoptions(threshold='nan', suppress=True) - - - def initialize(context): - # Portfolio assets list - context.assets = symbols('btc_usdt', 'eth_usdt', 'ltc_usdt', 'dash_usdt', - 'xmr_usdt') - context.nassets = len(context.assets) - # Set the time window that will be used to compute expected return - # and asset correlations - context.window = 180 - # Set the number of days between each portfolio rebalancing - context.rebalance_period = 30 - context.i = 0 - - - def handle_data(context, data): - # Only rebalance at the beggining of the algorithm execution and - # every multiple of the rebalance period - if context.i == 0 or context.i%context.rebalance_period == 0: - n = context.window - prices = data.history(context.assets, fields='price', - bar_count=n+1, frequency='1d') - pr = np.asmatrix(prices) - t_prices = prices.iloc[1:n+1] - t_val = t_prices.values - tminus_prices = prices.iloc[0:n] - tminus_val = tminus_prices.values - # Compute daily returns (r) - r = np.asmatrix(t_val/tminus_val-1) - # Compute the expected returns of each asset with the average - # daily return for the selected time window - m = np.asmatrix(np.mean(r, axis=0)) - # ### - stds = np.std(r, axis=0) - # Compute excess returns matrix (xr) - xr = r - m - # Matrix algebra to get variance-covariance matrix - cov_m = np.dot(np.transpose(xr),xr)/n - # Compute asset correlation matrix (informative only) - corr_m = cov_m/np.dot(np.transpose(stds),stds) - - # Define portfolio optimization parameters - n_portfolios = 50000 - results_array = np.zeros((3+context.nassets,n_portfolios)) - for p in xrange(n_portfolios): - weights = np.random.random(context.nassets) - weights /= np.sum(weights) - w = np.asmatrix(weights) - p_r = np.sum(np.dot(w,np.transpose(m)))*365 - p_std = np.sqrt(np.dot(np.dot(w,cov_m),np.transpose(w)))*np.sqrt(365) - - #store results in results array - results_array[0,p] = p_r - results_array[1,p] = p_std - #store Sharpe Ratio (return / volatility) - risk free rate element - #excluded for simplicity - results_array[2,p] = results_array[0,p] / results_array[1,p] - i = 0 - for iw in weights: - results_array[3+i,p] = weights[i] - i += 1 - - #convert results array to Pandas DataFrame - results_frame = pd.DataFrame(np.transpose(results_array), - columns=['r','stdev','sharpe']+context.assets) - #locate position of portfolio with highest Sharpe Ratio - max_sharpe_port = results_frame.iloc[results_frame['sharpe'].idxmax()] - #locate positon of portfolio with minimum standard deviation - min_vol_port = results_frame.iloc[results_frame['stdev'].idxmin()] - - #order optimal weights for each asset - for asset in context.assets: - if data.can_trade(asset): - order_target_percent(asset, max_sharpe_port[asset]) - - #create scatter plot coloured by Sharpe Ratio - plt.scatter(results_frame.stdev,results_frame.r,c=results_frame.sharpe,cmap='RdYlGn') - plt.xlabel('Volatility') - plt.ylabel('Returns') - plt.colorbar() - #plot red star to highlight position of portfolio with highest Sharpe Ratio - plt.scatter(max_sharpe_port[1],max_sharpe_port[0],marker='o',color='b',s=200) - #plot green star to highlight position of minimum variance portfolio - plt.show() - print(max_sharpe_port) - record(pr=pr,r=r, m=m, stds=stds ,max_sharpe_port=max_sharpe_port, corr_m=corr_m) - context.i += 1 - - - def analyze(context=None, results=None): - # Form DataFrame with selected data - data = results[['pr','r','m','stds','max_sharpe_port','corr_m','portfolio_value']] - - # Save results in CSV file - filename = os.path.splitext(os.path.basename(__file__))[0] - data.to_csv(filename + '.csv') - - - # Bitcoin data is available from 2015-3-2. Dates vary for other tokens. - start = datetime(2017, 1, 1, 0, 0, 0, 0, pytz.utc) - end = datetime(2017, 8, 16, 0, 0, 0, 0, pytz.utc) - results = run_algorithm(initialize=initialize, - handle_data=handle_data, - analyze=analyze, - start=start, - end=end, - exchange_name='poloniex', - capital_base=100000, ) +.. literalinclude:: ../../catalyst/examples/portfolio_optimization.py + :language: python .. image:: https://cdn-images-1.medium.com/max/1600/0*EjjiKZHlYF3sn7yQ. :align: center From d40585f56e286d923479a7b160dcac463e7720fa Mon Sep 17 00:00:00 2001 From: embaral Date: Thu, 22 Feb 2018 14:34:00 +0200 Subject: [PATCH 03/12] DOC: added an option "catalyst live --help" to the documentation. --- docs/source/live-trading.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/live-trading.rst b/docs/source/live-trading.rst index a2898d61..df449c05 100644 --- a/docs/source/live-trading.rst +++ b/docs/source/live-trading.rst @@ -175,5 +175,14 @@ Here is the breakdown of the new arguments: simulated in Catalyst instead of processed on the exchange. It defaults to ``True``. + +The `catalyst live` command offers additional parameters. +You can learn more by running the following from the command line: + +.. code-block:: bash + +catalyst live --help + + Here is a complete algorithm for reference: `Buy Low and Sell High `_ From 127878413e1a439ceb43864b5988de45a4f0a318 Mon Sep 17 00:00:00 2001 From: embaral Date: Thu, 22 Feb 2018 14:45:14 +0200 Subject: [PATCH 04/12] DOC: added an option "catalyst live --help" to the documentation. --- docs/source/live-trading.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/live-trading.rst b/docs/source/live-trading.rst index e65796f5..7d5f2394 100644 --- a/docs/source/live-trading.rst +++ b/docs/source/live-trading.rst @@ -190,7 +190,7 @@ You can learn more by running the following from the command line: .. code-block:: bash -catalyst live --help + catalyst live --help Here is a complete algorithm for reference: From fea2ed104e8aaf9796f815be984ef95a04ccbb5f Mon Sep 17 00:00:00 2001 From: lenak25 Date: Thu, 22 Feb 2018 16:50:27 +0200 Subject: [PATCH 05/12] BUG: fix issue #236: handle properly empty candles received from exchanges --- catalyst/exchange/exchange.py | 48 ++++++++++++++++++++--------------- catalyst/support/issue_236.py | 32 +++++++++++++++++++++++ 2 files changed, 59 insertions(+), 21 deletions(-) create mode 100644 catalyst/support/issue_236.py diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 3b57256b..a0a247fb 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -1,4 +1,5 @@ import abc +import pytz from abc import ABCMeta, abstractmethod, abstractproperty from datetime import timedelta from time import sleep @@ -514,32 +515,37 @@ class Exchange: series = dict() for asset in candles: - first_candle = candles[asset][0] - asset_series = self.get_series_from_candles( - candles=candles[asset], - start_dt=first_candle['last_traded'], - end_dt=end_dt, - data_frequency=frequency, - field=field, - ) - - delta_candle_size = candle_size * 60 if unit == 'H' else candle_size - # Checking to make sure that the dates match - delta = get_delta(delta_candle_size, data_frequency) - adj_end_dt = end_dt - delta - last_traded = asset_series.index[-1] - - if last_traded < adj_end_dt: - raise LastCandleTooEarlyError( - last_traded=last_traded, - end_dt=adj_end_dt, - exchange=self.name, + if candles[asset]: + first_candle = candles[asset][0] + asset_series = self.get_series_from_candles( + candles=candles[asset], + start_dt=first_candle['last_traded'], + end_dt=end_dt, + data_frequency=frequency, + field=field, ) + delta_candle_size = candle_size * 60 if unit == 'H' else candle_size + # Checking to make sure that the dates match + delta = get_delta(delta_candle_size, data_frequency) + adj_end_dt = end_dt - delta + last_traded = asset_series.index[-1] + + if last_traded < adj_end_dt: + raise LastCandleTooEarlyError( + last_traded=last_traded, + end_dt=adj_end_dt, + exchange=self.name, + ) + else: # empty candle received + # because other assets are tz-aware, we need its tz to be set as well + asset_series = pd.Series([], index=pd.DatetimeIndex([], tz=pytz.utc)) + + series[asset] = asset_series df = pd.DataFrame(series) - df.dropna(inplace=True) + #df.dropna(inplace=True) # commented out due to issue 236 return df diff --git a/catalyst/support/issue_236.py b/catalyst/support/issue_236.py new file mode 100644 index 00000000..c3a437a9 --- /dev/null +++ b/catalyst/support/issue_236.py @@ -0,0 +1,32 @@ +from catalyst.api import symbol +from catalyst.utils.run_algo import run_algorithm + +coins = ['dash', 'btc', 'dash', 'etc', 'eth', 'ltc', 'nxt', 'rep', 'str', 'xmr', 'xrp', 'zec'] +symbols = None + + +def initialize(context): + pass + + +def _handle_data(context, data): + global symbols + if symbols is None: symbols = [symbol(c + '_usdt') for c in coins] + + print'getting history for: %s' % [s.symbol for s in symbols] + history = data.history(symbols, + ['close', 'volume'], + bar_count=1, # EXCEPTION, Change to 2 + frequency='5T') + #print 'history: %s' % history.shape + +run_algorithm(initialize=initialize, + handle_data=_handle_data, + analyze=lambda _, results: True, + exchange_name='poloniex', + base_currency='usdt', + algo_namespace='issue-236', + live=True, + data_frequency='minute', + capital_base=3000, + simulate_orders=True) \ No newline at end of file From 50310576f90bfb87a789114685ac0177ec794650 Mon Sep 17 00:00:00 2001 From: lenak25 Date: Thu, 22 Feb 2018 17:42:14 +0200 Subject: [PATCH 06/12] BUG:fix an issue with wrong timestamps seen at tests.exchange.test_suites.test_suite_bundle.TestSuiteBundle#test_validate_bundles (which issue #230 uncovered) --- catalyst/exchange/utils/exchange_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index fb24f1c8..4f82356c 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -722,12 +722,14 @@ def get_candles_df(candles, field, freq, bar_count, end_dt, values = [candle[field] for candle in candles[asset]] series = pd.Series(values, index=dates) + """ series = series.reindex( periods, method='ffill', fill_value=previous_value, ) series.sort_index(inplace=True) + """ all_series[asset] = series df = pd.DataFrame(all_series) From bfd7e4b2dd98a9878c20291469894bb0f07347fc Mon Sep 17 00:00:00 2001 From: Victor Date: Wed, 14 Feb 2018 09:27:53 -0700 Subject: [PATCH 07/12] Update python3.6-environment.yml --- etc/python3.6-environment.yml | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml index 446198e0..0e5eabbf 100644 --- a/etc/python3.6-environment.yml +++ b/etc/python3.6-environment.yml @@ -2,28 +2,28 @@ name: catalyst channels: - defaults dependencies: -- ca-certificates=2017.08.26=ha1e5d58_0 -- certifi=2018.1.18=py36_0 -- intel-openmp=2018.0.0=h8158457_8 -- libcxx=4.0.1=h579ed51_0 -- libcxxabi=4.0.1=hebd6815_0 -- libedit=3.1=hb4e282d_0 -- libffi=3.2.1=h475c297_4 -- libgfortran=3.0.1=h93005f0_2 -- mkl=2018.0.1=hfbd8650_4 -- ncurses=6.0=hd04f020_2 -- numpy=1.14.0=py36h8a80b8c_1 -- openssl=1.0.2n=hdbc3d79_0 -- pip=9.0.1=py36h1555ced_4 -- python=3.6.4=hc167b69_1 -- readline=7.0=hc1231fa_4 -- scipy=1.0.0=py36h1de22e9_0 +- ca-certificates=2017.08.26 +- certifi=2018.1.18 +- intel-openmp=2018.0.0 +- libcxx=4.0.1 +- libcxxabi=4.0.1 +- libedit=3.1 +- libffi=3.2.1 +- libgfortran=3.0.1 +- mkl=2018.0.1 +- ncurses=6.0 +- numpy=1.14.0 +- openssl=1.0.2n +- pip=9.0.1 +- python=3.6.4 +- readline=7.0 +- scipy=1.0.0 - setuptools=38.4.0=py36_0 -- sqlite=3.22.0=h3efe00b_0 -- tk=8.6.7=h35a86e2_3 -- wheel=0.30.0=py36h5eb2c71_1 -- xz=5.2.3=h0278029_2 -- zlib=1.2.11=hf3cbc9b_2 +- sqlite=3.22.0 +- tk=8.6.7 +- wheel=0.30.0 +- xz=5.2.3 +- zlib=1.2.11 - pip: - aiodns==1.1.1 - aiohttp==3.0.1 From 8fe3ab344e89dade33ba306c15708aa8cf3f5f96 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 22 Feb 2018 12:54:29 -0700 Subject: [PATCH 08/12] MAINT: conda environment updates --- etc/python2.7-environment.yml | 2 ++ etc/python3.6-environment.yml | 9 ++------- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/etc/python2.7-environment.yml b/etc/python2.7-environment.yml index 2037515b..b2e2486c 100644 --- a/etc/python2.7-environment.yml +++ b/etc/python2.7-environment.yml @@ -1,9 +1,11 @@ name: catalyst channels: - defaults +- conda-forge dependencies: - certifi=2016.2.28=py27_0 - mkl=2017.0.3 +- matplotlib=2.1.2=py36_0 - numpy=1.13.1=py27_0 - openssl=1.0.2l - pip=9.0.1=py27_1 diff --git a/etc/python3.6-environment.yml b/etc/python3.6-environment.yml index 0e5eabbf..c93f3c82 100644 --- a/etc/python3.6-environment.yml +++ b/etc/python3.6-environment.yml @@ -1,22 +1,17 @@ name: catalyst channels: - defaults +- conda-forge dependencies: - ca-certificates=2017.08.26 - certifi=2018.1.18 - intel-openmp=2018.0.0 -- libcxx=4.0.1 -- libcxxabi=4.0.1 -- libedit=3.1 -- libffi=3.2.1 -- libgfortran=3.0.1 - mkl=2018.0.1 -- ncurses=6.0 - numpy=1.14.0 - openssl=1.0.2n +- matplotlib=2.1.2=py36_0 - pip=9.0.1 - python=3.6.4 -- readline=7.0 - scipy=1.0.0 - setuptools=38.4.0=py36_0 - sqlite=3.22.0 From b4bd557273dd253b9adf4e02d68389a082d81f8f Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Fri, 23 Feb 2018 00:38:51 +0200 Subject: [PATCH 09/12] BUG: fixes for issues #204 #237 -modified parameters for cancel_orders -update portfolio after any change in the orders before sync --- catalyst/examples/dual_moving_average.py | 7 +++---- catalyst/exchange/ccxt/ccxt_exchange.py | 6 ++++-- catalyst/exchange/exchange.py | 6 ++++-- catalyst/exchange/exchange_algorithm.py | 24 ++++++++++++++++++------ 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index 00eb01af..88a6fad5 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -4,8 +4,7 @@ import pandas as pd from logbook import Logger from catalyst import run_algorithm -from catalyst.api import (record, symbol, order_target_percent, - get_open_orders) +from catalyst.api import (record, symbol, order_target_percent,) from catalyst.exchange.utils.stats_utils import extract_transactions NAMESPACE = 'dual_moving_average' @@ -21,7 +20,7 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages short_window = 2 - long_window = 3 + long_window = 5 # Skip as many bars as long_window to properly compute the average context.i += 1 @@ -63,7 +62,7 @@ def handle_data(context, data): # Since we are using limit orders, some orders may not execute immediately # we wait until all orders are executed before considering more trades. - orders = get_open_orders(context.asset) + orders = context.blotter.open_orders if len(orders) > 0: return diff --git a/catalyst/exchange/ccxt/ccxt_exchange.py b/catalyst/exchange/ccxt/ccxt_exchange.py index 40f4b4d3..875661e9 100644 --- a/catalyst/exchange/ccxt/ccxt_exchange.py +++ b/catalyst/exchange/ccxt/ccxt_exchange.py @@ -980,7 +980,8 @@ class CCXT(Exchange): ) raise ExchangeRequestError(error=e) - def cancel_order(self, order_param, asset_or_symbol=None): + def cancel_order(self, order_param, + asset_or_symbol=None, params={}): order_id = order_param.id \ if isinstance(order_param, Order) else order_param @@ -992,7 +993,8 @@ class CCXT(Exchange): try: symbol = self.get_symbol(asset_or_symbol) \ if asset_or_symbol is not None else None - self.api.cancel_order(id=order_id, symbol=symbol) + self.api.cancel_order(id=order_id, + symbol=symbol, params= params) except (ExchangeError, NetworkError) as e: log.warn( diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index a0a247fb..20cbe967 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -705,7 +705,7 @@ class Exchange: balances=balances, amount=cash, ) - if is_lower and not open_orders: + if is_lower: raise NotEnoughCashError( currency=self.base_currency, exchange=self.name, @@ -932,7 +932,8 @@ class Exchange: """ @abstractmethod - def cancel_order(self, order_param, symbol_or_asset=None): + def cancel_order(self, order_param, + symbol_or_asset=None, params={}): """Cancel an open order. Parameters @@ -941,6 +942,7 @@ class Exchange: The order_id or order object to cancel. symbol_or_asset: str|TradingPair The catalyst symbol, some exchanges need this + params: """ pass diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index c827af4a..5bf103fa 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -388,6 +388,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.stats_minutes = 1 self._last_orders = [] + self._last_open_orders = [] self.trading_client = None super(ExchangeTradingAlgorithmLive, self).__init__(*args, **kwargs) @@ -791,12 +792,17 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.nullify_frame_stats(now=data.current_dt) self.performance_needs_update = False - orders = list(self.perf_tracker.todays_performance.orders_by_id.keys()) - if orders != self._last_orders: + last_orders_list = list(self.blotter.orders.keys()) + open_orders_list = list(self.blotter.open_orders.keys()) + + if last_orders_list != self._last_orders or \ + open_orders_list != self._last_open_orders: self.performance_needs_update = True - # Saving current orders to detect changes in the next frame - self._last_orders = copy.deepcopy(orders) + # Saving current order positions + # to detect changes in the next frame + self._last_orders = copy.deepcopy(last_orders_list) + self._last_open_orders = copy.deepcopy(open_orders_list) if self.performance_needs_update: self.perf_tracker.update_performance() @@ -1011,13 +1017,19 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): args=(order_id,)) @api_method - def cancel_order(self, order_param, exchange_name): + def cancel_order(self, order_param, exchange_name, + symbol=None, params={}): """Cancel an open order. Parameters ---------- order_param : str or Order The order_id or order object to cancel. + + exchange_name: name of exchange from + which you want to cancel the order + symbol: + params: """ exchange = self.exchanges[exchange_name] @@ -1031,4 +1043,4 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): sleeptime=self.attempts['retry_sleeptime'], retry_exceptions=(ExchangeRequestError,), cleanup=lambda: log.warn('cancelling order again.'), - args=(order_id,)) + args=(order_id, symbol, params)) From 388535b09c64c95f565397b2736b8576a7c73cab Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 22 Feb 2018 22:09:51 -0700 Subject: [PATCH 10/12] BUG: reverts changed introduced in 00f232e2d7426b8593295ee3e4e6488a39a4866b --- catalyst/examples/dual_moving_average.py | 39 +++++++++--------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/catalyst/examples/dual_moving_average.py b/catalyst/examples/dual_moving_average.py index 88a6fad5..f11e6b77 100644 --- a/catalyst/examples/dual_moving_average.py +++ b/catalyst/examples/dual_moving_average.py @@ -19,8 +19,8 @@ def initialize(context): def handle_data(context, data): # define the windows for the moving averages - short_window = 2 - long_window = 5 + short_window = 50 + long_window = 200 # Skip as many bars as long_window to properly compute the average context.i += 1 @@ -149,27 +149,16 @@ def analyze(context, perf): if __name__ == '__main__': + run_algorithm( - capital_base=1000, - data_frequency='minute', - initialize=initialize, - handle_data=handle_data, - analyze=analyze, - exchange_name='bitfinex', - algo_namespace=NAMESPACE, - base_currency='usd', - simulate_orders=True, - live=True, - ) - # run_algorithm( - # capital_base=1000, - # data_frequency='minute', - # initialize=initialize, - # handle_data=handle_data, - # analyze=analyze, - # exchange_name='bitfinex', - # algo_namespace=NAMESPACE, - # base_currency='usd', - # start=pd.to_datetime('2017-9-22', utc=True), - # end=pd.to_datetime('2017-9-23', utc=True), - # ) + capital_base=1000, + data_frequency='minute', + initialize=initialize, + handle_data=handle_data, + analyze=analyze, + exchange_name='bitfinex', + algo_namespace=NAMESPACE, + base_currency='usd', + start=pd.to_datetime('2017-9-22', utc=True), + end=pd.to_datetime('2017-9-23', utc=True), + ) From 8587fee0cea0c8bcd51ccbfc36fc68fce2a6b2f1 Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Sun, 25 Feb 2018 11:29:26 +0200 Subject: [PATCH 11/12] BUG: revert previous changes #249 --- catalyst/exchange/exchange.py | 8 ++------ catalyst/exchange/exchange_algorithm.py | 19 +++++++++---------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 20cbe967..310d12fc 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -662,20 +662,16 @@ class Exchange: return df - def _check_low_balance(self, currency, balances, amount, open_orders=None): + def _check_low_balance(self, currency, balances, amount): free = balances[currency]['free'] if currency in balances else 0.0 - if open_orders: - # TODO: make sure that this works - free += sum([order.amount for order in open_orders]) - if free < amount: return free, True else: return free, False - def sync_positions(self, positions, open_orders=None, cash=None, + def sync_positions(self, positions, cash=None, check_balances=False): """ Update the portfolio cash and position balances based on the diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 5bf103fa..98966a82 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -638,6 +638,15 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if asset_orders: orders += asset_orders + required_cash = self.portfolio.cash if not orders else None + cash, positions_value = exchange.sync_positions( + positions=exchange_positions, + check_balances=check_balances, + cash=required_cash, + ) + total_cash += cash + total_positions_value += positions_value + # Applying modifications to the original positions for position in exchange_positions: tracker.update_position( @@ -647,16 +656,6 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): last_sale_price=position.last_sale_price, ) - required_cash = self.portfolio.cash if not orders else None - cash, positions_value = exchange.sync_positions( - positions=exchange_positions, - open_orders=orders, - check_balances=check_balances, - cash=required_cash, - ) - total_cash += cash - total_positions_value += positions_value - if not check_balances: total_cash = self.portfolio.cash From e5870ea60a9b99aae256fa006620d84063a25dad Mon Sep 17 00:00:00 2001 From: AvishaiW Date: Mon, 26 Feb 2018 09:21:00 +0200 Subject: [PATCH 12/12] BUG: fix #252 #253 and split state into paper and live --- catalyst/exchange/exchange_algorithm.py | 51 ++++++++++++++++------- catalyst/exchange/utils/exchange_utils.py | 23 +++++----- catalyst/exchange/utils/stats_utils.py | 6 ++- 3 files changed, 53 insertions(+), 27 deletions(-) diff --git a/catalyst/exchange/exchange_algorithm.py b/catalyst/exchange/exchange_algorithm.py index 98966a82..b9c319f1 100644 --- a/catalyst/exchange/exchange_algorithm.py +++ b/catalyst/exchange/exchange_algorithm.py @@ -375,13 +375,23 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): if error: log.warning(error) - self.pnl_stats = get_algo_df(self.algo_namespace, 'pnl_stats') + # in order to save paper & live files separately + self.mode_name = 'paper' if kwargs['simulate_orders'] else 'live' - self.custom_signals_stats = \ - get_algo_df(self.algo_namespace, 'custom_signals_stats') + self.pnl_stats = get_algo_df( + self.algo_namespace, + 'pnl_stats_{}'.format(self.mode_name), + ) - self.exposure_stats = \ - get_algo_df(self.algo_namespace, 'exposure_stats') + self.custom_signals_stats = get_algo_df( + self.algo_namespace, + 'custom_signals_stats_{}'.format(self.mode_name) + ) + + self.exposure_stats = get_algo_df( + self.algo_namespace, + 'exposure_stats_{}'.format(self.mode_name) + ) self.is_running = True @@ -515,7 +525,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): """ self.state = get_algo_object( algo_name=self.algo_namespace, - key='context.state', + key='context.state_{}'.format(self.mode_name), ) if self.state is None: self.state = {} @@ -538,7 +548,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): # Unpacking the perf_tracker and positions if available cum_perf = get_algo_object( algo_name=self.algo_namespace, - key='cumulative_performance', + key='cumulative_performance_{}'.format(self.mode_name), ) if cum_perf is not None: tracker.cumulative_performance = cum_perf @@ -549,7 +559,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): todays_perf = get_algo_object( algo_name=self.algo_namespace, key=today.strftime('%Y-%m-%d'), - rel_path='daily_performance', + rel_path='daily_performance_{}'.format(self.mode_name), ) if todays_perf is not None: # Ensure single common position tracker @@ -686,7 +696,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): ) self.pnl_stats = pd.concat([self.pnl_stats, df]) - save_algo_df(self.algo_namespace, 'pnl_stats', self.pnl_stats) + save_algo_df( + self.algo_namespace, + 'pnl_stats_{}'.format(self.mode_name), + self.pnl_stats, + ) def add_custom_signals_stats(self, period_stats): """ @@ -707,8 +721,11 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): ) self.custom_signals_stats = pd.concat([self.custom_signals_stats, df]) - save_algo_df(self.algo_namespace, 'custom_signals_stats', - self.custom_signals_stats) + save_algo_df( + self.algo_namespace, + 'custom_signals_stats_{}'.format(self.mode_name), + self.custom_signals_stats, + ) def add_exposure_stats(self, period_stats): """ @@ -735,7 +752,9 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): self.exposure_stats = pd.concat([self.exposure_stats, df]) save_algo_df( - self.algo_namespace, 'exposure_stats', self.exposure_stats + self.algo_namespace, + 'exposure_stats_{}'.format(self.mode_name), + self.exposure_stats ) def nullify_frame_stats(self, now): @@ -759,6 +778,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): obj=self.frame_stats, rel_path='frame_stats' ) + error = remove_old_files( algo_name=self.algo_namespace, today=now, @@ -843,7 +863,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): log.debug('saving cumulative performance object') save_algo_object( algo_name=self.algo_namespace, - key='cumulative_performance', + key='cumulative_performance_{}'.format(self.mode_name), obj=self.perf_tracker.cumulative_performance, ) log.debug('saving todays performance object') @@ -851,12 +871,12 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): algo_name=self.algo_namespace, key=today.strftime('%Y-%m-%d'), obj=self.perf_tracker.todays_performance, - rel_path='daily_performance' + rel_path='daily_performance_{}'.format(self.mode_name) ) log.debug('saving context.state object') save_algo_object( algo_name=self.algo_namespace, - key='context.state', + key='context.state_{}'.format(self.mode_name), obj=self.state) def _process_stats(self, data): @@ -912,6 +932,7 @@ class ExchangeTradingAlgorithmLive(ExchangeTradingAlgorithmBase): csv_bytes = stats_to_algo_folder( stats=self.frame_stats, algo_namespace=self.algo_namespace, + folder_name='stats_{}'.format(self.mode_name), recorded_cols=recorded_cols, ) except Exception as e: diff --git a/catalyst/exchange/utils/exchange_utils.py b/catalyst/exchange/utils/exchange_utils.py index 4f82356c..5c40a26d 100644 --- a/catalyst/exchange/utils/exchange_utils.py +++ b/catalyst/exchange/utils/exchange_utils.py @@ -420,7 +420,7 @@ def clear_frame_stats_directory(algo_name): return error -def remove_old_files(algo_name, today, rel_path): +def remove_old_files(algo_name, today, rel_path, environ=None): """ remove old files from a directory to avoid overloading the disk @@ -430,27 +430,30 @@ def remove_old_files(algo_name, today, rel_path): algo_name: str today: Timestamp rel_path: str + environ: Returns ------- error: str """ + error = None - algo_folder = get_algo_folder(algo_name) + algo_folder = get_algo_folder(algo_name, environ) folder = os.path.join(algo_folder, rel_path) + ensure_directory(folder) # run on all files in the folder for f in os.listdir(folder): - creation_unix = os.path.getctime(f) - creation_time = pd.to_datetime(creation_unix, unit='s', ) + try: + creation_unix = os.path.getctime(os.path.join(folder, f)) + creation_time = pd.to_datetime(creation_unix, unit='s', ) - # if the file is older than 30 days erase it - if today - pd.DateOffset(30) > creation_time: - try: - os.unlink(f) - except OSError: - error = 'unable to erase files in {}'.format(folder) + # if the file is older than 30 days erase it + if today - pd.DateOffset(30) > creation_time: + os.unlink(f) + except OSError: + error = 'unable to erase files in {}'.format(folder) return error diff --git a/catalyst/exchange/utils/stats_utils.py b/catalyst/exchange/utils/stats_utils.py index 6e2aab0b..3db79d3b 100644 --- a/catalyst/exchange/utils/stats_utils.py +++ b/catalyst/exchange/utils/stats_utils.py @@ -396,7 +396,8 @@ def email_error(algo_name, dt, e, environ=None): )}) -def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): +def stats_to_algo_folder(stats, algo_namespace, + folder_name, recorded_cols=None): """ Saves the performance stats to the algo local folder. @@ -404,6 +405,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): ---------- stats: list[Object] algo_namespace: str + folder_name: str recorded_cols: list[str] Returns @@ -416,7 +418,7 @@ def stats_to_algo_folder(stats, algo_namespace, recorded_cols=None): timestr = time.strftime('%Y%m%d') folder = get_algo_folder(algo_namespace) - stats_folder = os.path.join(folder, 'stats') + stats_folder = os.path.join(folder, folder_name) ensure_directory(stats_folder) filename = os.path.join(stats_folder, '{}.csv'.format(timestr))