From 069639a325ff8bf26ed3e8561a3503e133498516 Mon Sep 17 00:00:00 2001 From: Frederic Fortier Date: Mon, 21 Aug 2017 14:30:55 -0400 Subject: [PATCH] Bug fixes and housekeeping --- catalyst/exchange/algorithm_exchange.py | 35 ++++++----------- catalyst/exchange/bitfinex.py | 46 ++++++++++------------- catalyst/exchange/data_portal_exchange.py | 5 +-- catalyst/exchange/exchange.py | 23 ++++++------ catalyst/exchange/exchange_clock.py | 7 +--- catalyst/exchange/exchange_order.py | 39 ------------------- catalyst/exchange/exchange_portfolio.py | 12 +++--- catalyst/exchange/exchange_utils.py | 9 +++-- 8 files changed, 58 insertions(+), 118 deletions(-) delete mode 100644 catalyst/exchange/exchange_order.py diff --git a/catalyst/exchange/algorithm_exchange.py b/catalyst/exchange/algorithm_exchange.py index a7198227..8de05186 100644 --- a/catalyst/exchange/algorithm_exchange.py +++ b/catalyst/exchange/algorithm_exchange.py @@ -10,31 +10,30 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from datetime import time, timedelta -from time import sleep -import logbook import signal import sys +from datetime import timedelta +from time import sleep + +import logbook import pandas as pd import catalyst.protocol as zp from catalyst.algorithm import TradingAlgorithm -from catalyst.exchange.exchange_clock import ExchangeClock -from catalyst.gens.tradesimulation import AlgorithmSimulator from catalyst.errors import OrderInBeforeTradingStart -from catalyst.utils.input_validation import error_keywords -from catalyst.utils.api_support import ( - api_method, - disallowed_in_before_trading_start) - -from catalyst.utils.calendars.trading_calendar import days_at_time +from catalyst.exchange.exchange_clock import ExchangeClock from catalyst.exchange.exchange_errors import ( ExchangeRequestError, ExchangePortfolioDataError, ExchangeTransactionError ) -from catalyst.finance.performance.period import calc_period_stats from catalyst.exchange.exchange_utils import save_algo_object, get_algo_object +from catalyst.finance.performance.period import calc_period_stats +from catalyst.gens.tradesimulation import AlgorithmSimulator +from catalyst.utils.api_support import ( + api_method, + disallowed_in_before_trading_start) +from catalyst.utils.input_validation import error_keywords log = logbook.Logger("ExchangeTradingAlgorithm") @@ -104,20 +103,13 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): execution_closes = \ self.trading_calendar.execution_time_from_close(market_closes) - # FIXME generalize these values - before_trading_start_minutes = days_at_time( - self.sim_params.sessions, - time(8, 45), - "US/Eastern" - ) - signal.signal(signal.SIGINT, self.signal_handler) return ExchangeClock( self.sim_params.sessions, execution_opens, execution_closes, - before_trading_start_minutes, + None, minute_emission=minutely_emission, time_skew=self.exchange.time_skew ) @@ -136,9 +128,6 @@ class ExchangeTradingAlgorithm(TradingAlgorithm): universe_func=self._calculate_universe ) - # self.perf_tracker.cumulative_performance.keep_transactions = True - # self.perf_tracker.cumulative_performance.keep_orders = True - return self.trading_client.transform() def updated_portfolio(self): diff --git a/catalyst/exchange/bitfinex.py b/catalyst/exchange/bitfinex.py index 211876e6..81808bd1 100644 --- a/catalyst/exchange/bitfinex.py +++ b/catalyst/exchange/bitfinex.py @@ -1,34 +1,30 @@ -import re -import pytz -import six import base64 import hashlib import hmac import json +import re import time -import requests + import pandas as pd -from datetime import timedelta, datetime -from catalyst.protocol import Portfolio, Account +import pytz +import requests +import six +from catalyst.assets._assets import Asset +from logbook import Logger + # from websocket import create_connection from catalyst.exchange.exchange import Exchange -from logbook import Logger -from catalyst.assets._assets import Asset -from catalyst.finance.order import ORDER_STATUS -from catalyst.exchange.exchange_order import ExchangeOrder -from catalyst.finance.execution import (MarketOrder, - LimitOrder, - StopOrder, - StopLimitOrder) -from catalyst.exchange.exchange_portfolio import ExchangePortfolio -from catalyst.exchange.exchange_utils import get_exchange_symbols -from catalyst.errors import ( - IncompatibleHistoryFrequency, -) from catalyst.exchange.exchange_errors import ( ExchangeRequestError, InvalidHistoryFrequencyError ) +from catalyst.exchange.exchange_utils import get_exchange_symbols +from catalyst.finance.execution import (MarketOrder, + LimitOrder, + StopOrder, + StopLimitOrder) +from catalyst.finance.order import Order, ORDER_STATUS +from catalyst.protocol import Account # Trying to account for REST api instability # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request @@ -159,7 +155,7 @@ class Bitfinex(Exchange): # TODO: zipline likes rounded dates to match statistics, is this ok? date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) date = pytz.utc.localize(date) - order = ExchangeOrder( + order = Order( dt=date, asset=self.assets[order_status['symbol']], amount=amount, @@ -170,9 +166,8 @@ class Bitfinex(Exchange): commission=commission ) order.status = status - order.executed_price = executed_price - return order + return order, executed_price def update_portfolio(self): """ @@ -494,7 +489,7 @@ class Bitfinex(Exchange): ) order_id = exchange_order['id'] - order = ExchangeOrder( + order = Order( dt=date, asset=asset, amount=amount, @@ -540,8 +535,7 @@ class Bitfinex(Exchange): orders = list() for order_status in order_statuses: - # TODO: filter by asset - order = self._create_order(order_status) + order, = self._create_order(order_status) if asset is None or asset == order.sid: orders.append(order) @@ -584,7 +578,7 @@ class Bitfinex(Exchange): The order_id or order object to cancel. """ order_id = order_param.id \ - if isinstance(order_param, ExchangeOrder) else order_param + if isinstance(order_param, Order) else order_param try: response = self._request('order/cancel', {'order_id': order_id}) diff --git a/catalyst/exchange/data_portal_exchange.py b/catalyst/exchange/data_portal_exchange.py index a747123f..77a7cb76 100644 --- a/catalyst/exchange/data_portal_exchange.py +++ b/catalyst/exchange/data_portal_exchange.py @@ -11,12 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pandas as pd - from time import sleep -from catalyst.data.data_portal import DataPortal from logbook import Logger + +from catalyst.data.data_portal import DataPortal from catalyst.exchange.exchange_errors import ( ExchangeRequestError, ExchangeBarDataError diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index c7d888d3..4e06717a 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -1,20 +1,18 @@ import abc -import json -from abc import ABCMeta, abstractmethod, abstractproperty import collections +from abc import ABCMeta, abstractmethod, abstractproperty +from datetime import timedelta import pandas as pd from catalyst.assets._assets import Asset -from catalyst.finance.order import ORDER_STATUS -from catalyst.finance.transaction import Transaction -from catalyst.data.data_portal import BASE_FIELDS +from logbook import Logger +from catalyst.data.data_portal import BASE_FIELDS from catalyst.errors import ( - MultipleSymbolsFound, SymbolNotFound, ) -from datetime import timedelta -from logbook import Logger +from catalyst.finance.order import ORDER_STATUS +from catalyst.finance.transaction import Transaction log = Logger('Exchange') @@ -90,7 +88,7 @@ class Exchange: if self.portfolio.open_orders: for order_id in list(self.portfolio.open_orders): log.debug('found open order: {}'.format(order_id)) - order = self.get_order(order_id) + order, executed_price = self.get_order(order_id) log.debug('got updated order {}'.format(order)) if order.status == ORDER_STATUS.FILLED: @@ -98,14 +96,13 @@ class Exchange: asset=order.asset, amount=order.amount, dt=pd.Timestamp.utcnow(), - price=order.executed_price, + price=executed_price, order_id=order.id, commission=order.commission ) transactions.append(transaction) - # TODO: use the transaction to pass the executed price - self.portfolio.execute_order(order) + self.portfolio.execute_order(order, transaction) elif order.status == ORDER_STATUS.CANCELLED: self.portfolio.remove_order(order) else: @@ -219,6 +216,8 @@ class Exchange: ------- order : Order The order object. + execution_price: float + The execution price per share of the order """ pass diff --git a/catalyst/exchange/exchange_clock.py b/catalyst/exchange/exchange_clock.py index 4698297c..0c528bc0 100644 --- a/catalyst/exchange/exchange_clock.py +++ b/catalyst/exchange/exchange_clock.py @@ -13,16 +13,13 @@ from time import sleep -from logbook import Logger import pandas as pd - from catalyst.gens.sim_engine import ( BAR, SESSION_START, - SESSION_END, - MINUTE_END, - BEFORE_TRADING_START_BAR + MINUTE_END ) +from logbook import Logger log = Logger('ExchangeClock') diff --git a/catalyst/exchange/exchange_order.py b/catalyst/exchange/exchange_order.py deleted file mode 100644 index 5ebc667c..00000000 --- a/catalyst/exchange/exchange_order.py +++ /dev/null @@ -1,39 +0,0 @@ -import math - -import catalyst.protocol as zp -from catalyst.assets import Asset -from catalyst.finance.order import Order, ORDER_STATUS -from catalyst.utils.input_validation import expect_types - - -class ExchangeOrder(Order): - @expect_types(asset=Asset) - def __init__(self, dt, asset, amount, stop=None, limit=None, filled=0, - commission=0, id=None, executed_price=None): - """ - @dt - datetime.datetime that the order was placed - @asset - asset for the order. - @amount - the number of shares to buy/sell - a positive sign indicates a buy - a negative sign indicates a sell - @filled - how many shares of the order have been filled so far - """ - - # get a string representation of the uuid. - self.id = self.make_id() if id is None else id - self.dt = dt - self.reason = None - self.created = dt - self.asset = asset - self.amount = amount - self.filled = filled - self.commission = commission - self._status = ORDER_STATUS.OPEN - self.stop = stop - self.limit = limit - self.stop_reached = False - self.limit_reached = False - self.direction = math.copysign(1, self.amount) - self.type = zp.DATASOURCE_TYPE.ORDER - self.broker_order_id = None - self.executed_price = executed_price diff --git a/catalyst/exchange/exchange_portfolio.py b/catalyst/exchange/exchange_portfolio.py index 1accc45f..f0e076ce 100644 --- a/catalyst/exchange/exchange_portfolio.py +++ b/catalyst/exchange/exchange_portfolio.py @@ -1,8 +1,8 @@ import numpy as np -from catalyst.protocol import Portfolio, Positions, Position -from catalyst.finance.order import BUY from logbook import Logger +from catalyst.protocol import Portfolio, Positions, Position + log = Logger('ExchangePortfolio') @@ -36,7 +36,7 @@ class ExchangePortfolio(Portfolio): order_position.amount += order.amount log.debug('open order added to portfolio') - def execute_order(self, order): + def execute_order(self, order, transaction): log.debug('executing order {}'.format(order.id)) del self.open_orders[order.id] @@ -48,16 +48,16 @@ class ExchangePortfolio(Portfolio): 'Trying to execute order for a position not held: %s' % order.id ) - self.capital_used += order.amount * order.executed_price + self.capital_used += order.amount * transaction.price if order.amount > 0: if order_position.cost_basis > 0: order_position.cost_basis = np.average( - [order_position.cost_basis, order.executed_price], + [order_position.cost_basis, transaction.price], weights=[order_position.amount, order.amount] ) else: - order_position.cost_basis = order.executed_price + order_position.cost_basis = transaction.price log.debug('updated portfolio with executed order') diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index d6a69802..d1aeacd5 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -1,10 +1,11 @@ -import os -import urllib import json +import os import pickle -from catalyst.utils.paths import data_root, ensure_directory +import urllib + from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \ - ExchangeSymbolsNotFound, AlgoPickleNotFound + ExchangeSymbolsNotFound +from catalyst.utils.paths import data_root, ensure_directory SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ 'live-trading/catalyst/exchange/symbols/{exchange}.json'