diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index d81d49b7..5e35346d 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -327,7 +327,7 @@ class Bittrex(Exchange): try: end_daily = cached_symbols[exchange_symbol]['end_daily'] except KeyError as e: - end_daily ='N/A' + end_daily = 'N/A' try: end_minute = cached_symbols[exchange_symbol]['end_minute'] @@ -336,13 +336,44 @@ class Bittrex(Exchange): symbol_map[exchange_symbol] = dict( symbol=symbol, - start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d"), - end_daily = end_daily, - end_minute = end_minute, + start_date=pd.to_datetime(market['Created'], + utc=True).strftime("%Y-%m-%d"), + end_daily=end_daily, + end_minute=end_minute, ) - if(filename is None): + if (filename is None): filename = get_exchange_symbols_filename(self.name) - with open(filename,'w') as f: - json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':')) + with open(filename, 'w') as f: + json.dump(symbol_map, f, sort_keys=True, indent=2, + separators=(',', ':')) + + def get_orderbook(self, asset, type='all'): + if type == 'all': + type = 'both' + elif type == 'bid': + type = 'buy' + elif type == 'ask': + type = 'sell' + else: + raise ValueError('invalid type') + + exchange_symbol = asset.exchange_symbol + data = self.api.getorderbook(market=exchange_symbol, type=type) + + result = dict() + for exchange_type in data: + if exchange_type == 'buy': + type = 'bid' + elif exchange_type == 'sell': + type = 'ask' + + result[type] = [] + for entry in data[exchange_type]: + result[type].append(dict( + rate=entry['Rate'], + quantity=entry['Quantity'] + )) + + return result diff --git a/catalyst/exchange/bundle_utils.py b/catalyst/exchange/bundle_utils.py index 14a0870c..d38f3a8c 100644 --- a/catalyst/exchange/bundle_utils.py +++ b/catalyst/exchange/bundle_utils.py @@ -220,6 +220,21 @@ def get_ffill_candles(candles, bar_count, end_dt, data_frequency, return all_dates, all_candles +def get_trailing_candles_dt(asset, start_dt, end_dt, data_frequency): + missing_start = None + + if asset.end_minute is not None and start_dt < asset.end_minute: + if asset.end_minute < end_dt: + delta = get_delta(1, data_frequency) + + missing_start = asset.end_minute + delta + + else: + missing_start = start_dt + + return missing_start + + def range_in_bundle(asset, start_dt, end_dt, reader): """ Evaluate whether price data of an asset is included has been ingested in diff --git a/catalyst/exchange/exchange.py b/catalyst/exchange/exchange.py index 882aeb2b..665dd0ce 100644 --- a/catalyst/exchange/exchange.py +++ b/catalyst/exchange/exchange.py @@ -12,8 +12,8 @@ from logbook import Logger from catalyst.data.data_portal import BASE_FIELDS from catalyst.exchange import bundle_utils -from catalyst.exchange.bundle_utils import get_ffill_candles, get_start_dt, \ - get_delta +from catalyst.exchange.bundle_utils import get_start_dt, \ + get_delta, get_trailing_candles_dt from catalyst.exchange.exchange_bundle import ExchangeBundle from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \ InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange, \ @@ -24,6 +24,7 @@ from catalyst.exchange.exchange_portfolio import ExchangePortfolio from catalyst.exchange.exchange_utils import get_exchange_symbols from catalyst.finance.order import ORDER_STATUS from catalyst.finance.transaction import Transaction +from catalyst.utils.deprecate import deprecated log = Logger('Exchange') @@ -418,6 +419,7 @@ class Exchange: return value + @deprecated def get_history(self, assets, end_dt, bar_count, data_frequency, fallback_exchange=True): """ @@ -449,6 +451,7 @@ class Exchange: ) return candles + @deprecated def get_asset_history(self, asset, end, bar_count, data_frequency, fallback_exchange=True): """ @@ -586,14 +589,32 @@ class Exchange: writer = bundle.get_writer(start_dt, end_dt, data_frequency) for asset in missing_assets: - bundle.ingest_chunk( - bar_count=adj_bar_count, - end_dt=end_dt, - data_frequency=data_frequency, + # TODO: use this only for data too recent to be in a bundle + trailing_candles_dt = get_trailing_candles_dt( asset=asset, - writer=writer + start_dt=start_dt, + end_dt=end_dt, + data_frequency=data_frequency ) + if trailing_candles_dt is not None: + # The get_history method supports multiple asset + candles = self.get_candles( + data_frequency=data_frequency, + assets=[asset], + bar_count=bar_count, + start_dt=trailing_candles_dt, + end_dt=end_dt + ) + + bundle.ingest_candles( + candles=candles, + bar_count=adj_bar_count, + end_dt=end_dt, + data_frequency=data_frequency, + writer=writer + ) + reader = bundle.get_reader(data_frequency) values = reader.load_raw_arrays( fields=[field], @@ -892,3 +913,11 @@ class Exchange: :return: """ pass + + @abc.abstractmethod + def get_orderbook(self): + """ + Retrieve the account parameters. + :return: + """ + pass diff --git a/catalyst/exchange/exchange_bundle.py b/catalyst/exchange/exchange_bundle.py index e6163580..bff7bebb 100644 --- a/catalyst/exchange/exchange_bundle.py +++ b/catalyst/exchange/exchange_bundle.py @@ -254,7 +254,7 @@ class ExchangeBundle: invalid_data_behavior='raise' ) - def ingest_chunk(self, bar_count, end_dt, data_frequency, asset, + def ingest_candles(self, candles, bar_count, end_dt, data_frequency, writer, previous_candle=dict()): """ Retrieve the specified OHLCV chunk and write it to the bundle @@ -268,14 +268,6 @@ class ExchangeBundle: :return: """ - # The get_history method supports multiple asset - candles = self.exchange.get_history( - assets=[asset], - end_dt=end_dt, - bar_count=bar_count, - data_frequency=data_frequency, - fallback_exchange=False - ) num_candles = 0 data = [] diff --git a/catalyst/exchange/poloniex/poloniex.py b/catalyst/exchange/poloniex/poloniex.py index 151fecd8..b8a6e522 100644 --- a/catalyst/exchange/poloniex/poloniex.py +++ b/catalyst/exchange/poloniex/poloniex.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd import pytz import requests -#import six +# import six from six import iteritems from catalyst.assets._assets import TradingPair from logbook import Logger @@ -32,7 +32,6 @@ from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ download_exchange_symbols from catalyst.finance.transaction import Transaction - log = Logger('Poloniex') @@ -52,7 +51,6 @@ class Poloniex(Exchange): self.max_requests_per_minute = 20 self.request_cpt = dict() - def sanitize_curency_symbol(self, exchange_symbol): """ Helper method used to build the universal pair. @@ -63,28 +61,27 @@ class Poloniex(Exchange): """ return exchange_symbol.lower() - def _create_order(self, order_status): """ Create a Catalyst order object from the Exchange order dictionary :param order_status: :return: Order """ - #if order_status['is_cancelled']: + # if order_status['is_cancelled']: # status = ORDER_STATUS.CANCELLED - #elif not order_status['is_live']: + # elif not order_status['is_live']: # log.info('found executed order {}'.format(order_status)) # status = ORDER_STATUS.FILLED - #else: + # else: status = ORDER_STATUS.OPEN amount = float(order_status['amount']) - #filled = float(order_status['executed_amount']) + # filled = float(order_status['executed_amount']) filled = None if order_status['type'] == 'sell': amount = -amount - #filled = -filled + # filled = -filled price = float(order_status['rate']) order_type = order_status['type'] @@ -93,24 +90,25 @@ class Poloniex(Exchange): limit_price = None # TODO: is this comprehensive enough? - #if order_type.endswith('limit'): + # if order_type.endswith('limit'): # limit_price = price - #elif order_type.endswith('stop'): + # elif order_type.endswith('stop'): # stop_price = price - #executed_price = float(order_status['avg_execution_price']) + # executed_price = float(order_status['avg_execution_price']) executed_price = price # TODO: bitfinex does not specify comission. I could calculate it but not sure if it's worth it. commission = None - #date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) - #date = pytz.utc.localize(date) + # date = pd.Timestamp.utcfromtimestamp(float(order_status['timestamp'])) + # date = pytz.utc.localize(date) date = None order = Order( dt=date, - asset=self.assets[order_status['symbol']], # No such field in Poloniex + asset=self.assets[order_status['symbol']], + # No such field in Poloniex amount=amount, stop=stop_price, limit=limit_price, @@ -121,7 +119,6 @@ class Poloniex(Exchange): order.status = status return order, executed_price - def get_balances(self): log.debug('retrieving wallets balances') @@ -143,7 +140,6 @@ class Poloniex(Exchange): return std_balances - @property def account(self): account = Account() @@ -192,17 +188,18 @@ class Poloniex(Exchange): """ # TODO: use BcolzMinuteBarReader to read from cache - if(data_frequency == '5m' or data_frequency == 'minute'): #TODO: Polo does not have '1m' + if ( + data_frequency == '5m' or data_frequency == 'minute'): # TODO: Polo does not have '1m' frequency = 300 - elif(data_frequency == '15m'): + elif (data_frequency == '15m'): frequency = 900 - elif(data_frequency == '30m'): + elif (data_frequency == '30m'): frequency = 1800 - elif(data_frequency == '2h'): + elif (data_frequency == '2h'): frequency = 7200 - elif(data_frequency == '4h'): + elif (data_frequency == '4h'): frequency = 14400 - elif(data_frequency == '1D' or data_frequency == 'daily'): + elif (data_frequency == '1D' or data_frequency == 'daily'): frequency = 86400 else: raise InvalidHistoryFrequencyError( @@ -216,13 +213,14 @@ class Poloniex(Exchange): for asset in asset_list: end = int(time.time()) - if(bar_count is None): + if (bar_count is None): start = end - 2 * frequency else: start = end - bar_count * frequency - try: - response = self.api.returnchartdata(self.get_symbol(asset),frequency, start, end) + try: + response = self.api.returnchartdata(self.get_symbol(asset), + frequency, start, end) except Exception as e: raise ExchangeRequestError(error=e) @@ -240,7 +238,7 @@ class Poloniex(Exchange): close=np.float64(candle['close']), volume=np.float64(candle['volume']), price=np.float64(candle['close']), - last_traded=pd.Timestamp.utcfromtimestamp( candle['date'] ) + last_traded=pd.Timestamp.utcfromtimestamp(candle['date']) ) return ohlc @@ -257,7 +255,6 @@ class Poloniex(Exchange): return ohlc_map[assets] \ if isinstance(assets, TradingPair) else ohlc_map - def create_order(self, asset, amount, is_buy, style): """ Creating order on the exchange. @@ -270,14 +267,15 @@ class Poloniex(Exchange): """ exchange_symbol = self.get_symbol(asset) - if isinstance(style, ExchangeLimitOrder) or isinstance(style, ExchangeStopLimitOrder): + if isinstance(style, ExchangeLimitOrder) or isinstance(style, + ExchangeStopLimitOrder): if isinstance(style, ExchangeStopLimitOrder): log.warn('{} will ignore the stop price'.format(self.name)) price = style.get_limit_price(is_buy) try: - if(is_buy): + if (is_buy): response = self.api.buy(exchange_symbol, amount, price) else: response = self.api.sell(exchange_symbol, -amount, price) @@ -286,7 +284,7 @@ class Poloniex(Exchange): date = pd.Timestamp.utcnow() - if('orderNumber' in response): + if ('orderNumber' in response): order_id = str(response['orderNumber']) order = Order( dt=date, @@ -298,13 +296,14 @@ class Poloniex(Exchange): ) return order else: - log.warn('{} order failed: {}'.format('buy' if is_buy else 'sell', response['error'])) + log.warn( + '{} order failed: {}'.format('buy' if is_buy else 'sell', + response['error'])) return None else: raise InvalidOrderStyle(exchange=self.name, style=style.__class__.__name__) - def get_open_orders(self, asset='all'): """Retrieve all of the current open orders. @@ -331,7 +330,7 @@ class Poloniex(Exchange): """ try: - if(asset=='all'): + if (asset == 'all'): response = self.api.returnopenorders('all') else: response = self.api.returnopenorders(self.get_symbol(asset)) @@ -346,15 +345,15 @@ class Poloniex(Exchange): print(self.portfolio.open_orders) - #TODO: Need to handle openOrders for 'all' + # TODO: Need to handle openOrders for 'all' orders = list() for order_status in response: - order, executed_price = self._create_order(order_status) # will Throw error b/c Polo doesn't track order['symbol'] + order, executed_price = self._create_order( + order_status) # will Throw error b/c Polo doesn't track order['symbol'] if asset is None or asset == order.sid: orders.append(order) return orders - def get_order(self, order_id): """Lookup an order based on the order id returned from one of the @@ -387,11 +386,10 @@ class Poloniex(Exchange): raise ExchangeRequestError(error=e) for o in response: - if(int(o['orderNumber'])==int(order_id)): + if (int(o['orderNumber']) == int(order_id)): return order - + return None - def cancel_order(self, order_param): """Cancel an open order. @@ -402,7 +400,7 @@ class Poloniex(Exchange): The order_id or order object to cancel. """ - if(isinstance(order_param, Order)): + if (isinstance(order_param, Order)): order = order_param else: order = self._portfolio.open_orders[order_param] @@ -413,20 +411,20 @@ class Poloniex(Exchange): raise ExchangeRequestError(error=e) if 'error' in response: - log.info('Unable to cancel order {order_id} on exchange {exchange} {error}.'.format( - order_id=order.id, - exchange=self.name, - error=response['error'] + log.info( + 'Unable to cancel order {order_id} on exchange {exchange} {error}.'.format( + order_id=order.id, + exchange=self.name, + error=response['error'] )) - #raise OrderCancelError( + # raise OrderCancelError( # order_id=order.id, # exchange=self.name, # error=response['error'] - #) - - self.portfolio.remove_order(order) + # ) + self.portfolio.remove_order(order) def tickers(self, assets): """ @@ -435,7 +433,7 @@ class Poloniex(Exchange): :param assets: :return: - """ + """ symbols = self.get_symbols(assets) log.debug('fetching tickers {}'.format(symbols)) @@ -454,21 +452,21 @@ class Poloniex(Exchange): ticks = dict() for index, symbol in enumerate(symbols): - ticks[assets[index]] = dict( timestamp=pd.Timestamp.utcnow(), bid=float(response[symbol]['highestBid']), ask=float(response[symbol]['lowestAsk']), last_price=float(response[symbol]['last']), - low=float(response[symbol]['lowestAsk']), #TODO: Polo does not provide low - high=float(response[symbol]['highestBid']), #TODO: Polo does not provide high + low=float(response[symbol]['lowestAsk']), + # TODO: Polo does not provide low + high=float(response[symbol]['highestBid']), + # TODO: Polo does not provide high volume=float(response[symbol]['baseVolume']), ) log.debug('got tickers {}'.format(ticks)) return ticks - def generate_symbols_json(self, filename=None, source_dates=False): symbol_map = {} @@ -480,10 +478,11 @@ class Poloniex(Exchange): response = self.api.returnticker() for exchange_symbol in response: - base, market = self.sanitize_curency_symbol(exchange_symbol).split('_') - symbol = '{market}_{base}'.format( market=market, base=base ) + base, market = self.sanitize_curency_symbol(exchange_symbol).split( + '_') + symbol = '{market}_{base}'.format(market=market, base=base) - if(source_dates): + if (source_dates): start_date = self.get_symbol_start_date(exchange_symbol) else: try: @@ -494,7 +493,7 @@ class Poloniex(Exchange): try: end_daily = cached_symbols[exchange_symbol]['end_daily'] except KeyError as e: - end_daily ='N/A' + end_daily = 'N/A' try: end_minute = cached_symbols[exchange_symbol]['end_minute'] @@ -502,28 +501,28 @@ class Poloniex(Exchange): end_minute = 'N/A' symbol_map[exchange_symbol] = dict( - symbol = symbol, - start_date = start_date, - end_daily = end_daily, - end_minute = end_minute, + symbol=symbol, + start_date=start_date, + end_daily=end_daily, + end_minute=end_minute, ) - if(filename is None): + if (filename is None): filename = get_exchange_symbols_filename(self.name) - with open(filename,'w') as f: - json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':')) + with open(filename, 'w') as f: + json.dump(symbol_map, f, sort_keys=True, indent=2, + separators=(',', ':')) def get_symbol_start_date(self, symbol): try: - r = self.api.returnchartdata(symbol,86400,pd.to_datetime('2010-1-1').value // 10 ** 9) + r = self.api.returnchartdata(symbol, 86400, pd.to_datetime( + '2010-1-1').value // 10 ** 9) except Exception as e: raise ExchangeRequestError(error=e) return time.strftime('%Y-%m-%d', time.gmtime(int(r[0]['date']))) - - def check_open_orders(self): """ Need to override this function for Poloniex: @@ -549,22 +548,23 @@ class Poloniex(Exchange): except Exception as e: raise ExchangeRequestError(error=e) - if(order_open): + if (order_open): delta = pd.Timestamp.utcnow() - order.dt log.info( 'order {order_id} still open after {delta}'.format( order_id=order_id, - delta=delta ) - ) + delta=delta) + ) try: response = self.api.returnordertrades(order_id) except Exception as e: raise ExchangeRequestError(error=e) - if('error' in response): - if(not order_open): - raise OrphanOrderReverseError(order_id=order_id, exchange=self.name) + if ('error' in response): + if (not order_open): + raise OrphanOrderReverseError(order_id=order_id, + exchange=self.name) else: for tx in response: """ @@ -576,25 +576,29 @@ class Poloniex(Exchange): When an order if fully filled, we flush the dict of transactions associated with that order. """ - if(not filter(lambda item: item['order_id'] == tx['tradeID'], self.transactions[order_id])): - log.debug('Got new transaction for order {}: amount {}, price {}'.format( - order_id, tx['amount'], tx['rate'])) - tx['amount']=float(tx['amount']) - if(tx['type']=='sell'): + if (not filter( + lambda item: item['order_id'] == tx['tradeID'], + self.transactions[order_id])): + log.debug( + 'Got new transaction for order {}: amount {}, price {}'.format( + order_id, tx['amount'], tx['rate'])) + tx['amount'] = float(tx['amount']) + if (tx['type'] == 'sell'): tx['amount'] = -tx['amount'] transaction = Transaction( asset=order.asset, amount=tx['amount'], dt=pd.to_datetime(tx['date'], utc=True), price=float(tx['rate']), - order_id=tx['tradeID'], # it's a misnomer, but keeping it for compatibility + order_id=tx['tradeID'], + # it's a misnomer, but keeping it for compatibility commission=float(tx['fee']) ) self.transactions[order_id].append(transaction) self.portfolio.execute_transaction(transaction) transactions.append(transaction) - if(not order_open): + if (not order_open): """ Since transactions have been executed individually the only thing left to do is remove them from list of open_orders @@ -603,3 +607,25 @@ class Poloniex(Exchange): del self.transactions[order_id] return transactions + + def get_orderbook(self, asset, type='all'): + exchange_symbol = asset.exchange_symbol + data = self.api.returnOrderBook(market=exchange_symbol) + + result = dict() + for exchange_type in data: + if exchange_type == 'bids': + type = 'bid' + elif exchange_type == 'asks': + type = 'ask' + else: + continue + + result[type] = [] + for entry in data[exchange_type]: + if len(entry) == 2: + result[type].append(dict( + rate=float(entry[0]), + quantity=float(entry[1]) + )) + return result diff --git a/tests/exchange/test_bitfinex.py b/tests/exchange/test_bitfinex.py index c8f969fd..dda5bab9 100644 --- a/tests/exchange/test_bitfinex.py +++ b/tests/exchange/test_bitfinex.py @@ -56,8 +56,8 @@ class BitfinexTestCase(BaseExchangeTestCase): def test_tickers(self): log.info('retrieving tickers') tickers = self.exchange.tickers([ - self.exchange.get_asset('eth_usd'), - self.exchange.get_asset('btc_usd') + self.exchange.get_asset('eth_btc'), + self.exchange.get_asset('etc_btc') ]) pass diff --git a/tests/exchange/test_bittrex.py b/tests/exchange/test_bittrex.py index 5d90d660..f1becbcc 100644 --- a/tests/exchange/test_bittrex.py +++ b/tests/exchange/test_bittrex.py @@ -67,8 +67,8 @@ class BittrexTestCase(BaseExchangeTestCase): def test_tickers(self): log.info('retrieving tickers') tickers = self.exchange.tickers([ - self.exchange.get_asset('ubq_btc'), - self.exchange.get_asset('neo_btc') + self.exchange.get_asset('eth_btc'), + self.exchange.get_asset('etc_btc') ]) assert len(tickers) == 2 pass @@ -81,3 +81,9 @@ class BittrexTestCase(BaseExchangeTestCase): def test_get_account(self): log.info('testing account data') pass + + def test_orderbook(self): + log.info('testing order book for bittrex') + asset = self.exchange.get_asset('eth_btc') + orderbook = self.exchange.get_orderbook(asset) + pass diff --git a/tests/exchange/test_poloniex.py b/tests/exchange/test_poloniex.py new file mode 100644 index 00000000..1da3313d --- /dev/null +++ b/tests/exchange/test_poloniex.py @@ -0,0 +1,90 @@ +from catalyst.exchange.bittrex.bittrex import Bittrex +from catalyst.exchange.poloniex.poloniex import Poloniex +from catalyst.finance.order import Order +from base import BaseExchangeTestCase +from logbook import Logger +from catalyst.exchange.exchange_utils import get_exchange_auth + +log = Logger('test_poloniex') + + +class PoloniexTestCase(BaseExchangeTestCase): + @classmethod + def setup(self): + print ('creating poloniex object') + auth = get_exchange_auth('poloniex') + self.exchange = Poloniex( + key=auth['key'], + secret=auth['secret'], + base_currency='btc' + ) + + def test_order(self): + log.info('creating order') + asset = self.exchange.get_asset('neo_btc') + order_id = self.exchange.order( + asset=asset, + limit_price=0.0005, + amount=1, + ) + log.info('order created {}'.format(order_id)) + assert order_id is not None + pass + + def test_open_orders(self): + log.info('retrieving open orders') + asset = self.exchange.get_asset('neo_btc') + orders = self.exchange.get_open_orders(asset) + pass + + def test_get_order(self): + log.info('retrieving order') + order = self.exchange.get_order( + u'2c584020-9caf-4af5-bde0-332c0bba17e2') + assert isinstance(order, Order) + pass + + def test_cancel_order(self, ): + log.info('cancel order') + self.exchange.cancel_order(u'dc7bcca2-5219-4145-8848-8a593d2a72f9') + pass + + def test_get_candles(self): + log.info('retrieving candles') + ohlcv_neo = self.exchange.get_candles( + data_frequency='5m', + assets=self.exchange.get_asset('neo_btc') + ) + ohlcv_neo_ubq = self.exchange.get_candles( + data_frequency='5m', + assets=[ + self.exchange.get_asset('neo_btc'), + self.exchange.get_asset('ubq_btc') + ], + bar_count=14 + ) + pass + + def test_tickers(self): + log.info('retrieving tickers') + tickers = self.exchange.tickers([ + self.exchange.get_asset('eth_btc'), + self.exchange.get_asset('etc_btc') + ]) + assert len(tickers) == 2 + pass + + def test_get_balances(self): + log.info('testing wallet balances') + balances = self.exchange.get_balances() + pass + + def test_get_account(self): + log.info('testing account data') + pass + + def test_orderbook(self): + log.info('testing order book for bittrex') + asset = self.exchange.get_asset('eth_btc') + orderbook = self.exchange.get_orderbook(asset) + pass