diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index 6c23fc93..d4d368a9 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -585,9 +585,21 @@ class Bitfinex(Exchange): except KeyError as e: start_date = time.strftime('%Y-%m-%d') + try: + end_daily = cached_symbols[symbol]['end_daily'] + except KeyError as e: + end_daily ='N/A' + + try: + end_minute = cached_symbols[symbol]['end_minute'] + except KeyError as e: + end_minute = 'N/A' + symbol_map[symbol]= dict( - symbol = symbol[:-3]+'_'+symbol[-3:], - start_date = start_date + symbol = symbol[:-3]+'_'+symbol[-3:], + start_date = start_date, + end_daily = end_daily, + end_minute = end_minute, ) if(filename is None): diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index 8ed92d85..e59281fa 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -12,8 +12,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ CreateOrderError from catalyst.finance.execution import LimitOrder, StopLimitOrder from catalyst.finance.order import Order, ORDER_STATUS -from catalyst.exchange.exchange_utils import get_exchange_symbols_filename - +from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \ + download_exchange_symbols log = Logger('Bittrex') @@ -309,6 +309,11 @@ class Bittrex(Exchange): def generate_symbols_json(self, filename=None): symbol_map = {} + + fn, r = download_exchange_symbols(self.name) + with open(fn) as data_file: + cached_symbols = json.load(data_file) + markets = self.api.getmarkets() for market in markets: exchange_symbol = market['MarketName'] @@ -316,9 +321,22 @@ class Bittrex(Exchange): market=self.sanitize_curency_symbol(market['MarketCurrency']), base=self.sanitize_curency_symbol(market['BaseCurrency']) ) + + try: + end_daily = cached_symbols[exchange_symbol]['end_daily'] + except KeyError as e: + end_daily ='N/A' + + try: + end_minute = cached_symbols[exchange_symbol]['end_minute'] + except KeyError as e: + end_minute = 'N/A' + symbol_map[exchange_symbol] = dict( symbol=symbol, - start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d") + start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d"), + end_daily = end_daily, + end_minute = end_minute, ) if(filename is None): diff --git a/catalyst/exchange/bundle_utils.py b/catalyst/exchange/bundle_utils.py index e4d2d12e..e4577706 100644 --- a/catalyst/exchange/bundle_utils.py +++ b/catalyst/exchange/bundle_utils.py @@ -1,3 +1,4 @@ +import datetime, requests import datetime import os from logging import Logger @@ -7,10 +8,79 @@ from catalyst.utils.paths import data_path log = Logger('test_exchange_bundle') +EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex'] +API_URL = 'http://data.enigma.co/api/v1' def get_date_from_ms(ms): return datetime.datetime.fromtimestamp(ms / 1000.0) +def get_history(exchange_name, data_frequency, symbol, start_ms = None, end_ms = None): + """ + History API provides OHLCV data for any of the supported exchanges up to yesterday. + + :param exchange_name: string + Required: The name identifier of the exchange (e.g. bitfinex, bittrex, poloniex). + :param data_frequency: string + Required: The bar frequency (minute or daily) + *** currently only 'daily' is supported *** + :param symbol: string + Required: The trading pair symbol. + :param start: float + Optional: The start date in milliseconds. + :param end: float + Optional: The end date in milliseconds. + + :return ohlcv: list[dict[string, float]] + Each row contains the following dictionary for the resulting bars: + 'ts' : int, the timestamp in seconds + 'open' : float + 'high' : float + 'low' : float + 'close' : float + 'volume' : float + + Notes + ===== + Using milliseconds for the start and end dates for ease of use in the + function query parameters. + + Sometimes, one minute goes by without completing a trade of the given + trading pair on the given exchange. To minimize the payload size, we + don't return identical sequential bars. Post-processing code will + forward fill missing bars outside of this function. + """ + + if exchange_name not in EXCHANGE_NAMES: + raise ValueError('get_history function only supports the following exchanges: {}'.format(list(EXCHANGE_NAMES))) + + if data_frequency != 'daily': + raise ValueError('get_history currently only supports daily data.') + + url = '{api_url}/candles?exchange={exchange}&market={symbol}&freq={data_frequency}'.format( + api_url=API_URL, + exchange=exchange_name, + symbol=symbol, + data_frequency=data_frequency, + ) + + if start_ms: + url += '&start={}'.format(int(start_ms/1000)) + + if end_ms: + url += '&end={}'.format(int(end_ms/1000)) + + try: + response = requests.get(url) + except Exception as e: + raise ValueError(e) + + data = response.json() + + if 'error' in response: + raise ValueError(e) + + return data + def get_history_mock(exchange_name, data_frequency, symbol, start_ms, end_ms, exchanges): @@ -80,6 +150,7 @@ def get_history_mock(exchange_name, data_frequency, symbol, start_ms, end_ms, return ohlcv + def fetch_candles_chunk(exchange, assets, data_frequency, end_dt, bar_count): calc_start_dt = end_dt - datetime.timedelta(minutes=bar_count) candles = exchange.get_candles( diff --git a/catalyst/exchange/poloniex/poloniex.py b/catalyst/exchange/poloniex/poloniex.py index 7d0f9986..0260b823 100644 --- a/catalyst/exchange/poloniex/poloniex.py +++ b/catalyst/exchange/poloniex/poloniex.py @@ -487,9 +487,21 @@ class Poloniex(Exchange): except KeyError as e: start_date = time.strftime('%Y-%m-%d') + try: + end_daily = cached_symbols[exchange_symbol]['end_daily'] + except KeyError as e: + end_daily ='N/A' + + try: + end_minute = cached_symbols[exchange_symbol]['end_minute'] + except KeyError as e: + end_minute = 'N/A' + symbol_map[exchange_symbol] = dict( - symbol = symbol, - start_date = start_date + symbol = symbol, + start_date = start_date, + end_daily = end_daily, + end_minute = end_minute, ) if(filename is None):