From df8ba90236c43a9b982884a24ba5c11f3c7f7d16 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Thu, 21 Sep 2017 12:43:29 -0600 Subject: [PATCH 1/3] {exchange}/symbols.json moved to AWS --- catalyst/exchange/exchange_utils.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 2638c1c9..1bf74a68 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -9,9 +9,8 @@ from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \ ExchangeSymbolsNotFound from catalyst.utils.paths import data_root, ensure_directory -# TODO: move to aws -SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \ - 'master/catalyst/exchange/{exchange}/symbols.json' +SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \ + '{exchange}/symbols.json' def get_exchange_folder(exchange_name, environ=None): From 8cabb33372cdaea8566bcc9926a756ff7d0ab758 Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 22 Sep 2017 09:54:51 -0600 Subject: [PATCH 2/3] Autogeneration of symbols.json for bitfinex --- catalyst/exchange/bitfinex/bitfinex.py | 13 +++++++++++++ catalyst/exchange/exchange_utils.py | 10 ++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index fe32991a..01ba45d2 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -23,6 +23,7 @@ from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \ ExchangeStopLimitOrder, ExchangeStopOrder from catalyst.finance.order import Order, ORDER_STATUS from catalyst.protocol import Account +from catalyst.exchange.exchange_utils import get_exchange_symbols_filename # Trying to account for REST api instability # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request @@ -527,3 +528,15 @@ class Bitfinex(Exchange): log.debug('got tickers {}'.format(ticks)) return ticks + + def generate_symbols_json(self, filename=None): + response = self._request('symbols', None) + symbols={} + for symbol in response.json(): + symbols[symbol]= {"symbol":symbol[:-3]+'_'+symbol[-3:], "start_date": "2010-01-01"} + + if(filename is None): + filename = get_exchange_symbols_filename(self.name) + + with open(filename,'w') as f: + json.dump(symbols, f, sort_keys=True, indent=2, separators=(',',':')) diff --git a/catalyst/exchange/exchange_utils.py b/catalyst/exchange/exchange_utils.py index 1bf74a68..ebb33023 100644 --- a/catalyst/exchange/exchange_utils.py +++ b/catalyst/exchange/exchange_utils.py @@ -24,18 +24,20 @@ def get_exchange_folder(exchange_name, environ=None): return exchange_folder -def download_exchange_symbols(exchange_name, environ=None): +def get_exchange_symbols_filename(exchange_name, environ=None): exchange_folder = get_exchange_folder(exchange_name, environ) - filename = os.path.join(exchange_folder, 'symbols.json') + return os.path.join(exchange_folder, 'symbols.json') + +def download_exchange_symbols(exchange_name, environ=None): + filename = get_exchange_symbols_filename(exchange_name) url = SYMBOLS_URL.format(exchange=exchange_name) response = urllib.urlretrieve(url=url, filename=filename) return response def get_exchange_symbols(exchange_name, environ=None): - exchange_folder = get_exchange_folder(exchange_name, environ) - filename = os.path.join(exchange_folder, 'symbols.json') + filename = get_exchange_symbols_filename(exchange_name) if not os.path.isfile(filename): download_exchange_symbols(exchange_name, environ) From daf3c4d285d12b6c9a09e23c88514395535a8e0c Mon Sep 17 00:00:00 2001 From: Victor Grau Serrat Date: Fri, 22 Sep 2017 10:55:49 -0600 Subject: [PATCH 3/3] Autogeneration of symbols.json for bittrex --- catalyst/exchange/bitfinex/bitfinex.py | 6 ++-- catalyst/exchange/bittrex/bittrex.py | 47 ++++++++++++-------------- 2 files changed, 25 insertions(+), 28 deletions(-) diff --git a/catalyst/exchange/bitfinex/bitfinex.py b/catalyst/exchange/bitfinex/bitfinex.py index 01ba45d2..cb33c3ee 100644 --- a/catalyst/exchange/bitfinex/bitfinex.py +++ b/catalyst/exchange/bitfinex/bitfinex.py @@ -530,13 +530,13 @@ class Bitfinex(Exchange): return ticks def generate_symbols_json(self, filename=None): + symbol_map = {} response = self._request('symbols', None) - symbols={} for symbol in response.json(): - symbols[symbol]= {"symbol":symbol[:-3]+'_'+symbol[-3:], "start_date": "2010-01-01"} + symbol_map[symbol]= {"symbol":symbol[:-3]+'_'+symbol[-3:], "start_date": "2010-01-01"} if(filename is None): filename = get_exchange_symbols_filename(self.name) with open(filename,'w') as f: - json.dump(symbols, f, sort_keys=True, indent=2, separators=(',',':')) + json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':')) diff --git a/catalyst/exchange/bittrex/bittrex.py b/catalyst/exchange/bittrex/bittrex.py index f56073c5..6c0dfbc1 100644 --- a/catalyst/exchange/bittrex/bittrex.py +++ b/catalyst/exchange/bittrex/bittrex.py @@ -12,6 +12,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \ CreateOrderError from catalyst.finance.execution import LimitOrder, StopLimitOrder from catalyst.finance.order import Order, ORDER_STATUS +from catalyst.exchange.exchange_utils import get_exchange_symbols_filename + log = Logger('Bittrex') @@ -50,31 +52,6 @@ class Bittrex(Exchange): """ return exchange_symbol.lower() - def fetch_symbol_map(self): - """ - Since Bittrex gives us a complete dictionary of symbols, - we can build the symbol map ad-hoc as opposed to maintaining - a static file. We must be careful with mapping any unconventional - symbol name as appropriate. - - :return symbol_map: - """ - symbol_map = dict() - - markets = self.api.getmarkets() - for market in markets: - exchange_symbol = market['MarketName'] - symbol = '{market}_{base}'.format( - market=self.sanitize_curency_symbol(market['MarketCurrency']), - base=self.sanitize_curency_symbol(market['BaseCurrency']) - ) - symbol_map[exchange_symbol] = dict( - symbol=symbol, - start_date=pd.to_datetime(market['Created'], utc=True) - ) - - return symbol_map - def get_balances(self): try: log.debug('retrieving wallet balances') @@ -316,3 +293,23 @@ class Bittrex(Exchange): def get_account(self): log.info('retrieving account data') pass + + def generate_symbols_json(self, filename=None): + symbol_map = {} + markets = self.api.getmarkets() + for market in markets: + exchange_symbol = market['MarketName'] + symbol = '{market}_{base}'.format( + market=self.sanitize_curency_symbol(market['MarketCurrency']), + base=self.sanitize_curency_symbol(market['BaseCurrency']) + ) + symbol_map[exchange_symbol] = dict( + symbol=symbol, + start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d") + ) + + if(filename is None): + filename = get_exchange_symbols_filename(self.name) + + with open(filename,'w') as f: + json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':'))