Merge branch 'aws-symbols-json' into develop

catalyst/exchange/bitfinex/bitfinex.py

@@ -23,6 +23,7 @@ from catalyst.exchange.exchange_execution import ExchangeLimitOrder, \
     ExchangeStopLimitOrder, ExchangeStopOrder
 from catalyst.finance.order import Order, ORDER_STATUS
 from catalyst.protocol import Account
+from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
 
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
@@ -527,3 +528,15 @@ class Bitfinex(Exchange):
 
         log.debug('got tickers {}'.format(ticks))
         return ticks
+
+    def generate_symbols_json(self, filename=None):
+        symbol_map = {}
+        response = self._request('symbols', None)
+        for symbol in response.json():
+            symbol_map[symbol]= {"symbol":symbol[:-3]+'_'+symbol[-3:], "start_date": "2010-01-01"}
+
+        if(filename is None):
+            filename = get_exchange_symbols_filename(self.name)
+
+        with open(filename,'w') as f:
+            json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':'))

catalyst/exchange/bittrex/bittrex.py

@@ -12,6 +12,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     CreateOrderError
 from catalyst.finance.execution import LimitOrder, StopLimitOrder
 from catalyst.finance.order import Order, ORDER_STATUS
+from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
+
 
 log = Logger('Bittrex')
 
@@ -50,31 +52,6 @@ class Bittrex(Exchange):
         """
         return exchange_symbol.lower()
 
-    def fetch_symbol_map(self):
-        """
-        Since Bittrex gives us a complete dictionary of symbols,
-        we can build the symbol map ad-hoc as opposed to maintaining
-        a static file. We must be careful with mapping any unconventional
-        symbol name as appropriate.
-
-        :return symbol_map:
-        """
-        symbol_map = dict()
-
-        markets = self.api.getmarkets()
-        for market in markets:
-            exchange_symbol = market['MarketName']
-            symbol = '{market}_{base}'.format(
-                market=self.sanitize_curency_symbol(market['MarketCurrency']),
-                base=self.sanitize_curency_symbol(market['BaseCurrency'])
-            )
-            symbol_map[exchange_symbol] = dict(
-                symbol=symbol,
-                start_date=pd.to_datetime(market['Created'], utc=True)
-            )
-
-        return symbol_map
-
     def get_balances(self):
         try:
             log.debug('retrieving wallet balances')
@@ -316,3 +293,23 @@ class Bittrex(Exchange):
     def get_account(self):
         log.info('retrieving account data')
         pass
+
+    def generate_symbols_json(self, filename=None):
+        symbol_map = {}
+        markets = self.api.getmarkets()
+        for market in markets:
+            exchange_symbol = market['MarketName']
+            symbol = '{market}_{base}'.format(
+                market=self.sanitize_curency_symbol(market['MarketCurrency']),
+                base=self.sanitize_curency_symbol(market['BaseCurrency'])
+            )
+            symbol_map[exchange_symbol] = dict(
+                symbol=symbol,
+                start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d")
+            )
+
+        if(filename is None):
+            filename = get_exchange_symbols_filename(self.name)
+
+        with open(filename,'w') as f:
+            json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':'))

catalyst/exchange/exchange_utils.py

@@ -9,9 +9,8 @@ from catalyst.exchange.exchange_errors import ExchangeAuthNotFound, \
     ExchangeSymbolsNotFound
 from catalyst.utils.paths import data_root, ensure_directory
 
-# TODO: move to aws
-SYMBOLS_URL = 'https://raw.githubusercontent.com/enigmampc/catalyst/' \
-              'master/catalyst/exchange/{exchange}/symbols.json'
+SYMBOLS_URL = 'https://s3.amazonaws.com/enigmaco/catalyst-exchanges/' \
+              '{exchange}/symbols.json'
 
 
 def get_exchange_folder(exchange_name, environ=None):
@@ -25,18 +24,20 @@ def get_exchange_folder(exchange_name, environ=None):
     return exchange_folder
 
 
-def download_exchange_symbols(exchange_name, environ=None):
+def get_exchange_symbols_filename(exchange_name, environ=None):
     exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'symbols.json')
+    return os.path.join(exchange_folder, 'symbols.json')
 
+
+def download_exchange_symbols(exchange_name, environ=None):
+    filename = get_exchange_symbols_filename(exchange_name)
     url = SYMBOLS_URL.format(exchange=exchange_name)
     response = urllib.urlretrieve(url=url, filename=filename)
     return response
 
 
 def get_exchange_symbols(exchange_name, environ=None):
-    exchange_folder = get_exchange_folder(exchange_name, environ)
-    filename = os.path.join(exchange_folder, 'symbols.json')
+    filename = get_exchange_symbols_filename(exchange_name)
 
     if not os.path.isfile(filename):
         download_exchange_symbols(exchange_name, environ)