Merge remote-tracking branch 'origin/concurrent-exchanges' into concurrent-exchanges

# Conflicts:
#	catalyst/exchange/bundle_utils.py

catalyst/exchange/bitfinex/bitfinex.py

@@ -585,9 +585,21 @@ class Bitfinex(Exchange):
                 except KeyError as e:
                     start_date = time.strftime('%Y-%m-%d')
 
+            try:
+                end_daily = cached_symbols[symbol]['end_daily']
+            except KeyError as e:
+                end_daily ='N/A'
+
+            try:
+                end_minute = cached_symbols[symbol]['end_minute']
+            except KeyError as e:
+                end_minute = 'N/A'
+
             symbol_map[symbol]= dict(
-                symbol = symbol[:-3]+'_'+symbol[-3:], 
-                start_date = start_date
+                symbol     = symbol[:-3]+'_'+symbol[-3:], 
+                start_date = start_date,
+                end_daily  = end_daily,
+                end_minute = end_minute,
             )
 
         if(filename is None):

catalyst/exchange/bittrex/bittrex.py

@@ -12,8 +12,8 @@ from catalyst.exchange.exchange_errors import InvalidHistoryFrequencyError, \
     CreateOrderError
 from catalyst.finance.execution import LimitOrder, StopLimitOrder
 from catalyst.finance.order import Order, ORDER_STATUS
-from catalyst.exchange.exchange_utils import get_exchange_symbols_filename
-
+from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
+    download_exchange_symbols
 
 log = Logger('Bittrex')
 
@@ -309,6 +309,11 @@ class Bittrex(Exchange):
 
     def generate_symbols_json(self, filename=None):
         symbol_map = {}
+
+        fn, r = download_exchange_symbols(self.name)
+        with open(fn) as data_file:
+            cached_symbols = json.load(data_file)
+
         markets = self.api.getmarkets()
         for market in markets:
             exchange_symbol = market['MarketName']
@@ -316,9 +321,22 @@ class Bittrex(Exchange):
                 market=self.sanitize_curency_symbol(market['MarketCurrency']),
                 base=self.sanitize_curency_symbol(market['BaseCurrency'])
             )
+
+            try:
+                end_daily = cached_symbols[exchange_symbol]['end_daily']
+            except KeyError as e:
+                end_daily ='N/A'
+
+            try:
+                end_minute = cached_symbols[exchange_symbol]['end_minute']
+            except KeyError as e:
+                end_minute = 'N/A'
+
             symbol_map[exchange_symbol] = dict(
                 symbol=symbol,
-                start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d")
+                start_date=pd.to_datetime(market['Created'], utc=True).strftime("%Y-%m-%d"),
+                end_daily  = end_daily,
+                end_minute = end_minute,
             )
 
         if(filename is None):

catalyst/exchange/bundle_utils.py

@@ -1,3 +1,4 @@
+import datetime, requests
 import datetime
 import os
 from logging import Logger
@@ -7,10 +8,79 @@ from catalyst.utils.paths import data_path
 
 log = Logger('test_exchange_bundle')
 
+EXCHANGE_NAMES = ['bitfinex', 'bittrex', 'poloniex']
+API_URL = 'http://data.enigma.co/api/v1'
 
 def get_date_from_ms(ms):
     return datetime.datetime.fromtimestamp(ms / 1000.0)
 
+def get_history(exchange_name, data_frequency, symbol, start_ms = None, end_ms = None):
+    """
+    History API provides OHLCV data for any of the supported exchanges up to yesterday.
+
+    :param exchange_name: string
+        Required: The name identifier of the exchange (e.g. bitfinex, bittrex, poloniex).
+    :param data_frequency: string
+        Required: The bar frequency (minute or daily)
+        *** currently only 'daily' is supported ***
+    :param symbol: string
+        Required: The trading pair symbol.
+    :param start: float
+        Optional: The start date in milliseconds.
+    :param end: float
+        Optional: The end date in milliseconds.
+
+    :return ohlcv: list[dict[string, float]]
+        Each row contains the following dictionary for the resulting bars:
+        'ts'     : int, the timestamp in seconds
+        'open'   : float
+        'high'   : float
+        'low'    : float
+        'close'  : float
+        'volume' : float
+
+    Notes
+    =====
+    Using milliseconds for the start and end dates for ease of use in the
+    function query parameters.
+
+    Sometimes, one minute goes by without completing a trade of the given
+    trading pair on the given exchange. To minimize the payload size, we
+    don't return identical sequential bars. Post-processing code will
+    forward fill missing bars outside of this function.
+    """
+
+    if exchange_name not in EXCHANGE_NAMES:
+        raise ValueError('get_history function only supports the following exchanges: {}'.format(list(EXCHANGE_NAMES)))
+
+    if data_frequency != 'daily':
+        raise ValueError('get_history currently only supports daily data.')
+
+    url = '{api_url}/candles?exchange={exchange}&market={symbol}&freq={data_frequency}'.format(
+        api_url=API_URL,
+        exchange=exchange_name,
+        symbol=symbol,
+        data_frequency=data_frequency,
+        )
+
+    if start_ms:
+        url += '&start={}'.format(int(start_ms/1000))
+
+    if end_ms:
+        url += '&end={}'.format(int(end_ms/1000))
+
+    try:
+        response = requests.get(url)
+    except Exception as e:
+        raise ValueError(e)
+
+    data = response.json()
+
+    if 'error' in response:
+        raise ValueError(e)
+
+    return data
+
 
 def get_history_mock(exchange_name, data_frequency, symbol, start_ms, end_ms,
                      exchanges):
@@ -80,6 +150,7 @@ def get_history_mock(exchange_name, data_frequency, symbol, start_ms, end_ms,
     return ohlcv
 
 
+
 def fetch_candles_chunk(exchange, assets, data_frequency, end_dt, bar_count):
     calc_start_dt = end_dt - datetime.timedelta(minutes=bar_count)
     candles = exchange.get_candles(

catalyst/exchange/poloniex/poloniex.py

@@ -487,9 +487,21 @@ class Poloniex(Exchange):
                 except KeyError as e:
                     start_date = time.strftime('%Y-%m-%d')
 
+            try:
+                end_daily = cached_symbols[exchange_symbol]['end_daily']
+            except KeyError as e:
+                end_daily ='N/A'
+
+            try:
+                end_minute = cached_symbols[exchange_symbol]['end_minute']
+            except KeyError as e:
+                end_minute = 'N/A'
+
             symbol_map[exchange_symbol] = dict(
-                symbol = symbol,
-                start_date = start_date
+                symbol     = symbol,
+                start_date = start_date,
+                end_daily  = end_daily,
+                end_minute = end_minute,
             )
 
         if(filename is None):