Making some adjustments to the ingestion method after discussion with Victor

catalyst/examples/buy_low_sell_high_neo_with_interface.py

@@ -36,7 +36,7 @@ def _handle_data(context, data):
     prices = data.history(
         context.asset,
         fields='price',
-        bar_count=50,
+        bar_count=1,
         frequency='1m'
     )
     rsi = talib.RSI(prices.values, timeperiod=14)[-1]
@@ -148,27 +148,27 @@ def analyze(context, stats):
     pass
 
 
-run_algorithm(
-    initialize=initialize,
-    handle_data=handle_data,
-    analyze=analyze,
-    exchange_name='bitfinex',
-    live=True,
-    algo_namespace=algo_namespace,
-    base_currency='btc',
-    live_graph=False
-)
-
-# Backtest
 # run_algorithm(
-#     capital_base=250,
-#     start=pd.to_datetime('2017-09-08', utc=True),
-#     end=pd.to_datetime('2017-09-15', utc=True),
-#     data_frequency='minute',
 #     initialize=initialize,
 #     handle_data=handle_data,
 #     analyze=analyze,
 #     exchange_name='bitfinex',
+#     live=True,
 #     algo_namespace=algo_namespace,
-#     base_currency='btc'
+#     base_currency='btc',
+#     live_graph=False
 # )
+
+# Backtest
+run_algorithm(
+    capital_base=250,
+    start=pd.to_datetime('2017-10-01', utc=True),
+    end=pd.to_datetime('2017-10-15', utc=True),
+    data_frequency='minute',
+    initialize=initialize,
+    handle_data=handle_data,
+    analyze=analyze,
+    exchange_name='bitfinex',
+    algo_namespace=algo_namespace,
+    base_currency='btc'
+)

catalyst/exchange/bitfinex/bitfinex.py

@@ -26,7 +26,6 @@ from catalyst.protocol import Account
 from catalyst.exchange.exchange_utils import get_exchange_symbols_filename, \
     download_exchange_symbols
 
-
 # Trying to account for REST api instability
 # https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request
 requests.adapters.DEFAULT_RETRIES = 20
@@ -50,7 +49,9 @@ class Bitfinex(Exchange):
         self._portfolio = portfolio
         self.minute_writer = None
         self.minute_reader = None
-        self.num_candles_limit = 10000
+
+        # The candle limit for each request
+        self.num_candles_limit = 1000
 
         # Max is 90 but playing it safe
         # https://www.bitfinex.com/posts/188
@@ -576,7 +577,7 @@ class Bitfinex(Exchange):
         response = self._request('symbols', None)
 
         for symbol in response.json():
-            if(source_dates):
+            if (source_dates):
                 start_date = self.get_symbol_start_date(symbol)
             else:
                 try:
@@ -587,25 +588,26 @@ class Bitfinex(Exchange):
             try:
                 end_daily = cached_symbols[symbol]['end_daily']
             except KeyError as e:
-                end_daily ='N/A'
+                end_daily = 'N/A'
 
             try:
                 end_minute = cached_symbols[symbol]['end_minute']
             except KeyError as e:
                 end_minute = 'N/A'
 
-            symbol_map[symbol]= dict(
-                symbol     = symbol[:-3]+'_'+symbol[-3:], 
-                start_date = start_date,
-                end_daily  = end_daily,
-                end_minute = end_minute,
+            symbol_map[symbol] = dict(
+                symbol=symbol[:-3] + '_' + symbol[-3:],
+                start_date=start_date,
+                end_daily=end_daily,
+                end_minute=end_minute,
             )
 
-        if(filename is None):
+        if (filename is None):
             filename = get_exchange_symbols_filename(self.name)
 
-        with open(filename,'w') as f:
-            json.dump(symbol_map, f, sort_keys=True, indent=2, separators=(',',':'))
+        with open(filename, 'w') as f:
+            json.dump(symbol_map, f, sort_keys=True, indent=2,
+                      separators=(',', ':'))
 
     def get_symbol_start_date(self, symbol):
 
@@ -634,10 +636,10 @@ class Bitfinex(Exchange):
             arbitrarily set the ref. date to 15 days ago to be safe with
             +/- 31 days
         """
-        if(len(response.json())):
+        if (len(response.json())):
             startmonth = response.json()[-1][0]
         else:
-            startmonth = int((time.time()-15*24*3600)*1000)
+            startmonth = int((time.time() - 15 * 24 * 3600) * 1000)
 
         """
             Query again with daily resolution setting the start and end around
@@ -646,8 +648,9 @@ class Bitfinex(Exchange):
         url = '{url}/v2/candles/trade:1D:{symbol}/hist?start={start}&end={end}'.format(
             url=self.url,
             symbol=symbol_v2,
-            start=startmonth - 3600 *24 *31 *1000,
-            end=min(startmonth + 3600 *24 *31 *1000, int(time.time()*1000))
+            start=startmonth - 3600 * 24 * 31 * 1000,
+            end=min(startmonth + 3600 * 24 * 31 * 1000,
+                    int(time.time() * 1000))
         )
 
         try:
@@ -656,9 +659,5 @@ class Bitfinex(Exchange):
         except Exception as e:
             raise ExchangeRequestError(error=e)
 
-        return time.strftime('%Y-%m-%d', time.gmtime(int(response.json()[-1][0]/1000)))
-
-   
-
-
-
+        return time.strftime('%Y-%m-%d',
+                             time.gmtime(int(response.json()[-1][0] / 1000)))

catalyst/exchange/bundle_utils.py

@@ -29,6 +29,21 @@ def get_seconds_from_date(date):
     return int((date - epoch).total_seconds())
 
 
+def get_bcolz_chunk(exchange_name, data_frequency, symbol, period_a, period_b):
+    """
+
+    :param exchange_name:
+    :param data_frequency:
+    :param symbol:
+    :param period_a:
+        Example: 2017
+    :param period_b:
+        Example: 10
+
+    Note:
+        Filename: bitfinex-daily-neo_eth-2017-10.tar.gz
+    :return:
+    """
 def get_history(exchange_name, data_frequency, symbol, start=None, end=None):
     """
     History API provides OHLCV data for any of the supported exchanges up to yesterday.
@@ -55,7 +70,7 @@ def get_history(exchange_name, data_frequency, symbol, start=None, end=None):
 
     Notes
     =====
-    Using milliseconds for the start and end dates for ease of use in the
+    Using seconds for the start and end dates for ease of use in the
     function query parameters.
 
     Sometimes, one minute goes by without completing a trade of the given

catalyst/exchange/exchange.py

@@ -559,6 +559,7 @@ class Exchange:
         if freq_match:
             candle_size = int(freq_match.group(1))
             unit = freq_match.group(2)
+
         else:
             raise InvalidHistoryFrequencyError(frequency)
 

catalyst/exchange/exchange_bundle.py

@@ -13,6 +13,7 @@ from catalyst.exchange.bundle_utils import get_ffill_candles, get_start_dt, \
     get_periods, range_in_bundle
 from catalyst.exchange.exchange_utils import get_exchange_folder
 from catalyst.utils.cli import maybe_show_progress
+from catalyst.utils.deprecate import deprecated
 from catalyst.utils.paths import ensure_directory
 
 
@@ -208,6 +209,7 @@ class ExchangeBundle:
 
         return missing_assets
 
+    @deprecated
     def ingest_chunk(self, bar_count, end_dt, data_frequency, asset,
                      writer, previous_candle=dict()):
         """

tests/exchange/test_bitfinex.py

@@ -47,6 +47,10 @@ class BitfinexTestCase(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
+        ohlcv_neo = self.exchange.get_candles(
+            data_frequency='1m',
+            assets=self.exchange.get_asset('neo_btc')
+        )
         pass
 
     def test_tickers(self):

tests/exchange/test_bundle.py

@@ -14,8 +14,8 @@ class ExchangeBundleTestCase:
         exchange_name = 'bitfinex'
 
         # start = pd.to_datetime('2017-09-01', utc=True)
-        end = pd.Timestamp.utcnow() - timedelta(minutes=5)
-        start = end - timedelta(minutes=30)
+        start = pd.to_datetime('2017-10-01', utc=True)
+        end = pd.to_datetime('2017-10-06', utc=True)
 
         exchange_bundle = ExchangeBundle(get_exchange(exchange_name))