BUG: fixed a get_candles issue with the Poloniex exchange

Fully tested on v0.3.8

catalyst/examples/simple_universe.py

@@ -0,0 +1,129 @@
+"""
+Requires Catalyst version 0.3.0 or above
+Tested on Catalyst version 0.3.3
+
+These example aims to provide and easy way for users to learn how to collect data from the different exchanges.
+You simply need to specify the exchange and the market that you want to focus on.
+You will all see how to create a universe and filter it base on the exchange and the market you desire.
+
+The example prints out the closing price of all the pairs for a given market-exchange every 30 minutes.
+The example also contains the ohlcv minute data for the past seven days which could be used to create indicators
+Use this as the backbone to create your own trading strategies.
+
+Variables lookback date and date are used to ensure data for a coin existed on the lookback period specified.
+"""
+
+import numpy as np
+import pandas as pd
+from datetime import timedelta
+from catalyst import run_algorithm
+from catalyst.exchange.exchange_utils import get_exchange_symbols
+
+from catalyst.api import (
+    symbols,
+)
+
+
+def initialize(context):
+    context.i = -1  # counts the minutes
+    context.exchange = context.exchanges.values()[0].name.lower()  # exchange name
+    context.base_currency = context.exchanges.values()[0].base_currency.lower()  # market base currency
+
+
+def handle_data(context, data):
+    context.i += 1
+    lookback_days = 7  # 7 days
+
+    # current date formatted into a string
+    today = data.current_dt
+    date, time = today.strftime('%Y-%m-%d %H:%M:%S').split(' ')
+    lookback_date = today - timedelta(days=lookback_days)  # subtract the amount of days specified in lookback
+    lookback_date = lookback_date.strftime('%Y-%m-%d %H:%M:%S').split(' ')[0]  # get only the date as a string
+
+    # update universe everyday
+    new_day = 60 * 24  # assuming data_frequency='minute'
+    if not context.i % new_day:
+        context.universe = universe(context, lookback_date, date)
+
+    # get data every 30 minutes
+    minutes = 30
+    one_day_in_minutes = 1440  # 1440 assumes data_frequency='minute'
+    lookback = one_day_in_minutes / minutes * lookback_days  # get N lookback_days of history data
+    if not context.i % minutes and context.universe:
+        # we iterate for every pair in the current universe
+        for coin in context.coins:
+            pair = str(coin.symbol)
+
+            # 30 minute interval ohlcv data (the standard data required for candlestick or indicators/signals)
+            # 30T means 30 minutes re-sampling of one minute data. change to your desire time interval.
+            opened = fill(data.history(coin, 'open', bar_count=lookback, frequency='30T')).values
+            high = fill(data.history(coin, 'high', bar_count=lookback, frequency='30T')).values
+            low = fill(data.history(coin, 'low', bar_count=lookback, frequency='30T')).values
+            close = fill(data.history(coin, 'price', bar_count=lookback, frequency='30T')).values
+            volume = fill(data.history(coin, 'volume', bar_count=lookback, frequency='30T')).values
+
+            # close[-1] is the equivalent to current price
+            # displays the minute price for each pair every 30 minutes
+            print(today, pair, opened[-1], high[-1], low[-1], close[-1], volume[-1])
+
+            # ----------------------------------------------------------------------------------------------------------
+            # -------------------------------------- Insert Your Strategy Here -----------------------------------------
+            # ----------------------------------------------------------------------------------------------------------
+
+
+def analyze(context=None, results=None):
+    pass
+
+
+# Get the universe for a given exchange and a given base_currency market
+# Example: Poloniex BTC Market
+def universe(context, lookback_date, current_date):
+    json_symbols = get_exchange_symbols(context.exchange)  # get all the pairs for the exchange
+    universe_df = pd.DataFrame.from_dict(json_symbols).transpose().astype(str)  # convert into a dataframe
+    universe_df['base_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[1],
+                                                                       axis=1)
+    universe_df['market_currency'] = universe_df.apply(lambda row: row.symbol.split('_')[0],
+                                                                         axis=1)
+
+    # Filter all the exchange pairs to only the ones for a give base currency
+    universe_df = universe_df[universe_df['base_currency'] == context.base_currency]
+
+    # Filter all the pairs to ensure that pair existed in the current date range
+    universe_df = universe_df[universe_df.start_date < lookback_date]
+    universe_df = universe_df[universe_df.end_daily >= current_date]
+    context.coins = symbols(*universe_df.symbol)  # convert all the pairs to symbols
+
+    # print(universe_df.symbol.tolist())
+    return universe_df.symbol.tolist()
+
+
+# Replace all NA, NAN or infinite values with its nearest value
+def fill(series):
+    if isinstance(series, pd.Series):
+        return series.replace([np.inf, -np.inf], np.nan).ffill().bfill()
+    elif isinstance(series, np.ndarray):
+        return pd.Series(series).replace([np.inf, -np.inf], np.nan).ffill().bfill().values
+    else:
+        return series
+
+
+if __name__ == '__main__':
+    start_date = pd.to_datetime('2017-01-01', utc=True)
+    end_date = pd.to_datetime('2017-11-13', utc=True)
+
+    performance = run_algorithm(start=start_date, end=end_date,
+                                capital_base=100.0,  # amount of base_currency, not always in dollars unless usd
+                                initialize=initialize,
+                                handle_data=handle_data,
+                                analyze=analyze,
+                                exchange_name='poloniex',
+                                data_frequency='minute',
+                                base_currency='btc',
+                                live=False,
+                                live_graph=False,
+                                algo_namespace='simple_universe')
+
+"""
+Run in Terminal (inside catalyst environment):
+python simple_universe.py
+"""

catalyst/exchange/poloniex/poloniex.py

@@ -226,10 +226,9 @@ class Poloniex(Exchange):
         ohlc_map = dict()
 
         for asset in asset_list:
+            delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
+            end = int(delta.total_seconds())
 
-            # TODO: what's wrong with this?
-            # end = int(time.mktime(end_dt.timetuple()))
-            end = int(time.time())
             if bar_count is None:
                 start = end - 2 * frequency
             else:

tests/exchange/test_data_portal.py

@@ -113,3 +113,40 @@ class TestExchangeDataPortal:
         )
 
         log.info('found history window: {}'.format(data))
+
+    def test_validate_resample(self):
+        symbol = ['eth_btc']
+        exchange_name = 'poloniex'
+        exchange = get_exchange(exchange_name, base_currency=symbol)
+
+        assets = exchange.get_assets(symbols=symbol)
+
+        date = rnd_history_date_days(
+            max_days=10,
+            last_dt=pd.to_datetime('2017-11-1', utc=True)
+        )
+        bar_count = rnd_bar_count(max_bars=10)
+        sample_minutes = 15
+        sample_data = self.data_portal_backtest.get_history_window(
+            assets=assets,
+            end_dt=date,
+            bar_count=bar_count,
+            frequency='{}T'.format(sample_minutes),
+            field='close',
+            data_frequency='daily'
+        )
+        minute_data = self.data_portal_backtest.get_history_window(
+            assets=assets,
+            end_dt=date,
+            bar_count=bar_count * sample_minutes,
+            frequency='1T',
+            field='close',
+            data_frequency='daily'
+        )
+        resampled_minute_data = minute_data.resample(
+            '{}T'.format(sample_minutes))
+
+        print(sample_data.tail(10))
+        print(resampled_minute_data.tail(10))
+        print(minute_data.tail(10))
+        pass

tests/exchange/test_poloniex.py

@@ -1,9 +1,10 @@
-from catalyst.exchange.bittrex.bittrex import Bittrex
 from catalyst.exchange.poloniex.poloniex import Poloniex
 from catalyst.finance.order import Order
 from base import BaseExchangeTestCase
 from logbook import Logger
 from catalyst.exchange.exchange_utils import get_exchange_auth
+import pandas as pd
+from test_utils import output_df
 
 log = Logger('test_poloniex')
 
@@ -51,18 +52,19 @@ class TestPoloniex(BaseExchangeTestCase):
 
     def test_get_candles(self):
         log.info('retrieving candles')
-        ohlcv_neo = self.exchange.get_candles(
+        assets = self.exchange.get_asset('eth_btc')
-            freq='5T',
+        ohlcv = self.exchange.get_candles(
-            assets=self.exchange.get_asset('eth_btc')
+            end_dt=pd.to_datetime('2017-11-01', utc=True),
-        )
+            freq='30T',
-        ohlcv_neo_ubq = self.exchange.get_candles(
+            assets=assets,
-            freq='5T',
+            bar_count=200
-            assets=[
-                self.exchange.get_asset('neos_btc'),
-                self.exchange.get_asset('via_btc')
-            ],
-            bar_count=14
         )
+        df = pd.DataFrame(ohlcv)
+        df.set_index('last_traded', drop=True, inplace=True)
+        log.info(df.tail(25))
+
+        path = output_df(df, assets, 'candles')
+        log.info('saved candles: {}'.format(path))
         pass
 
     def test_tickers(self):

tests/exchange/test_utils.py

@@ -4,11 +4,20 @@ from random import randint
 import pandas as pd
 
 
-def rnd_history_date_days(max_days=30):
+def rnd_history_date_days(max_days=30, last_dt=None):
-    now = pd.Timestamp.utcnow()
+    if last_dt is None:
+        last_dt = pd.Timestamp.utcnow()
+
     days = randint(0, max_days)
 
-    return now - timedelta(days=days)
+    return last_dt - timedelta(days=days)
+
+
+def rnd_history_date_minutes(max_minutes=1440):
+    now = pd.Timestamp.utcnow()
+    days = randint(0, max_minutes)
+
+    return now - timedelta(minutes=days)
 
 
 def rnd_bar_count(max_bars=21):