Resolve the start and end time issue on candles

libcryptomarket/core/candle/__init__.py

@@ -4,8 +4,6 @@ from time import sleep
 import pandas as pd
 import ccxt
 
-from .exchanges import *        # noqa
-
 FREQUENCY_TO_SEC_DICT = {
     '1m': 60,
     '5m': 300,
@@ -24,6 +22,7 @@ FREQUENCY_TO_SEC_DICT = {
 FREQUENCY_TO_SEC_DICT.update(dict(
     [(value, value) for value in FREQUENCY_TO_SEC_DICT.values()]))
 
+from .exchanges import *        # noqa
 
 def candles(source, symbol, start_time, end_time, frequency):
     """Return candles of a given period and frequency.
@@ -32,7 +31,7 @@ def candles(source, symbol, start_time, end_time, frequency):
     :param symbol: `str` symbol.
     :param start_time: `datetime` start time.
     :param end_time: `datetime` end time.
-    :param frequency: `int` frequency in seconds.
+    :param frequency: `str` frequency.
     """
 
     source = source.lower()
@@ -51,12 +50,11 @@ def candles(source, symbol, start_time, end_time, frequency):
             source.__class__.__name__))
 
     # Initialization
-    frequency = describe['timeframes'][frequency]
-
     all_data = []
     last_start_time = None
 
-    while start_time < end_time:
+    while (start_time <
+           end_time - pd.DateOffset(seconds=FREQUENCY_TO_SEC_DICT[frequency])):
         sleep(describe['rateLimit'] / 1000)
         data = func(source=exchange, symbol=symbol, start_time=start_time,
                     end_time=end_time, frequency=frequency)
@@ -68,9 +66,10 @@ def candles(source, symbol, start_time, end_time, frequency):
                 data["start_time"].iloc[0] >= last_start_time):
             break
 
-        all_data.append(data)
         start_time = data["end_time"].iloc[-1]
 
+        all_data.append(data)
+
     if len(all_data) == 0:
         raise ValueError("Start time cannot be after end time.")
     elif len(all_data) == 1:
@@ -85,7 +84,7 @@ def latest_candles(source, symbols, frequency, frequency_count, end_time=None):
     :param source: `str` exchange name.
     :param symbols: `list` list of symbols, or `str` symbol name.
     :param frequency: `int` frequency in seconds.
-    :param frequency: `int` frequency count.
+    :param frequency: `str` frequency.
     :param end_time: `datetime` end time. Default is None which will use
                      current time.
     """
@@ -98,7 +97,7 @@ def latest_candles(source, symbols, frequency, frequency_count, end_time=None):
     closest_end_time = pd.Timestamp(end_time).floor(
         timedelta(seconds=FREQUENCY_TO_SEC_DICT[frequency]))
     start_time = closest_end_time - timedelta(
-        seconds=FREQUENCY_TO_SEC_DICT[frequency] * (frequency_count + 1))
+        seconds=FREQUENCY_TO_SEC_DICT[frequency] * frequency_count + 1)
 
     all_data = []
     for symbol in symbols:
@@ -111,6 +110,6 @@ def latest_candles(source, symbols, frequency, frequency_count, end_time=None):
         all_data.append(data.set_index(['start_time', 'end_time']))
 
     if len(all_data) == 1:
-        return all_data[0].iloc[1:, :]
+        return all_data[0]
     else:
-        return pd.concat(all_data, axis=1, keys=symbols).iloc[1:, :]
+        return pd.concat(all_data, axis=1, keys=symbols)

libcryptomarket/core/candle/exchanges.py

@@ -1,5 +1,7 @@
 import pandas as pd
 
+from libcryptomarket.core.candle import FREQUENCY_TO_SEC_DICT
+
 
 def poloniex_candles(source, symbol, start_time, end_time, frequency):
     """Poloniex candles.
@@ -7,8 +9,8 @@ def poloniex_candles(source, symbol, start_time, end_time, frequency):
     data = source.public_get_returnchartdata(params={
         "currencyPair": symbol,
         "start": round(start_time.timestamp()),
-        "end": round(end_time.timestamp()),
-        "period": frequency
+        "end": round(end_time.timestamp()) - FREQUENCY_TO_SEC_DICT[frequency],
+        "period": source.describe()['timeframes'][frequency]
     })
 
     data = pd.DataFrame(data).rename(columns={
@@ -19,19 +21,22 @@ def poloniex_candles(source, symbol, start_time, end_time, frequency):
 
     data.loc[:, 'start_time'] = data['start_time'].apply(
         lambda x: pd.Timestamp.utcfromtimestamp(x))
-    data['end_time'] = data['start_time'].shift(-1)
+    data['end_time'] = data['start_time'] + pd.DateOffset(
+        seconds=FREQUENCY_TO_SEC_DICT[frequency])
 
-    return data.iloc[:-1, :]
+    return data
 
 
 def bitfinex_candles(source, symbol, start_time, end_time, frequency):
     """Bitfinex candles.
     """
     data = source.request(
-        path='candles/trade:{}:{}/hist'.format(frequency, symbol),
+        path='candles/trade:{}:{}/hist'.format(
+            source.describe()['timeframes'][frequency], symbol),
         params={
             "start": round(start_time.timestamp() * 1000),
-            "end": round(end_time.timestamp() * 1000),
+            "end": round((end_time.timestamp() -
+                          FREQUENCY_TO_SEC_DICT[frequency]) * 1000),
             "sort": 1
         })
 
@@ -40,20 +45,19 @@ def bitfinex_candles(source, symbol, start_time, end_time, frequency):
 
     data.loc[:, 'start_time'] = data['start_time'].apply(
         lambda x: pd.Timestamp.utcfromtimestamp(x / 1000))
-    data['end_time'] = data['start_time'].shift(-1)
+    data['end_time'] = data['start_time'] + pd.DateOffset(
+        seconds=FREQUENCY_TO_SEC_DICT[frequency])
 
-    return data.iloc[:-1, :]
+    return data
 
 
 def gdax_candles(source, symbol, start_time, end_time, frequency):
     """GDAX candles.
     """
-    end_time += pd.DateOffset(seconds=1)
-
     data = source.request(
         path='products/{}/candles'.format(symbol),
         params={
-            "granularity": frequency,
+            "granularity": source.describe()['timeframes'][frequency],
             "start": start_time.isoformat(),
             "end": end_time.isoformat(),
         })
@@ -67,6 +71,7 @@ def gdax_candles(source, symbol, start_time, end_time, frequency):
     data.loc[:, 'start_time'] = data['start_time'].apply(
         lambda x: pd.Timestamp.utcfromtimestamp(x))
     data = data.sort_values(['start_time'])
-    data['end_time'] = data['start_time'].shift(-1)
+    data['end_time'] = data['start_time'] + pd.DateOffset(
+        seconds=FREQUENCY_TO_SEC_DICT[frequency])
 
-    return data.iloc[:-1, :]
+    return data

notebooks/Candles.ipynb

@@ -2,21 +2,30 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 7,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-01-31T15:32:12.633315Z",
-     "start_time": "2018-01-31T15:32:11.476421Z"
+     "end_time": "2018-02-01T03:07:19.370253Z",
+     "start_time": "2018-02-01T03:07:19.320425Z"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
    "source": [
     "%load_ext autoreload\n",
     "%autoreload 2\n",
     "\n",
     "import pandas as pd\n",
     "\n",
-    "from libcryptomarket.core import candles, latest_candles"
+    "from libcryptomarket.core import candles, latest_candles, FREQUENCY_TO_SEC_DICT"
    ]
   },
   {
@@ -28,11 +37,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 22,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-01-31T15:32:18.992459Z",
-     "start_time": "2018-01-31T15:32:12.637992Z"
+     "end_time": "2018-02-01T11:08:13.573415Z",
+     "start_time": "2018-02-01T11:08:06.960801Z"
     },
     "scrolled": true
    },
@@ -68,11 +77,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 24,
    "metadata": {
     "ExecuteTime": {
-     "end_time": "2018-01-31T15:33:04.405264Z",
-     "start_time": "2018-01-31T15:33:00.256283Z"
+     "end_time": "2018-02-01T11:08:52.923410Z",
+     "start_time": "2018-02-01T11:08:40.223787Z"
     }
    },
    "outputs": [
@@ -80,7 +89,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Running exchange poloniex for instrument ['BTC_LTC', 'BTC_ETH']\n"
+      "Running exchange poloniex for instrument ['BTC_LTC', 'BTC_ETH']\n",
+      "Running exchange bitfinex for instrument ['tBTCUSD', 'tETHUSD']\n",
+      "Running exchange gdax for instrument ['BTC-USD', 'ETH-USD']\n"
      ]
     }
    ],
@@ -88,9 +99,10 @@
     "for source, symbols in [\n",
     "        (\"poloniex\", [\"BTC_LTC\", \"BTC_ETH\"]), \n",
     "        (\"bitfinex\", [\"tBTCUSD\", \"tETHUSD\"]),\n",
+    "        (\"gdax\", [\"BTC-USD\", \"ETH-USD\"])\n",
     "    ]:\n",
     "    print(\"Running exchange {} for instrument {}\".format(source, symbols))\n",
-    "    data = latest_candles(source=source, symbols=symbols, frequency=\"30m\", frequency_count=1)\n",
+    "    data = latest_candles(source=source, symbols=symbols, frequency=\"5m\", frequency_count=1)\n",
     "    assert data.shape[0] == 1"
    ]
   },

setup.py

@@ -36,8 +36,7 @@ setup(
     author="Gavin Chan",
     author_email='gavincyi@gmail.com',
     url='https://github.com/gavincyi/libcryptomarket',
-    packages=['libcryptomarket', 'libcryptomarket.api',
-              'libcryptomarket.core'],
+    packages=find_packages(),
     include_package_data=True,
     install_requires=requirements,
     license="GNU General Public License v3",