[23] Integrate with ccxt

This commit is contained in:
Gavin.Chan
2018-02-27 14:23:28 +00:00
parent 354d249d8f
commit cc2927ed52
12 changed files with 481 additions and 112 deletions
+10
View File
@@ -1,6 +1,16 @@
# -*- coding: utf-8 -*-
# pylint: disable-msg=W0401
# flake8: noqa
"""Top-level package for libcryptomarket."""
from libcryptomarket.exchange import *
from libcryptomarket.instrument import instruments
from libcryptomarket.order_book import order_book
from libcryptomarket.candle import (
candles, latest_candles, FREQUENCY_TO_SEC_DICT)
import libcryptomarket.candle.inject
__author__ = """Gavin Chan"""
__email__ = 'gavincyi@gmail.com'
@@ -4,6 +4,7 @@ from time import sleep
import pandas as pd
import ccxt
FREQUENCY_TO_SEC_DICT = {
'1m': 60,
'5m': 300,
@@ -22,8 +23,6 @@ FREQUENCY_TO_SEC_DICT = {
FREQUENCY_TO_SEC_DICT.update(dict(
[(value, value) for value in FREQUENCY_TO_SEC_DICT.values()]))
from .exchanges import * # noqa
def candles(source, symbol, start_time, end_time, frequency, **kwargs):
r"""Return candles of a given period and frequency.
+232
View File
@@ -0,0 +1,232 @@
import inspect
from time import sleep
from datetime import datetime, timedelta
import pandas as pd
import ccxt
import libcryptomarket.exchange
from libcryptomarket.candle import FREQUENCY_TO_SEC_DICT
def _fetch_candles(self, symbol, start_time, end_time, frequency,
**kwargs):
r"""Return candles of a given period and frequency.
:param symbol: `str` symbol.
:param start_time: `datetime` start time.
:param end_time: `datetime` end time.
:param frequency: `str` frequency.
:param \**kwargs:
See below
:Keyword Arguments:
* *quote_currency* (``str``) --
Quote currency symbol, e.g. BTC.
"""
self.load_markets()
# Get the exchange market id
symbol = self.market_id(symbol)
# Initialization
all_data = []
last_start_time = None
while (start_time <
end_time - pd.DateOffset(seconds=FREQUENCY_TO_SEC_DICT[frequency])):
sleep(self.describe()['rateLimit'] / 1000)
data = self._fetch_single_candles(
symbol=symbol, start_time=start_time, end_time=end_time,
frequency=frequency, **kwargs)
if len(data) == 0:
break
if (last_start_time is not None and
data["start_time"].iloc[0] >= last_start_time):
break
all_data.append(data)
if data["end_time"].iloc[-1] > start_time:
start_time = data["end_time"].iloc[-1]
else:
break
if len(all_data) == 0:
raise ValueError("Start time cannot be after end time.")
elif len(all_data) == 1:
return all_data[0]
else:
return pd.concat(all_data)
def _fetch_latest_candles(self, symbols, frequency, frequency_count,
end_time=None, **kwargs):
"""Return the latest candles based on the frequency and its count.
:param symbols: `list` list of symbols, or `str` symbol name.
:param frequency: `int` frequency in seconds.
:param frequency: `str` frequency.
:param end_time: `datetime` end time. Default is None which will use
current time.
:param \**kwargs:
See below
:Keyword Arguments:
* *quote_currency* (``str``) --
Quote currency symbol, e.g. BTC.
"""
if isinstance(symbols, str):
symbols = [symbols]
if end_time is None:
end_time = datetime.utcnow()
closest_end_time = pd.Timestamp(end_time).floor(
timedelta(seconds=FREQUENCY_TO_SEC_DICT[frequency]))
start_time = closest_end_time - timedelta(
seconds=FREQUENCY_TO_SEC_DICT[frequency] * frequency_count + 1)
all_data = []
for symbol in symbols:
data = self.fetch_candles(
symbol=symbol,
start_time=start_time,
end_time=closest_end_time,
frequency=frequency,
**kwargs)
data = data[data['end_time'] <= closest_end_time]
all_data.append(data.set_index(['start_time', 'end_time']))
if len(all_data) == 1:
return all_data[0]
else:
return pd.concat(all_data, axis=1, keys=symbols)
###############################################################################
# Patch
###############################################################################
for exchange in dir(libcryptomarket.exchange):
instance = getattr(ccxt, exchange)
try:
if inspect.isclass(instance) and issubclass(instance, ccxt.Exchange):
setattr(instance, 'fetch_candles', _fetch_candles)
setattr(instance, 'fetch_latest_candles', _fetch_latest_candles)
except Exception as e:
raise e
###############################################################################
# Poloniex patching
###############################################################################
def _poloniex_single_candles(
self, symbol, start_time, end_time, frequency, **kwargs):
"""Poloniex candles.
"""
data = self.public_get_returnchartdata(params={
"currencyPair": symbol,
"start": round(start_time.timestamp()),
"end": round(end_time.timestamp()) - FREQUENCY_TO_SEC_DICT[frequency],
"period": self.describe()['timeframes'][frequency]
})
data = pd.DataFrame(data).rename(columns={
'date': 'start_time',
'quoteVolume': 'quote_volume',
'weightedAverage': 'weighted_average'
})
data.loc[:, 'start_time'] = data['start_time'].apply(
lambda x: pd.Timestamp.utcfromtimestamp(x))
data['end_time'] = data['start_time'] + pd.DateOffset(
seconds=FREQUENCY_TO_SEC_DICT[frequency])
if 'quote_currency' in kwargs.keys():
base_currency = symbol.split('_')[1]
if kwargs['quote_currency'] == base_currency:
data.loc[:, "open"] = (1 / data.loc[:, "open"]).apply(
lambda x: round(x, 8))
data.loc[:, "close"] = (1 / data.loc[:, "close"]).apply(
lambda x: round(x, 8))
data.loc[:, "weighted_average"] = (
(1 / data.loc[:, "weighted_average"]).apply(
lambda x: round(x, 8)))
high_prices = (1 / data.loc[:, "low"]).apply(
lambda x: round(x, 8))
low_prices = (1 / data.loc[:, "high"]).apply(
lambda x: round(x, 8))
data.loc[:, "high"] = high_prices
data.loc[:, "low"] = low_prices
return data
setattr(ccxt.poloniex, '_fetch_single_candles', _poloniex_single_candles)
###############################################################################
# Bitfinex patching
###############################################################################
def _bitfinex_single_candles(self, symbol, start_time, end_time, frequency,
**kwargs):
"""Bitfinex candles.
"""
data = self.request(
path='candles/trade:{}:{}/hist'.format(
self.describe()['timeframes'][frequency], symbol),
params={
"start": round(start_time.timestamp() * 1000),
"end": round((end_time.timestamp() -
FREQUENCY_TO_SEC_DICT[frequency]) * 1000),
"sort": 1
})
data = pd.DataFrame(data, columns=["start_time", "open", "close", "high",
"low", "volume"])
data.loc[:, 'start_time'] = data['start_time'].apply(
lambda x: pd.Timestamp.utcfromtimestamp(x / 1000))
data['end_time'] = data['start_time'] + pd.DateOffset(
seconds=FREQUENCY_TO_SEC_DICT[frequency])
return data
setattr(ccxt.bitfinex, '_fetch_single_candles', _bitfinex_single_candles)
###############################################################################
# GDAX patching
###############################################################################
def _gdax_single_candles(self, symbol, start_time, end_time, frequency,
**kwargs):
"""GDAX candles.
"""
data = self.request(
path='products/{}/candles'.format(symbol),
params={
"granularity": self.describe()['timeframes'][frequency],
"start": start_time.isoformat(),
"end": end_time.isoformat(),
})
if len(data) == 0:
return data
data = pd.DataFrame(data, columns=["start_time", "low", "high", "open",
"close", "volume"])
data.loc[:, 'start_time'] = data['start_time'].apply(
lambda x: pd.Timestamp.utcfromtimestamp(x))
data = data.sort_values(['start_time'])
data['end_time'] = data['start_time'] + pd.DateOffset(
seconds=FREQUENCY_TO_SEC_DICT[frequency])
return data
setattr(ccxt.gdax, '_fetch_single_candles', _gdax_single_candles)
-6
View File
@@ -1,6 +0,0 @@
# pylint: disable-msg=W0401
# flake8: noqa
from libcryptomarket.core.instrument import instruments
from libcryptomarket.core.order_book import order_book
from libcryptomarket.core.candle import (
candles, latest_candles, FREQUENCY_TO_SEC_DICT)
-60
View File
@@ -1,60 +0,0 @@
import pandas as pd
def order_book(source, symbol, depth=5):
"""Return the order book.
:param source: Source, an Exchange API object.
:param symbol: Symbol.
:param depth: Depth of the order book.
"""
source_name = source.__class__.__name__.lower().replace("api", "")
if source_name == "poloniex":
return _order_book_poloniex(source, symbol, depth)
elif source_name == "bittrex":
return _order_book_bittrex(source, symbol, depth)
else:
raise ValueError("Source (%s [%s]) does not support order book"
% (source, source_name))
def _order_book_poloniex(source, symbol, depth=5):
"""Return the order book from Poloniex
:param source: Source, an Exchange API object.
:param symbol: Symbol.
:param depth: Depth of the order book.
"""
if symbol == "all":
raise ValueError("Currently not support all symbol order book query.")
response = source.return_order_book(currencyPair=symbol, depth=depth)
response.raise_for_status()
data = response.json()
data = [pd.DataFrame(
data[side],
columns=pd.MultiIndex.from_product(
[[side], ['price', 'quantity']]),
index=range(1, depth + 1)) for side in ['bids', 'asks']]
data = pd.concat(data, axis=1).astype('float64')
return data
def _order_book_bittrex(source, symbol, depth=5):
"""Return the order book from Poloniex
:param source: Source, an Exchange API object.
:param symbol: Symbol.
:param depth: Depth of the order book.
"""
response = source.get_order_book(market=symbol, type="both")
response.raise_for_status()
data = response.json()['result']
data = pd.concat([pd.DataFrame(data['buy']), pd.DataFrame(data['sell'])],
axis=1,
keys=['bids', 'asks'])
data.index = data.index + 1
return data
+3
View File
@@ -0,0 +1,3 @@
# pylint: disable-msg=W0401
# flake8: noqa
from ccxt import *
+16
View File
@@ -0,0 +1,16 @@
import ccxt
def order_book(source, symbol, depth=None, params=None):
"""Return the order book.
:param source: Source, an Exchange API object.
:param symbol: Symbol.
:param depth: Depth of the order book.
"""
exchange = getattr(ccxt, source.lower())()
if depth is not None:
raise ValueError("Sorry that currently depth is not supported.")
return exchange.fetch_order_book(symbol=symbol, params=params or {})
+40 -42
View File
@@ -2,30 +2,22 @@
"cells": [
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2018-02-01T03:07:19.370253Z",
"start_time": "2018-02-01T03:07:19.320425Z"
"end_time": "2018-02-25T15:25:54.826490Z",
"start_time": "2018-02-25T15:25:54.202778Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
"from datetime import datetime\n",
"\n",
"import pandas as pd\n",
"\n",
"from libcryptomarket.core import candles, latest_candles, FREQUENCY_TO_SEC_DICT"
"# from libcryptomarket.core import candles, latest_candles, FREQUENCY_TO_SEC_DICT"
]
},
{
@@ -37,11 +29,11 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 14,
"metadata": {
"ExecuteTime": {
"end_time": "2018-02-01T11:08:13.573415Z",
"start_time": "2018-02-01T11:08:06.960801Z"
"end_time": "2018-02-27T14:08:40.924686Z",
"start_time": "2018-02-27T14:08:32.436437Z"
},
"scrolled": true
},
@@ -50,20 +42,22 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Running exchange poloniex for instrument BTC_LTC\n",
"Running exchange bitfinex for instrument tBTCUSD\n",
"Running exchange gdax for instrument BTC-USD\n"
"Running exchange <ccxt.poloniex.poloniex object at 0x7f6a1f15d3c8> for instrument LTC/BTC\n",
"Running exchange <ccxt.bitfinex2.bitfinex2 object at 0x7f6a1f15d400> for instrument BTC/USD\n",
"Running exchange <ccxt.gdax.gdax object at 0x7f6a1f151898> for instrument BTC/USD\n"
]
}
],
"source": [
"for source, symbol in [\n",
" (\"poloniex\", \"BTC_LTC\"), \n",
" (\"bitfinex\", \"tBTCUSD\"),\n",
" (\"gdax\", \"BTC-USD\")]:\n",
"import libcryptomarket\n",
"\n",
"for source, symbol, frequency in [\n",
" (libcryptomarket.poloniex(), \"LTC/BTC\", \"1d\"), \n",
" (libcryptomarket.bitfinex2(), \"BTC/USD\", \"1d\"),\n",
" (libcryptomarket.gdax(), \"BTC/USD\", \"1d\")]:\n",
" print(\"Running exchange {} for instrument {}\".format(source, symbol))\n",
" data = candles(source=source, symbol=symbol, \n",
" start_time=pd.Timestamp(\"2017-12-15\"), end_time=pd.Timestamp(\"2017-12-31\"), frequency=\"1d\")\n",
" data = source.fetch_candles(\n",
" symbol=symbol, start_time=pd.Timestamp(\"2017-12-15\"), end_time=pd.Timestamp(\"2017-12-31\"), frequency=frequency)\n",
" assert data[\"start_time\"].iloc[0] == pd.Timestamp(\"2017-12-15\")\n",
" assert data[\"end_time\"].iloc[-1] == pd.Timestamp(\"2017-12-31\")"
]
@@ -77,11 +71,11 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 20,
"metadata": {
"ExecuteTime": {
"end_time": "2018-02-01T11:08:52.923410Z",
"start_time": "2018-02-01T11:08:40.223787Z"
"end_time": "2018-02-27T14:12:49.988940Z",
"start_time": "2018-02-27T14:12:33.771357Z"
}
},
"outputs": [
@@ -89,20 +83,22 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Running exchange poloniex for instrument ['BTC_LTC', 'BTC_ETH']\n",
"Running exchange bitfinex for instrument ['tBTCUSD', 'tETHUSD']\n",
"Running exchange gdax for instrument ['BTC-USD', 'ETH-USD']\n"
"Running exchange <ccxt.poloniex.poloniex object at 0x7f6a1f20db70> for instrument ['LTC/BTC', 'ETH/BTC']\n",
"Running exchange <ccxt.bitfinex2.bitfinex2 object at 0x7f6a1f03be10> for instrument ['BTC/USD', 'ETH/USD']\n",
"Running exchange <ccxt.gdax.gdax object at 0x7f6a1efec400> for instrument ['BTC/USD', 'ETH/USD']\n"
]
}
],
"source": [
"import libcryptomarket\n",
"\n",
"for source, symbols in [\n",
" (\"poloniex\", [\"BTC_LTC\", \"BTC_ETH\"]), \n",
" (\"bitfinex\", [\"tBTCUSD\", \"tETHUSD\"]),\n",
" (\"gdax\", [\"BTC-USD\", \"ETH-USD\"])\n",
" (libcryptomarket.poloniex(), [\"LTC/BTC\", \"ETH/BTC\"]), \n",
" (libcryptomarket.bitfinex2(), [\"BTC/USD\", \"ETH/USD\"]),\n",
" (libcryptomarket.gdax(), [\"BTC/USD\", \"ETH/USD\"])\n",
" ]:\n",
" print(\"Running exchange {} for instrument {}\".format(source, symbols))\n",
" data = latest_candles(source=source, symbols=symbols, frequency=\"5m\", frequency_count=1)\n",
" data = source.fetch_latest_candles(source=source, symbols=symbols, frequency=\"5m\", frequency_count=1)\n",
" assert data.shape[0] == 1"
]
},
@@ -115,11 +111,11 @@
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": 27,
"metadata": {
"ExecuteTime": {
"end_time": "2018-02-05T14:32:57.924368Z",
"start_time": "2018-02-05T14:32:55.837626Z"
"end_time": "2018-02-27T14:16:10.738926Z",
"start_time": "2018-02-27T14:16:06.651033Z"
}
},
"outputs": [
@@ -127,16 +123,18 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Running exchange poloniex for instrument ['USDT_BTC']\n"
"Running exchange <ccxt.poloniex.poloniex object at 0x7f6a1f0d5eb8> for instrument ['BTC/USDT']\n"
]
}
],
"source": [
"import libcryptomarket\n",
"\n",
"for source, symbols in [\n",
" (\"poloniex\", [\"USDT_BTC\", ]), \n",
" (libcryptomarket.poloniex(), [\"BTC/USDT\", ]), \n",
" ]:\n",
" print(\"Running exchange {} for instrument {}\".format(source, symbols))\n",
" data = latest_candles(source=source, symbols=symbols, frequency=\"5m\", frequency_count=1, quote_currency=\"BTC\")\n",
" data = source.fetch_latest_candles(symbols=symbols, frequency=\"5m\", frequency_count=1, quote_currency=\"BTC\")\n",
" assert data.shape[0] == 1"
]
},
+177
View File
@@ -0,0 +1,177 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2018-02-19T00:53:44.009602Z",
"start_time": "2018-02-19T00:53:43.270385Z"
}
},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"import pandas as pd\n",
"\n",
"from libcryptomarket.core import order_book"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"ExecuteTime": {
"end_time": "2018-02-19T01:16:10.084818Z",
"start_time": "2018-02-19T01:16:08.636986Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"{'asks': [[0.08806319, 74.903],\n",
" [0.0880632, 8.0],\n",
" [0.08806385, 69.565],\n",
" [0.08806386, 3.46965772],\n",
" [0.08806389, 66.78351839],\n",
" [0.08806458, 8.88892227],\n",
" [0.08806789, 0.12091826],\n",
" [0.0882145, 2.25359],\n",
" [0.08821516, 54.61112212],\n",
" [0.08821517, 227.75833027],\n",
" [0.08821816, 0.3],\n",
" [0.08823728, 0.00590621],\n",
" [0.0882497, 0.0021669],\n",
" [0.08828145, 0.11410314],\n",
" [0.08831127, 0.11398609],\n",
" [0.08834685, 5.68478826],\n",
" [0.08836529, 34.0632],\n",
" [0.08840216, 4.05],\n",
" [0.08840839, 0.45244575],\n",
" [0.08846982, 0.22753128],\n",
" [0.08849, 0.05],\n",
" [0.08849985, 0.06866587],\n",
" [0.0885, 0.09051559],\n",
" [0.08851171, 5.78455985],\n",
" [0.0885299, 0.1],\n",
" [0.08852991, 52.4109342],\n",
" [0.08852993, 4.3],\n",
" [0.08855261, 0.3094033],\n",
" [0.0885753, 0.00682119],\n",
" [0.08867609, 81.73041587],\n",
" [0.0886761, 0.0232845],\n",
" [0.0886787, 0.0524102],\n",
" [0.0886876, 0.0244448],\n",
" [0.08868846, 0.02322458],\n",
" [0.08869742, 0.11345586],\n",
" [0.0887, 1.009013],\n",
" [0.088777, 0.05025],\n",
" [0.08881776, 0.0124816],\n",
" [0.08883, 0.158],\n",
" [0.08883927, 12.497],\n",
" [0.08883928, 340.632],\n",
" [0.0888515, 0.00217467],\n",
" [0.08885441, 0.11282501],\n",
" [0.08885442, 0.0022565],\n",
" [0.08887284, 0.00498195],\n",
" [0.08888873, 0.001183],\n",
" [0.08891778, 0.0124816],\n",
" [0.0889555, 0.006],\n",
" [0.08897037, 2.5],\n",
" [0.089, 0.01997]],\n",
" 'bids': [[0.08792195, 0.02713255],\n",
" [0.08790009, 5.22382779],\n",
" [0.08790006, 0.00248732],\n",
" [0.08790005, 7.4],\n",
" [0.08787895, 1.14258325],\n",
" [0.0878521, 0.00218435],\n",
" [0.08782086, 0.00569341],\n",
" [0.0878, 2.9925],\n",
" [0.08779018, 2.05152775],\n",
" [0.08777526, 0.05696365],\n",
" [0.08777524, 0.00287234],\n",
" [0.08777328, 1.21637425],\n",
" [0.08776902, 0.2278708],\n",
" [0.08775456, 0.0048503],\n",
" [0.08773207, 0.006],\n",
" [0.0877, 0.84925887],\n",
" [0.08765172, 5.67135807],\n",
" [0.08760894, 0.006854],\n",
" [0.0876063, 0.00683824],\n",
" [0.08756815, 0.02975783],\n",
" [0.087559, 15.7857],\n",
" [0.0875584, 0.00218142],\n",
" [0.08755837, 0.49874994],\n",
" [0.0875497, 4.0],\n",
" [0.087544, 56.44903468],\n",
" [0.08754399, 10.0],\n",
" [0.08752531, 0.028909],\n",
" [0.08742471, 5.4347113],\n",
" [0.08738839, 0.31325678],\n",
" [0.08738086, 0.0379],\n",
" [0.08737386, 0.02837897],\n",
" [0.0873598, 0.00569341],\n",
" [0.08735837, 0.49874994],\n",
" [0.08735423, 157.93071263],\n",
" [0.08735421, 0.00854971],\n",
" [0.0873542, 5.2],\n",
" [0.08735, 0.05],\n",
" [0.0873391, 34.0632],\n",
" [0.08733193, 0.0031275],\n",
" [0.08730518, 2.54749794],\n",
" [0.08728562, 0.04791179],\n",
" [0.08728529, 1.44332751],\n",
" [0.08724949, 0.03252064],\n",
" [0.08723311, 5.0],\n",
" [0.08722376, 0.065117],\n",
" [0.08721564, 0.04661645],\n",
" [0.08720485, 0.229],\n",
" [0.0871709, 0.00219089],\n",
" [0.08716448, 5.73065903],\n",
" [0.08714296, 15.9665893]],\n",
" 'datetime': '2018-02-19T01:16:10.720Z',\n",
" 'timestamp': 1519002970072}"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"order_book(source=\"poloniex\", symbol=\"ETH/BTC\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
+2 -2
View File
@@ -12,9 +12,9 @@ with open('HISTORY.rst') as history_file:
history = history_file.read()
requirements = [
'ccxt>=1.10.0',
'ccxt==1.10.1198',
'pandas>=0.20.0',
'requests',
'requests==2.11.0',
'setuptools_scm',
]