[#20] Support extracting candles to csv script

This commit is contained in:
Gavin.Chan
2018-01-28 00:13:15 +00:00
parent 2131c6bf93
commit 6d000c285f
9 changed files with 333 additions and 236 deletions
+1
View File
@@ -65,3 +65,4 @@ target/
.editorconfig
.github/
.idea/
.ipynb_checkpoints
+1
View File
@@ -48,6 +48,7 @@ clean-test: ## remove test and coverage artifacts
rm -fr htmlcov/
autopep8: ## autopep8 to clean
autopep8 --aggressive --in-place --recursive libcryptomarket/*/*/*.py
autopep8 --aggressive --in-place --recursive libcryptomarket/*/*.py
autopep8 --aggressive --in-place --recursive libcryptomarket/*.py
autopep8 --aggressive --in-place --recursive tests/*.py
+75
View File
@@ -0,0 +1,75 @@
import argparse
import logging
import pandas as pd
from libcryptomarket.core import candles, FREQUENCY_TO_SEC_DICT
LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
def get_args():
"""Get input arguments.
"""
parser = argparse.ArgumentParser(description=(
'Query historical candles to files.'))
parser.add_argument('--exchange', action='store', dest='exchange',
help='Exchange name.', required=True)
parser.add_argument('--symbols', action='store', dest='symbols',
help='List of symbols',
type=str, nargs='+', required=True)
parser.add_argument('--frequency', action='store', dest='frequency',
help='Frequency.',
choices=list(FREQUENCY_TO_SEC_DICT.keys()),
required=True)
parser.add_argument('--start-time', action='store', dest='start_time',
help='Start time in format of \'YYYY-MM-DD\'',
required=True)
parser.add_argument('--end-time', action='store', dest='end_time',
help='End time in format of \'YYYY-MM-DD\'',
required=True)
parser.add_argument('--output', action='store', dest='output',
help='Output filename', required=True)
return parser.parse_args()
def main():
"""Main.
"""
args = get_args()
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
start_time = pd.Timestamp(args.start_time)
logging.info('Start time: %s', start_time)
end_time = pd.Timestamp(args.end_time)
logging.info('End time: %s', end_time)
logging.info('Starting querying to exchange %s with frequency %s...',
args.exchange, args.frequency)
all_data = {}
for symbol in args.symbols:
logging.info('Querying symbol %s...', symbol)
data = candles(source=args.exchange,
symbol=symbol,
start_time=start_time,
end_time=end_time,
frequency=args.frequency)
all_data[symbol] = data.set_index(['start_time'], ['end_time'])
logging.info('Cleaning the data...')
all_data = pd.concat(all_data, axis=1, names=['symbol', 'value'])
all_data = all_data.stack(level=0)
logging.info('Exporting to path (%s)...', args.output)
all_data.to_csv(args.output)
logging.info('Exported all the historical prices')
if __name__ == '__main__':
main()
+2 -2
View File
@@ -1,6 +1,6 @@
# pylint: disable-msg=W0401
# flake8: noqa
from libcryptomarket.core.historical import historical_ticker
from libcryptomarket.core.instrument import instruments
from libcryptomarket.core.order_book import order_book
from libcryptomarket.core.candle import candles, latest_candles
from libcryptomarket.core.candle import (
candles, latest_candles, FREQUENCY_TO_SEC_DICT)
@@ -1,8 +1,29 @@
from datetime import datetime, timedelta
from time import sleep
import pandas as pd
import ccxt
from .exchanges import * # noqa
FREQUENCY_TO_SEC_DICT = {
'1m': 60,
'5m': 300,
'15m': 900,
'30m': 1800,
'1h': 3600,
'3h': 10800,
'6h': 21600,
'12h': 43200,
'1d': 86400,
'1w': 86400 * 7,
'2w': 86400 * 7 * 2,
'1M': 86400 * 30,
}
FREQUENCY_TO_SEC_DICT.update(dict(
[(value, value) for value in FREQUENCY_TO_SEC_DICT.values()]))
def candles(source, symbol, start_time, end_time, frequency):
"""Return candles of a given period and frequency.
@@ -14,32 +35,49 @@ def candles(source, symbol, start_time, end_time, frequency):
:param frequency: `int` frequency in seconds.
"""
if source.lower() == 'poloniex':
source = getattr(ccxt, source.lower())()
source = source.lower()
func_name = "%s_candles" % source
func = globals().get(func_name)
data = source.public_get_returnchartdata(params={
"currencyPair": symbol,
"start": round(start_time.timestamp()),
"end": round(end_time.timestamp()),
"period": frequency
})
if source == "bitfinex":
# Always use version 2 for bitfinex
source += "2"
data = pd.DataFrame(data).rename(columns={
'date': 'start_time',
'quoteVolume': 'quote_volume',
'weightedAverage': 'weighted_average'
})
exchange = getattr(ccxt, source.lower())()
describe = exchange.describe()
data.loc[:, 'start_time'] = data['start_time'].apply(
lambda x : pd.Timestamp.fromtimestamp(x).tz_localize('UTC'))
data['end_time'] = data['start_time'] + pd.DateOffset(
seconds=frequency)
return data
else:
if func is None:
raise ValueError("Source {} is not implemented".format(
source.__class__.__name__))
# Initialization
frequency = describe['timeframes'][frequency]
all_data = []
last_start_time = None
while start_time < end_time:
sleep(describe['rateLimit'] / 1000)
data = func(source=exchange, symbol=symbol, start_time=start_time,
end_time=end_time, frequency=frequency)
if len(data) == 0:
break
if (last_start_time is not None and
data["start_time"].iloc[0] >= last_start_time):
break
all_data.append(data)
start_time = data["end_time"].iloc[-1]
if len(all_data) == 0:
raise ValueError("Start time cannot be after end time.")
elif len(all_data) == 1:
return all_data[0]
else:
return pd.concat(all_data)
def latest_candles(source, symbols, frequency, frequency_count, end_time=None):
"""Return the latest candles based on the frequency and its count.
@@ -58,9 +96,9 @@ def latest_candles(source, symbols, frequency, frequency_count, end_time=None):
end_time = datetime.utcnow()
closest_end_time = pd.Timestamp(end_time).floor(
timedelta(seconds=frequency))
timedelta(seconds=FREQUENCY_TO_SEC_DICT[frequency]))
start_time = closest_end_time - timedelta(
seconds=frequency * frequency_count)
seconds=FREQUENCY_TO_SEC_DICT[frequency] * frequency_count)
all_data = []
for symbol in symbols:
+72
View File
@@ -0,0 +1,72 @@
import pandas as pd
def poloniex_candles(source, symbol, start_time, end_time, frequency):
"""Poloniex candles.
"""
data = source.public_get_returnchartdata(params={
"currencyPair": symbol,
"start": round(start_time.timestamp()),
"end": round(end_time.timestamp()),
"period": frequency
})
data = pd.DataFrame(data).rename(columns={
'date': 'start_time',
'quoteVolume': 'quote_volume',
'weightedAverage': 'weighted_average'
})
data.loc[:, 'start_time'] = data['start_time'].apply(
lambda x: pd.Timestamp.utcfromtimestamp(x))
data['end_time'] = data['start_time'].shift(-1)
return data.iloc[:-1, :]
def bitfinex_candles(source, symbol, start_time, end_time, frequency):
"""Bitfinex candles.
"""
data = source.request(
path='candles/trade:{}:{}/hist'.format(frequency, symbol),
params={
"start": round(start_time.timestamp() * 1000),
"end": round(end_time.timestamp() * 1000),
"sort": 1
})
data = pd.DataFrame(data, columns=["start_time", "open", "close", "high",
"low", "volume"])
data.loc[:, 'start_time'] = data['start_time'].apply(
lambda x: pd.Timestamp.utcfromtimestamp(x / 1000))
data['end_time'] = data['start_time'].shift(-1)
return data.iloc[:-1, :]
def gdax_candles(source, symbol, start_time, end_time, frequency):
"""GDAX candles.
"""
end_time += pd.DateOffset(seconds=1)
data = source.request(
path='products/{}/candles'.format(symbol),
params={
"granularity": frequency,
"start": start_time.isoformat(),
"end": end_time.isoformat(),
})
if len(data) == 0:
return data
data = pd.DataFrame(data, columns=["start_time", "low", "high", "open",
"close", "volume"])
data.loc[:, 'start_time'] = data['start_time'].apply(
lambda x: pd.Timestamp.utcfromtimestamp(x))
data = data.sort_values(['start_time'])
data['end_time'] = data['start_time'].shift(-1)
return data.iloc[:-1, :]
-212
View File
@@ -1,212 +0,0 @@
from functools import partial
from datetime import datetime, timedelta
from time import sleep
import pandas as pd
def historical_ticker(source, symbol, period, start_time=None, end_time=None,
**kwargs):
"""Return historical ticker.
:param source: Source, an Exchange API object.
:param symbol: Symbol, string object.
:param period: Period or frequency, followed with exchange protocol, string
object.
:param start_time: Start time, datetime object.
:param end_time: Start time, datetime object.
:param wait_sec: Seconds to wait between queries, int. Optional.
"""
# Validation
if start_time is not None and not isinstance(start_time, datetime):
raise ValueError("Start time is not a datetime object.")
if end_time is not None and not isinstance(end_time, datetime):
raise ValueError("End time is not a datetime object.")
# Source object name
source_name = source.__class__.__name__.lower().replace("api", "")
if source_name == "poloniex":
return _historical_ticker_poloniex(
source=source, symbol=symbol, period=period,
start_time=start_time, end_time=end_time,
**kwargs)
elif source_name == "gdax":
return _historical_ticker_gdax(
source=source, symbol=symbol, period=period,
start_time=start_time, end_time=end_time,
**kwargs)
elif source_name == "bitfinex":
return _historical_ticker_bitfinex(
source=source, symbol=symbol, period=period,
start_time=start_time, end_time=end_time,
**kwargs)
else:
raise ValueError("Source (%s [%s]) does not support historical ticker"
% (source, source_name))
def _historical_ticker_poloniex(source, symbol, period, start_time, end_time,
**kwargs):
"""Return historical ticker in Poloniex.
:param source: Source, an Exchange API object.
:param symbol: Symbol, string object.
:param period: Period or frequency, followed with exchange protocol, string
object.
:param start_time: Start time, datetime object.
:param end_time: Start time, datetime object.
"""
# Exchange validation
if start_time is None and end_time is None:
raise ValueError("Start time and end time cannot be both None.")
request_func = partial(source.return_chart_data, currencyPair=symbol,
period=period)
if start_time is not None:
request_func = partial(request_func,
start=start_time.timestamp())
if end_time is not None:
request_func = partial(request_func,
end=end_time.timestamp())
data = request_func()
data.raise_for_status()
data = pd.DataFrame(data.json())
data['date'] = data['date'].apply(
lambda x: pd.to_datetime(x, unit='s'))
data = data.set_index(['date'])
data.index.name = 'datetime'
data.columns.name = symbol
return data
def _historical_ticker_gdax(source, symbol, period, start_time, end_time,
**kwargs):
"""Return historical ticker in GDAX.
:param source: Source, an Exchange API object.
:param symbol: Symbol, string object.
:param period: Period or frequency, followed with exchange protocol, string
object.
:param start_time: Start time, datetime object.
:param end_time: Start time, datetime object.
:param wait_sec: Seconds to wait between queries, int. Optional.
"""
# Exchange validation
if (start_time is None) + (end_time is None) not in [0, 2]:
# Both start and end time must be provided
raise ValueError("Start and end time must be both provided")
request_func = partial(source.products_candles, product_id=symbol,
granularity=period)
if start_time is None and end_time is None:
# Just get the latest 300 ticks
data = request_func()
data.raise_for_status()
data = pd.DataFrame(data.json())
else:
# Safety net
last_datetime = start_time.timestamp()
data = []
while start_time <= end_time:
tmp_data = request_func(
start=start_time.isoformat(),
end=(start_time +
timedelta(seconds=period * 200)).isoformat())
tmp_data.raise_for_status()
tmp_data = pd.DataFrame(tmp_data.json())
# Append into data list
data.append(tmp_data)
# Check to exit
if last_datetime >= tmp_data.iloc[0, 0]:
# Same as the previous query
break
else:
last_datetime = tmp_data.iloc[0, 0]
start_time = datetime.fromtimestamp(last_datetime)
sleep(kwargs.get("wait_sec", 0.33))
if len(data) > 1:
data = pd.concat(data, axis=0)
else:
data = data[0]
data.columns = ['datetime', 'low', 'high', 'open', 'close', 'volume']
data['datetime'] = pd.to_datetime(data['datetime'], unit='s')
data = data.set_index('datetime').sort_index()
data = data[~data.index.duplicated(keep='first')]
return data
def _historical_ticker_bitfinex(source, symbol, period, start_time, end_time,
**kwargs):
"""Return historical ticker in Bitfinex.
:param source: Source, an Exchange API object.
:param symbol: Symbol, string object.
:param period: Period or frequency, followed with exchange protocol, string
object.
:param start_time: Start time, datetime object.
:param end_time: Start time, datetime object.
:param wait_sec: Seconds to wait between queries, int. Optional.
"""
request_func = partial(source.candles, symbol=symbol, timeframe=period,
section="hist", sort=1, limit=1000)
if start_time is None and end_time is None:
data = request_func()
data.raise_for_status()
data = pd.DataFrame(data.json())
else:
# Safety net
last_datetime = (
0 if start_time is None else start_time.timestamp() * 1000)
data = []
while start_time is None or end_time is None or start_time <= end_time:
f = request_func
if start_time is not None:
f = partial(f, start=round(start_time.timestamp() * 1000))
if end_time is not None:
f = partial(f, end=round(end_time.timestamp() * 1000))
tmp_data = f()
tmp_data.raise_for_status()
tmp_data = tmp_data.json()
if len(tmp_data) == 0:
break
elif last_datetime >= tmp_data[-1][0]:
print(last_datetime)
print(tmp_data[-1][0])
break
else:
last_datetime = tmp_data[-1][0]
start_time = datetime.fromtimestamp(
round(last_datetime / 1000 + 1))
data.append(pd.DataFrame(tmp_data))
sleep(kwargs.get("wait_sec", 1))
data = pd.concat(data, axis=0)
data.columns = ["datetime", "open", "close", "high", "low", "volume"]
data["datetime"] = pd.to_datetime(data["datetime"], unit="ms")
data = data.set_index("datetime").sort_index()
data = data[~data.index.duplicated(keep='first')]
return data
+120
View File
@@ -0,0 +1,120 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2018-01-27T23:56:10.234256Z",
"start_time": "2018-01-27T23:56:09.703438Z"
}
},
"outputs": [],
"source": [
"%load_ext autoreload\n",
"%autoreload 2\n",
"\n",
"import pandas as pd\n",
"\n",
"from libcryptomarket.core import candles, latest_candles"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Candles"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {
"ExecuteTime": {
"end_time": "2018-01-28T00:06:20.147882Z",
"start_time": "2018-01-28T00:06:14.812410Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Running exchange poloniex for instrument BTC_LTC\n",
"Running exchange bitfinex for instrument tBTCUSD\n",
"Running exchange gdax for instrument BTC-USD\n"
]
}
],
"source": [
"for source, symbol in [\n",
" (\"poloniex\", \"BTC_LTC\"), \n",
" (\"bitfinex\", \"tBTCUSD\"),\n",
" (\"gdax\", \"BTC-USD\")]:\n",
" print(\"Running exchange {} for instrument {}\".format(source, symbol))\n",
" data = candles(source=source, symbol=symbol, \n",
" start_time=pd.Timestamp(\"2017-12-15\"), end_time=pd.Timestamp(\"2017-12-31\"), frequency=\"1d\")\n",
" assert data[\"start_time\"].iloc[0] == pd.Timestamp(\"2017-12-15\")\n",
" assert data[\"end_time\"].iloc[-1] == pd.Timestamp(\"2017-12-31\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Latest candles"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for source, symbols in [\n",
" (\"poloniex\", [\"BTC_LTC\", \"BTC_ETH\"]), \n",
" (\"bitfinex\", [\"tBTCUSD\", \"tETHUSD\"])]:\n",
" print(\"Running exchange {} for instrument {}\".format(source, symbols))\n",
" data = latest_candles(source=source, symbols=symbols, frequency=\"30m\", frequency_count=1)\n",
" assert data.shape[0] == 1"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"ExecuteTime": {
"end_time": "2018-01-28T00:02:19.164726Z",
"start_time": "2018-01-28T00:02:14.937305Z"
}
},
"outputs": [],
"source": [
"data = candles(source=\"gdax\", symbol=\"BTC-USD\", \n",
" start_time=pd.Timestamp(\"2017-12-15\"), end_time=pd.Timestamp(\"2017-12-31\"), frequency=\"1d\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
+2
View File
@@ -53,6 +53,8 @@ setup(
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
],
entry_points={'console_scripts': [
'request-candles=libcryptomarket.cli.candles:main']},
test_suite='tests',
tests_require=test_requirements,
setup_requires=setup_requirements,