diff --git a/.gitignore b/.gitignore index 8282dcf..1c6cc45 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,4 @@ target/ .editorconfig .github/ .idea/ +.ipynb_checkpoints \ No newline at end of file diff --git a/Makefile b/Makefile index d1a2ed2..eebc730 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ clean-test: ## remove test and coverage artifacts rm -fr htmlcov/ autopep8: ## autopep8 to clean + autopep8 --aggressive --in-place --recursive libcryptomarket/*/*/*.py autopep8 --aggressive --in-place --recursive libcryptomarket/*/*.py autopep8 --aggressive --in-place --recursive libcryptomarket/*.py autopep8 --aggressive --in-place --recursive tests/*.py diff --git a/libcryptomarket/cli/candles.py b/libcryptomarket/cli/candles.py new file mode 100644 index 0000000..7b50ae9 --- /dev/null +++ b/libcryptomarket/cli/candles.py @@ -0,0 +1,75 @@ +import argparse +import logging + +import pandas as pd + +from libcryptomarket.core import candles, FREQUENCY_TO_SEC_DICT + +LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + + +def get_args(): + """Get input arguments. + """ + parser = argparse.ArgumentParser(description=( + 'Query historical candles to files.')) + parser.add_argument('--exchange', action='store', dest='exchange', + help='Exchange name.', required=True) + parser.add_argument('--symbols', action='store', dest='symbols', + help='List of symbols', + type=str, nargs='+', required=True) + parser.add_argument('--frequency', action='store', dest='frequency', + help='Frequency.', + choices=list(FREQUENCY_TO_SEC_DICT.keys()), + required=True) + parser.add_argument('--start-time', action='store', dest='start_time', + help='Start time in format of \'YYYY-MM-DD\'', + required=True) + parser.add_argument('--end-time', action='store', dest='end_time', + help='End time in format of \'YYYY-MM-DD\'', + required=True) + parser.add_argument('--output', action='store', dest='output', + help='Output filename', required=True) + + return parser.parse_args() + + +def main(): + """Main. + """ + args = get_args() + logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) + + start_time = pd.Timestamp(args.start_time) + logging.info('Start time: %s', start_time) + + end_time = pd.Timestamp(args.end_time) + logging.info('End time: %s', end_time) + + logging.info('Starting querying to exchange %s with frequency %s...', + args.exchange, args.frequency) + + all_data = {} + + for symbol in args.symbols: + logging.info('Querying symbol %s...', symbol) + data = candles(source=args.exchange, + symbol=symbol, + start_time=start_time, + end_time=end_time, + frequency=args.frequency) + all_data[symbol] = data.set_index(['start_time'], ['end_time']) + + logging.info('Cleaning the data...') + + all_data = pd.concat(all_data, axis=1, names=['symbol', 'value']) + all_data = all_data.stack(level=0) + + logging.info('Exporting to path (%s)...', args.output) + all_data.to_csv(args.output) + + logging.info('Exported all the historical prices') + + +if __name__ == '__main__': + main() diff --git a/libcryptomarket/core/__init__.py b/libcryptomarket/core/__init__.py index 9ef2f9a..4c590fa 100644 --- a/libcryptomarket/core/__init__.py +++ b/libcryptomarket/core/__init__.py @@ -1,6 +1,6 @@ # pylint: disable-msg=W0401 # flake8: noqa -from libcryptomarket.core.historical import historical_ticker from libcryptomarket.core.instrument import instruments from libcryptomarket.core.order_book import order_book -from libcryptomarket.core.candle import candles, latest_candles +from libcryptomarket.core.candle import ( + candles, latest_candles, FREQUENCY_TO_SEC_DICT) diff --git a/libcryptomarket/core/candle.py b/libcryptomarket/core/candle/__init__.py similarity index 52% rename from libcryptomarket/core/candle.py rename to libcryptomarket/core/candle/__init__.py index 3d04c15..ec11bf5 100644 --- a/libcryptomarket/core/candle.py +++ b/libcryptomarket/core/candle/__init__.py @@ -1,8 +1,29 @@ from datetime import datetime, timedelta +from time import sleep import pandas as pd import ccxt +from .exchanges import * # noqa + +FREQUENCY_TO_SEC_DICT = { + '1m': 60, + '5m': 300, + '15m': 900, + '30m': 1800, + '1h': 3600, + '3h': 10800, + '6h': 21600, + '12h': 43200, + '1d': 86400, + '1w': 86400 * 7, + '2w': 86400 * 7 * 2, + '1M': 86400 * 30, +} + +FREQUENCY_TO_SEC_DICT.update(dict( + [(value, value) for value in FREQUENCY_TO_SEC_DICT.values()])) + def candles(source, symbol, start_time, end_time, frequency): """Return candles of a given period and frequency. @@ -14,32 +35,49 @@ def candles(source, symbol, start_time, end_time, frequency): :param frequency: `int` frequency in seconds. """ - if source.lower() == 'poloniex': - source = getattr(ccxt, source.lower())() + source = source.lower() + func_name = "%s_candles" % source + func = globals().get(func_name) - data = source.public_get_returnchartdata(params={ - "currencyPair": symbol, - "start": round(start_time.timestamp()), - "end": round(end_time.timestamp()), - "period": frequency - }) + if source == "bitfinex": + # Always use version 2 for bitfinex + source += "2" - data = pd.DataFrame(data).rename(columns={ - 'date': 'start_time', - 'quoteVolume': 'quote_volume', - 'weightedAverage': 'weighted_average' - }) + exchange = getattr(ccxt, source.lower())() + describe = exchange.describe() - data.loc[:, 'start_time'] = data['start_time'].apply( - lambda x : pd.Timestamp.fromtimestamp(x).tz_localize('UTC')) - data['end_time'] = data['start_time'] + pd.DateOffset( - seconds=frequency) - - return data - else: + if func is None: raise ValueError("Source {} is not implemented".format( source.__class__.__name__)) + # Initialization + frequency = describe['timeframes'][frequency] + + all_data = [] + last_start_time = None + + while start_time < end_time: + sleep(describe['rateLimit'] / 1000) + data = func(source=exchange, symbol=symbol, start_time=start_time, + end_time=end_time, frequency=frequency) + + if len(data) == 0: + break + + if (last_start_time is not None and + data["start_time"].iloc[0] >= last_start_time): + break + + all_data.append(data) + start_time = data["end_time"].iloc[-1] + + if len(all_data) == 0: + raise ValueError("Start time cannot be after end time.") + elif len(all_data) == 1: + return all_data[0] + else: + return pd.concat(all_data) + def latest_candles(source, symbols, frequency, frequency_count, end_time=None): """Return the latest candles based on the frequency and its count. @@ -58,9 +96,9 @@ def latest_candles(source, symbols, frequency, frequency_count, end_time=None): end_time = datetime.utcnow() closest_end_time = pd.Timestamp(end_time).floor( - timedelta(seconds=frequency)) + timedelta(seconds=FREQUENCY_TO_SEC_DICT[frequency])) start_time = closest_end_time - timedelta( - seconds=frequency * frequency_count) + seconds=FREQUENCY_TO_SEC_DICT[frequency] * frequency_count) all_data = [] for symbol in symbols: diff --git a/libcryptomarket/core/candle/exchanges.py b/libcryptomarket/core/candle/exchanges.py new file mode 100644 index 0000000..103f061 --- /dev/null +++ b/libcryptomarket/core/candle/exchanges.py @@ -0,0 +1,72 @@ +import pandas as pd + + +def poloniex_candles(source, symbol, start_time, end_time, frequency): + """Poloniex candles. + """ + data = source.public_get_returnchartdata(params={ + "currencyPair": symbol, + "start": round(start_time.timestamp()), + "end": round(end_time.timestamp()), + "period": frequency + }) + + data = pd.DataFrame(data).rename(columns={ + 'date': 'start_time', + 'quoteVolume': 'quote_volume', + 'weightedAverage': 'weighted_average' + }) + + data.loc[:, 'start_time'] = data['start_time'].apply( + lambda x: pd.Timestamp.utcfromtimestamp(x)) + data['end_time'] = data['start_time'].shift(-1) + + return data.iloc[:-1, :] + + +def bitfinex_candles(source, symbol, start_time, end_time, frequency): + """Bitfinex candles. + """ + data = source.request( + path='candles/trade:{}:{}/hist'.format(frequency, symbol), + params={ + "start": round(start_time.timestamp() * 1000), + "end": round(end_time.timestamp() * 1000), + "sort": 1 + }) + + data = pd.DataFrame(data, columns=["start_time", "open", "close", "high", + "low", "volume"]) + + data.loc[:, 'start_time'] = data['start_time'].apply( + lambda x: pd.Timestamp.utcfromtimestamp(x / 1000)) + data['end_time'] = data['start_time'].shift(-1) + + return data.iloc[:-1, :] + + +def gdax_candles(source, symbol, start_time, end_time, frequency): + """GDAX candles. + """ + end_time += pd.DateOffset(seconds=1) + + data = source.request( + path='products/{}/candles'.format(symbol), + params={ + "granularity": frequency, + "start": start_time.isoformat(), + "end": end_time.isoformat(), + }) + + if len(data) == 0: + return data + + data = pd.DataFrame(data, columns=["start_time", "low", "high", "open", + "close", "volume"]) + + data.loc[:, 'start_time'] = data['start_time'].apply( + lambda x: pd.Timestamp.utcfromtimestamp(x)) + data = data.sort_values(['start_time']) + data['end_time'] = data['start_time'].shift(-1) + + return data.iloc[:-1, :] diff --git a/libcryptomarket/core/historical.py b/libcryptomarket/core/historical.py deleted file mode 100644 index cf5a232..0000000 --- a/libcryptomarket/core/historical.py +++ /dev/null @@ -1,212 +0,0 @@ -from functools import partial -from datetime import datetime, timedelta -from time import sleep - -import pandas as pd - - -def historical_ticker(source, symbol, period, start_time=None, end_time=None, - **kwargs): - """Return historical ticker. - - :param source: Source, an Exchange API object. - :param symbol: Symbol, string object. - :param period: Period or frequency, followed with exchange protocol, string - object. - :param start_time: Start time, datetime object. - :param end_time: Start time, datetime object. - :param wait_sec: Seconds to wait between queries, int. Optional. - """ - # Validation - if start_time is not None and not isinstance(start_time, datetime): - raise ValueError("Start time is not a datetime object.") - - if end_time is not None and not isinstance(end_time, datetime): - raise ValueError("End time is not a datetime object.") - - # Source object name - source_name = source.__class__.__name__.lower().replace("api", "") - - if source_name == "poloniex": - return _historical_ticker_poloniex( - source=source, symbol=symbol, period=period, - start_time=start_time, end_time=end_time, - **kwargs) - elif source_name == "gdax": - return _historical_ticker_gdax( - source=source, symbol=symbol, period=period, - start_time=start_time, end_time=end_time, - **kwargs) - elif source_name == "bitfinex": - return _historical_ticker_bitfinex( - source=source, symbol=symbol, period=period, - start_time=start_time, end_time=end_time, - **kwargs) - else: - raise ValueError("Source (%s [%s]) does not support historical ticker" - % (source, source_name)) - - -def _historical_ticker_poloniex(source, symbol, period, start_time, end_time, - **kwargs): - """Return historical ticker in Poloniex. - - :param source: Source, an Exchange API object. - :param symbol: Symbol, string object. - :param period: Period or frequency, followed with exchange protocol, string - object. - :param start_time: Start time, datetime object. - :param end_time: Start time, datetime object. - """ - # Exchange validation - if start_time is None and end_time is None: - raise ValueError("Start time and end time cannot be both None.") - - request_func = partial(source.return_chart_data, currencyPair=symbol, - period=period) - - if start_time is not None: - request_func = partial(request_func, - start=start_time.timestamp()) - if end_time is not None: - request_func = partial(request_func, - end=end_time.timestamp()) - data = request_func() - data.raise_for_status() - data = pd.DataFrame(data.json()) - data['date'] = data['date'].apply( - lambda x: pd.to_datetime(x, unit='s')) - data = data.set_index(['date']) - data.index.name = 'datetime' - data.columns.name = symbol - - return data - - -def _historical_ticker_gdax(source, symbol, period, start_time, end_time, - **kwargs): - """Return historical ticker in GDAX. - - :param source: Source, an Exchange API object. - :param symbol: Symbol, string object. - :param period: Period or frequency, followed with exchange protocol, string - object. - :param start_time: Start time, datetime object. - :param end_time: Start time, datetime object. - :param wait_sec: Seconds to wait between queries, int. Optional. - """ - # Exchange validation - if (start_time is None) + (end_time is None) not in [0, 2]: - # Both start and end time must be provided - raise ValueError("Start and end time must be both provided") - - request_func = partial(source.products_candles, product_id=symbol, - granularity=period) - - if start_time is None and end_time is None: - # Just get the latest 300 ticks - data = request_func() - data.raise_for_status() - data = pd.DataFrame(data.json()) - - else: - # Safety net - last_datetime = start_time.timestamp() - - data = [] - - while start_time <= end_time: - tmp_data = request_func( - start=start_time.isoformat(), - end=(start_time + - timedelta(seconds=period * 200)).isoformat()) - tmp_data.raise_for_status() - tmp_data = pd.DataFrame(tmp_data.json()) - - # Append into data list - data.append(tmp_data) - - # Check to exit - if last_datetime >= tmp_data.iloc[0, 0]: - # Same as the previous query - break - else: - last_datetime = tmp_data.iloc[0, 0] - start_time = datetime.fromtimestamp(last_datetime) - - sleep(kwargs.get("wait_sec", 0.33)) - - if len(data) > 1: - data = pd.concat(data, axis=0) - else: - data = data[0] - - data.columns = ['datetime', 'low', 'high', 'open', 'close', 'volume'] - data['datetime'] = pd.to_datetime(data['datetime'], unit='s') - data = data.set_index('datetime').sort_index() - data = data[~data.index.duplicated(keep='first')] - - return data - - -def _historical_ticker_bitfinex(source, symbol, period, start_time, end_time, - **kwargs): - """Return historical ticker in Bitfinex. - - :param source: Source, an Exchange API object. - :param symbol: Symbol, string object. - :param period: Period or frequency, followed with exchange protocol, string - object. - :param start_time: Start time, datetime object. - :param end_time: Start time, datetime object. - :param wait_sec: Seconds to wait between queries, int. Optional. - """ - request_func = partial(source.candles, symbol=symbol, timeframe=period, - section="hist", sort=1, limit=1000) - - if start_time is None and end_time is None: - data = request_func() - data.raise_for_status() - data = pd.DataFrame(data.json()) - else: - # Safety net - last_datetime = ( - 0 if start_time is None else start_time.timestamp() * 1000) - - data = [] - - while start_time is None or end_time is None or start_time <= end_time: - f = request_func - - if start_time is not None: - f = partial(f, start=round(start_time.timestamp() * 1000)) - - if end_time is not None: - f = partial(f, end=round(end_time.timestamp() * 1000)) - - tmp_data = f() - tmp_data.raise_for_status() - tmp_data = tmp_data.json() - - if len(tmp_data) == 0: - break - elif last_datetime >= tmp_data[-1][0]: - print(last_datetime) - print(tmp_data[-1][0]) - break - else: - last_datetime = tmp_data[-1][0] - start_time = datetime.fromtimestamp( - round(last_datetime / 1000 + 1)) - data.append(pd.DataFrame(tmp_data)) - - sleep(kwargs.get("wait_sec", 1)) - - data = pd.concat(data, axis=0) - - data.columns = ["datetime", "open", "close", "high", "low", "volume"] - data["datetime"] = pd.to_datetime(data["datetime"], unit="ms") - data = data.set_index("datetime").sort_index() - data = data[~data.index.duplicated(keep='first')] - - return data diff --git a/notebooks/Candles.ipynb b/notebooks/Candles.ipynb new file mode 100644 index 0000000..7c9d0ac --- /dev/null +++ b/notebooks/Candles.ipynb @@ -0,0 +1,120 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2018-01-27T23:56:10.234256Z", + "start_time": "2018-01-27T23:56:09.703438Z" + } + }, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "import pandas as pd\n", + "\n", + "from libcryptomarket.core import candles, latest_candles" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Candles" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "ExecuteTime": { + "end_time": "2018-01-28T00:06:20.147882Z", + "start_time": "2018-01-28T00:06:14.812410Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running exchange poloniex for instrument BTC_LTC\n", + "Running exchange bitfinex for instrument tBTCUSD\n", + "Running exchange gdax for instrument BTC-USD\n" + ] + } + ], + "source": [ + "for source, symbol in [\n", + " (\"poloniex\", \"BTC_LTC\"), \n", + " (\"bitfinex\", \"tBTCUSD\"),\n", + " (\"gdax\", \"BTC-USD\")]:\n", + " print(\"Running exchange {} for instrument {}\".format(source, symbol))\n", + " data = candles(source=source, symbol=symbol, \n", + " start_time=pd.Timestamp(\"2017-12-15\"), end_time=pd.Timestamp(\"2017-12-31\"), frequency=\"1d\")\n", + " assert data[\"start_time\"].iloc[0] == pd.Timestamp(\"2017-12-15\")\n", + " assert data[\"end_time\"].iloc[-1] == pd.Timestamp(\"2017-12-31\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Latest candles" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for source, symbols in [\n", + " (\"poloniex\", [\"BTC_LTC\", \"BTC_ETH\"]), \n", + " (\"bitfinex\", [\"tBTCUSD\", \"tETHUSD\"])]:\n", + " print(\"Running exchange {} for instrument {}\".format(source, symbols))\n", + " data = latest_candles(source=source, symbols=symbols, frequency=\"30m\", frequency_count=1)\n", + " assert data.shape[0] == 1" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2018-01-28T00:02:19.164726Z", + "start_time": "2018-01-28T00:02:14.937305Z" + } + }, + "outputs": [], + "source": [ + "data = candles(source=\"gdax\", symbol=\"BTC-USD\", \n", + " start_time=pd.Timestamp(\"2017-12-15\"), end_time=pd.Timestamp(\"2017-12-31\"), frequency=\"1d\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/setup.py b/setup.py index 5f928af..971d3bd 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,8 @@ setup( 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', ], + entry_points={'console_scripts': [ + 'request-candles=libcryptomarket.cli.candles:main']}, test_suite='tests', tests_require=test_requirements, setup_requires=setup_requirements,