mirror of
https://github.com/wassname/catalyst.git
synced 2026-07-07 17:47:24 +08:00
Unit tested exchange loader extension and backtest data portal refactoring
This commit is contained in:
@@ -48,7 +48,12 @@ class Bitfinex(Exchange):
|
||||
self._portfolio = portfolio
|
||||
self.minute_writer = None
|
||||
self.minute_reader = None
|
||||
self.num_candles_limit = 100
|
||||
self.num_candles_limit = 1000
|
||||
|
||||
# Max is 90 but playing it safe
|
||||
# https://www.bitfinex.com/posts/188
|
||||
self.max_requests_per_minute = 20
|
||||
self.request_cpt = dict()
|
||||
|
||||
def _request(self, operation, data, version='v1'):
|
||||
payload_object = {
|
||||
@@ -176,6 +181,7 @@ class Bitfinex(Exchange):
|
||||
def get_balances(self):
|
||||
log.debug('retrieving wallets balances')
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('balances', None)
|
||||
balances = response.json()
|
||||
except Exception as e:
|
||||
@@ -295,6 +301,7 @@ class Bitfinex(Exchange):
|
||||
url += '/last'
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
response = requests.get(url)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
@@ -377,6 +384,7 @@ class Bitfinex(Exchange):
|
||||
|
||||
date = pd.Timestamp.utcnow()
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('order/new', req)
|
||||
order_status = response.json()
|
||||
except Exception as e:
|
||||
@@ -418,6 +426,7 @@ class Bitfinex(Exchange):
|
||||
orders for this asset.
|
||||
"""
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('orders', None)
|
||||
order_statuses = response.json()
|
||||
except Exception as e:
|
||||
@@ -452,6 +461,7 @@ class Bitfinex(Exchange):
|
||||
The order object.
|
||||
"""
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request(
|
||||
'order/status', {'order_id': int(order_id)})
|
||||
order_status = response.json()
|
||||
@@ -477,6 +487,7 @@ class Bitfinex(Exchange):
|
||||
if isinstance(order_param, Order) else order_param
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
response = self._request('order/cancel', {'order_id': order_id})
|
||||
status = response.json()
|
||||
except Exception as e:
|
||||
@@ -501,6 +512,7 @@ class Bitfinex(Exchange):
|
||||
log.debug('fetching tickers {}'.format(symbols))
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
response = requests.get(
|
||||
'{url}/v2/tickers?symbols={symbols}'.format(
|
||||
url=self.url,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"neobtc": {
|
||||
"symbol": "neo_btc",
|
||||
"start_date": "2017-09-07",
|
||||
"precision":5
|
||||
"precision": 5
|
||||
},
|
||||
"neousd": {
|
||||
"symbol": "neo_usd",
|
||||
@@ -30,19 +30,19 @@
|
||||
},
|
||||
"ethusd": {
|
||||
"symbol": "eth_usd",
|
||||
"start_date": "2010-01-01"
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"ethbtc": {
|
||||
"symbol": "eth_btc",
|
||||
"start_date": "2010-01-01"
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"etcbtc": {
|
||||
"symbol": "etc_btc",
|
||||
"start_date": "2010-01-01"
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"etcusd": {
|
||||
"symbol": "etc_usd",
|
||||
"start_date": "2010-01-01"
|
||||
"start_date": "2017-01-01"
|
||||
},
|
||||
"rrtusd": {
|
||||
"symbol": "rrt_usd",
|
||||
|
||||
@@ -26,6 +26,11 @@ class Bittrex(Exchange):
|
||||
self.base_currency = base_currency
|
||||
self._portfolio = portfolio
|
||||
|
||||
# Not sure what the rate limit is but trying to play it safe
|
||||
# https://bitcoin.stackexchange.com/questions/53778/bittrex-api-rate-limit
|
||||
self.max_requests_per_minute = 60
|
||||
self.request_cpt = dict()
|
||||
|
||||
self.minute_writer = None
|
||||
self.minute_reader = None
|
||||
|
||||
@@ -62,6 +67,7 @@ class Bittrex(Exchange):
|
||||
"""
|
||||
symbol_map = dict()
|
||||
|
||||
self.ask_request()
|
||||
markets = self.api.getmarkets()
|
||||
for market in markets:
|
||||
exchange_symbol = market['MarketName']
|
||||
@@ -79,6 +85,7 @@ class Bittrex(Exchange):
|
||||
def get_balances(self):
|
||||
try:
|
||||
log.debug('retrieving wallet balances')
|
||||
self.ask_request()
|
||||
balances = self.api.getbalances()
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
@@ -99,6 +106,7 @@ class Bittrex(Exchange):
|
||||
|
||||
price = style.get_limit_price(is_buy)
|
||||
try:
|
||||
self.ask_request()
|
||||
if is_buy:
|
||||
order_status = self.api.buylimit(exchange_symbol, amount,
|
||||
price)
|
||||
@@ -139,6 +147,7 @@ class Bittrex(Exchange):
|
||||
def get_open_orders(self, asset):
|
||||
symbol = self.get_symbol(asset)
|
||||
try:
|
||||
self.ask_request()
|
||||
open_orders = self.api.getopenorders(symbol)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
@@ -182,6 +191,7 @@ class Bittrex(Exchange):
|
||||
def get_order(self, order_id):
|
||||
log.info('retrieving order {}'.format(order_id))
|
||||
try:
|
||||
self.ask_request()
|
||||
order_status = self.api.getorder(order_id)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
@@ -197,6 +207,7 @@ class Bittrex(Exchange):
|
||||
log.info('cancelling order {}'.format(order_id))
|
||||
|
||||
try:
|
||||
self.ask_request()
|
||||
status = self.api.cancel(order_id)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
@@ -208,7 +219,8 @@ class Bittrex(Exchange):
|
||||
error=status['message']
|
||||
)
|
||||
|
||||
def get_candles(self, data_frequency, assets, bar_count=None, start_date=None):
|
||||
def get_candles(self, data_frequency, assets, bar_count=None,
|
||||
start_date=None):
|
||||
"""
|
||||
Supported Intervals
|
||||
-------------------
|
||||
@@ -299,6 +311,7 @@ class Bittrex(Exchange):
|
||||
for asset in assets:
|
||||
symbol = self.get_symbol(asset)
|
||||
try:
|
||||
self.ask_request()
|
||||
ticker = self.api.getticker(symbol)
|
||||
except Exception as e:
|
||||
raise ExchangeRequestError(error=e)
|
||||
|
||||
@@ -12,21 +12,25 @@
|
||||
# limitations under the License.
|
||||
|
||||
import abc
|
||||
import os
|
||||
from time import sleep
|
||||
|
||||
import os
|
||||
import collections
|
||||
import pandas as pd
|
||||
from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.data.bundles.core import load, from_bundle_ingest_dirname, \
|
||||
BundleData, minute_path, five_minute_path, daily_path
|
||||
from catalyst.data.data_portal import DataPortal
|
||||
from catalyst.data.five_minute_bars import BcolzFiveMinuteBarReader
|
||||
from catalyst.data.minute_bars import BcolzMinuteBarReader
|
||||
from catalyst.data.us_equity_pricing import BcolzDailyBarReader
|
||||
from catalyst.exchange.exchange_errors import (
|
||||
ExchangeRequestError,
|
||||
ExchangeBarDataError
|
||||
)
|
||||
from catalyst.data.bundles.core import load
|
||||
from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root
|
||||
ExchangeBarDataError,
|
||||
BundleNotFoundError)
|
||||
from catalyst.utils.paths import data_path
|
||||
|
||||
log = Logger('DataPortalExchange')
|
||||
|
||||
@@ -145,8 +149,8 @@ class DataPortalExchangeBase(DataPortal):
|
||||
try:
|
||||
if isinstance(assets, TradingPair):
|
||||
exchange = self.exchanges[assets.exchange]
|
||||
return exchange.get_spot_value(
|
||||
assets, field, dt, data_frequency)
|
||||
return self.get_exchange_spot_value(
|
||||
exchange, assets, field, dt, data_frequency)
|
||||
|
||||
else:
|
||||
exchange_assets = dict()
|
||||
@@ -156,20 +160,29 @@ class DataPortalExchangeBase(DataPortal):
|
||||
|
||||
exchange_assets[asset.exchange].append(asset)
|
||||
|
||||
spot_values = []
|
||||
for exchange_name in exchange_assets:
|
||||
exchange = self.exchanges[exchange_name]
|
||||
assets = exchange_assets[exchange_name]
|
||||
exchange_spot_values = self.get_exchange_spot_value(
|
||||
exchange,
|
||||
assets,
|
||||
field,
|
||||
dt,
|
||||
data_frequency
|
||||
)
|
||||
spot_values += exchange_spot_values
|
||||
if len(exchange_assets.keys()) == 1:
|
||||
exchange = self.exchanges[exchange_assets.keys()[0]]
|
||||
return self.get_exchange_spot_value(
|
||||
exchange, assets, field, dt, data_frequency)
|
||||
|
||||
return spot_values
|
||||
else:
|
||||
spot_values = []
|
||||
for exchange_name in exchange_assets:
|
||||
exchange = self.exchanges[exchange_name]
|
||||
assets = exchange_assets[exchange_name]
|
||||
exchange_spot_values = self.get_exchange_spot_value(
|
||||
exchange,
|
||||
assets,
|
||||
field,
|
||||
dt,
|
||||
data_frequency
|
||||
)
|
||||
if len(assets) == 1:
|
||||
spot_values.append(exchange_spot_values)
|
||||
else:
|
||||
spot_values += exchange_spot_values
|
||||
|
||||
return spot_values
|
||||
|
||||
except ExchangeRequestError as e:
|
||||
log.warn(
|
||||
@@ -239,10 +252,53 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
|
||||
|
||||
super(DataPortalExchangeBacktest, self).__init__(*args, **kwargs)
|
||||
|
||||
self.minute_readers = dict()
|
||||
self.daily_bar_readers = dict()
|
||||
self.minute_bar_readers = dict()
|
||||
self.five_minute_bar_readers = dict()
|
||||
for exchange_name in self.exchanges:
|
||||
root = get_exchange_minute_writer_root(exchange_name)
|
||||
self.minute_readers[exchange_name] = BcolzMinuteBarReader(root)
|
||||
name = 'exchange_{}'.format(exchange_name)
|
||||
time_folder = \
|
||||
DataPortalExchangeBacktest.find_most_recent_time(name)
|
||||
|
||||
if time_folder is None:
|
||||
raise BundleNotFoundError(exchange=exchange_name)
|
||||
|
||||
self.daily_bar_readers[exchange_name] = \
|
||||
BcolzDailyBarReader(
|
||||
daily_path(name, time_folder),
|
||||
)
|
||||
|
||||
self.five_minute_bar_readers[exchange_name] = \
|
||||
BcolzFiveMinuteBarReader(
|
||||
five_minute_path(name, time_folder),
|
||||
)
|
||||
|
||||
self.minute_bar_readers[exchange_name] = \
|
||||
BcolzMinuteBarReader(
|
||||
minute_path(name, time_folder),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def find_most_recent_time(bundle_name):
|
||||
try:
|
||||
bundle_folders = os.listdir(
|
||||
data_path([bundle_name]),
|
||||
)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
most_recent_bundle = dict()
|
||||
for folder in bundle_folders:
|
||||
date = from_bundle_ingest_dirname(folder)
|
||||
if not most_recent_bundle or date > \
|
||||
most_recent_bundle[most_recent_bundle.keys()[0]]:
|
||||
most_recent_bundle = dict()
|
||||
most_recent_bundle[folder] = date
|
||||
|
||||
if most_recent_bundle:
|
||||
return most_recent_bundle.keys()[0]
|
||||
else:
|
||||
return None
|
||||
|
||||
def get_exchange_history_window(self,
|
||||
exchange,
|
||||
@@ -267,7 +323,11 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
|
||||
data_frequency):
|
||||
|
||||
if data_frequency == 'minute':
|
||||
reader = self.minute_readers[exchange.name]
|
||||
reader = self.minute_bar_readers[exchange.name]
|
||||
elif data_frequency == '5-minute':
|
||||
reader = self.five_minute_bar_readers[exchange.name]
|
||||
elif data_frequency == 'daily':
|
||||
reader = self.daily_bar_readers[exchange.name]
|
||||
else:
|
||||
raise ValueError('Unsupported frequency')
|
||||
|
||||
@@ -284,4 +344,7 @@ class DataPortalExchangeBacktest(DataPortalExchangeBase):
|
||||
log.warn('minute data not found: {}'.format(e))
|
||||
values.append(None)
|
||||
|
||||
return values
|
||||
if len(assets) == 1:
|
||||
return values[0]
|
||||
else:
|
||||
return values
|
||||
|
||||
@@ -2,6 +2,7 @@ import abc
|
||||
import collections
|
||||
import random
|
||||
from abc import ABCMeta, abstractmethod, abstractproperty
|
||||
from datetime import timedelta
|
||||
from time import sleep
|
||||
|
||||
import numpy as np
|
||||
@@ -10,9 +11,6 @@ from catalyst.assets._assets import TradingPair
|
||||
from logbook import Logger
|
||||
|
||||
from catalyst.data.data_portal import BASE_FIELDS
|
||||
from catalyst.errors import (
|
||||
SymbolNotFound,
|
||||
)
|
||||
from catalyst.exchange.exchange_errors import MismatchingBaseCurrencies, \
|
||||
InvalidOrderStyle, BaseCurrencyNotFoundError, SymbolNotFoundOnExchange
|
||||
from catalyst.exchange.exchange_execution import ExchangeStopLimitOrder, \
|
||||
@@ -35,7 +33,10 @@ class Exchange:
|
||||
self.minute_writer = None
|
||||
self.minute_reader = None
|
||||
self.base_currency = None
|
||||
self.num_candles_limit = 100
|
||||
|
||||
self.num_candles_limit = None
|
||||
self.max_requests_per_minute = None
|
||||
self.request_cpt = None
|
||||
|
||||
@property
|
||||
def positions(self):
|
||||
@@ -64,6 +65,50 @@ class Exchange:
|
||||
def time_skew(self):
|
||||
pass
|
||||
|
||||
def ask_request(self):
|
||||
"""
|
||||
Asks permission to issue a request to the exchange.
|
||||
The primary purpose is to avoid hitting rate limits.
|
||||
|
||||
The application will pause if the maximum requests per minute
|
||||
permitted by the exchange is exceeded.
|
||||
|
||||
:return boolean:
|
||||
|
||||
"""
|
||||
now = pd.Timestamp.utcnow()
|
||||
if not self.request_cpt:
|
||||
self.request_cpt = dict()
|
||||
self.request_cpt[now] = 0
|
||||
return True
|
||||
|
||||
cpt_date = self.request_cpt.keys()[0]
|
||||
cpt = self.request_cpt[cpt_date]
|
||||
|
||||
if now > cpt_date + timedelta(minutes=1):
|
||||
self.request_cpt = dict()
|
||||
self.request_cpt[now] = 0
|
||||
return True
|
||||
|
||||
if cpt >= self.max_requests_per_minute:
|
||||
delta = now - cpt_date
|
||||
|
||||
sleep_period = 60 - delta.total_seconds()
|
||||
|
||||
# log.debug(
|
||||
# 'max requests {} reached, sleeping for {} seconds'.format(
|
||||
# self.max_requests_per_minute,
|
||||
# sleep_period
|
||||
# ))
|
||||
sleep(sleep_period)
|
||||
|
||||
now = pd.Timestamp.utcnow()
|
||||
self.request_cpt = dict()
|
||||
self.request_cpt[now] = 0
|
||||
return True
|
||||
else:
|
||||
self.request_cpt[cpt_date] += 1
|
||||
|
||||
def get_symbol(self, asset):
|
||||
"""
|
||||
Get the exchange specific symbol of the given asset.
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
from datetime import timedelta
|
||||
|
||||
import pandas as pd
|
||||
from logbook import Logger
|
||||
from time import sleep
|
||||
|
||||
from catalyst.data.minute_bars import BcolzMinuteOverlappingData
|
||||
import os
|
||||
import pandas as pd
|
||||
from catalyst.data.bundles.base_pricing import BaseCryptoPricingBundle
|
||||
|
||||
from catalyst import get_calendar
|
||||
from logbook import Logger, INFO
|
||||
|
||||
from catalyst.data.five_minute_bars import BcolzFiveMinuteOverlappingData
|
||||
from catalyst.data.minute_bars import BcolzMinuteOverlappingData, \
|
||||
BcolzMinuteBarReader
|
||||
from catalyst.exchange.bitfinex.bitfinex import Bitfinex
|
||||
from catalyst.exchange.bittrex.bittrex import Bittrex
|
||||
from catalyst.exchange.exchange_errors import ExchangeNotFoundError
|
||||
@@ -32,15 +40,111 @@ def fetch_candles_chunk(exchange, assets, data_frequency, end_dt, bar_count):
|
||||
asset_candles = candles[asset]
|
||||
|
||||
asset_df = pd.DataFrame(asset_candles)
|
||||
asset_df.set_index('last_traded', inplace=True, drop=True)
|
||||
asset_df.sort_index(inplace=True)
|
||||
if not asset_df.empty:
|
||||
asset_df.set_index('last_traded', inplace=True, drop=True)
|
||||
asset_df.sort_index(inplace=True)
|
||||
|
||||
series[asset] = asset_df
|
||||
series[asset] = asset_df
|
||||
|
||||
return series
|
||||
|
||||
|
||||
def exchange_bundle(exchange_name, symbols, start=None, end=None):
|
||||
def process_bar_data(exchange, assets, writer, data_frequency,
|
||||
show_progress, start, end):
|
||||
open_calendar = get_calendar('OPEN')
|
||||
|
||||
writer.calendar = open_calendar
|
||||
writer.minutes_per_day = 1440
|
||||
writer.write_metadata = True
|
||||
|
||||
delta = end - start
|
||||
if data_frequency == 'minute':
|
||||
delta_periods = delta.total_seconds() / 60
|
||||
frequency = '1m'
|
||||
|
||||
elif data_frequency == '5-minute':
|
||||
delta_periods = delta.total_seconds() / 60 / 5
|
||||
frequency = '5m'
|
||||
|
||||
elif data_frequency == 'daily':
|
||||
delta_periods = delta.total_seconds() / 60 / 60 / 24
|
||||
frequency = '1d'
|
||||
|
||||
else:
|
||||
raise ValueError('frequency not supported')
|
||||
|
||||
if delta_periods > exchange.num_candles_limit:
|
||||
bar_count = exchange.num_candles_limit
|
||||
|
||||
chunks = []
|
||||
last_chunk_date = end
|
||||
while last_chunk_date > start + timedelta(minutes=bar_count):
|
||||
# TODO: account for the partial last bar
|
||||
chunk = dict(end=last_chunk_date, bar_count=bar_count)
|
||||
chunks.append(chunk)
|
||||
|
||||
last_chunk_date = \
|
||||
last_chunk_date - timedelta(minutes=(bar_count + 1))
|
||||
|
||||
chunks.reverse()
|
||||
|
||||
else:
|
||||
chunks = [dict(end=end, bar_count=delta_periods)]
|
||||
|
||||
with maybe_show_progress(
|
||||
chunks,
|
||||
show_progress,
|
||||
label='Fetching {exchange} {frequency} candles: '.format(
|
||||
exchange=exchange.name,
|
||||
frequency=data_frequency
|
||||
)) as it:
|
||||
|
||||
for chunk in it:
|
||||
assets_candles_dict = fetch_candles_chunk(
|
||||
exchange=exchange,
|
||||
assets=assets,
|
||||
data_frequency=frequency,
|
||||
end_dt=chunk['end'],
|
||||
bar_count=chunk['bar_count']
|
||||
)
|
||||
log.debug('requests counter {}'.format(exchange.request_cpt))
|
||||
|
||||
if not assets_candles_dict.keys():
|
||||
log.debug(
|
||||
'no data: {symbols} on {exchange}, date {end}'.format(
|
||||
symbols=assets,
|
||||
exchange=exchange.name,
|
||||
end=chunk['end']
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
data = []
|
||||
for asset in assets_candles_dict:
|
||||
df = assets_candles_dict[asset]
|
||||
sid = asset.sid
|
||||
data.append((sid, df))
|
||||
|
||||
try:
|
||||
log.debug(
|
||||
'writing chunk {start} to {end}'.format(
|
||||
start=chunk['end'] - timedelta(
|
||||
minutes=chunk['bar_count']),
|
||||
end=chunk['end']
|
||||
)
|
||||
)
|
||||
writer.write(
|
||||
data=data,
|
||||
show_progress=False,
|
||||
invalid_data_behavior='raise'
|
||||
)
|
||||
except (BcolzMinuteOverlappingData,
|
||||
BcolzFiveMinuteOverlappingData) as e:
|
||||
log.warn('chunk already exists {}: {}'.format(chunk, e))
|
||||
|
||||
|
||||
def exchange_bundle(exchange_name, symbols=None, start=None, end=None,
|
||||
log_level=INFO):
|
||||
"""Create a data bundle ingest function for the specified exchange.
|
||||
|
||||
Parameters
|
||||
@@ -68,24 +172,27 @@ def exchange_bundle(exchange_name, symbols, start=None, end=None):
|
||||
.. code-block:: python
|
||||
|
||||
from catalyst.data.bundles import register
|
||||
from catalyst.exchange.exchange_bundle import exchange_bundle
|
||||
|
||||
symbols = (
|
||||
'btc_usd',
|
||||
'eth_btc',
|
||||
'etc_btc',
|
||||
'neo_btc',
|
||||
)
|
||||
register('bitfinex_bundle', exchange_bundle('bitfinex', symbols))
|
||||
register('exchange_bitfinex', exchange_bundle('bitfinex', symbols))
|
||||
|
||||
Notes
|
||||
-----
|
||||
The sids for each symbol will be the index into the symbols sequence.
|
||||
"""
|
||||
# strict this in memory so that we can reiterate over it
|
||||
symbols = tuple(symbols)
|
||||
log.level = log_level
|
||||
|
||||
def ingest(environ,
|
||||
asset_db_writer,
|
||||
minute_bar_writer, # unused
|
||||
minute_bar_writer,
|
||||
five_minute_bar_writer,
|
||||
daily_bar_writer,
|
||||
adjustment_writer,
|
||||
calendar,
|
||||
@@ -93,16 +200,23 @@ def exchange_bundle(exchange_name, symbols, start=None, end=None):
|
||||
end_session,
|
||||
cache,
|
||||
show_progress,
|
||||
is_compile,
|
||||
output_dir,
|
||||
# pass these as defaults to make them 'nonlocal' in py2
|
||||
start=start,
|
||||
end=end):
|
||||
|
||||
log.info('ingesting bundle {}'.format(output_dir))
|
||||
|
||||
# TODO: I don't understand this session vs dates idea
|
||||
if start is None:
|
||||
start = start_session
|
||||
if end is None:
|
||||
end = None
|
||||
end = end_session
|
||||
|
||||
now = pd.Timestamp.utcnow()
|
||||
if end > now:
|
||||
log.info('adjusting the end date to now {}'.format(now))
|
||||
end = now
|
||||
|
||||
log.info('ingesting data from {} to {}'.format(start, end))
|
||||
|
||||
@@ -124,61 +238,59 @@ def exchange_bundle(exchange_name, symbols, start=None, end=None):
|
||||
else:
|
||||
raise ExchangeNotFoundError(exchange_name=exchange_name)
|
||||
|
||||
assets = exchange.get_assets(symbols)
|
||||
|
||||
delta = end - start
|
||||
delta_minutes = delta.total_seconds() / 60
|
||||
if delta_minutes > exchange.num_candles_limit:
|
||||
bar_count = exchange.num_candles_limit
|
||||
|
||||
chunks = []
|
||||
last_chunk_date = end
|
||||
while last_chunk_date > start + timedelta(minutes=bar_count):
|
||||
# TODO: account for the partial last bar
|
||||
chunk = dict(end=last_chunk_date, bar_count=bar_count)
|
||||
chunks.append(chunk)
|
||||
|
||||
last_chunk_date = \
|
||||
last_chunk_date - timedelta(minutes=(bar_count + 1))
|
||||
|
||||
chunks.reverse()
|
||||
|
||||
if symbols is not None:
|
||||
assets = exchange.get_assets(symbols)
|
||||
else:
|
||||
chunks = [dict(end=end, bar_count=delta_minutes)]
|
||||
assets = exchange.assets
|
||||
|
||||
with maybe_show_progress(
|
||||
chunks,
|
||||
show_progress,
|
||||
label='Fetching {} candles: '.format(exchange_name)) as it:
|
||||
earliest_trade = None
|
||||
for asset in assets:
|
||||
if earliest_trade is None or earliest_trade > asset.start_date:
|
||||
earliest_trade = asset.start_date
|
||||
|
||||
for chunk in it:
|
||||
asset_df = fetch_candles_chunk(
|
||||
exchange=exchange,
|
||||
assets=assets,
|
||||
data_frequency='1m',
|
||||
end_dt=chunk['end'],
|
||||
bar_count=chunk['bar_count']
|
||||
)
|
||||
if earliest_trade > start:
|
||||
log.info(
|
||||
'adjusting start date to earliest trade date found {}'.format(
|
||||
earliest_trade
|
||||
))
|
||||
start = earliest_trade
|
||||
|
||||
data = []
|
||||
for asset in asset_df:
|
||||
df = asset_df[asset]
|
||||
sid = asset.sid
|
||||
data.append((sid, df))
|
||||
if start >= end:
|
||||
raise ValueError('start date cannot be after end date')
|
||||
|
||||
try:
|
||||
log.debug(
|
||||
'writing chunk: {sid} start: {start} end: {end}'.format(
|
||||
sid=sid,
|
||||
start=chunk['end'] - timedelta(
|
||||
minutes=chunk['bar_count']),
|
||||
end=chunk['end']
|
||||
)
|
||||
)
|
||||
minute_bar_writer.write(data, show_progress=show_progress)
|
||||
except KeyError:
|
||||
minute_bar_writer.write(data, show_progress=show_progress)
|
||||
except BcolzMinuteOverlappingData as e:
|
||||
log.warn('Unable to write chunk {}: {}'.format(chunk, e))
|
||||
if daily_bar_writer is not None:
|
||||
process_bar_data(
|
||||
exchange=exchange,
|
||||
assets=assets,
|
||||
writer=daily_bar_writer,
|
||||
data_frequency='daily',
|
||||
show_progress=show_progress,
|
||||
start=start,
|
||||
end=end
|
||||
)
|
||||
|
||||
if five_minute_bar_writer is not None:
|
||||
process_bar_data(
|
||||
exchange=exchange,
|
||||
assets=assets,
|
||||
writer=five_minute_bar_writer,
|
||||
data_frequency='5-minute',
|
||||
show_progress=show_progress,
|
||||
start=start,
|
||||
end=end
|
||||
)
|
||||
|
||||
if minute_bar_writer is not None:
|
||||
process_bar_data(
|
||||
exchange=exchange,
|
||||
assets=assets,
|
||||
writer=minute_bar_writer,
|
||||
data_frequency='minute',
|
||||
show_progress=show_progress,
|
||||
start=start,
|
||||
end=end
|
||||
)
|
||||
|
||||
return ingest
|
||||
|
||||
|
||||
|
||||
@@ -140,3 +140,10 @@ class SymbolNotFoundOnExchange(ZiplineError):
|
||||
"""
|
||||
msg = ('Symbol {symbol} not found on exchange {exchange}. '
|
||||
'Choose from: {supported_symbols}').strip()
|
||||
|
||||
|
||||
class BundleNotFoundError(ZiplineError):
|
||||
msg = ('Unable to find bundle data for exchange {exchange}. '
|
||||
'Please ingest data using the command '
|
||||
'`catalyst ingest -b exchange_{exchange}`. '
|
||||
'See catalyst documentation for details.').strip()
|
||||
|
||||
@@ -1,14 +1,13 @@
|
||||
from datetime import timedelta
|
||||
|
||||
import os
|
||||
from datetime import timedelta
|
||||
from logging import Logger, DEBUG
|
||||
|
||||
import pandas as pd
|
||||
from logging import Logger
|
||||
|
||||
from catalyst import get_calendar
|
||||
|
||||
from catalyst.data.minute_bars import BcolzMinuteBarWriter
|
||||
from catalyst.exchange.exchange_bundle import exchange_bundle
|
||||
from catalyst.exchange.exchange_utils import get_exchange_minute_writer_root
|
||||
from catalyst.utils.paths import ensure_directory, data_root
|
||||
|
||||
log = Logger('test_exchange_bundle')
|
||||
|
||||
@@ -17,44 +16,52 @@ class ExchangeBundleTestCase:
|
||||
def test_ingest(self):
|
||||
exchange_name = 'bitfinex'
|
||||
|
||||
start = pd.Timestamp.utcnow() - timedelta(days=2)
|
||||
start = pd.Timestamp.utcnow() - timedelta(days=365)
|
||||
end = pd.Timestamp.utcnow()
|
||||
open_calendar = get_calendar('OPEN')
|
||||
|
||||
root = get_exchange_minute_writer_root(exchange_name)
|
||||
filename = os.path.join(root, 'metadata.json')
|
||||
root = data_root(os.environ)
|
||||
output_dir = '{root}/exchange_{exchange}/test'.format(
|
||||
root=root,
|
||||
exchange=exchange_name
|
||||
)
|
||||
ensure_directory(output_dir)
|
||||
|
||||
filename = os.path.join(output_dir, 'metadata.json')
|
||||
|
||||
start_session = start.floor('1d')
|
||||
if os.path.isfile(filename):
|
||||
minute_bar_writer = BcolzMinuteBarWriter.open(root, end)
|
||||
minute_bar_writer = BcolzMinuteBarWriter.open(output_dir, end)
|
||||
else:
|
||||
# TODO: need to be able to write more precise numbers
|
||||
minute_bar_writer = BcolzMinuteBarWriter(
|
||||
rootdir=root,
|
||||
rootdir=output_dir,
|
||||
calendar=open_calendar,
|
||||
minutes_per_day=1440,
|
||||
start_session=start.floor('1d'),
|
||||
start_session=start_session,
|
||||
end_session=end,
|
||||
write_metadata=True
|
||||
)
|
||||
|
||||
ingest = exchange_bundle(
|
||||
exchange_name=exchange_name,
|
||||
symbols=['btc_usd']
|
||||
symbols=['eth_btc'],
|
||||
log_level=DEBUG
|
||||
)
|
||||
|
||||
ingest(
|
||||
environ=os.environ,
|
||||
asset_db_writer=None, # TODO: nice to have
|
||||
minute_bar_writer=minute_bar_writer,
|
||||
daily_bar_writer=None, # TODO: add later
|
||||
adjustment_writer=None, # Not applicable to crypto
|
||||
calendar=open_calendar,
|
||||
start_session=start,
|
||||
end_session=end,
|
||||
cache=dict(),
|
||||
show_progress=True,
|
||||
output_dir=exchange_name, # TODO: not sure
|
||||
start=start,
|
||||
end=end
|
||||
)
|
||||
ingest(environ=os.environ,
|
||||
asset_db_writer=None,
|
||||
minute_bar_writer=minute_bar_writer,
|
||||
five_minute_bar_writer=None,
|
||||
daily_bar_writer=None,
|
||||
adjustment_writer=None,
|
||||
calendar=open_calendar,
|
||||
start_session=start_session,
|
||||
end_session=end,
|
||||
cache=dict(),
|
||||
show_progress=True,
|
||||
is_compile=False,
|
||||
output_dir=output_dir,
|
||||
start=start,
|
||||
end=end)
|
||||
pass
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from datetime import timedelta
|
||||
|
||||
import os
|
||||
import pandas as pd
|
||||
from catalyst import get_calendar
|
||||
from logbook import Logger
|
||||
@@ -10,7 +11,9 @@ from catalyst.exchange.bitfinex.bitfinex import Bitfinex
|
||||
from catalyst.exchange.bittrex.bittrex import Bittrex
|
||||
from catalyst.exchange.data_portal_exchange import DataPortalExchangeBacktest, \
|
||||
DataPortalExchangeLive
|
||||
from catalyst.exchange.exchange_bundle import exchange_bundle
|
||||
from catalyst.exchange.exchange_utils import get_exchange_auth
|
||||
from catalyst.utils.run_algo import load_extensions
|
||||
|
||||
log = Logger('test_bitfinex')
|
||||
|
||||
@@ -44,13 +47,14 @@ class ExchangeDataPortalTestCase:
|
||||
first_trading_day=pd.to_datetime('today', utc=True)
|
||||
)
|
||||
self.data_portal_backtest = DataPortalExchangeBacktest(
|
||||
exchanges=dict(bitfinex=self.bitfinex, bittrex=self.bittrex),
|
||||
exchanges=dict(bitfinex=self.bitfinex),
|
||||
asset_finder=asset_finder,
|
||||
trading_calendar=open_calendar,
|
||||
first_trading_day=pd.to_datetime('today', utc=True)
|
||||
)
|
||||
|
||||
def test_get_history_window_live(self):
|
||||
|
||||
asset_finder = self.data_portal_live.asset_finder
|
||||
|
||||
assets = [
|
||||
@@ -79,13 +83,14 @@ class ExchangeDataPortalTestCase:
|
||||
pass
|
||||
|
||||
def test_get_spot_value_backtest(self):
|
||||
|
||||
asset_finder = self.data_portal_backtest.asset_finder
|
||||
|
||||
assets = [
|
||||
asset_finder.lookup_symbol('btc_usd', self.bitfinex),
|
||||
asset_finder.lookup_symbol('neo_btc', self.bitfinex),
|
||||
]
|
||||
|
||||
date = pd.Timestamp.utcnow() - timedelta(hours=2)
|
||||
date = pd.Timestamp.utcnow() - timedelta(hours=8)
|
||||
value = self.data_portal_backtest.get_spot_value(
|
||||
assets, 'close', date, 'minute')
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user