wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-08-16 11:18:05 +08:00
Code Issues Packages Projects Releases Wiki Activity

USDT_BTC benchmark

Browse Source 
This commit:
 * Adds a crypto_benchmark that can create benchmarks for symbols found on POLO
 * Changes default trading calendars to OPEN
 * Properly computes daily bar data from five minute POLO bars
 * Allows trading of one hundredth of a coin, later we plan to integrate per the
   ratio of a full coin to its base denomination.
This commit is contained in:
Conner Fromknecht
2017-06-23 18:44:45 -07:00
parent 9c03012aa4
commit b4b19a7555
12 changed files with 177 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/data/bundles/poloniex.py
+79 -72
View File
@@ -84,88 +84,95 @@ def poloniex_cryptoassets(symbols, start=None, end=None):
('symbol', 'object'),
]))
day_offset = pd.Timedelta(days=1)
def compute_daily_bars(five_min_bars):
# filter and copy the entry at the beginning of each session
daily_bars = five_min_bars[
five_min_bars.index.isin(calendar.all_sessions)
].copy()
# iterate through session starts doing:
# 1. filter five_min_bars to get all entries in one day
# 2. compute daily bar entry
# 3. record in rid-th row of daily_bars
for rid, start_date in enumerate(daily_bars.index):
# compute beginning of next session
end_date = start_date + day_offset
# filter for entries session entries
day_data = five_min_bars[
(five_min_bars.index >= start_date) &
(five_min_bars.index < end_date)
]
# compute and record daily bar
daily_bars.iloc[rid] = (
day_data.open.iloc[0], # first open price
day_data.high.max(), # max of high prices
day_data.low.min(), # min of low prices
day_data.close.iloc[-1], # last close price
day_data.volume.sum(), # sum of all volumes
)
# scale to allow trading 100-ths of a coin
daily_bars.loc[:, 'open'] /= 100.0
daily_bars.loc[:, 'high'] /= 100.0
daily_bars.loc[:, 'low'] /= 100.0
daily_bars.loc[:, 'close'] /= 100.0
daily_bars.loc[:, 'volume'] *= 100.0
return daily_bars
def _pricing_iter():
sid = 0
for symbol in symbols:
#def to_dataframe(self, start, end, currencyPair=None):
csv_fn = '/var/tmp/catalyst/data/poloniex/crypto_prices-' + symbol + '.csv' # TODO: DIR as parameter
#last_date = self._get_start_date(csv_fn)
#if last_date + 300 < end or not os.path.exists(csv_fn):
# get latest data
#self.append_data_single_pair(currencyPair)
# CSV holds the latest snapshot
data = pd.read_csv(csv_fn, names=['date', 'open', 'high', 'low', 'close', 'volume'])
data['date'] = pd.to_datetime(data['date'], utc=True, unit='s')
data.set_index('date', inplace=True)
#df = df.resample('D').mean()
df = data.loc[data.index.isin(calendar.schedule.index)]
offset = DateOffset(days=1)
for start_date in df.index:
end_date = start_date + offset
day_data = data[start_date:end_date]
df[start_date]['open'] = day_data[0]['open']
df[start_date]['high'] = day_data['high'].max()
df[start_date]['low'] = day_data['low'].min()
df[start_date]['close'] = day_data[-1]['close']
df[start_date]['volume'] = day_data['volume'].sum()
# the start date is the date of the first trade and
# the end date is the date of the last trade
start_date = df.index[0]
end_date = df.index[-1]
# The auto_close date is the day after the last trade.
ac_date = end_date + pd.Timedelta(days=1)
metadata.iloc[sid] = start_date, end_date, ac_date, symbol
yield sid, df
sid += 1
'''
print 'Ingesting symbols: {0}'.format(symbols)
with maybe_show_progress(
symbols,
show_progress,
label='Downloading Yahoo pricing data: ') as it, \
requests.Session() as session:
symbols,
show_progress,
show_percent=True,
item_show_func=lambda s: 'building {0}'.format(s)
if s is not None
else 'DONE',
info_sep=' | ',
label='Compiling daily bar pricing datasets:',
) as it:
for symbol in it:
path = _cachpath(symbol, 'ohlcv')
try:
df = cache[path]
except KeyError:
df = cache[path] = DataReader(
symbol,
'yahoo',
start,
end,
session=session,
).sort_index()
#def to_dataframe(self, start, end, currencyPair=None):
csv_fn = '/var/tmp/catalyst/data/poloniex/crypto_prices-' +\
symbol + '.csv'
#last_date = self._get_start_date(csv_fn)
#if last_date + 300 < end or not os.path.exists(csv_fn):
# get latest data
#self.append_data_single_pair(currencyPair)
# CSV holds the latest snapshot
columns = ['date', 'open', 'high', 'low', 'close', 'volume']
five_min_bars = pd.read_csv(csv_fn, names=columns)
five_min_bars.set_index('date', inplace=True)
five_min_bars.index = pd.to_datetime(
five_min_bars.index,
utc=True,
unit='s',
)
daily_bars = compute_daily_bars(five_min_bars)
# the start date is the date of the first trade and
# the end date is the date of the last trade
start_date = df.index[0]
end_date = df.index[-1]
start_date = daily_bars.index[0].tz_localize(None)
end_date = daily_bars.index[-1].tz_localize(None)
# The auto_close date is the day after the last trade.
ac_date = end_date + pd.Timedelta(days=1)
ac_date = end_date + day_offset
metadata.iloc[sid] = start_date, end_date, ac_date, symbol
df.rename(
columns={
'Open': 'open',
'High': 'high',
'Low': 'low',
'Close': 'close',
'Volume': 'volume',
},
inplace=True,
)
yield sid, df
yield sid, daily_bars
sid += 1
'''
daily_bar_writer.write(_pricing_iter(), show_progress=show_progress)
daily_bar_writer.write(_pricing_iter())
symbol_map = pd.Series(metadata.symbol.index, metadata.symbol)
@@ -178,7 +185,7 @@ def poloniex_cryptoassets(symbols, start=None, end=None):
adjustment_writer.write()
return ingest
# bundle used when creating test data
register(
catalyst/data/loader.py
+61 -12
View File
@@ -17,6 +17,7 @@ from collections import OrderedDict
import logbook
import pandas as pd
import numpy as np
from pandas_datareader.data import DataReader
import datetime
import pytz
@@ -253,7 +254,6 @@ def ensure_crypto_benchmark_data(symbol, first_date, last_date, now,
if data is not None:
print 'benchmark data:\n', data.head()
return data
# If no cached data was found or it was missing any dates then download the
@@ -269,33 +269,82 @@ def ensure_crypto_benchmark_data(symbol, first_date, last_date, now,
def dateparse(time_in_secs):
return datetime.datetime.fromtimestamp(float(time_in_secs), pytz.utc)
def compute_daily_bars(five_min_bars, schedule):
# filter and copy the entry at the beginning of each session
daily_bars = five_min_bars[
five_min_bars.index.isin(schedule)
].copy()
day_offset = pd.Timedelta(days=1)
# iterate through session starts doing:
# 1. filter five_min_bars to get all entries in one day
# 2. compute daily bar entry
# 3. record in rid-th row of daily_bars
for rid, start_date in enumerate(daily_bars.index):
# compute beginning of next session
end_date = start_date + day_offset
# filter for entries session entries
day_data = five_min_bars[
(five_min_bars.index >= start_date) &
(five_min_bars.index < end_date)
]
# compute and record daily bar
daily_bars.iloc[rid] = (
day_data.open.iloc[0], # first open price
day_data.high.max(), # max of high prices
day_data.low.min(), # min of low prices
day_data.close.iloc[-1], # last close prices
day_data.volume.sum(), # sum of all volumes
)
# scale to allow trading 100-ths of a coin
daily_bars.loc[:, 'open'] /= 100.0
daily_bars.loc[:, 'high'] /= 100.0
daily_bars.loc[:, 'low'] /= 100.0
daily_bars.loc[:, 'close'] /= 100.0
daily_bars.loc[:, 'volume'] *= 100.0
return daily_bars
try:
data = pd.read_csv(
# load five minute bars from csv cache
five_min_bars = pd.read_csv(
source_filename,
names=['date', 'open', 'high', 'low', 'close', 'volume'],
index_col=[0],
parse_dates=True,
date_parser=dateparse,
)
data = data[['close']]
five_min_bars.index = pd.to_datetime(five_min_bars.index, utc=True, unit='s')
print 'loaded benchmark data:\n', data.index
# compute daily bars for open calendar
open_calendar = get_calendar('OPEN')
daily_bars = compute_daily_bars(
five_min_bars,
open_calendar.all_sessions,
)
data = data[
(data.index >= (first_date-trading_day)) &
(data.index <= last_date)
# filter daily bars to include first_date and last_date
daily_bars = daily_bars[
(daily_bars.index >= (first_date - trading_day)) &
(daily_bars.index <= last_date)
]
data = data.pct_change(1).iloc[1:]
print 'writing benchmark data:\n', data.head()
# select close column and compute percent change between days
daily_close = daily_bars[['close']]
daily_close = daily_close.pct_change(1).iloc[1:]
data.to_csv(get_data_filepath(filename, environ))
# write to benchmark csv cache
daily_close.to_csv(get_data_filepath(filename, environ))
except (OSError, IOError, HTTPError):
logger.exception('Failed to cache the new benchmark returns')
raise
if not has_data_for_dates(data, first_date, last_date):
if not has_data_for_dates(daily_close, first_date, last_date):
logger.warn("Still don't have expected data after redownload!")
return data
return daily_close
def ensure_benchmark_data(symbol, first_date, last_date, now, trading_day,