wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-08-15 12:15:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

Purge 5-min implementation

Browse Source
This commit is contained in:
Victor Grau Serrat
2017-09-28 11:03:47 -06:00
parent 15fa98420d
commit 3b681d197d
25 changed files with 100 additions and 1776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/data/_minute_bar_internal.pyx
-78
View File
@@ -35,17 +35,6 @@ def minute_value(ndarray[long_t, ndim=1] market_opens,
return market_opens[q] + r
@cython.cdivision(True)
def five_minute_value(ndarray[long_t, ndim=1] market_opens,
Py_ssize_t pos,
short five_minutes_per_day):
cdef short q, r
q = cython.cdiv(pos, five_minutes_per_day)
r = cython.cmod(pos, five_minutes_per_day)
return market_opens[q] + r
def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
ndarray[long_t, ndim=1] market_closes,
long_t minute_val,
@@ -99,26 +88,6 @@ def find_position_of_minute(ndarray[long_t, ndim=1] market_opens,
return (market_open_loc * minutes_per_day) + delta
def find_position_of_five_minute(ndarray[long_t, ndim=1] market_opens,
ndarray[long_t, ndim=1] market_closes,
long_t five_minute_val,
short five_minutes_per_day,
bool forward_fill):
cdef Py_ssize_t market_open_loc, market_open, delta
market_open_loc = \
searchsorted(market_opens, five_minute_val, side='right') - 1
market_open = market_opens[market_open_loc]
market_close = market_closes[market_open_loc]
if not forward_fill and ((five_minute_val - market_open) >= five_minutes_per_day):
raise ValueError("Given five minutes is not between an open and a close")
delta = int_min(five_minute_val - market_open, market_close - market_open)
return (market_open_loc * five_minutes_per_day) + delta
def find_last_traded_position_internal(
ndarray[long_t, ndim=1] market_opens,
ndarray[long_t, ndim=1] market_closes,
@@ -189,50 +158,3 @@ def find_last_traded_position_internal(
# found a trade event
return -1
def find_last_traded_five_minute_position_internal(
ndarray[long_t, ndim=1] market_opens,
ndarray[long_t, ndim=1] market_closes,
long_t end_five_minute,
long_t start_five_minute,
volumes,
short five_minutes_per_day):
cdef Py_ssize_t minute_pos, current_minute, q
five_minute_pos = int_min(
find_position_of_five_minute(
market_opens,
market_closes,
end_five_minute,
five_minutes_per_day,
True,
),
len(volumes) - 1,
)
while five_minute_pos >= 0:
current_five_minute = five_minute_value(
market_opens, five_minute_pos, five_minutes_per_day
)
q = cython.cdiv(five_minute_pos, five_minutes_per_day)
if current_five_minute > market_closes[q]:
five_minute_pos = find_position_of_five_minute(
market_opens,
market_closes,
market_closes[q],
five_minutes_per_day,
False,
)
continue
if current_five_minute < start_five_minute:
return -1
if volumes[five_minute_pos] != 0:
return five_minute_pos
five_minute_pos -= 1
# we've gone to the beginning of this asset's range, and still haven't
# found a trade event
return -1
catalyst/data/bundles/base.py
+1 -26
View File
@@ -60,10 +60,6 @@ class BaseBundle(object):
def minutes_per_day(self):
raise NotImplementedError()
@lazyval
def five_minutes_per_day(self):
raise NotImplementedError()
@lazyval
def frequencies(self):
raise NotImplementedError()
@@ -115,7 +111,6 @@ class BaseBundle(object):
environ,
asset_db_writer,
minute_bar_writer,
five_minute_bar_writer,
daily_bar_writer,
adjustment_writer,
calendar,
@@ -162,7 +157,7 @@ class BaseBundle(object):
# Post-process metadata using cached symbol frames, and write to
# disk. This metadata must be written before any attempt to write
# either minute or 5-minute data.
# minute data.
metadata = self._post_process_metadata(
raw_metadata,
cache,
@@ -170,26 +165,6 @@ class BaseBundle(object):
)
asset_db_writer.write(metadata)
# Compile 5-minute symbol data if bundle supports 5-minute mode and
# persist the dataset to disk.
'''
if '5-minute' in self.frequencies:
five_minute_bar_writer.write(
self._fetch_symbol_iter(
api_key,
cache,
symbol_map,
calendar,
start_session,
end_session,
'5-minute',
retries,
),
length=len(symbol_map),
show_progress=show_progress,
)
'''
# Compile minute symbol data if bundle supports minute mode and
# persist the dataset to disk.
if 'minute' in self.frequencies:
catalyst/data/bundles/base_pricing.py
-8
View File
@@ -47,10 +47,6 @@ class BaseCryptoPricingBundle(BasePricingBundle):
def minutes_per_day(self):
return 1440
@lazyval
def five_minutes_per_day(self):
return 288
@property
def splits(self):
return []
@@ -68,10 +64,6 @@ class BaseEquityPricingBundle(BasePricingBundle):
def minutes_per_day(self):
return 390
@lazyval
def five_minutes_per_day(self):
return 78
@property
def splits(self):
return self._splits
catalyst/data/bundles/core.py
+1 -31
View File
@@ -17,10 +17,6 @@ from ..us_equity_pricing import (
SQLiteAdjustmentReader,
SQLiteAdjustmentWriter,
)
from ..five_minute_bars import (
BcolzFiveMinuteBarReader,
BcolzFiveMinuteBarWriter,
)
from ..minute_bars import (
BcolzMinuteBarReader,
BcolzMinuteBarWriter,
@@ -54,11 +50,6 @@ def minute_path(bundle_name, timestr, environ=None):
environ=environ,
)
def five_minute_path(bundle_name, timestr, environ=None):
return pth.data_path(
five_minute_relative(bundle_name, timestr, environ),
environ=environ,
)
def daily_path(bundle_name, timestr, environ=None):
return pth.data_path(
@@ -92,8 +83,6 @@ def cache_relative(bundle_name, timestr, environ=None):
def daily_relative(bundle_name, timestr, environ=None):
return bundle_name, timestr, 'daily_equities.bcolz'
def five_minute_relative(bundle_name, timestr, environ=None):
return bundle_name, timestr, 'five_minute.bcolz'
def minute_relative(bundle_name, timestr, environ=None):
return bundle_name, timestr, 'minute_equities.bcolz'
@@ -206,14 +195,13 @@ RegisteredBundle = namedtuple(
'start_session',
'end_session',
'minutes_per_day',
'five_minutes_per_day',
'ingest',
'create_writers']
)
BundleData = namedtuple(
'BundleData',
'asset_finder minute_bar_reader five_minute_bar_reader daily_bar_reader '
'asset_finder minute_bar_reader daily_bar_reader '
'adjustment_reader',
)
@@ -303,7 +291,6 @@ def _make_bundle_core():
bundle.ingest,
calendar_name=bundle.calendar_name,
minutes_per_day=bundle.minutes_per_day,
five_minutes_per_day=bundle.five_minutes_per_day,
start_session=start_session,
end_session=end_session,
create_writers=create_writers,
@@ -316,7 +303,6 @@ def _make_bundle_core():
start_session=None,
end_session=None,
minutes_per_day=1440,
five_minutes_per_day=288,
create_writers=True):
"""Register a data bundle ingest function.
@@ -397,7 +383,6 @@ def _make_bundle_core():
start_session=start_session,
end_session=end_session,
minutes_per_day=minutes_per_day,
five_minutes_per_day=five_minutes_per_day,
ingest=f,
create_writers=create_writers,
)
@@ -496,16 +481,6 @@ def _make_bundle_core():
# that it can compute the adjustment ratios for the dividends.
daily_bar_writer.write(())
five_minute_bar_writer = BcolzFiveMinuteBarWriter(
wd.ensure_dir(*five_minute_relative(
name, timestr, environ=environ)
),
calendar,
start_session,
end_session,
five_minutes_per_day=bundle.five_minutes_per_day,
)
minute_bar_writer = BcolzMinuteBarWriter(
wd.ensure_dir(*minute_relative(
name, timestr, environ=environ)
@@ -532,7 +507,6 @@ def _make_bundle_core():
)
else:
daily_bar_writer = None
five_minute_bar_writer = None
minute_bar_writer = None
asset_db_writer = None
adjustment_db_writer = None
@@ -544,7 +518,6 @@ def _make_bundle_core():
environ,
asset_db_writer,
minute_bar_writer,
five_minute_bar_writer,
daily_bar_writer,
adjustment_db_writer,
calendar,
@@ -631,9 +604,6 @@ def _make_bundle_core():
minute_bar_reader=BcolzMinuteBarReader(
minute_path(name, timestr, environ=environ),
),
five_minute_bar_reader=BcolzFiveMinuteBarReader(
five_minute_path(name, timestr, environ=environ),
),
daily_bar_reader=BcolzDailyBarReader(
daily_path(name, timestr, environ=environ),
),
catalyst/data/bundles/poloniex.py
-1
View File
@@ -148,7 +148,6 @@ class PoloniexBundle(BaseCryptoPricingBundle):
data_frequency):
period_map = {
'daily': 86400,
# '5-minute': 300,
}
try:
catalyst/data/data_portal.py
-31
View File
@@ -42,7 +42,6 @@ from catalyst.assets.roll_finder import (
)
from catalyst.data.dispatch_bar_reader import (
AssetDispatchMinuteBarReader,
AssetDispatchFiveMinuteBarReader,
AssetDispatchSessionBarReader
)
from catalyst.data.resample import (
@@ -120,10 +119,6 @@ class DataPortal(object):
daily data backtests or daily history calls in a minute backetest.
If a daily bar reader is not provided but a minute bar reader is,
the minutes will be rolled up to serve the daily requests.
five_minute_reader : BcolzFiveMinuteBarReader, optional
The five minute bar reader for equities. This will be used to service
5-minute data backtests or five-minute history calls. This can be used
to serve daily calls if no daily bar reader is provided.
minute_reader : BcolzMinuteBarReader, optional
The minute bar reader for equities. This will be used to service
minute data backtests or minute history calls. This can be used
@@ -150,7 +145,6 @@ class DataPortal(object):
trading_calendar,
first_trading_day,
daily_reader=None,
five_minute_reader=None,
minute_reader=None,
future_daily_reader=None,
future_minute_reader=None,
@@ -202,7 +196,6 @@ class DataPortal(object):
reader.last_available_dt
for reader in [
minute_reader,
five_minute_reader,
future_minute_reader,
]
if reader is not None
@@ -214,8 +207,6 @@ class DataPortal(object):
aligned_minute_reader = self._ensure_reader_aligned(
minute_reader)
aligned_five_minute_reader = self._ensure_reader_aligned(
five_minute_reader)
aligned_session_reader = self._ensure_reader_aligned(
daily_reader)
aligned_future_minute_reader = self._ensure_reader_aligned(
@@ -229,13 +220,10 @@ class DataPortal(object):
}
aligned_minute_readers = {}
aligned_five_minute_readers = {}
aligned_session_readers = {}
if aligned_minute_reader is not None:
aligned_minute_readers[Equity] = aligned_minute_reader
if aligned_five_minute_reader is not None:
aligned_five_minute_readers[Equity] = aligned_five_minute_reader
if aligned_session_reader is not None:
aligned_session_readers[Equity] = aligned_session_reader
@@ -267,13 +255,6 @@ class DataPortal(object):
self._last_available_minute,
)
_dispatch_five_minute_reader = AssetDispatchFiveMinuteBarReader(
self.trading_calendar,
self.asset_finder,
aligned_five_minute_readers,
self._last_available_minute,
)
_dispatch_session_reader = AssetDispatchSessionBarReader(
self.trading_calendar,
self.asset_finder,
@@ -283,7 +264,6 @@ class DataPortal(object):
self._pricing_readers = {
'minute': _dispatch_minute_reader,
'5-minute': _dispatch_five_minute_reader,
'daily': _dispatch_session_reader,
}
@@ -719,17 +699,6 @@ class DataPortal(object):
spot_value=result
)
def _get_five_minute_spot_value(self, asset, column, dt, ffill=False):
return self._get_minutely_spot_value(
asset,
column,
dt,
ffill,
'5-minute',
)
def _get_minute_spot_value(self, asset, column, dt, ffill=False):
return self._get_minutely_spot_value(
asset,
catalyst/data/dispatch_bar_reader.py
-6
View File
@@ -135,12 +135,6 @@ class AssetDispatchMinuteBarReader(AssetDispatchBarReader):
def _dt_window_size(self, start_dt, end_dt):
return len(self.trading_calendar.minutes_in_range(start_dt, end_dt))
class AssetDispatchFiveMinuteBarReader(AssetDispatchBarReader):
def _dt_window_size(self, start_dt, end_dt):
return len(self.trading_calendar.five_minutes_in_range(start_dt, end_dt))
class AssetDispatchSessionBarReader(AssetDispatchBarReader):
def _dt_window_size(self, start_dt, end_dt):
catalyst/data/five_minute_bars.py
-1385
View File
File diff suppressed because it is too large Load Diff