mirror of
https://github.com/wassname/catalyst.git
synced 2026-08-11 11:16:15 +08:00
Changed zipline -> catalyst import paths
* Updated cython build scripts * Updated setup.py to to install catalyst package * Updated momentum example to use catalyst package * catalyst executable now supports loading pipelines from multiple bundles
This commit is contained in:
Conner Fromknecht
@@ -0,0 +1,5 @@
|
||||
from .test_source import SpecificEquityTrades
|
||||
|
||||
__all__ = [
|
||||
'SpecificEquityTrades',
|
||||
]
|
||||
@@ -0,0 +1,199 @@
|
||||
#
|
||||
# Copyright 2015 Quantopian, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from catalyst.errors import (
|
||||
InvalidBenchmarkAsset,
|
||||
BenchmarkAssetNotAvailableTooEarly,
|
||||
BenchmarkAssetNotAvailableTooLate
|
||||
)
|
||||
|
||||
|
||||
class BenchmarkSource(object):
|
||||
def __init__(self,
|
||||
benchmark_asset,
|
||||
trading_calendar,
|
||||
sessions,
|
||||
data_portal,
|
||||
emission_rate="daily",
|
||||
benchmark_returns=None):
|
||||
self.benchmark_asset = benchmark_asset
|
||||
self.sessions = sessions
|
||||
self.emission_rate = emission_rate
|
||||
self.data_portal = data_portal
|
||||
|
||||
if len(sessions) == 0:
|
||||
self._precalculated_series = pd.Series()
|
||||
elif benchmark_asset is not None:
|
||||
|
||||
self._validate_benchmark(benchmark_asset)
|
||||
|
||||
self._precalculated_series = \
|
||||
self._initialize_precalculated_series(
|
||||
benchmark_asset,
|
||||
trading_calendar,
|
||||
self.sessions,
|
||||
self.data_portal
|
||||
)
|
||||
elif benchmark_returns is not None:
|
||||
daily_series = benchmark_returns[sessions[0]:sessions[-1]]
|
||||
|
||||
if self.emission_rate == "minute":
|
||||
# we need to take the env's benchmark returns, which are daily,
|
||||
# and resample them to minute
|
||||
minutes = trading_calendar.minutes_for_sessions_in_range(
|
||||
sessions[0],
|
||||
sessions[-1]
|
||||
)
|
||||
|
||||
minute_series = daily_series.reindex(
|
||||
index=minutes,
|
||||
method="ffill"
|
||||
)
|
||||
|
||||
self._precalculated_series = minute_series
|
||||
else:
|
||||
self._precalculated_series = daily_series
|
||||
else:
|
||||
raise Exception("Must provide either benchmark_asset or "
|
||||
"benchmark_returns.")
|
||||
|
||||
def get_value(self, dt):
|
||||
return self._precalculated_series.loc[dt]
|
||||
|
||||
def get_range(self, start_dt, end_dt):
|
||||
return self._precalculated_series.loc[start_dt:end_dt]
|
||||
|
||||
def _validate_benchmark(self, benchmark_asset):
|
||||
# check if this security has a stock dividend. if so, raise an
|
||||
# error suggesting that the user pick a different asset to use
|
||||
# as benchmark.
|
||||
stock_dividends = \
|
||||
self.data_portal.get_stock_dividends(self.benchmark_asset,
|
||||
self.sessions)
|
||||
|
||||
if len(stock_dividends) > 0:
|
||||
raise InvalidBenchmarkAsset(
|
||||
sid=str(self.benchmark_asset),
|
||||
dt=stock_dividends[0]["ex_date"]
|
||||
)
|
||||
|
||||
if benchmark_asset.start_date > self.sessions[0]:
|
||||
# the asset started trading after the first simulation day
|
||||
raise BenchmarkAssetNotAvailableTooEarly(
|
||||
sid=str(self.benchmark_asset),
|
||||
dt=self.sessions[0],
|
||||
start_dt=benchmark_asset.start_date
|
||||
)
|
||||
|
||||
if benchmark_asset.end_date < self.sessions[-1]:
|
||||
# the asset stopped trading before the last simulation day
|
||||
raise BenchmarkAssetNotAvailableTooLate(
|
||||
sid=str(self.benchmark_asset),
|
||||
dt=self.sessions[-1],
|
||||
end_dt=benchmark_asset.end_date
|
||||
)
|
||||
|
||||
def _initialize_precalculated_series(self, asset, trading_calendar,
|
||||
trading_days, data_portal):
|
||||
"""
|
||||
Internal method that pre-calculates the benchmark return series for
|
||||
use in the simulation.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
asset: Asset to use
|
||||
|
||||
trading_calendar: TradingCalendar
|
||||
|
||||
trading_days: pd.DateTimeIndex
|
||||
|
||||
data_portal: DataPortal
|
||||
|
||||
Notes
|
||||
-----
|
||||
If the benchmark asset started trading after the simulation start,
|
||||
or finished trading before the simulation end, exceptions are raised.
|
||||
|
||||
If the benchmark asset started trading the same day as the simulation
|
||||
start, the first available minute price on that day is used instead
|
||||
of the previous close.
|
||||
|
||||
We use history to get an adjusted price history for each day's close,
|
||||
as of the look-back date (the last day of the simulation). Prices are
|
||||
fully adjusted for dividends, splits, and mergers.
|
||||
|
||||
Returns
|
||||
-------
|
||||
A pd.Series, indexed by trading day, whose values represent the %
|
||||
change from close to close.
|
||||
"""
|
||||
if self.emission_rate == "minute":
|
||||
minutes = trading_calendar.minutes_for_sessions_in_range(
|
||||
self.sessions[0], self.sessions[-1]
|
||||
)
|
||||
benchmark_series = data_portal.get_history_window(
|
||||
[asset],
|
||||
minutes[-1],
|
||||
bar_count=len(minutes) + 1,
|
||||
frequency="1m",
|
||||
field="price",
|
||||
data_frequency=self.emission_rate,
|
||||
ffill=True
|
||||
)[asset]
|
||||
|
||||
return benchmark_series.pct_change()[1:]
|
||||
else:
|
||||
start_date = asset.start_date
|
||||
if start_date < trading_days[0]:
|
||||
# get the window of close prices for benchmark_asset from the
|
||||
# last trading day of the simulation, going up to one day
|
||||
# before the simulation start day (so that we can get the %
|
||||
# change on day 1)
|
||||
benchmark_series = data_portal.get_history_window(
|
||||
[asset],
|
||||
trading_days[-1],
|
||||
bar_count=len(trading_days) + 1,
|
||||
frequency="1d",
|
||||
field="price",
|
||||
data_frequency=self.emission_rate,
|
||||
ffill=True
|
||||
)[asset]
|
||||
return benchmark_series.pct_change()[1:]
|
||||
elif start_date == trading_days[0]:
|
||||
# Attempt to handle case where stock data starts on first
|
||||
# day, in this case use the open to close return.
|
||||
benchmark_series = data_portal.get_history_window(
|
||||
[asset],
|
||||
trading_days[-1],
|
||||
bar_count=len(trading_days),
|
||||
frequency="1d",
|
||||
field="price",
|
||||
data_frequency=self.emission_rate,
|
||||
ffill=True
|
||||
)[asset]
|
||||
|
||||
# get a minute history window of the first day
|
||||
first_open = data_portal.get_spot_value(
|
||||
asset, 'open', trading_days[0], 'daily')
|
||||
first_close = data_portal.get_spot_value(
|
||||
asset, 'close', trading_days[0], 'daily')
|
||||
|
||||
first_day_return = (first_close - first_open) / first_open
|
||||
|
||||
returns = benchmark_series.pct_change()[:]
|
||||
returns[0] = first_day_return
|
||||
return returns
|
||||
@@ -0,0 +1,590 @@
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from collections import namedtuple
|
||||
import hashlib
|
||||
from textwrap import dedent
|
||||
import warnings
|
||||
|
||||
from logbook import Logger
|
||||
import numpy
|
||||
import pandas as pd
|
||||
from pandas import read_csv
|
||||
import pytz
|
||||
import requests
|
||||
from six import StringIO, iteritems, with_metaclass
|
||||
|
||||
from catalyst.errors import (
|
||||
MultipleSymbolsFound,
|
||||
SymbolNotFound,
|
||||
ZiplineError
|
||||
)
|
||||
from catalyst.protocol import (
|
||||
DATASOURCE_TYPE,
|
||||
Event
|
||||
)
|
||||
from catalyst.assets import Equity
|
||||
|
||||
logger = Logger('Requests Source Logger')
|
||||
|
||||
|
||||
def roll_dts_to_midnight(dts, trading_day):
|
||||
if len(dts) == 0:
|
||||
return dts
|
||||
|
||||
return pd.DatetimeIndex(
|
||||
(dts.tz_convert('US/Eastern') - pd.Timedelta(hours=16)).date,
|
||||
tz='UTC',
|
||||
) + trading_day
|
||||
|
||||
|
||||
class FetcherEvent(Event):
|
||||
pass
|
||||
|
||||
|
||||
class FetcherCSVRedirectError(ZiplineError):
|
||||
msg = dedent(
|
||||
"""\
|
||||
Attempt to fetch_csv from a redirected url. {url}
|
||||
must be changed to {new_url}
|
||||
"""
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.url = kwargs["url"]
|
||||
self.new_url = kwargs["new_url"]
|
||||
self.extra = kwargs["extra"]
|
||||
|
||||
super(FetcherCSVRedirectError, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
# The following optional arguments are supported for
|
||||
# requests backed data sources.
|
||||
# see http://docs.python-requests.org/en/latest/api/#main-interface
|
||||
# for a full list.
|
||||
ALLOWED_REQUESTS_KWARGS = {
|
||||
'params',
|
||||
'headers',
|
||||
'auth',
|
||||
'cert'
|
||||
}
|
||||
|
||||
|
||||
# The following optional arguments are supported for pandas' read_csv
|
||||
# function, and may be passed as kwargs to the datasource below.
|
||||
# see http://pandas.pydata.org/
|
||||
# pandas-docs/stable/generated/pandas.io.parsers.read_csv.html
|
||||
ALLOWED_READ_CSV_KWARGS = {
|
||||
'sep',
|
||||
'dialect',
|
||||
'doublequote',
|
||||
'escapechar',
|
||||
'quotechar',
|
||||
'quoting',
|
||||
'skipinitialspace',
|
||||
'lineterminator',
|
||||
'header',
|
||||
'index_col',
|
||||
'names',
|
||||
'prefix',
|
||||
'skiprows',
|
||||
'skipfooter',
|
||||
'skip_footer',
|
||||
'na_values',
|
||||
'true_values',
|
||||
'false_values',
|
||||
'delimiter',
|
||||
'converters',
|
||||
'dtype',
|
||||
'delim_whitespace',
|
||||
'as_recarray',
|
||||
'na_filter',
|
||||
'compact_ints',
|
||||
'use_unsigned',
|
||||
'buffer_lines',
|
||||
'warn_bad_lines',
|
||||
'error_bad_lines',
|
||||
'keep_default_na',
|
||||
'thousands',
|
||||
'comment',
|
||||
'decimal',
|
||||
'keep_date_col',
|
||||
'nrows',
|
||||
'chunksize',
|
||||
'encoding',
|
||||
'usecols'
|
||||
}
|
||||
|
||||
SHARED_REQUESTS_KWARGS = {
|
||||
'stream': True,
|
||||
'allow_redirects': False,
|
||||
}
|
||||
|
||||
|
||||
def mask_requests_args(url, validating=False, params_checker=None, **kwargs):
|
||||
requests_kwargs = {key: val for (key, val) in iteritems(kwargs)
|
||||
if key in ALLOWED_REQUESTS_KWARGS}
|
||||
if params_checker is not None:
|
||||
url, s_params = params_checker(url)
|
||||
if s_params:
|
||||
if 'params' in requests_kwargs:
|
||||
requests_kwargs['params'].update(s_params)
|
||||
else:
|
||||
requests_kwargs['params'] = s_params
|
||||
|
||||
# Giving the connection 30 seconds. This timeout does not
|
||||
# apply to the download of the response body.
|
||||
# (Note that Quandl links can take >10 seconds to return their
|
||||
# first byte on occasion)
|
||||
requests_kwargs['timeout'] = 1.0 if validating else 30.0
|
||||
requests_kwargs.update(SHARED_REQUESTS_KWARGS)
|
||||
|
||||
request_pair = namedtuple("RequestPair", ("requests_kwargs", "url"))
|
||||
return request_pair(requests_kwargs, url)
|
||||
|
||||
|
||||
class PandasCSV(with_metaclass(ABCMeta, object)):
|
||||
|
||||
def __init__(self,
|
||||
pre_func,
|
||||
post_func,
|
||||
asset_finder,
|
||||
trading_day,
|
||||
start_date,
|
||||
end_date,
|
||||
date_column,
|
||||
date_format,
|
||||
timezone,
|
||||
symbol,
|
||||
mask,
|
||||
symbol_column,
|
||||
data_frequency,
|
||||
**kwargs):
|
||||
|
||||
self.start_date = start_date
|
||||
self.end_date = end_date
|
||||
self.date_column = date_column
|
||||
self.date_format = date_format
|
||||
self.timezone = timezone
|
||||
self.mask = mask
|
||||
self.symbol_column = symbol_column or "symbol"
|
||||
self.data_frequency = data_frequency
|
||||
|
||||
invalid_kwargs = set(kwargs) - ALLOWED_READ_CSV_KWARGS
|
||||
if invalid_kwargs:
|
||||
raise TypeError(
|
||||
"Unexpected keyword arguments: %s" % invalid_kwargs,
|
||||
)
|
||||
|
||||
self.pandas_kwargs = self.mask_pandas_args(kwargs)
|
||||
|
||||
self.symbol = symbol
|
||||
|
||||
self.finder = asset_finder
|
||||
self.trading_day = trading_day
|
||||
|
||||
self.pre_func = pre_func
|
||||
self.post_func = post_func
|
||||
|
||||
@property
|
||||
def fields(self):
|
||||
return self.df.columns.tolist()
|
||||
|
||||
def get_hash(self):
|
||||
return self.namestring
|
||||
|
||||
@abstractmethod
|
||||
def fetch_data(self):
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def parse_date_str_series(format_str, tz, date_str_series, data_frequency,
|
||||
trading_day):
|
||||
"""
|
||||
Efficient parsing for a 1d Pandas/numpy object containing string
|
||||
representations of dates.
|
||||
|
||||
Note: pd.to_datetime is significantly faster when no format string is
|
||||
passed, and in pandas 0.12.0 the %p strptime directive is not correctly
|
||||
handled if a format string is explicitly passed, but AM/PM is handled
|
||||
properly if format=None.
|
||||
|
||||
Moreover, we were previously ignoring this parameter unintentionally
|
||||
because we were incorrectly passing it as a positional. For all these
|
||||
reasons, we ignore the format_str parameter when parsing datetimes.
|
||||
"""
|
||||
|
||||
# Explicitly ignoring this parameter. See note above.
|
||||
if format_str is not None:
|
||||
logger.warn(
|
||||
"The 'format_str' parameter to fetch_csv is deprecated. "
|
||||
"Ignoring and defaulting to pandas default date parsing."
|
||||
)
|
||||
format_str = None
|
||||
|
||||
tz_str = str(tz)
|
||||
if tz_str == pytz.utc.zone:
|
||||
parsed = pd.to_datetime(
|
||||
date_str_series.values,
|
||||
format=format_str,
|
||||
utc=True,
|
||||
errors='coerce',
|
||||
)
|
||||
else:
|
||||
parsed = pd.to_datetime(
|
||||
date_str_series.values,
|
||||
format=format_str,
|
||||
errors='coerce',
|
||||
).tz_localize(tz_str).tz_convert('UTC')
|
||||
|
||||
if data_frequency == 'daily':
|
||||
parsed = roll_dts_to_midnight(parsed, trading_day)
|
||||
return parsed
|
||||
|
||||
def mask_pandas_args(self, kwargs):
|
||||
pandas_kwargs = {key: val for (key, val) in iteritems(kwargs)
|
||||
if key in ALLOWED_READ_CSV_KWARGS}
|
||||
if 'usecols' in pandas_kwargs:
|
||||
usecols = pandas_kwargs['usecols']
|
||||
if usecols and self.date_column not in usecols:
|
||||
# make a new list so we don't modify user's,
|
||||
# and to ensure it is mutable
|
||||
with_date = list(usecols)
|
||||
with_date.append(self.date_column)
|
||||
pandas_kwargs['usecols'] = with_date
|
||||
|
||||
# No strings in the 'symbol' column should be interpreted as NaNs
|
||||
pandas_kwargs.setdefault('keep_default_na', False)
|
||||
pandas_kwargs.setdefault('na_values', {'symbol': []})
|
||||
|
||||
return pandas_kwargs
|
||||
|
||||
def _lookup_unconflicted_symbol(self, symbol):
|
||||
"""
|
||||
Attempt to find a unique asset whose symbol is the given string.
|
||||
|
||||
If multiple assets have held the given symbol, return a 0.
|
||||
|
||||
If no asset has held the given symbol, return a NaN.
|
||||
"""
|
||||
try:
|
||||
uppered = symbol.upper()
|
||||
except AttributeError:
|
||||
# The mapping fails because symbol was a non-string
|
||||
return numpy.nan
|
||||
|
||||
try:
|
||||
return self.finder.lookup_symbol(uppered, as_of_date=None)
|
||||
except MultipleSymbolsFound:
|
||||
# Fill conflicted entries with zeros to mark that they need to be
|
||||
# resolved by date.
|
||||
return 0
|
||||
except SymbolNotFound:
|
||||
# Fill not found entries with nans.
|
||||
return numpy.nan
|
||||
|
||||
def load_df(self):
|
||||
df = self.fetch_data()
|
||||
|
||||
if self.pre_func:
|
||||
df = self.pre_func(df)
|
||||
|
||||
# Batch-convert the user-specifed date column into timestamps.
|
||||
df['dt'] = self.parse_date_str_series(
|
||||
self.date_format,
|
||||
self.timezone,
|
||||
df[self.date_column],
|
||||
self.data_frequency,
|
||||
self.trading_day,
|
||||
).values
|
||||
|
||||
# ignore rows whose dates we couldn't parse
|
||||
df = df[df['dt'].notnull()]
|
||||
|
||||
if self.symbol is not None:
|
||||
df['sid'] = self.symbol
|
||||
elif self.finder:
|
||||
|
||||
df.sort_values(by=self.symbol_column, inplace=True)
|
||||
|
||||
# Pop the 'sid' column off of the DataFrame, just in case the user
|
||||
# has assigned it, and throw a warning
|
||||
try:
|
||||
df.pop('sid')
|
||||
warnings.warn(
|
||||
"Assignment of the 'sid' column of a DataFrame is "
|
||||
"not supported by Fetcher. The 'sid' column has been "
|
||||
"overwritten.",
|
||||
category=UserWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
except KeyError:
|
||||
# There was no 'sid' column, so no warning is necessary
|
||||
pass
|
||||
|
||||
# Fill entries for any symbols that don't require a date to
|
||||
# uniquely identify. Entries for which multiple securities exist
|
||||
# are replaced with zeroes, while entries for which no asset
|
||||
# exists are replaced with NaNs.
|
||||
unique_symbols = df[self.symbol_column].unique()
|
||||
sid_series = pd.Series(
|
||||
data=map(self._lookup_unconflicted_symbol, unique_symbols),
|
||||
index=unique_symbols,
|
||||
name='sid',
|
||||
)
|
||||
df = df.join(sid_series, on=self.symbol_column)
|
||||
|
||||
# Fill any zero entries left in our sid column by doing a lookup
|
||||
# using both symbol and the row date.
|
||||
conflict_rows = df[df['sid'] == 0]
|
||||
for row_idx, row in conflict_rows.iterrows():
|
||||
try:
|
||||
asset = self.finder.lookup_symbol(
|
||||
row[self.symbol_column],
|
||||
# Replacing tzinfo here is necessary because of the
|
||||
# timezone metadata bug described below.
|
||||
row['dt'].replace(tzinfo=pytz.utc),
|
||||
|
||||
# It's possible that no asset comes back here if our
|
||||
# lookup date is from before any asset held the
|
||||
# requested symbol. Mark such cases as NaN so that
|
||||
# they get dropped in the next step.
|
||||
) or numpy.nan
|
||||
except SymbolNotFound:
|
||||
asset = numpy.nan
|
||||
|
||||
# Assign the resolved asset to the cell
|
||||
df.ix[row_idx, 'sid'] = asset
|
||||
|
||||
# Filter out rows containing symbols that we failed to find.
|
||||
length_before_drop = len(df)
|
||||
df = df[df['sid'].notnull()]
|
||||
no_sid_count = length_before_drop - len(df)
|
||||
if no_sid_count:
|
||||
logger.warn(
|
||||
"Dropped {} rows from fetched csv.".format(no_sid_count),
|
||||
no_sid_count,
|
||||
extra={'syslog': True},
|
||||
)
|
||||
else:
|
||||
df['sid'] = df['symbol']
|
||||
|
||||
# Dates are localized to UTC when they come out of
|
||||
# parse_date_str_series, but we need to re-localize them here because
|
||||
# of a bug that wasn't fixed until
|
||||
# https://github.com/pydata/pandas/pull/7092.
|
||||
# We should be able to remove the call to tz_localize once we're on
|
||||
# pandas 0.14.0
|
||||
|
||||
# We don't set 'dt' as the index until here because the Symbol parsing
|
||||
# operations above depend on having a unique index for the dataframe,
|
||||
# and the 'dt' column can contain multiple dates for the same entry.
|
||||
df.drop_duplicates(["sid", "dt"])
|
||||
df.set_index(['dt'], inplace=True)
|
||||
df = df.tz_localize('UTC')
|
||||
df.sort_index(inplace=True)
|
||||
|
||||
cols_to_drop = [self.date_column]
|
||||
if self.symbol is None:
|
||||
cols_to_drop.append(self.symbol_column)
|
||||
df = df[df.columns.drop(cols_to_drop)]
|
||||
|
||||
if self.post_func:
|
||||
df = self.post_func(df)
|
||||
|
||||
return df
|
||||
|
||||
def __iter__(self):
|
||||
asset_cache = {}
|
||||
for dt, series in self.df.iterrows():
|
||||
if dt < self.start_date:
|
||||
continue
|
||||
|
||||
if dt > self.end_date:
|
||||
return
|
||||
|
||||
event = FetcherEvent()
|
||||
# when dt column is converted to be the dataframe's index
|
||||
# the dt column is dropped. So, we need to manually copy
|
||||
# dt into the event.
|
||||
event.dt = dt
|
||||
for k, v in series.iteritems():
|
||||
# convert numpy integer types to
|
||||
# int. This assumes we are on a 64bit
|
||||
# platform that will not lose information
|
||||
# by casting.
|
||||
# TODO: this is only necessary on the
|
||||
# amazon qexec instances. would be good
|
||||
# to figure out how to use the numpy dtypes
|
||||
# without this check and casting.
|
||||
if isinstance(v, numpy.integer):
|
||||
v = int(v)
|
||||
|
||||
setattr(event, k, v)
|
||||
|
||||
# If it has start_date, then it's already an Asset
|
||||
# object from asset_for_symbol, and we don't have to
|
||||
# transform it any further. Checking for start_date is
|
||||
# faster than isinstance.
|
||||
if event.sid in asset_cache:
|
||||
event.sid = asset_cache[event.sid]
|
||||
elif hasattr(event.sid, 'start_date'):
|
||||
# Clone for user algo code, if we haven't already.
|
||||
asset_cache[event.sid] = event.sid
|
||||
elif self.finder and isinstance(event.sid, int):
|
||||
asset = self.finder.retrieve_asset(event.sid,
|
||||
default_none=True)
|
||||
if asset:
|
||||
# Clone for user algo code.
|
||||
event.sid = asset_cache[asset] = asset
|
||||
elif self.mask:
|
||||
# When masking drop all non-mappable values.
|
||||
continue
|
||||
elif self.symbol is None:
|
||||
# If the event's sid property is an int we coerce
|
||||
# it into an Equity.
|
||||
event.sid = asset_cache[event.sid] = Equity(event.sid)
|
||||
|
||||
event.type = DATASOURCE_TYPE.CUSTOM
|
||||
event.source_id = self.namestring
|
||||
yield event
|
||||
|
||||
|
||||
class PandasRequestsCSV(PandasCSV):
|
||||
# maximum 100 megs to prevent DDoS
|
||||
MAX_DOCUMENT_SIZE = (1024 * 1024) * 100
|
||||
|
||||
# maximum number of bytes to read in at a time
|
||||
CONTENT_CHUNK_SIZE = 4096
|
||||
|
||||
def __init__(self,
|
||||
url,
|
||||
pre_func,
|
||||
post_func,
|
||||
asset_finder,
|
||||
trading_day,
|
||||
start_date,
|
||||
end_date,
|
||||
date_column,
|
||||
date_format,
|
||||
timezone,
|
||||
symbol,
|
||||
mask,
|
||||
symbol_column,
|
||||
data_frequency,
|
||||
special_params_checker=None,
|
||||
**kwargs):
|
||||
|
||||
# Peel off extra requests kwargs, forwarding the remaining kwargs to
|
||||
# the superclass.
|
||||
# Also returns possible https updated url if sent to http quandl ds
|
||||
# If url hasn't changed, will just return the original.
|
||||
self._requests_kwargs, self.url =\
|
||||
mask_requests_args(url,
|
||||
params_checker=special_params_checker,
|
||||
**kwargs)
|
||||
|
||||
remaining_kwargs = {
|
||||
k: v for k, v in iteritems(kwargs)
|
||||
if k not in self.requests_kwargs
|
||||
}
|
||||
|
||||
self.namestring = type(self).__name__
|
||||
|
||||
super(PandasRequestsCSV, self).__init__(
|
||||
pre_func,
|
||||
post_func,
|
||||
asset_finder,
|
||||
trading_day,
|
||||
start_date,
|
||||
end_date,
|
||||
date_column,
|
||||
date_format,
|
||||
timezone,
|
||||
symbol,
|
||||
mask,
|
||||
symbol_column,
|
||||
data_frequency,
|
||||
**remaining_kwargs
|
||||
)
|
||||
|
||||
self.fetch_size = None
|
||||
self.fetch_hash = None
|
||||
|
||||
self.df = self.load_df()
|
||||
|
||||
self.special_params_checker = special_params_checker
|
||||
|
||||
@property
|
||||
def requests_kwargs(self):
|
||||
return self._requests_kwargs
|
||||
|
||||
def fetch_url(self, url):
|
||||
info = "checking {url} with {params}"
|
||||
logger.info(info.format(url=url, params=self.requests_kwargs))
|
||||
# setting decode_unicode=True sometimes results in a
|
||||
# UnicodeEncodeError exception, so instead we'll use
|
||||
# pandas logic for decoding content
|
||||
try:
|
||||
response = requests.get(url, **self.requests_kwargs)
|
||||
except requests.exceptions.ConnectionError:
|
||||
raise Exception('Could not connect to %s' % url)
|
||||
|
||||
if not response.ok:
|
||||
raise Exception('Problem reaching %s' % url)
|
||||
elif response.is_redirect:
|
||||
# On the offchance we don't catch a redirect URL
|
||||
# in validation, this will catch it.
|
||||
new_url = response.headers['location']
|
||||
raise FetcherCSVRedirectError(
|
||||
url=url,
|
||||
new_url=new_url,
|
||||
extra={
|
||||
'old_url': url,
|
||||
'new_url': new_url
|
||||
}
|
||||
)
|
||||
|
||||
content_length = 0
|
||||
logger.info('{} connection established in {:.1f} seconds'.format(
|
||||
url, response.elapsed.total_seconds()))
|
||||
|
||||
# use the decode_unicode flag to ensure that the output of this is
|
||||
# a string, and not bytes.
|
||||
for chunk in response.iter_content(self.CONTENT_CHUNK_SIZE,
|
||||
decode_unicode=True):
|
||||
if content_length > self.MAX_DOCUMENT_SIZE:
|
||||
raise Exception('Document size too big.')
|
||||
if chunk:
|
||||
content_length += len(chunk)
|
||||
yield chunk
|
||||
|
||||
return
|
||||
|
||||
def fetch_data(self):
|
||||
# create a data frame directly from the full text of
|
||||
# the response from the returned file-descriptor.
|
||||
data = self.fetch_url(self.url)
|
||||
fd = StringIO()
|
||||
|
||||
if isinstance(data, str):
|
||||
fd.write(data)
|
||||
else:
|
||||
for chunk in data:
|
||||
fd.write(chunk)
|
||||
|
||||
self.fetch_size = fd.tell()
|
||||
|
||||
fd.seek(0)
|
||||
|
||||
try:
|
||||
# see if pandas can parse csv data
|
||||
frames = read_csv(fd, **self.pandas_kwargs)
|
||||
|
||||
frames_hash = hashlib.md5(str(fd.getvalue()).encode('utf-8'))
|
||||
self.fetch_hash = frames_hash.hexdigest()
|
||||
except pd.parser.CParserError:
|
||||
# could not parse the data, raise exception
|
||||
raise Exception('Error parsing remote CSV data.')
|
||||
finally:
|
||||
fd.close()
|
||||
|
||||
return frames
|
||||
@@ -0,0 +1,248 @@
|
||||
#
|
||||
# Copyright 2013 Quantopian, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
A source to be used in testing.
|
||||
"""
|
||||
|
||||
import pytz
|
||||
|
||||
from six.moves import filter
|
||||
from datetime import datetime, timedelta
|
||||
import itertools
|
||||
|
||||
from six.moves import range
|
||||
|
||||
from catalyst.protocol import (
|
||||
Event,
|
||||
DATASOURCE_TYPE
|
||||
)
|
||||
from catalyst.gens.utils import hash_args
|
||||
|
||||
|
||||
def create_trade(sid, price, amount, datetime, source_id="test_factory"):
|
||||
|
||||
trade = Event()
|
||||
|
||||
trade.source_id = source_id
|
||||
trade.type = DATASOURCE_TYPE.TRADE
|
||||
trade.sid = sid
|
||||
trade.dt = datetime
|
||||
trade.price = price
|
||||
trade.close_price = price
|
||||
trade.open_price = price
|
||||
trade.low = price * .95
|
||||
trade.high = price * 1.05
|
||||
trade.volume = amount
|
||||
|
||||
return trade
|
||||
|
||||
|
||||
def date_gen(start,
|
||||
end,
|
||||
trading_calendar,
|
||||
delta=timedelta(minutes=1),
|
||||
repeats=None):
|
||||
"""
|
||||
Utility to generate a stream of dates.
|
||||
"""
|
||||
daily_delta = not (delta.total_seconds()
|
||||
% timedelta(days=1).total_seconds())
|
||||
cur = start
|
||||
if daily_delta:
|
||||
# if we are producing daily timestamps, we
|
||||
# use midnight
|
||||
cur = cur.replace(hour=0, minute=0, second=0,
|
||||
microsecond=0)
|
||||
|
||||
def advance_current(cur):
|
||||
"""
|
||||
Advances the current dt skipping non market days and minutes.
|
||||
"""
|
||||
cur = cur + delta
|
||||
|
||||
currently_executing = \
|
||||
(daily_delta and (cur in trading_calendar.all_sessions)) or \
|
||||
(trading_calendar.is_open_on_minute(cur))
|
||||
|
||||
if currently_executing:
|
||||
return cur
|
||||
else:
|
||||
if daily_delta:
|
||||
return trading_calendar.minute_to_session_label(cur)
|
||||
else:
|
||||
return trading_calendar.open_and_close_for_session(
|
||||
trading_calendar.minute_to_session_label(cur)
|
||||
)[0]
|
||||
|
||||
# yield count trade events, all on trading days, and
|
||||
# during trading hours.
|
||||
while cur < end:
|
||||
if repeats:
|
||||
for j in range(repeats):
|
||||
yield cur
|
||||
else:
|
||||
yield cur
|
||||
|
||||
cur = advance_current(cur)
|
||||
|
||||
|
||||
class SpecificEquityTrades(object):
|
||||
"""
|
||||
Yields all events in event_list that match the given sid_filter.
|
||||
If no event_list is specified, generates an internal stream of events
|
||||
to filter. Returns all events if filter is None.
|
||||
|
||||
Configuration options:
|
||||
|
||||
count : integer representing number of trades
|
||||
sids : list of values representing simulated internal sids
|
||||
start : start date
|
||||
delta : timedelta between internal events
|
||||
filter : filter to remove the sids
|
||||
"""
|
||||
def __init__(self, env, trading_calendar, *args, **kwargs):
|
||||
# We shouldn't get any positional arguments.
|
||||
assert len(args) == 0
|
||||
|
||||
self.env = env
|
||||
self.trading_calendar = trading_calendar
|
||||
|
||||
# Default to None for event_list and filter.
|
||||
self.event_list = kwargs.get('event_list')
|
||||
self.filter = kwargs.get('filter')
|
||||
if self.event_list is not None:
|
||||
# If event_list is provided, extract parameters from there
|
||||
# This isn't really clean and ultimately I think this
|
||||
# class should serve a single purpose (either take an
|
||||
# event_list or autocreate events).
|
||||
self.count = kwargs.get('count', len(self.event_list))
|
||||
self.start = kwargs.get('start', self.event_list[0].dt)
|
||||
self.end = kwargs.get('end', self.event_list[-1].dt)
|
||||
self.delta = delta = kwargs.get('delta')
|
||||
if delta is None:
|
||||
self.delta = self.event_list[1].dt - self.event_list[0].dt
|
||||
self.concurrent = kwargs.get('concurrent', False)
|
||||
|
||||
self.identifiers = kwargs.get(
|
||||
'sids',
|
||||
set(event.sid for event in self.event_list)
|
||||
)
|
||||
assets_by_identifier = {}
|
||||
for identifier in self.identifiers:
|
||||
assets_by_identifier[identifier] = env.asset_finder.\
|
||||
lookup_generic(identifier, datetime.now())[0]
|
||||
self.sids = [asset.sid for asset in assets_by_identifier.values()]
|
||||
for event in self.event_list:
|
||||
event.sid = assets_by_identifier[event.sid].sid
|
||||
|
||||
else:
|
||||
# Unpack config dictionary with default values.
|
||||
self.count = kwargs.get('count', 500)
|
||||
self.start = kwargs.get(
|
||||
'start',
|
||||
datetime(2008, 6, 6, 15, tzinfo=pytz.utc))
|
||||
self.end = kwargs.get(
|
||||
'end',
|
||||
datetime(2008, 6, 6, 15, tzinfo=pytz.utc))
|
||||
self.delta = kwargs.get(
|
||||
'delta',
|
||||
timedelta(minutes=1))
|
||||
self.concurrent = kwargs.get('concurrent', False)
|
||||
|
||||
self.identifiers = kwargs.get('sids', [1, 2])
|
||||
assets_by_identifier = {}
|
||||
for identifier in self.identifiers:
|
||||
assets_by_identifier[identifier] = env.asset_finder.\
|
||||
lookup_generic(identifier, datetime.now())[0]
|
||||
self.sids = [asset.sid for asset in assets_by_identifier.values()]
|
||||
|
||||
# Hash_value for downstream sorting.
|
||||
self.arg_string = hash_args(*args, **kwargs)
|
||||
|
||||
self.generator = self.create_fresh_generator()
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def next(self):
|
||||
return self.generator.next()
|
||||
|
||||
def __next__(self):
|
||||
return next(self.generator)
|
||||
|
||||
def rewind(self):
|
||||
self.generator = self.create_fresh_generator()
|
||||
|
||||
def get_hash(self):
|
||||
return self.__class__.__name__ + "-" + self.arg_string
|
||||
|
||||
def update_source_id(self, gen):
|
||||
for event in gen:
|
||||
event.source_id = self.get_hash()
|
||||
yield event
|
||||
|
||||
def create_fresh_generator(self):
|
||||
|
||||
if self.event_list:
|
||||
event_gen = (event for event in self.event_list)
|
||||
unfiltered = self.update_source_id(event_gen)
|
||||
|
||||
# Set up iterators for each expected field.
|
||||
else:
|
||||
if self.concurrent:
|
||||
# in this context the count is the number of
|
||||
# trades per sid, not the total.
|
||||
date_generator = date_gen(
|
||||
start=self.start,
|
||||
end=self.end,
|
||||
delta=self.delta,
|
||||
repeats=len(self.sids),
|
||||
trading_calendar=self.trading_calendar,
|
||||
)
|
||||
else:
|
||||
date_generator = date_gen(
|
||||
start=self.start,
|
||||
end=self.end,
|
||||
delta=self.delta,
|
||||
trading_calendar=self.trading_calendar,
|
||||
)
|
||||
|
||||
source_id = self.get_hash()
|
||||
|
||||
unfiltered = (
|
||||
create_trade(
|
||||
sid=sid,
|
||||
price=float(i % 10) + 1.0,
|
||||
amount=(i * 50) % 900 + 100,
|
||||
datetime=date,
|
||||
source_id=source_id,
|
||||
) for (i, date), sid in itertools.product(
|
||||
enumerate(date_generator), self.sids
|
||||
)
|
||||
)
|
||||
|
||||
# If we specified a sid filter, filter out elements that don't
|
||||
# match the filter.
|
||||
if self.filter:
|
||||
filtered = filter(
|
||||
lambda event: event.sid in self.filter, unfiltered)
|
||||
|
||||
# Otherwise just use all events.
|
||||
else:
|
||||
filtered = unfiltered
|
||||
|
||||
# Return the filtered event stream.
|
||||
return filtered
|
||||
Reference in New Issue
Block a user