ENH: make BcolzMinuteBarWriter.write take iterable

Updates the BcolzMinuteBarWriter.write api to allow users to pass their
data as a stream instead of requiring that they loop over their data
externally. This matches the API presented by BcolzDailyBarWriter.
This commit is contained in:
Joe Jevnik
2016-04-29 16:14:48 -04:00
parent e73ce0bf2b
commit efac476976
12 changed files with 172 additions and 170 deletions
+6 -11
View File
@@ -349,7 +349,7 @@ def make_trade_data_for_asset_info(dates,
)
if writer:
writer.write(sid, df)
writer.write_sid(sid, df)
trade_data[sid] = df
@@ -424,8 +424,8 @@ def write_minute_data(env, tempdir, minutes, sids):
def create_minute_bar_data(minutes, sids):
length = len(minutes)
return {
sid: pd.DataFrame(
for sid_idx, sid in enumerate(sids):
yield sid, pd.DataFrame(
{
'open': np.arange(length) + 10 + sid_idx,
'high': np.arange(length) + 15 + sid_idx,
@@ -435,8 +435,6 @@ def create_minute_bar_data(minutes, sids):
},
index=minutes,
)
for sid_idx, sid in enumerate(sids)
}
def create_daily_bar_data(trading_days, sids):
@@ -492,20 +490,17 @@ def create_data_portal(env, tempdir, sim_params, sids, adjustment_reader=None):
)
def write_bcolz_minute_data(env, days, path, df_dict):
def write_bcolz_minute_data(env, days, path, data):
market_opens = env.open_and_closes.market_open.loc[days]
market_closes = env.open_and_closes.market_close.loc[days]
writer = BcolzMinuteBarWriter(
BcolzMinuteBarWriter(
days[0],
path,
market_opens,
market_closes,
US_EQUITIES_MINUTES_PER_DAY
)
for sid, df in iteritems(df_dict):
writer.write(sid, df)
).write(data)
def create_minute_df_for_asset(env,
+3 -5
View File
@@ -6,7 +6,7 @@ from contextlib2 import ExitStack
from logbook import NullHandler, Logger
from nose_parameterized import parameterized
from pandas.util.testing import assert_series_equal
from six import with_metaclass, iteritems
from six import with_metaclass
from toolz import flip
import numpy as np
import pandas as pd
@@ -681,7 +681,7 @@ class WithBcolzMinuteBarReader(WithTradingEnvironment, WithTmpDir):
Methods
-------
make_minute_bar_data() -> dict[int -> pd.DataFrame]
make_minute_bar_data() -> iterable[(int, pd.DataFrame)]
A class method that returns a dict mapping sid to dataframe
which will be written to the bcolz files that the class's
``BcolzMinuteBarReader`` will read from. By default this creates
@@ -734,9 +734,7 @@ class WithBcolzMinuteBarReader(WithTradingEnvironment, WithTmpDir):
cls.env.open_and_closes.market_close.loc[days],
US_EQUITIES_MINUTES_PER_DAY
)
cls.bcolz_minute_bar_data = cls.make_minute_bar_data()
for sid, df in iteritems(cls.bcolz_minute_bar_data):
writer.write(sid, df)
writer.write(cls.make_minute_bar_data())
cls.bcolz_minute_bar_reader = BcolzMinuteBarReader(p)