wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-08-07 11:20:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

BUG: fix replacement of NaN with None

Browse Source 
TST: finish test with expected data

STY: alphabetize imports

MAINT: simplify condition - remove unnecessary statement
This commit is contained in:
Maya Tydykov
2016-05-23 15:53:55 -04:00
parent 8e630bff77
commit c94f3d0c9b
2 changed files with 30 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/pipeline/test_blaze.py
+19 -7
View File
@@ -30,7 +30,6 @@ from zipline.pipeline.loaders.blaze import (
NoDeltasWarning,
)
from zipline.pipeline.loaders.blaze.core import (
NonNumpyField,
NonPipelineField,
no_deltas_rules,
)
@@ -277,11 +276,11 @@ class BlazeToPipelineTestCase(TestCase):
)
def test_cols_with_missing_vals(self):
dates = (pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-03')) * 3
dates = (pd.Timestamp('2014-01-01'), pd.Timestamp('2014-01-03'))
df = pd.DataFrame({
'sid': self.sids * 2,
'value': (np.NaN, 0., 1., 2., 3., 2.,),
'str_value': (None, "b", "c", "d", "e", "f"),
'sid': self.sids[:-1],
'value': (0., 1.,),
'str_value': ("a", "b",),
'asof_date': dates,
'timestamp': dates,
})
@@ -299,7 +298,21 @@ class BlazeToPipelineTestCase(TestCase):
fields = OrderedDict(expr.dshape.measure.fields)
with tmp_asset_finder() as finder:
expected = pd.DataFrame()
expected = pd.DataFrame(
np.array([["a", 0],
[None, np.NaN],
[None, np.NaN],
["a", 0],
[None, np.NaN],
[None, np.NaN],
["a", 0],
["b", 1],
[None, np.NaN]]),
columns=['str_value', 'value'],
index=pd.MultiIndex.from_product(
(self.dates, finder.retrieve_all(self.sids))
)
)
self._test_id(
df,
var * Record(fields),
@@ -369,7 +382,6 @@ class BlazeToPipelineTestCase(TestCase):
dates,
finder,
).run_pipeline(p, dates[0], dates[-1])
import pdb; pdb.set_trace()
assert_frame_equal(
result,
_utc_localize_index_level_0(expected),
zipline/pipeline/loaders/blaze/core.py
+11 -11
View File
@@ -177,7 +177,7 @@ from zipline.utils.input_validation import (
ensure_timezone,
optionally,
)
from zipline.utils.numpy_utils import repeat_last_axis, categorical_dtype
from zipline.utils.numpy_utils import categorical_dtype, repeat_last_axis
from zipline.utils.pandas_utils import sort_values
from zipline.utils.preprocess import preprocess
@@ -314,7 +314,7 @@ def new_dataset(expr, deltas, missing_values):
if isinstance(type_, Option):
type_ = type_.ty
type_ = type_.to_numpy_dtype()
if not isinstance(type_, String) and not can_represent_dtype(type_):
if not can_represent_dtype(type_):
raise NotPipelineCompatible()
col = Column(
type_,
@@ -1006,15 +1006,15 @@ class BlazeLoader(dict):
)
else:
last_in_group = last_in_group.reindex(dates)
import pdb; pdb.set_trace()
# str_cols = df.columns[df.dtypes == categorical_dtype]
#
# # Unstack will fill all missing values with NaN; we need to fix
# # this for strings.
# for col in str_cols:
# last_in_group.iloc[
# :, last_in_group.columns.get_level_values(0) == col
# ].fillna('None')
# Unstack will fill all missing values with NaN; we need to fix
# this for strings.
if not df.empty:
str_cols = df.columns[df.dtypes == categorical_dtype]
for col in str_cols:
last_in_group[col] = last_in_group[col].where(pd.notnull(
last_in_group[col]), None)
return last_in_group
sparse_deltas = last_in_date_group(non_novel_deltas, reindex=False)