wassname/catalyst
1
0
Fork 0
mirror of https://github.com/wassname/catalyst.git synced 2026-07-09 03:43:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

BLD:updating the forward fill to set volume to zero and others values to the previous close value

Browse Source
This commit is contained in:
lenak25
2018-03-01 15:51:54 +02:00
parent c4b10bae39
commit b95cf465fc
2 changed files with 125 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalyst/exchange/utils/exchange_utils.py
+20 -15
View File
@@ -716,25 +716,30 @@ def save_asset_data(folder, df, decimals=8):
)
def get_candles_df(candles, field, freq, bar_count, end_dt,
previous_value=None):
def forward_fill_df_if_needed(df, periods):
df = df.reindex(periods)
df['volume'] = df['volume'].fillna(0.0)# volume should always be 0 (if there were no trades in this interval)
df['close'] = df.fillna(method='pad') # ie pull the last close into this close
# now copy the close that was pulled down from the last timestep into this row, across into o/h/l
df['open'] = df['open'].fillna(df['close'])
df['low'] = df['low'].fillna(df['close'])
df['high'] = df['high'].fillna(df['close'])
return df
def transform_candles_to_df(candles):
return pd.DataFrame(candles).set_index('last_traded')
def get_candles_df(candles, field, freq, bar_count, end_dt=None):
all_series = dict()
for asset in candles:
asset_df = transform_candles_to_df(candles[asset])
periods = pd.date_range(end=end_dt, periods=bar_count, freq=freq)
asset_df = forward_fill_df_if_needed(asset_df, periods)
dates = [candle['last_traded'] for candle in candles[asset]]
values = [candle[field] for candle in candles[asset]]
series = pd.Series(values, index=dates)
"""
series = series.reindex(
periods,
method='ffill',
fill_value=previous_value,
)
series.sort_index(inplace=True)
"""
all_series[asset] = series
all_series[asset] = pd.Series(asset_df[field])
df = pd.DataFrame(all_series)
df.dropna(inplace=True)
tests/exchange/test_exchange_utils.py
+105
View File
@@ -0,0 +1,105 @@
from catalyst.exchange.utils.exchange_utils import transform_candles_to_df, forward_fill_df_if_needed, get_candles_df
from catalyst.testing.fixtures import WithLogger, ZiplineTestCase
from pandas import Timestamp, Series, DataFrame
import numpy as np
class TestExchangeUtils(WithLogger, ZiplineTestCase):
@classmethod
def get_specific_field_from_df(cls, df, field, asset):
new_df = DataFrame(df[field])
new_df.columns = [asset]
new_df.index.name = None
return new_df
def test_transform_candles_to_series(self):
asset = 'btc_usdt'
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')}]
expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
{'high': 593.0, 'volume': 0.0, 'low': 593.0,
'close': 593.0, 'open': 593.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}
]
periods = [Timestamp('2018-03-01 09:45:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:50:00+0000', tz='UTC'),
Timestamp('2018-03-01 09:55:00+0000', tz='UTC')]
observed_df = forward_fill_df_if_needed(transform_candles_to_df(candles), periods)
expected_df = transform_candles_to_df(expected)
assert (expected_df.equals(observed_df))
for field in ['volume', 'open', 'close', 'high', 'low']:
assert(self.get_specific_field_from_df(observed_df, field, asset).equals(
get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2])))
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}]
expected = [{'high': 595.0, 'volume': 10.0, 'low': 594.0,
'close': 595.0, 'open': 594.0,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
{'high': 595.0, 'volume': 0.0, 'low': 595.0,
'close': 595.0, 'open': 595.0,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
{'high': 594.0, 'volume': 108.0, 'low': 592.0,
'close': 593.0, 'open': 592.0,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}
]
df = transform_candles_to_df(candles)
observed_df = forward_fill_df_if_needed(df, periods)
assert (transform_candles_to_df(expected).equals(observed_df))
for field in ['volume', 'open', 'close', 'high', 'low']:
assert(self.get_specific_field_from_df(observed_df, field, asset).equals(
get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2])))
candles = [{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}]
expected = [{'high': np.NaN, 'volume': 0.0, 'low': np.NaN,
'close': np.NaN, 'open': np.NaN,
'last_traded': Timestamp('2018-03-01 09:45:00+0000', tz='UTC')},
{'high': 595, 'volume': 10, 'low': 594,
'close': 595, 'open': 594,
'last_traded': Timestamp('2018-03-01 09:50:00+0000', tz='UTC')},
{'high': 594, 'volume': 108, 'low': 592,
'close': 593, 'open': 592,
'last_traded': Timestamp('2018-03-01 09:55:00+0000', tz='UTC')}
]
df = transform_candles_to_df(candles)
observed_df = forward_fill_df_if_needed(df, periods)
assert (transform_candles_to_df(expected).equals(observed_df))
# Not the same due to dropna - commenting out for now
"""
for field in ['volume', 'open', 'close', 'high', 'low']:
assert(self.get_specific_field_from_df(observed_df, field, asset).equals(
get_candles_df({asset:candles}, field, '5T', 3, end_dt=periods[2])))
"""