options_backtester/backtester/examples/data_cleanup.ipynb

In [1]:

import os
import sys

import pandas as pd

backtester_dir = os.path.realpath(os.path.join(os.getcwd(), '..', '..'))
sys.path.append(backtester_dir) # Add backtester base dir to $PYTHONPATH

In [2]:

data_dir = os.path.join(backtester_dir, 'data')
store_path = os.path.join(data_dir, 'options_data_full_v2.h5')
store = pd.HDFStore(store_path, complevel=9, complib='blosc', fletcher32=True)

First we move the data from a h5 store with multiple keys (one per year) to another with a single key.

In [ ]:

old_data = os.path.join(data_dir, 'options_data_compressed_v2.h5')
sizes = {'optionroot': 20, 'optionalias': 20}
underlying_categories = pd.CategoricalDtype(categories=['SPX', 'SPXW', 'SPXPM'], ordered=False)

keys = ('spx_{}'.format(year) for year in range(1990, 2019))
offset = 0

for k in keys:
    df = pd.read_hdf(old_data, key=k)
    df['underlying'] = df['underlying'].astype(underlying_categories)
    df.drop(columns='exchange', inplace=True)
    
    df.index += offset
    offset += len(df)
    store.append('/SPX', df, index=False, data_columns=['quotedate', 'expiration'], min_itemsize=sizes)
    
os.path.getsize(store_path) / 1024**2

In [4]:

store.close()

Data cleaning

We will remove contracts where bid and ask columns go to 0 unexpectedly, that is, where the bid/ask price of the contract is greater than 0 for the previous and following days.
We will also remove contracts that have a missing quotedate before expiration.

In [3]:

full_data = os.path.join(data_dir, 'options_data_full_v2.h5')
df = pd.read_hdf(full_data, key='/SPX')
df.head()

Out [3]:

	underlying	underlying_last	optionroot	type	expiration	quotedate	strike	last	bid	ask	volume	openinterest	impliedvol	delta	gamma	theta	vega	optionalias
0	SPX	359.69	SPX900120C00225000	call	1990-01-20	1990-01-02	225.0	0.0	135.5	135.5	0	820	0.0	0.0	0.0	0.0	0.0	SPX900120C00225000
1	SPX	359.69	SPX900120C00320000	call	1990-01-20	1990-01-02	320.0	0.0	40.9	40.9	0	1088	0.0	0.0	0.0	0.0	0.0	SPX900120C00320000
2	SPX	359.69	SPX900120C00325000	call	1990-01-20	1990-01-02	325.0	0.0	35.9	35.9	0	1252	0.0	0.0	0.0	0.0	0.0	SPX900120C00325000
3	SPX	359.69	SPX900120C00330000	call	1990-01-20	1990-01-02	330.0	30.5	30.9	30.9	25	8738	0.0	0.0	0.0	0.0	0.0	SPX900120C00330000
4	SPX	359.69	SPX900120C00335000	call	1990-01-20	1990-01-02	335.0	0.0	26.0	26.0	0	580	0.0	0.0	0.0	0.0	0.0	SPX900120C00335000

In [4]:

df['optionroot'].nunique()

Out [4]:

240554

In [5]:

len(df)

Out [5]:

16756680

In [6]:

to_clean = set()
threshold = 0.5

dates = pd.Series(index=df['quotedate'].unique())

for contract, group in df.groupby('optionroot'):
    both_zero = group.eval('bid == ask == 0.0')
    if both_zero.any():
        ask_diff_previous = group['ask'].diff().abs()
        ask_diff_next = group['ask'].diff(-1).abs()
        
        if ((ask_diff_previous > threshold) & (ask_diff_next > threshold) & both_zero).any():
            to_clean.add(contract)
        else:
            start_date, end_date = group['quotedate'].min(), group['quotedate'].max()
            if len(dates.loc[start_date:end_date]) != len(group):
                to_clean.add(contract)

In [7]:

len(to_clean)

Out [7]:

29394

In [12]:

len(to_clean) / df['optionroot'].nunique()

Out [12]:

0.1221929379681901

In [24]:

to_clean_mask = df[df['optionroot'].isin(to_clean)]
clean_df = df.drop(to_clean_mask.index)
clean_df.reset_index(drop=True, inplace=True)

min_sizes = {'optionroot': 20, 'optionalias': 20}
clean_file = os.path.join(data_dir, 'options_data_clean_v2.h5')
clean_df.to_hdf(clean_file,
                mode='w',
                key='/SPX',
                format='table',
                data_columns=['quotedate', 'expiration'],
                complevel=9,
                complib='blosc:lz4',
                fletcher32=True,
                min_itemsize=min_sizes)

In [26]:

clean_df.info(memory_usage='deep')

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 13909922 entries, 0 to 13909921
Data columns (total 18 columns):
underlying         category
underlying_last    float64
optionroot         object
type               category
expiration         datetime64[ns]
quotedate          datetime64[ns]
strike             float64
last               float64
bid                float64
ask                float64
volume             int64
openinterest       int64
impliedvol         float64
delta              float64
gamma              float64
theta              float64
vega               float64
optionalias        object
dtypes: category(2), datetime64[ns](2), float64(10), int64(2), object(2)
memory usage: 3.4 GB

In [27]:

os.path.getsize(clean_file) / 1024**2

Out [27]:

568.1449775695801

In [31]:

unique_contracts_per_day = df.groupby('quotedate').apply(lambda x: x['optionroot'].nunique())
unique_contracts_per_day.plot()

Out [31]:

<matplotlib.axes._subplots.AxesSubplot at 0x1c0d21510>

In [30]:

unique_contracts_per_day_clean = clean_df.groupby('quotedate').apply(lambda x: x['optionroot'].nunique())
unique_contracts_per_day_clean.plot()

Out [30]:

<matplotlib.axes._subplots.AxesSubplot at 0x1c0c37c10>

Previous data cleaning attempt

Our data has some zero values and missing entries that are inaccurate and could throw off our backtester, giving misleading results. We perform here a cleanup of the data.

In [ ]:

store_path = os.path.join("/Users/jrchatruc/Documents/backtester_options/allspx/", "options_data_v2_light.h5")
store = pd.HDFStore(store_path, complevel=9, complib="blosc", fletcher32=True)
underlying_dtype = pd.api.types.CategoricalDtype(
                categories=['SPX', 'SPXW', 'SPXPM'])

offset = 0
sizes = {"optionroot": 20, "optionalias": 20}
for year in range(1990, 2019):
    filename = "SPX_{}.csv".format(year)
    year_df = pd.read_csv(os.path.join("/Users/jrchatruc/Documents/backtester_options/allspx/", filename),
                          parse_dates=["expiration", "quotedate"])
    year_df.rename(columns={" exchange": "exchange"}, inplace=True)
    year_df = year_df.astype({"strike": "float", "optionalias": "object", "type": "category",
                              'underlying': underlying_dtype})
    year_df = year_df.drop('optionext', axis=1)
    year_df = year_df.drop('exchange', axis=1)
    year_df = year_df.reset_index(drop=True)
    year_df.index += offset
    offset += len(year_df)
    store.append("/SPX", year_df, index=False, data_columns=True, min_itemsize=sizes)
    print(year)

In [ ]:

%cd -q /Users/jrchatruc/Documents/backtester_options/allspx
!ptrepack --complevel=9 --complib=blosc options_data_v2_light.h5 options_data_v2_light_compressed.h5

In [95]:

data = HistoricalOptionsData("options_data_v2_light_compressed.h5", key="/SPX")
schema = data.schema
df = data._data

In [96]:

df.shape

Out [96]:

(16756680, 19)

In [97]:

df[(df['bid'] == 0) & (df['ask'] == 0)].shape[0] / df.shape[0]

Out [97]:

0.065681208926828

As can be seen, around 6.5\% of the rows have both their bid and ask equal to zero. Some of these are because these values are going progressively down until they reach around 0.1 or 0.05, at which point they (sometimes) get replaced with zero. This isn't a problematic case, but it is not the only one.

In [98]:

# A (mostly) innocuous contract, its bid and ask values are always very close to zero, so them dipping into zero
# is not a problem.
df[df['optionroot'] == 'SPX191220P00100000']

Out [98]:

	underlying	underlying_last	optionroot	type	expiration	quotedate	strike	last	bid	ask	volume	openinterest	impliedvol	delta	gamma	theta	vega	optionalias	dte
13359109	SPX	2276.98	SPX191220P00100000	put	2019-12-20	2017-01-06	100.0	0.15	0.10	0.15	20	773	0.3185	0.0000	0.0	0.0000	0.0000	SPX191220P00100000	1078
13388755	SPX	2275.32	SPX191220P00100000	put	2019-12-20	2017-01-11	100.0	0.15	0.10	0.15	0	783	0.5958	-0.0002	0.0	-0.2454	2.4561	SPX191220P00100000	1073
13400173	SPX	2270.44	SPX191220P00100000	put	2019-12-20	2017-01-12	100.0	0.15	0.10	0.15	1	783	0.5940	-0.0002	0.0	-0.2466	2.4588	SPX191220P00100000	1072
13411669	SPX	2274.64	SPX191220P00100000	put	2019-12-20	2017-01-13	100.0	0.11	0.10	0.15	0	785	0.5963	-0.0002	0.0	-0.2458	2.4539	SPX191220P00100000	1071
13423111	SPX	2267.89	SPX191220P00100000	put	2019-12-20	2017-01-17	100.0	0.10	0.10	0.15	0	805	0.5986	-0.0002	0.0	-0.2453	2.4442	SPX191220P00100000	1067
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
16740413	SPX	2351.10	SPX191220P00100000	put	2019-12-21	2018-12-24	100.0	0.05	0.05	0.10	530	14541	0.3500	0.0000	0.0	0.0000	0.0000	NaN	362
16744107	SPX	2467.70	SPX191220P00100000	put	2019-12-21	2018-12-26	100.0	0.05	0.00	0.10	0	15059	0.3219	0.0000	0.0	0.0000	0.0000	NaN	360
16747947	SPX	2488.83	SPX191220P00100000	put	2019-12-21	2018-12-27	100.0	0.05	0.00	0.05	0	15059	0.3314	0.0000	0.0	0.0000	0.0000	NaN	359
16751791	SPX	2485.74	SPX191220P00100000	put	2019-12-21	2018-12-28	100.0	0.05	0.00	0.05	500	15059	0.3256	0.0000	0.0	0.0000	0.0000	NaN	358
16755635	SPX	2506.85	SPX191220P00100000	put	2019-12-21	2018-12-31	100.0	0.05	0.00	0.05	1100	15559	0.3188	0.0000	0.0	0.0000	0.0000	NaN	355

497 rows × 19 columns

This next contract, however, is a problem. On first inspection it looks fine, but it actually contains a lot of zero values intermingled.

In [99]:

problematic = df[df['optionroot'] == 'SPX910622C00300000']
problematic

Out [99]:

	underlying	underlying_last	optionroot	type	expiration	quotedate	strike	last	bid	ask	volume	openinterest	impliedvol	delta	gamma	theta	vega	optionalias	dte
144	SPX	359.69	SPX910622C00300000	call	1991-06-22	1990-01-02	300.0	0.0	94.3	94.3	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	536
314	SPX	358.76	SPX910622C00300000	call	1991-06-22	1990-01-03	300.0	0.0	93.3	93.3	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	535
484	SPX	355.66	SPX910622C00300000	call	1991-06-22	1990-01-04	300.0	0.0	90.2	90.2	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	534
654	SPX	352.20	SPX910622C00300000	call	1991-06-22	1990-01-05	300.0	0.0	86.6	86.6	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	533
824	SPX	353.79	SPX910622C00300000	call	1991-06-22	1990-01-08	300.0	0.0	88.1	88.1	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	530
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
68132	SPX	380.13	SPX910622C00300000	call	1991-06-22	1991-06-17	300.0	81.5	80.4	80.9	1	3728	1.0531	0.9832	0.0010	-0.2442	0.0173	SPX910622C00300000	5
68330	SPX	378.59	SPX910622C00300000	call	1991-06-22	1991-06-18	300.0	0.0	78.5	79.0	0	3727	1.4422	0.9608	0.0016	-0.6967	0.0307	SPX910622C00300000	4
68528	SPX	375.09	SPX910622C00300000	call	1991-06-22	1991-06-19	300.0	75.5	75.0	76.0	756	3727	1.6634	0.9598	0.0017	-0.9583	0.0260	SPX910622C00300000	3
68726	SPX	375.42	SPX910622C00300000	call	1991-06-22	1991-06-20	300.0	0.0	75.0	76.0	0	3557	1.8106	0.9824	0.0011	-0.7040	0.0099	SPX910622C00300000	2
68924	SPX	377.75	SPX910622C00300000	call	1991-06-22	1991-06-21	300.0	76.0	77.5	78.5	132	3557	3.9547	0.9766	0.0012	-3.7758	0.0063	SPX910622C00300000	1

373 rows × 19 columns

In [100]:

problematic[4:20]

Out [100]:

	underlying	underlying_last	optionroot	type	expiration	quotedate	strike	last	bid	ask	volume	openinterest	impliedvol	delta	gamma	theta	vega	optionalias	dte
824	SPX	353.79	SPX910622C00300000	call	1991-06-22	1990-01-08	300.0	0.0	88.1	88.1	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	530
994	SPX	349.62	SPX910622C00300000	call	1991-06-22	1990-01-09	300.0	0.0	83.8	83.8	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	529
1164	SPX	347.31	SPX910622C00300000	call	1991-06-22	1990-01-10	300.0	0.0	81.4	81.4	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	528
1334	SPX	348.53	SPX910622C00300000	call	1991-06-22	1990-01-11	300.0	0.0	82.6	82.6	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	527
1504	SPX	339.93	SPX910622C00300000	call	1991-06-22	1990-01-12	300.0	0.0	0.0	0.0	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	526
1674	SPX	337.00	SPX910622C00300000	call	1991-06-22	1990-01-15	300.0	0.0	70.8	70.8	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	523
1844	SPX	340.75	SPX910622C00300000	call	1991-06-22	1990-01-16	300.0	0.0	74.5	74.5	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	522
2026	SPX	337.40	SPX910622C00300000	call	1991-06-22	1990-01-17	300.0	0.0	71.1	71.1	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	521
2208	SPX	338.19	SPX910622C00300000	call	1991-06-22	1990-01-18	300.0	0.0	0.0	0.0	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	520
2391	SPX	339.15	SPX910622C00300000	call	1991-06-22	1990-01-19	300.0	0.0	0.0	0.0	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	519
2548	SPX	330.38	SPX910622C00300000	call	1991-06-22	1990-01-22	300.0	0.0	63.8	64.5	0	0	0.0926	0.9776	0.0015	-22.3628	20.9095	SPX910622C00300000	516
2705	SPX	331.61	SPX910622C00300000	call	1991-06-22	1990-01-23	300.0	0.0	65.0	65.5	0	0	0.0904	0.9816	0.0013	-22.3658	17.7692	SPX910622C00300000	515
2862	SPX	330.26	SPX910622C00300000	call	1991-06-22	1990-01-24	300.0	0.0	63.5	64.2	0	0	0.0896	0.9806	0.0013	-22.3636	18.4703	SPX910622C00300000	514
3022	SPX	326.08	SPX910622C00300000	call	1991-06-22	1990-01-25	300.0	0.0	0.0	59.7	0	0	0.0000	0.0000	0.0000	0.0000	0.0000	SPX910622C00300000	513
3182	SPX	325.80	SPX910622C00300000	call	1991-06-22	1990-01-26	300.0	0.0	59.0	59.7	0	0	0.0887	0.9747	0.0017	-22.3457	22.7432	SPX910622C00300000	512
3344	SPX	325.20	SPX910622C00300000	call	1991-06-22	1990-01-29	300.0	0.0	58.2	59.3	0	0	0.0957	0.9644	0.0021	-22.3769	30.0972	SPX910622C00300000	509

In [101]:

problematic[problematic['bid'] == 0].shape

Out [101]:

(34, 19)

Of 373 total rows, 34 of them have their bid equal to zero even though the non-zero values in between are never even close to zero. These types of contracts happen all throughout the dataset and need to be addressed. We will drop contracts that have an entry with both their ask and bid equal to zero. This means of course that we will also be dropping some contracts with correct values, but we are willing to make that tradeoff.

Here's an example of a contract with all its bid and ask entries set to zero.

In [102]:

df[df['optionroot'] == 'SPX920718P00250000']

Out [102]:

	underlying	underlying_last	optionroot	type	expiration	quotedate	strike	last	bid	ask	volume	openinterest	impliedvol	delta	gamma	theta	vega	optionalias	dte
117271	SPX	409.76	SPX920718P00250000	put	1992-07-18	1992-06-12	250.0	0.000	0.0	0.0	0	0	0.5202	-0.0008	0.0	-0.0025	0.0035	SPX920718P00250000	36
117505	SPX	410.29	SPX920718P00250000	put	1992-07-18	1992-06-15	250.0	0.031	0.0	0.0	250	0	0.5463	-0.0008	0.0	-0.0028	0.0034	SPX920718P00250000	33
117743	SPX	408.32	SPX920718P00250000	put	1992-07-18	1992-06-16	250.0	0.000	0.0	0.0	0	0	0.5485	-0.0008	0.0	-0.0029	0.0033	SPX920718P00250000	32
117981	SPX	402.26	SPX920718P00250000	put	1992-07-18	1992-06-17	250.0	0.000	0.0	0.0	0	250	0.5408	-0.0008	0.0	-0.0029	0.0033	SPX920718P00250000	31
118219	SPX	400.96	SPX920718P00250000	put	1992-07-18	1992-06-18	250.0	0.000	0.0	0.0	0	250	0.5452	-0.0009	0.0	-0.0031	0.0033	SPX920718P00250000	30
118457	SPX	403.67	SPX920718P00250000	put	1992-07-18	1992-06-19	250.0	0.000	0.0	0.0	0	250	0.5618	-0.0008	0.0	-0.0032	0.0032	SPX920718P00250000	29
118645	SPX	403.40	SPX920718P00250000	put	1992-07-18	1992-06-22	250.0	0.000	0.0	0.0	0	250	0.5933	-0.0008	0.0	-0.0035	0.0030	SPX920718P00250000	26
118833	SPX	404.04	SPX920718P00250000	put	1992-07-18	1992-06-23	250.0	0.000	0.0	0.0	0	250	0.6072	-0.0008	0.0	-0.0037	0.0030	SPX920718P00250000	25
119021	SPX	403.84	SPX920718P00250000	put	1992-07-18	1992-06-24	250.0	0.000	0.0	0.0	0	250	0.6194	-0.0008	0.0	-0.0039	0.0029	SPX920718P00250000	24
119209	SPX	403.12	SPX920718P00250000	put	1992-07-18	1992-06-25	250.0	0.000	0.0	0.0	0	250	0.6311	-0.0008	0.0	-0.0040	0.0029	SPX920718P00250000	23
119397	SPX	403.45	SPX920718P00250000	put	1992-07-18	1992-06-26	250.0	0.000	0.0	0.0	0	250	0.6466	-0.0008	0.0	-0.0042	0.0028	SPX920718P00250000	22
119585	SPX	408.94	SPX920718P00250000	put	1992-07-18	1992-06-29	250.0	0.000	0.0	0.0	0	250	0.7150	-0.0008	0.0	-0.0049	0.0025	SPX920718P00250000	19
119773	SPX	408.14	SPX920718P00250000	put	1992-07-18	1992-06-30	250.0	0.000	0.0	0.0	0	250	0.7327	-0.0008	0.0	-0.0052	0.0025	SPX920718P00250000	18
119961	SPX	412.88	SPX920718P00250000	put	1992-07-18	1992-07-01	250.0	0.000	0.0	0.0	0	250	0.7704	-0.0008	0.0	-0.0056	0.0024	SPX920718P00250000	17
120149	SPX	411.79	SPX920718P00250000	put	1992-07-18	1992-07-02	250.0	0.000	0.0	0.0	0	250	0.7914	-0.0008	0.0	-0.0059	0.0023	SPX920718P00250000	16
120339	SPX	413.84	SPX920718P00250000	put	1992-07-18	1992-07-06	250.0	0.000	0.0	0.0	0	250	0.9286	-0.0008	0.0	-0.0080	0.0020	SPX920718P00250000	12
120529	SPX	409.16	SPX920718P00250000	put	1992-07-18	1992-07-07	250.0	0.000	0.0	0.0	0	250	0.9527	-0.0008	0.0	-0.0087	0.0019	SPX920718P00250000	11
120719	SPX	410.28	SPX920718P00250000	put	1992-07-18	1992-07-08	250.0	0.000	0.0	0.0	0	250	1.0074	-0.0008	0.0	-0.0097	0.0018	SPX920718P00250000	10
120909	SPX	414.22	SPX920718P00250000	put	1992-07-18	1992-07-09	250.0	0.000	0.0	0.0	0	250	1.0846	-0.0008	0.0	-0.0109	0.0017	SPX920718P00250000	9
121099	SPX	414.62	SPX920718P00250000	put	1992-07-18	1992-07-10	250.0	0.000	0.0	0.0	0	250	1.1581	-0.0008	0.0	-0.0124	0.0016	SPX920718P00250000	8
121289	SPX	414.87	SPX920718P00250000	put	1992-07-18	1992-07-13	250.0	0.000	0.0	0.0	0	250	1.5091	-0.0008	0.0	-0.0209	0.0012	SPX920718P00250000	5
121479	SPX	417.68	SPX920718P00250000	put	1992-07-18	1992-07-14	250.0	0.000	0.0	0.0	0	250	1.7409	-0.0008	0.0	-0.0273	0.0010	SPX920718P00250000	4
121669	SPX	417.10	SPX920718P00250000	put	1992-07-18	1992-07-15	250.0	0.000	0.0	0.0	0	250	2.0753	-0.0008	0.0	-0.0390	0.0009	SPX920718P00250000	3
121859	SPX	417.54	SPX920718P00250000	put	1992-07-18	1992-07-16	250.0	0.000	0.0	0.0	0	250	2.7500	-0.0008	0.0	-0.0683	0.0007	SPX920718P00250000	2
122049	SPX	415.62	SPX920718P00250000	put	1992-07-18	1992-07-17	250.0	0.000	0.0	0.0	0	250	5.4547	-0.0008	0.0	-0.2727	0.0003	SPX920718P00250000	1

Another issue are contracts with missing entries like the following, which stops showing up at 358 dte (it also has almost all rows with both ask and bid set to zero, so this is a particularly problematic contract).

In [109]:

df[df['optionroot'] == 'SXG031220P01450000']

Out [109]:

	underlying	underlying_last	optionroot	type	expiration	quotedate	strike	last	bid	ask	volume	openinterest	impliedvol	delta	gamma	theta	vega	optionalias	dte
1120981	SPX	1165.25	SXG031220P01450000	put	2003-12-20	2002-01-03	1450.0	0.0	0.0	0.0	0	0	0.1837	-0.7189	0.0011	-4.9932	550.1433	SXG031220P01450000	716
1121467	SPX	1172.50	SXG031220P01450000	put	2003-12-20	2002-01-04	1450.0	0.0	0.0	0.0	0	200	0.1780	-0.7198	0.0012	-4.3687	552.3557	SXG031220P01450000	715
1121957	SPX	1164.90	SXG031220P01450000	put	2003-12-20	2002-01-07	1450.0	0.0	0.0	0.0	0	200	0.1820	-0.7230	0.0011	-4.5666	544.5476	SXG031220P01450000	712
1122447	SPX	1160.70	SXG031220P01450000	put	2003-12-20	2002-01-08	1450.0	0.0	0.0	0.0	0	200	0.1830	-0.7265	0.0011	-4.3912	538.8742	SXG031220P01450000	711
1122940	SPX	1155.15	SXG031220P01450000	put	2003-12-20	2002-01-09	1450.0	0.0	0.0	0.0	0	200	0.1846	-0.7304	0.0011	-4.2222	532.0289	SXG031220P01450000	710
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
1265123	SPX	895.82	SXG031220P01450000	put	2003-12-20	2002-12-20	1450.0	0.0	0.0	0.0	0	2421	0.2241	-0.9648	0.0002	1.5314	43.4461	SXG031220P01450000	365
1265633	SPX	897.38	SXG031220P01450000	put	2003-12-20	2002-12-23	1450.0	0.0	0.0	0.0	0	2421	0.2170	-0.9666	0.0002	0.5313	36.1597	SXG031220P01450000	362
1266143	SPX	892.47	SXG031220P01450000	put	2003-12-20	2002-12-24	1450.0	0.0	0.0	0.0	0	2419	0.2175	-0.9667	0.0002	-0.1834	33.7203	SXG031220P01450000	361
1266653	SPX	889.66	SXG031220P01450000	put	2003-12-20	2002-12-26	1450.0	0.0	0.0	0.0	0	2419	0.2145	-0.9651	0.0002	-3.3717	28.2564	SXG031220P01450000	359
1267163	SPX	875.40	SXG031220P01450000	put	2003-12-20	2002-12-27	1450.0	550.0	571.3	574.3	0	2419	0.2282	-0.9625	0.0002	-3.8675	32.5230	SXG031220P01450000	358

249 rows × 19 columns

Once again, we will drop contracts with missing entries.

In [26]:

unique_contracts_per_day = df.groupby('quotedate').apply(lambda x: x['optionroot'].nunique())

Let's plot the number of contracts per day to compare after the cleanup.

In [28]:

unique_contracts_per_day.plot();

In [71]:

# to_clean is a set with the contracts with missing entries to then drop.
current_contracts = set()
to_clean = set()
for _, options in df.groupby('quotedate'):
    current_contracts = current_contracts.union(set(options['optionroot']))
    current_contracts = current_contracts.difference(set(options[options['dte'] <= 1]['optionroot']))
    missing_contracts = current_contracts.difference(set(options['optionroot']))
    to_clean = to_clean.union(missing_contracts)
to_clean = to_clean.union(current_contracts)

In [54]:

to_clean_df = df[df['optionroot'].isin(list(to_clean))]

In [55]:

new_df = df.drop(to_clean_df.index)

In [56]:

new_df.reset_index(inplace=True, drop=True)

In [57]:

# Drop the contracts that have a row with both their ask and bid zero.
contracts_with_zero = new_df[(new_df['ask'] == 0) & (new_df['bid'] == 0)]['optionroot'].unique()

In [58]:

final_df = new_df.drop(new_df[new_df['optionroot'].isin(contracts_with_zero)].index)

In [59]:

final_df.reset_index(inplace=True, drop=True)

In [72]:

final_df['optionroot'].nunique() / df['optionroot'].nunique()

Out [72]:

0.6339574482236837

In [64]:

final_df.shape

Out [64]:

(9245562, 19)

In [103]:

unique_contracts_per_day_final = final_df.groupby('quotedate').apply(lambda x: x['optionroot'].nunique())

In [105]:

unique_contracts_per_day_final.plot();

In [66]:

store_path = os.path.join("/Users/jrchatruc/Documents/backtester_options/allspx/", "options_data_v2_pruned.h5")
store = pd.HDFStore(store_path, complevel=9, complib="blosc", fletcher32=True)

In [68]:

sizes = {"optionroot": 20, "optionalias": 20}
store.append("/SPX", final_df, index=False, data_columns=True, min_itemsize=sizes)

In [69]:

store.close()