Volt/experiments/weather/make_wind_dataset.ipynb

In [1]:

import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
import requests
import matplotlib.pyplot as plt
import torch

In [2]:

base_url = 'https://www.ncei.noaa.gov/pub/data/uscrn/products/subhourly01/2021/'

In [3]:

grab = requests.get(base_url)

In [4]:

stn_names = {}
stn_lonlat = {}
stn_data = {}
ndata = 105120
stn_id = 0

soup = BeautifulSoup(grab.text, 'html.parser')
for link in soup.find_all('a'):
    url = link.get('href')
    if url[-4:] == '.txt':
        dat = pd.read_csv(base_url + url,
                          header=None, delim_whitespace=True)
        if dat.shape[0] == ndata:
            stn_names[stn_id] = url[17:-4]
            stn_lonlat[stn_id] = [dat.iloc[0, 6], dat.iloc[0, 7]]
            stn_data[stn_id] = dat.iloc[:, 21].to_numpy()
            stn_id += 1

In [5]:

full_dat = np.stack( list(stn_data.values()))
lonlat = np.array(list(stn_lonlat.values()))

In [6]:

full_dat[full_dat == -99.0] = 0.

In [7]:

plt.figure(dpi=150)
plt.scatter(lonlat[:, 0], lonlat[:, 1], c=full_dat.mean(-1))
plt.axvline(-128)

Out [7]:

<matplotlib.lines.Line2D at 0x7f765db6d250>

In [8]:

import pickle as pkl

In [9]:

dicts = [stn_names, stn_lonlat, stn_data]

In [10]:

pkl.dump(dicts, open("./wind_data.p", "wb"))