{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "Here I download and preprocess current data\n", "\n", "\n", "see\n", "- from https://catalogue-imos.aodn.org.au/geonetwork/srv/api/records/ae86e2f5-eaaf-459e-a405-e654d85adb9c\n", "- http://thredds.aodn.org.au/thredds/catalog/IMOS/ANMN/WA/WATR20/Velocity/catalog.html" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:39:41.682896Z", "start_time": "2020-10-26T03:39:40.104951Z" } }, "outputs": [], "source": [ "from pathlib import Path\n", "import xarray as xr\n", "import pandas as pd\n", "import numpy as np\n", "from urllib import request\n", "import os, shutil\n", "from matplotlib import pyplot as plt" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2020-10-25T07:01:29.004548Z", "start_time": "2020-10-25T07:01:29.001734Z" } }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 2, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:39:42.553735Z", "start_time": "2020-10-26T03:39:41.685439Z" } }, "outputs": [], "source": [ "from torchvision.datasets.utils import download_url, extract_archive, download_and_extract_archive" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:39:42.560919Z", "start_time": "2020-10-26T03:39:42.556898Z" } }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import uptide\n", "\n", "# https://en.wikipedia.org/wiki/Theory_of_tides#Harmonic_analysis\n", "default_tidal_constituents = [\n", " 'M2', 'S2', 'N2', 'K2', # Semi-diurnal\n", " 'K1', 'O1', 'P1', 'Q1', # Diurnal\n", " 'M4', 'M6', 'S4', 'MK3', # Short period\n", " 'MM', 'SSA', 'SA' # Long period\n", " ]\n", "\n", "def generate_tidal_periods(t:pd.Series, constituents:list=default_tidal_constituents):\n", " tide = uptide.Tides(constituents)\n", " t0 = t[0]\n", " td = t-t0\n", " td = td.dt.total_seconds().to_numpy().astype(int)\n", " tide.set_initial_time(t0)\n", "\n", " # calc tides\n", " amplitudes=np.ones_like(td)\n", " phases=np.zeros_like(td)\n", " eta = {}\n", " for name, f, amplitude, omega, phase, phi, u in zip(tide.constituents, tide.f, amplitudes, tide.omega,\n", " phases, tide.phi, tide.u):\n", " eta[name] = f*amplitude*np.cos(omega*td-phase+phi+u)\n", " df_eta = pd.DataFrame(eta, index=t)\n", " return df_eta" ] }, { "cell_type": "code", "execution_count": 30, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:59:16.671092Z", "start_time": "2020-10-26T03:59:16.655680Z" }, "scrolled": true }, "outputs": [], "source": [ "# 'ANMN Two Rocks, WA, 204m mooring, Jul2009 - Dec2009. Preprocessed with DepthPP.'\n", "\n", "def get_current_timeseries(\n", " cache_folder=Path(\"../data/raw/IMOS_ANMN/\"), \n", " outfile=Path('../data/processed/currents/MOS_ANMN-WA_AETVZ_WATR20_FV01_WATR20-1909-Continental-194_currents.nc')\n", " ):\n", " if not outfile.exists():\n", "\n", " files = [\n", " \"IMOS_ANMN-WA_AETVZ_20090715T080000Z_WATR20_FV01_WATR20-0907-Continental-194_END-20090716T181317Z_C-20191122T052830Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20100409T080000Z_WATR20_FV01_WATR20-1004-Continental-194_END-20100430T084500Z_C-20191122T053845Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20101222T080000Z_WATR20_FV01_WATR20-1012-Continental-194_END-20110518T051500Z_C-20200916T020035Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20110608T080000Z_WATR20_FV01_WATR20-1106-Continental-194_END-20111122T035000Z_C-20200916T025619Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20111221T060300Z_WATR20_FV01_WATR20-1112-Continental-194_END-20120704T050500Z_C-20200916T043212Z.nc\", \n", " \"IMOS_ANMN-WA_AETVZ_20120726T044000Z_WATR20_FV01_WATR20-1207-Continental-194_END-20130204T044000Z_C-20200916T032027Z.nc\",\n", "\n", " \"IMOS_ANMN-WA_AETVZ_20130221T080000Z_WATR20_FV01_WATR20-1302-Continental-194_END-20131003T035000Z_C-20180529T020609Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20131111T080000Z_WATR20_FV01_WATR20-1311-Continental-194_END-20140519T035000Z_C-20200114T033335Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20140710T080000Z_WATR20_FV01_WATR20-1407-Continental-194_END-20150121T021500Z_C-20180529T055902Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20150213T080000Z_WATR20_FV01_WATR20-1502-Continental-194_END-20150424T134002Z_C-20200114T035347Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20150914T080000Z_WATR20_FV01_WATR20-1509-Continental-194_END-20160331T043000Z_C-20180601T013623Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20160427T080000Z_WATR20_FV01_WATR20-1604-Continental-194_END-20160531T021800Z_C-20180531T071709Z.nc\",\n", " # \"IMOS_ANMN-WA_AETVZ_20170512T080000Z_WATR20_FV01_WATR20-1705-Continental-194_END-20170717T014558Z_C-20190805T004647Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20171204T080000Z_WATR20_FV01_WATR20-1712-Continental-194_END-20180618T030000Z_C-20180620T233149Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20180802T080000Z_WATR20_FV01_WATR20-1807-Continental-194_END-20190225T054500Z_C-20190227T001343Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20190307T080000Z_WATR20_FV01_WATR20-1903-Continental-194_END-20190911T003144Z_C-20200114T045053Z.nc\",\n", " \"IMOS_ANMN-WA_AETVZ_20190926T080000Z_WATR20_FV01_WATR20-1909-Continental-194_END-20200326T030000Z_C-20200420T064334Z.nc\",\n", " ]\n", " base=\"http://thredds.aodn.org.au/thredds/fileServer/IMOS/ANMN/WA/WATR20/Velocity/\"\n", "\n", " # Download files\n", " [download_url(base+f, cache_folder) for f in files]\n", "\n", " # load and merge\n", " xds=[xr.open_dataset(cache_folder/f) for f in files]\n", " vars=['VCUR', 'UCUR', 'WCUR', 'TEMP', 'PRES_REL', 'DEPTH', 'ROLL', 'PITCH']\n", " xds2= [x[vars].isel(HEIGHT_ABOVE_SENSOR=18) for x in xds]\n", " xd = xr.concat(xds2, dim='TIME')\n", " xd = xd.where(xd.DEPTH>150) # remove outliers\n", "\n", "\n", " xd['TIME'] = xd['TIME'].dt.round('10T')\n", " xd = xd.dropna(dim='TIME', subset=['VCUR', 'UCUR', 'WCUR'])\n", " # xd = xd.resample(TIME='30T').first()\n", " # Add tides, these are features that can be forecast\n", "\n", " # Generate tidal freqs\n", " t = xd.TIME.to_series()\n", " df_eta = generate_tidal_periods(t)\n", "\n", " # Add tidal freqs\n", " xd = xd.merge(df_eta)\n", "\n", " # Cache to nc\n", " xd.to_netcdf(outfile)\n", " print(f'wrote \"{outfile}\" with size {outfile.stat().st_size*1e-6:2.2f} MB')\n", " return outfile" ] }, { "cell_type": "code", "execution_count": 40, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T04:04:08.230047Z", "start_time": "2020-10-26T04:04:08.099310Z" } }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
VCURUCURWCURTEMPPRES_RELDEPTHROLLPITCHLATITUDELONGITUDE...O1P1Q1M4M6S4MK3MMSSASA
TIME
2009-07-15 08:00:00-0.3963910.089687-0.00967118.549999205.076004203.5508124.6-3.4-31.728650115.037217...0.2862880.116457-1.014973-0.146817-0.801534-0.5000000.3700820.132683-0.686775-0.395743
2009-07-15 08:10:00-0.4076200.085398-0.01987518.650000205.078003203.5527954.6-2.4-31.728650115.037217...0.2428100.159551-1.031149-0.304345-0.900573-0.6427880.4944170.134147-0.686601-0.395853
2009-07-15 08:20:00-0.3653140.1040380.00099118.730000205.076996203.5517884.8-2.7-31.728650115.037217...0.1989320.202343-1.045759-0.453239-0.942304-0.7660440.6106540.135610-0.686427-0.395963
2009-07-15 08:30:00-0.4066320.119376-0.00372918.799999205.067001203.5419014.7-2.4-31.728650115.037217...0.1547270.244751-1.058780-0.589276-0.924071-0.8660250.7168900.137073-0.686253-0.396072
2009-07-15 08:40:00-0.3837440.090066-0.00892118.860001205.065994203.5408944.9-2.9-31.728650115.037217...0.1102680.286697-1.070194-0.708598-0.847034-0.9396930.8113840.138535-0.686080-0.396182
..................................................................
2020-03-26 01:00:00-0.436635-0.784922-0.01214716.610001197.384003195.919662-2.93.0-31.728717115.042133...-0.7347410.1901390.9647920.8824840.7704440.5054391.028587-0.8819510.9905140.997626
2020-03-26 01:30:00-0.355067-0.845100-0.00520116.629999197.408005195.943497-2.73.0-31.728717115.042133...-0.6292570.3163170.8955450.9579140.9337740.0062920.851981-0.8804830.9904160.997601
2020-03-26 02:00:00-0.568277-0.816935-0.02494416.660000197.412994195.948425-2.62.9-31.728717115.042133...-0.5144700.4371130.8140670.7933950.584762-0.4945410.551159-0.8789960.9903160.997576
2020-03-26 02:30:00-0.306141-0.773147-0.02809616.719999197.419006195.954407-2.62.7-31.728717115.042133...-0.3920740.5504700.7214730.430136-0.085096-0.8628620.169980-0.8774890.9902170.997551
2020-03-26 03:00:00-0.218563-0.7572170.01323316.790001197.429001195.964340-2.82.8-31.728717115.042133...-0.2638810.6544600.619026-0.040868-0.708264-0.999980-0.235982-0.8759620.9901160.997526
\n", "

239075 rows × 25 columns

\n", "
" ], "text/plain": [ " VCUR UCUR WCUR TEMP PRES_REL \\\n", "TIME \n", "2009-07-15 08:00:00 -0.396391 0.089687 -0.009671 18.549999 205.076004 \n", "2009-07-15 08:10:00 -0.407620 0.085398 -0.019875 18.650000 205.078003 \n", "2009-07-15 08:20:00 -0.365314 0.104038 0.000991 18.730000 205.076996 \n", "2009-07-15 08:30:00 -0.406632 0.119376 -0.003729 18.799999 205.067001 \n", "2009-07-15 08:40:00 -0.383744 0.090066 -0.008921 18.860001 205.065994 \n", "... ... ... ... ... ... \n", "2020-03-26 01:00:00 -0.436635 -0.784922 -0.012147 16.610001 197.384003 \n", "2020-03-26 01:30:00 -0.355067 -0.845100 -0.005201 16.629999 197.408005 \n", "2020-03-26 02:00:00 -0.568277 -0.816935 -0.024944 16.660000 197.412994 \n", "2020-03-26 02:30:00 -0.306141 -0.773147 -0.028096 16.719999 197.419006 \n", "2020-03-26 03:00:00 -0.218563 -0.757217 0.013233 16.790001 197.429001 \n", "\n", " DEPTH ROLL PITCH LATITUDE LONGITUDE ... \\\n", "TIME ... \n", "2009-07-15 08:00:00 203.550812 4.6 -3.4 -31.728650 115.037217 ... \n", "2009-07-15 08:10:00 203.552795 4.6 -2.4 -31.728650 115.037217 ... \n", "2009-07-15 08:20:00 203.551788 4.8 -2.7 -31.728650 115.037217 ... \n", "2009-07-15 08:30:00 203.541901 4.7 -2.4 -31.728650 115.037217 ... \n", "2009-07-15 08:40:00 203.540894 4.9 -2.9 -31.728650 115.037217 ... \n", "... ... ... ... ... ... ... \n", "2020-03-26 01:00:00 195.919662 -2.9 3.0 -31.728717 115.042133 ... \n", "2020-03-26 01:30:00 195.943497 -2.7 3.0 -31.728717 115.042133 ... \n", "2020-03-26 02:00:00 195.948425 -2.6 2.9 -31.728717 115.042133 ... \n", "2020-03-26 02:30:00 195.954407 -2.6 2.7 -31.728717 115.042133 ... \n", "2020-03-26 03:00:00 195.964340 -2.8 2.8 -31.728717 115.042133 ... \n", "\n", " O1 P1 Q1 M4 M6 \\\n", "TIME \n", "2009-07-15 08:00:00 0.286288 0.116457 -1.014973 -0.146817 -0.801534 \n", "2009-07-15 08:10:00 0.242810 0.159551 -1.031149 -0.304345 -0.900573 \n", "2009-07-15 08:20:00 0.198932 0.202343 -1.045759 -0.453239 -0.942304 \n", "2009-07-15 08:30:00 0.154727 0.244751 -1.058780 -0.589276 -0.924071 \n", "2009-07-15 08:40:00 0.110268 0.286697 -1.070194 -0.708598 -0.847034 \n", "... ... ... ... ... ... \n", "2020-03-26 01:00:00 -0.734741 0.190139 0.964792 0.882484 0.770444 \n", "2020-03-26 01:30:00 -0.629257 0.316317 0.895545 0.957914 0.933774 \n", "2020-03-26 02:00:00 -0.514470 0.437113 0.814067 0.793395 0.584762 \n", "2020-03-26 02:30:00 -0.392074 0.550470 0.721473 0.430136 -0.085096 \n", "2020-03-26 03:00:00 -0.263881 0.654460 0.619026 -0.040868 -0.708264 \n", "\n", " S4 MK3 MM SSA SA \n", "TIME \n", "2009-07-15 08:00:00 -0.500000 0.370082 0.132683 -0.686775 -0.395743 \n", "2009-07-15 08:10:00 -0.642788 0.494417 0.134147 -0.686601 -0.395853 \n", "2009-07-15 08:20:00 -0.766044 0.610654 0.135610 -0.686427 -0.395963 \n", "2009-07-15 08:30:00 -0.866025 0.716890 0.137073 -0.686253 -0.396072 \n", "2009-07-15 08:40:00 -0.939693 0.811384 0.138535 -0.686080 -0.396182 \n", "... ... ... ... ... ... \n", "2020-03-26 01:00:00 0.505439 1.028587 -0.881951 0.990514 0.997626 \n", "2020-03-26 01:30:00 0.006292 0.851981 -0.880483 0.990416 0.997601 \n", "2020-03-26 02:00:00 -0.494541 0.551159 -0.878996 0.990316 0.997576 \n", "2020-03-26 02:30:00 -0.862862 0.169980 -0.877489 0.990217 0.997551 \n", "2020-03-26 03:00:00 -0.999980 -0.235982 -0.875962 0.990116 0.997526 \n", "\n", "[239075 rows x 25 columns]" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "xd.to_dataframe().drop(columns=['HEIGHT_ABOVE_SENSOR', 'NOMINAL_DEPTH'])#.columns#[['VCUR', 'UCUR', 'WCUR', 'TEMP', 'PRES_REL', 'DEPTH', 'ROLL', 'PITCH']]" ] }, { "cell_type": "code", "execution_count": 7, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:44:41.020269Z", "start_time": "2020-10-26T03:44:41.017322Z" } }, "outputs": [], "source": [ "# for x in xds:\n", "# x.DEPTH.plot()\n", "# plt.ylim(190, 210)\n", "\n", "# plt.show()\n", "# for x in xds:\n", "# x.plot.scatter('LONGITUDE', 'LONGITUDE')\n", "# plt.show()\n", "\n", "# xd['VCUR'].plot(alpha=0.5)\n", "# xd['UCUR'].plot(alpha=0.5)\n", "# xd['WCUR'].plot(alpha=0.5)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:51:16.821117Z", "start_time": "2020-10-26T03:51:16.606212Z" } }, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 18, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:51:17.614829Z", "start_time": "2020-10-26T03:51:17.204376Z" } }, "outputs": [ { "data": { "text/plain": [ "PosixPath('../data/processed/currents/MOS_ANMN-WA_AETVZ_WATR20_FV01_WATR20-1909-Continental-194_currents.nc')" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": 19, "metadata": { "ExecuteTime": { "end_time": "2020-10-26T03:51:18.335001Z", "start_time": "2020-10-26T03:51:18.328504Z" } }, "outputs": [ { "data": { "text/plain": [ "43.107293" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "seq2seq-time", "language": "python", "name": "seq2seq-time" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.8" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": false, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": { "height": "calc(100% - 180px)", "left": "10px", "top": "150px", "width": "219.011px" }, "toc_section_display": true, "toc_window_display": true } }, "nbformat": 4, "nbformat_minor": 2 }