1&&\"string\"==typeof v&&!d.checkClone&&De.test(v))return e.each((function(i){var o=e.eq(i);y&&(t[0]=v.call(this,i,o.html())),Re(o,t,n,r)}));if(p&&(a=(i=xe(t,e[0].ownerDocument,!1,e,r)).firstChild,1===i.childNodes.length&&(i=a),a||r)){for(u=(s=b.map(ve(i,\"script\"),Le)).length;f0&&ye(a,!u&&ve(e,\"script\")),s},cleanData:function(e){for(var t,n,r,i=b.event.special,o=0;void 0!==(n=e[o]);o++)if(X(n)){if(t=n[G.expando]){if(t.events)for(r in t.events)i[r]?b.event.remove(n,r):b.removeEvent(n,r,t.handle);n[G.expando]=void 0}n[Y.expando]&&(n[Y.expando]=void 0)}}}),b.fn.extend({detach:function(e){return Me(this,e,!0)},remove:function(e){return Me(this,e)},text:function(e){return B(this,(function(e){return void 0===e?b.text(this):this.empty().each((function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)}))}),null,e,arguments.length)},append:function(){return Re(this,arguments,(function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||qe(this,e).appendChild(e)}))},prepend:function(){return Re(this,arguments,(function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=qe(this,e);t.insertBefore(e,t.firstChild)}}))},before:function(){return Re(this,arguments,(function(e){this.parentNode&&this.parentNode.insertBefore(e,this)}))},after:function(){return Re(this,arguments,(function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)}))},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(b.cleanData(ve(e,!1)),e.textContent=\"\");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map((function(){return b.clone(this,e,t)}))},html:function(e){return B(this,(function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if(\"string\"==typeof e&&!Ne.test(e)&&!ge[(de.exec(e)||[\"\",\"\"])[1].toLowerCase()]){e=b.htmlPrefilter(e);try{for(;n3,ne.removeChild(t)),s}}))}();var ze=[\"Webkit\",\"Moz\",\"ms\"],Ue=v.createElement(\"div\").style,Xe={};function Ve(e){var t=b.cssProps[e]||Xe[e];return t||(e in Ue?e:Xe[e]=function(e){for(var t=e[0].toUpperCase()+e.slice(1),n=ze.length;n--;)if((e=ze[n]+t)in Ue)return e}(e)||e)}var Ge=/^(none|table(?!-c[ea]).+)/,Ye=/^--/,Qe={position:\"absolute\",visibility:\"hidden\",display:\"block\"},Je={letterSpacing:\"0\",fontWeight:\"400\"};function Ke(e,t,n){var r=ee.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||\"px\"):t}function Ze(e,t,n,r,i,o){var a=\"width\"===t?1:0,s=0,u=0;if(n===(r?\"border\":\"content\"))return 0;for(;a<4;a+=2)\"margin\"===n&&(u+=b.css(e,n+te[a],!0,i)),r?(\"content\"===n&&(u-=b.css(e,\"padding\"+te[a],!0,i)),\"margin\"!==n&&(u-=b.css(e,\"border\"+te[a]+\"Width\",!0,i))):(u+=b.css(e,\"padding\"+te[a],!0,i),\"padding\"!==n?u+=b.css(e,\"border\"+te[a]+\"Width\",!0,i):s+=b.css(e,\"border\"+te[a]+\"Width\",!0,i));return!r&&o>=0&&(u+=Math.max(0,Math.ceil(e[\"offset\"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u}function et(e,t,n){var r=We(e),i=(!d.boxSizingReliable()||n)&&\"border-box\"===b.css(e,\"boxSizing\",!1,r),o=i,a=$e(e,t,r),s=\"offset\"+t[0].toUpperCase()+t.slice(1);if(Ie.test(a)){if(!n)return a;a=\"auto\"}return(!d.boxSizingReliable()&&i||!d.reliableTrDimensions()&&k(e,\"tr\")||\"auto\"===a||!parseFloat(a)&&\"inline\"===b.css(e,\"display\",!1,r))&&e.getClientRects().length&&(i=\"border-box\"===b.css(e,\"boxSizing\",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+Ze(e,t,n||(i?\"border\":\"content\"),o,r,a)+\"px\"}function tt(e,t,n,r,i){return new tt.prototype.init(e,t,n,r,i)}b.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=$e(e,\"opacity\");return\"\"===n?\"1\":n}}}},cssNumber:{animationIterationCount:!0,columnCount:!0,fillOpacity:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=U(t),u=Ye.test(t),l=e.style;if(u||(t=Ve(s)),a=b.cssHooks[t]||b.cssHooks[s],void 0===n)return a&&\"get\"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];\"string\"===(o=typeof n)&&(i=ee.exec(n))&&i[1]&&(n=ae(e,t,i),o=\"number\"),null!=n&&n==n&&(\"number\"!==o||u||(n+=i&&i[3]||(b.cssNumber[s]?\"\":\"px\")),d.clearCloneStyle||\"\"!==n||0!==t.indexOf(\"background\")||(l[t]=\"inherit\"),a&&\"set\"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=U(t);return Ye.test(t)||(t=Ve(s)),(a=b.cssHooks[t]||b.cssHooks[s])&&\"get\"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=$e(e,t,r)),\"normal\"===i&&t in Je&&(i=Je[t]),\"\"===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),b.each([\"height\",\"width\"],(function(e,t){b.cssHooks[t]={get:function(e,n,r){if(n)return!Ge.test(b.css(e,\"display\"))||e.getClientRects().length&&e.getBoundingClientRect().width?et(e,t,r):Fe(e,Qe,(function(){return et(e,t,r)}))},set:function(e,n,r){var i,o=We(e),a=!d.scrollboxSize()&&\"absolute\"===o.position,s=(a||r)&&\"border-box\"===b.css(e,\"boxSizing\",!1,o),u=r?Ze(e,t,r,s,o):0;return s&&a&&(u-=Math.ceil(e[\"offset\"+t[0].toUpperCase()+t.slice(1)]-parseFloat(o[t])-Ze(e,t,\"border\",!1,o)-.5)),u&&(i=ee.exec(n))&&\"px\"!==(i[3]||\"px\")&&(e.style[t]=n,n=b.css(e,t)),Ke(0,n,u)}}})),b.cssHooks.marginLeft=_e(d.reliableMarginLeft,(function(e,t){if(t)return(parseFloat($e(e,\"marginLeft\"))||e.getBoundingClientRect().left-Fe(e,{marginLeft:0},(function(){return e.getBoundingClientRect().left})))+\"px\"})),b.each({margin:\"\",padding:\"\",border:\"Width\"},(function(e,t){b.cssHooks[e+t]={expand:function(n){for(var r=0,i={},o=\"string\"==typeof n?n.split(\" \"):[n];r<4;r++)i[e+te[r]+t]=o[r]||o[r-2]||o[0];return i}},\"margin\"!==e&&(b.cssHooks[e+t].set=Ke)})),b.fn.extend({css:function(e,t){return B(this,(function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=We(e),i=t.length;a1)}}),b.Tween=tt,tt.prototype={constructor:tt,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||b.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(b.cssNumber[n]?\"\":\"px\")},cur:function(){var e=tt.propHooks[this.prop];return e&&e.get?e.get(this):tt.propHooks._default.get(this)},run:function(e){var t,n=tt.propHooks[this.prop];return this.options.duration?this.pos=t=b.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):tt.propHooks._default.set(this),this}},tt.prototype.init.prototype=tt.prototype,tt.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=b.css(e.elem,e.prop,\"\"))&&\"auto\"!==t?t:0},set:function(e){b.fx.step[e.prop]?b.fx.step[e.prop](e):1!==e.elem.nodeType||!b.cssHooks[e.prop]&&null==e.elem.style[Ve(e.prop)]?e.elem[e.prop]=e.now:b.style(e.elem,e.prop,e.now+e.unit)}}},tt.propHooks.scrollTop=tt.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},b.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:\"swing\"},b.fx=tt.prototype.init,b.fx.step={};var nt,rt,it=/^(?:toggle|show|hide)$/,ot=/queueHooks$/;function at(){rt&&(!1===v.hidden&&e.requestAnimationFrame?e.requestAnimationFrame(at):e.setTimeout(at,b.fx.interval),b.fx.tick())}function st(){return e.setTimeout((function(){nt=void 0})),nt=Date.now()}function ut(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i[\"margin\"+(n=te[r])]=i[\"padding\"+n]=e;return t&&(i.opacity=i.width=e),i}function lt(e,t,n){for(var r,i=(ct.tweeners[t]||[]).concat(ct.tweeners[\"*\"]),o=0,a=i.length;o1)},removeAttr:function(e){return this.each((function(){b.removeAttr(this,e)}))}}),b.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return void 0===e.getAttribute?b.prop(e,t,n):(1===o&&b.isXMLDoc(e)||(i=b.attrHooks[t.toLowerCase()]||(b.expr.match.bool.test(t)?ft:void 0)),void 0!==n?null===n?void b.removeAttr(e,t):i&&\"set\"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+\"\"),n):i&&\"get\"in i&&null!==(r=i.get(e,t))?r:null==(r=b.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!d.radioValue&&\"radio\"===t&&k(e,\"input\")){var n=e.value;return e.setAttribute(\"type\",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(O);if(i&&1===e.nodeType)for(;n=i[r++];)e.removeAttribute(n)}}),ft={set:function(e,t,n){return!1===t?b.removeAttr(e,n):e.setAttribute(n,n),n}},b.each(b.expr.match.bool.source.match(/\\w+/g),(function(e,t){var n=pt[t]||b.find.attr;pt[t]=function(e,t,r){var i,o,a=t.toLowerCase();return r||(o=pt[a],pt[a]=i,i=null!=n(e,t,r)?a:null,pt[a]=o),i}}));var dt=/^(?:input|select|textarea|button)$/i,ht=/^(?:a|area)$/i;function gt(e){return(e.match(O)||[]).join(\" \")}function vt(e){return e.getAttribute&&e.getAttribute(\"class\")||\"\"}function yt(e){return Array.isArray(e)?e:\"string\"==typeof e&&e.match(O)||[]}b.fn.extend({prop:function(e,t){return B(this,b.prop,e,t,arguments.length>1)},removeProp:function(e){return this.each((function(){delete this[b.propFix[e]||e]}))}}),b.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&b.isXMLDoc(e)||(t=b.propFix[t]||t,i=b.propHooks[t]),void 0!==n?i&&\"set\"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&\"get\"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=b.find.attr(e,\"tabindex\");return t?parseInt(t,10):dt.test(e.nodeName)||ht.test(e.nodeName)&&e.href?0:-1}}},propFix:{for:\"htmlFor\",class:\"className\"}}),d.optSelected||(b.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),b.each([\"tabIndex\",\"readOnly\",\"maxLength\",\"cellSpacing\",\"cellPadding\",\"rowSpan\",\"colSpan\",\"useMap\",\"frameBorder\",\"contentEditable\"],(function(){b.propFix[this.toLowerCase()]=this})),b.fn.extend({addClass:function(e){var t,n,r,i,o,a,s,u=0;if(h(e))return this.each((function(t){b(this).addClass(e.call(this,t,vt(this)))}));if((t=yt(e)).length)for(;n=this[u++];)if(i=vt(n),r=1===n.nodeType&&\" \"+gt(i)+\" \"){for(a=0;o=t[a++];)r.indexOf(\" \"+o+\" \")<0&&(r+=o+\" \");i!==(s=gt(r))&&n.setAttribute(\"class\",s)}return this},removeClass:function(e){var t,n,r,i,o,a,s,u=0;if(h(e))return this.each((function(t){b(this).removeClass(e.call(this,t,vt(this)))}));if(!arguments.length)return this.attr(\"class\",\"\");if((t=yt(e)).length)for(;n=this[u++];)if(i=vt(n),r=1===n.nodeType&&\" \"+gt(i)+\" \"){for(a=0;o=t[a++];)for(;r.indexOf(\" \"+o+\" \")>-1;)r=r.replace(\" \"+o+\" \",\" \");i!==(s=gt(r))&&n.setAttribute(\"class\",s)}return this},toggleClass:function(e,t){var n=typeof e,r=\"string\"===n||Array.isArray(e);return\"boolean\"==typeof t&&r?t?this.addClass(e):this.removeClass(e):h(e)?this.each((function(n){b(this).toggleClass(e.call(this,n,vt(this),t),t)})):this.each((function(){var t,i,o,a;if(r)for(i=0,o=b(this),a=yt(e);t=a[i++];)o.hasClass(t)?o.removeClass(t):o.addClass(t);else void 0!==e&&\"boolean\"!==n||((t=vt(this))&&G.set(this,\"__className__\",t),this.setAttribute&&this.setAttribute(\"class\",t||!1===e?\"\":G.get(this,\"__className__\")||\"\"))}))},hasClass:function(e){var t,n,r=0;for(t=\" \"+e+\" \";n=this[r++];)if(1===n.nodeType&&(\" \"+gt(vt(n))+\" \").indexOf(t)>-1)return!0;return!1}});var mt=/\\r/g;b.fn.extend({val:function(e){var t,n,r,i=this[0];return arguments.length?(r=h(e),this.each((function(n){var i;1===this.nodeType&&(null==(i=r?e.call(this,n,b(this).val()):e)?i=\"\":\"number\"==typeof i?i+=\"\":Array.isArray(i)&&(i=b.map(i,(function(e){return null==e?\"\":e+\"\"}))),(t=b.valHooks[this.type]||b.valHooks[this.nodeName.toLowerCase()])&&\"set\"in t&&void 0!==t.set(this,i,\"value\")||(this.value=i))}))):i?(t=b.valHooks[i.type]||b.valHooks[i.nodeName.toLowerCase()])&&\"get\"in t&&void 0!==(n=t.get(i,\"value\"))?n:\"string\"==typeof(n=i.value)?n.replace(mt,\"\"):null==n?\"\":n:void 0}}),b.extend({valHooks:{option:{get:function(e){var t=b.find.attr(e,\"value\");return null!=t?t:gt(b.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a=\"select-one\"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r-1)&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),b.each([\"radio\",\"checkbox\"],(function(){b.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=b.inArray(b(e).val(),t)>-1}},d.checkOn||(b.valHooks[this].get=function(e){return null===e.getAttribute(\"value\")?\"on\":e.value})})),d.focusin=\"onfocusin\"in e;var xt=/^(?:focusinfocus|focusoutblur)$/,bt=function(e){e.stopPropagation()};b.extend(b.event,{trigger:function(t,n,r,i){var o,a,s,u,l,f,p,d,y=[r||v],m=c.call(t,\"type\")?t.type:t,x=c.call(t,\"namespace\")?t.namespace.split(\".\"):[];if(a=d=s=r=r||v,3!==r.nodeType&&8!==r.nodeType&&!xt.test(m+b.event.triggered)&&(m.indexOf(\".\")>-1&&(x=m.split(\".\"),m=x.shift(),x.sort()),l=m.indexOf(\":\")<0&&\"on\"+m,(t=t[b.expando]?t:new b.Event(m,\"object\"==typeof t&&t)).isTrigger=i?2:3,t.namespace=x.join(\".\"),t.rnamespace=t.namespace?new RegExp(\"(^|\\\\.)\"+x.join(\"\\\\.(?:.*\\\\.|)\")+\"(\\\\.|$)\"):null,t.result=void 0,t.target||(t.target=r),n=null==n?[t]:b.makeArray(n,[t]),p=b.event.special[m]||{},i||!p.trigger||!1!==p.trigger.apply(r,n))){if(!i&&!p.noBubble&&!g(r)){for(u=p.delegateType||m,xt.test(u+m)||(a=a.parentNode);a;a=a.parentNode)y.push(a),s=a;s===(r.ownerDocument||v)&&y.push(s.defaultView||s.parentWindow||e)}for(o=0;(a=y[o++])&&!t.isPropagationStopped();)d=a,t.type=o>1?u:p.bindType||m,(f=(G.get(a,\"events\")||Object.create(null))[t.type]&&G.get(a,\"handle\"))&&f.apply(a,n),(f=l&&a[l])&&f.apply&&X(a)&&(t.result=f.apply(a,n),!1===t.result&&t.preventDefault());return t.type=m,i||t.isDefaultPrevented()||p._default&&!1!==p._default.apply(y.pop(),n)||!X(r)||l&&h(r[m])&&!g(r)&&((s=r[l])&&(r[l]=null),b.event.triggered=m,t.isPropagationStopped()&&d.addEventListener(m,bt),r[m](),t.isPropagationStopped()&&d.removeEventListener(m,bt),b.event.triggered=void 0,s&&(r[l]=s)),t.result}},simulate:function(e,t,n){var r=b.extend(new b.Event,n,{type:e,isSimulated:!0});b.event.trigger(r,null,t)}}),b.fn.extend({trigger:function(e,t){return this.each((function(){b.event.trigger(e,t,this)}))},triggerHandler:function(e,t){var n=this[0];if(n)return b.event.trigger(e,t,n,!0)}}),d.focusin||b.each({focus:\"focusin\",blur:\"focusout\"},(function(e,t){var n=function(e){b.event.simulate(t,e.target,b.event.fix(e))};b.event.special[t]={setup:function(){var r=this.ownerDocument||this.document||this,i=G.access(r,t);i||r.addEventListener(e,n,!0),G.access(r,t,(i||0)+1)},teardown:function(){var r=this.ownerDocument||this.document||this,i=G.access(r,t)-1;i?G.access(r,t,i):(r.removeEventListener(e,n,!0),G.remove(r,t))}}}));var wt=e.location,Tt={guid:Date.now()},Ct=/\\?/;b.parseXML=function(t){var n;if(!t||\"string\"!=typeof t)return null;try{n=(new e.DOMParser).parseFromString(t,\"text/xml\")}catch(e){n=void 0}return n&&!n.getElementsByTagName(\"parsererror\").length||b.error(\"Invalid XML: \"+t),n};var Et=/\\[\\]$/,St=/\\r?\\n/g,kt=/^(?:submit|button|image|reset|file)$/i,At=/^(?:input|select|textarea|keygen)/i;function Nt(e,t,n,r){var i;if(Array.isArray(t))b.each(t,(function(t,i){n||Et.test(e)?r(e,i):Nt(e+\"[\"+(\"object\"==typeof i&&null!=i?t:\"\")+\"]\",i,n,r)}));else if(n||\"object\"!==x(t))r(e,t);else for(i in t)Nt(e+\"[\"+i+\"]\",t[i],n,r)}b.param=function(e,t){var n,r=[],i=function(e,t){var n=h(t)?t():t;r[r.length]=encodeURIComponent(e)+\"=\"+encodeURIComponent(null==n?\"\":n)};if(null==e)return\"\";if(Array.isArray(e)||e.jquery&&!b.isPlainObject(e))b.each(e,(function(){i(this.name,this.value)}));else for(n in e)Nt(n,e[n],t,i);return r.join(\"&\")},b.fn.extend({serialize:function(){return b.param(this.serializeArray())},serializeArray:function(){return this.map((function(){var e=b.prop(this,\"elements\");return e?b.makeArray(e):this})).filter((function(){var e=this.type;return this.name&&!b(this).is(\":disabled\")&&At.test(this.nodeName)&&!kt.test(e)&&(this.checked||!pe.test(e))})).map((function(e,t){var n=b(this).val();return null==n?null:Array.isArray(n)?b.map(n,(function(e){return{name:t.name,value:e.replace(St,\"\\r\\n\")}})):{name:t.name,value:n.replace(St,\"\\r\\n\")}})).get()}});var Dt=/%20/g,jt=/#.*$/,qt=/([?&])_=[^&]*/,Lt=/^(.*?):[ \\t]*([^\\r\\n]*)$/gm,Ht=/^(?:GET|HEAD)$/,Ot=/^\\/\\//,Pt={},Rt={},Mt=\"*/\".concat(\"*\"),It=v.createElement(\"a\");function Wt(e){return function(t,n){\"string\"!=typeof t&&(n=t,t=\"*\");var r,i=0,o=t.toLowerCase().match(O)||[];if(h(n))for(;r=o[i++];)\"+\"===r[0]?(r=r.slice(1)||\"*\",(e[r]=e[r]||[]).unshift(n)):(e[r]=e[r]||[]).push(n)}}function Ft(e,t,n,r){var i={},o=e===Rt;function a(s){var u;return i[s]=!0,b.each(e[s]||[],(function(e,s){var l=s(t,n,r);return\"string\"!=typeof l||o||i[l]?o?!(u=l):void 0:(t.dataTypes.unshift(l),a(l),!1)})),u}return a(t.dataTypes[0])||!i[\"*\"]&&a(\"*\")}function Bt(e,t){var n,r,i=b.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&b.extend(!0,e,r),e}It.href=wt.href,b.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:wt.href,type:\"GET\",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(wt.protocol),global:!0,processData:!0,async:!0,contentType:\"application/x-www-form-urlencoded; charset=UTF-8\",accepts:{\"*\":Mt,text:\"text/plain\",html:\"text/html\",xml:\"application/xml, text/xml\",json:\"application/json, text/javascript\"},contents:{xml:/\\bxml\\b/,html:/\\bhtml/,json:/\\bjson\\b/},responseFields:{xml:\"responseXML\",text:\"responseText\",json:\"responseJSON\"},converters:{\"* text\":String,\"text html\":!0,\"text json\":JSON.parse,\"text xml\":b.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Bt(Bt(e,b.ajaxSettings),t):Bt(b.ajaxSettings,e)},ajaxPrefilter:Wt(Pt),ajaxTransport:Wt(Rt),ajax:function(t,n){\"object\"==typeof t&&(n=t,t=void 0),n=n||{};var r,i,o,a,s,u,l,c,f,p,d=b.ajaxSetup({},n),h=d.context||d,g=d.context&&(h.nodeType||h.jquery)?b(h):b.event,y=b.Deferred(),m=b.Callbacks(\"once memory\"),x=d.statusCode||{},w={},T={},C=\"canceled\",E={readyState:0,getResponseHeader:function(e){var t;if(l){if(!a)for(a={};t=Lt.exec(o);)a[t[1].toLowerCase()+\" \"]=(a[t[1].toLowerCase()+\" \"]||[]).concat(t[2]);t=a[e.toLowerCase()+\" \"]}return null==t?null:t.join(\", \")},getAllResponseHeaders:function(){return l?o:null},setRequestHeader:function(e,t){return null==l&&(e=T[e.toLowerCase()]=T[e.toLowerCase()]||e,w[e]=t),this},overrideMimeType:function(e){return null==l&&(d.mimeType=e),this},statusCode:function(e){var t;if(e)if(l)E.always(e[E.status]);else for(t in e)x[t]=[x[t],e[t]];return this},abort:function(e){var t=e||C;return r&&r.abort(t),S(0,t),this}};if(y.promise(E),d.url=((t||d.url||wt.href)+\"\").replace(Ot,wt.protocol+\"//\"),d.type=n.method||n.type||d.method||d.type,d.dataTypes=(d.dataType||\"*\").toLowerCase().match(O)||[\"\"],null==d.crossDomain){u=v.createElement(\"a\");try{u.href=d.url,u.href=u.href,d.crossDomain=It.protocol+\"//\"+It.host!=u.protocol+\"//\"+u.host}catch(e){d.crossDomain=!0}}if(d.data&&d.processData&&\"string\"!=typeof d.data&&(d.data=b.param(d.data,d.traditional)),Ft(Pt,d,n,E),l)return E;for(f in(c=b.event&&d.global)&&0==b.active++&&b.event.trigger(\"ajaxStart\"),d.type=d.type.toUpperCase(),d.hasContent=!Ht.test(d.type),i=d.url.replace(jt,\"\"),d.hasContent?d.data&&d.processData&&0===(d.contentType||\"\").indexOf(\"application/x-www-form-urlencoded\")&&(d.data=d.data.replace(Dt,\"+\")):(p=d.url.slice(i.length),d.data&&(d.processData||\"string\"==typeof d.data)&&(i+=(Ct.test(i)?\"&\":\"?\")+d.data,delete d.data),!1===d.cache&&(i=i.replace(qt,\"$1\"),p=(Ct.test(i)?\"&\":\"?\")+\"_=\"+Tt.guid+++p),d.url=i+p),d.ifModified&&(b.lastModified[i]&&E.setRequestHeader(\"If-Modified-Since\",b.lastModified[i]),b.etag[i]&&E.setRequestHeader(\"If-None-Match\",b.etag[i])),(d.data&&d.hasContent&&!1!==d.contentType||n.contentType)&&E.setRequestHeader(\"Content-Type\",d.contentType),E.setRequestHeader(\"Accept\",d.dataTypes[0]&&d.accepts[d.dataTypes[0]]?d.accepts[d.dataTypes[0]]+(\"*\"!==d.dataTypes[0]?\", \"+Mt+\"; q=0.01\":\"\"):d.accepts[\"*\"]),d.headers)E.setRequestHeader(f,d.headers[f]);if(d.beforeSend&&(!1===d.beforeSend.call(h,E,d)||l))return E.abort();if(C=\"abort\",m.add(d.complete),E.done(d.success),E.fail(d.error),r=Ft(Rt,d,n,E)){if(E.readyState=1,c&&g.trigger(\"ajaxSend\",[E,d]),l)return E;d.async&&d.timeout>0&&(s=e.setTimeout((function(){E.abort(\"timeout\")}),d.timeout));try{l=!1,r.send(w,S)}catch(e){if(l)throw e;S(-1,e)}}else S(-1,\"No Transport\");function S(t,n,a,u){var f,p,v,w,T,C=n;l||(l=!0,s&&e.clearTimeout(s),r=void 0,o=u||\"\",E.readyState=t>0?4:0,f=t>=200&&t<300||304===t,a&&(w=function(e,t,n){for(var r,i,o,a,s=e.contents,u=e.dataTypes;\"*\"===u[0];)u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader(\"Content-Type\"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+\" \"+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(d,E,a)),!f&&b.inArray(\"script\",d.dataTypes)>-1&&(d.converters[\"text script\"]=function(){}),w=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];for(o=c.shift();o;)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if(\"*\"===o)o=u;else if(\"*\"!==u&&u!==o){if(!(a=l[u+\" \"+o]||l[\"* \"+o]))for(i in l)if((s=i.split(\" \"))[1]===o&&(a=l[u+\" \"+s[0]]||l[\"* \"+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e.throws)t=a(t);else try{t=a(t)}catch(e){return{state:\"parsererror\",error:a?e:\"No conversion from \"+u+\" to \"+o}}}return{state:\"success\",data:t}}(d,w,E,f),f?(d.ifModified&&((T=E.getResponseHeader(\"Last-Modified\"))&&(b.lastModified[i]=T),(T=E.getResponseHeader(\"etag\"))&&(b.etag[i]=T)),204===t||\"HEAD\"===d.type?C=\"nocontent\":304===t?C=\"notmodified\":(C=w.state,p=w.data,f=!(v=w.error))):(v=C,!t&&C||(C=\"error\",t<0&&(t=0))),E.status=t,E.statusText=(n||C)+\"\",f?y.resolveWith(h,[p,C,E]):y.rejectWith(h,[E,C,v]),E.statusCode(x),x=void 0,c&&g.trigger(f?\"ajaxSuccess\":\"ajaxError\",[E,d,f?p:v]),m.fireWith(h,[E,C]),c&&(g.trigger(\"ajaxComplete\",[E,d]),--b.active||b.event.trigger(\"ajaxStop\")))}return E},getJSON:function(e,t,n){return b.get(e,t,n,\"json\")},getScript:function(e,t){return b.get(e,void 0,t,\"script\")}}),b.each([\"get\",\"post\"],(function(e,t){b[t]=function(e,n,r,i){return h(n)&&(i=i||r,r=n,n=void 0),b.ajax(b.extend({url:e,type:t,dataType:i,data:n,success:r},b.isPlainObject(e)&&e))}})),b.ajaxPrefilter((function(e){var t;for(t in e.headers)\"content-type\"===t.toLowerCase()&&(e.contentType=e.headers[t]||\"\")})),b._evalUrl=function(e,t,n){return b.ajax({url:e,type:\"GET\",dataType:\"script\",cache:!0,async:!1,global:!1,converters:{\"text script\":function(){}},dataFilter:function(e){b.globalEval(e,t,n)}})},b.fn.extend({wrapAll:function(e){var t;return this[0]&&(h(e)&&(e=e.call(this[0])),t=b(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map((function(){for(var e=this;e.firstElementChild;)e=e.firstElementChild;return e})).append(this)),this},wrapInner:function(e){return h(e)?this.each((function(t){b(this).wrapInner(e.call(this,t))})):this.each((function(){var t=b(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)}))},wrap:function(e){var t=h(e);return this.each((function(n){b(this).wrapAll(t?e.call(this,n):e)}))},unwrap:function(e){return this.parent(e).not(\"body\").each((function(){b(this).replaceWith(this.childNodes)})),this}}),b.expr.pseudos.hidden=function(e){return!b.expr.pseudos.visible(e)},b.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},b.ajaxSettings.xhr=function(){try{return new e.XMLHttpRequest}catch(e){}};var $t={0:200,1223:204},_t=b.ajaxSettings.xhr();d.cors=!!_t&&\"withCredentials\"in _t,d.ajax=_t=!!_t,b.ajaxTransport((function(t){var n,r;if(d.cors||_t&&!t.crossDomain)return{send:function(i,o){var a,s=t.xhr();if(s.open(t.type,t.url,t.async,t.username,t.password),t.xhrFields)for(a in t.xhrFields)s[a]=t.xhrFields[a];for(a in t.mimeType&&s.overrideMimeType&&s.overrideMimeType(t.mimeType),t.crossDomain||i[\"X-Requested-With\"]||(i[\"X-Requested-With\"]=\"XMLHttpRequest\"),i)s.setRequestHeader(a,i[a]);n=function(e){return function(){n&&(n=r=s.onload=s.onerror=s.onabort=s.ontimeout=s.onreadystatechange=null,\"abort\"===e?s.abort():\"error\"===e?\"number\"!=typeof s.status?o(0,\"error\"):o(s.status,s.statusText):o($t[s.status]||s.status,s.statusText,\"text\"!==(s.responseType||\"text\")||\"string\"!=typeof s.responseText?{binary:s.response}:{text:s.responseText},s.getAllResponseHeaders()))}},s.onload=n(),r=s.onerror=s.ontimeout=n(\"error\"),void 0!==s.onabort?s.onabort=r:s.onreadystatechange=function(){4===s.readyState&&e.setTimeout((function(){n&&r()}))},n=n(\"abort\");try{s.send(t.hasContent&&t.data||null)}catch(e){if(n)throw e}},abort:function(){n&&n()}}})),b.ajaxPrefilter((function(e){e.crossDomain&&(e.contents.script=!1)})),b.ajaxSetup({accepts:{script:\"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript\"},contents:{script:/\\b(?:java|ecma)script\\b/},converters:{\"text script\":function(e){return b.globalEval(e),e}}}),b.ajaxPrefilter(\"script\",(function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type=\"GET\")})),b.ajaxTransport(\"script\",(function(e){var t,n;if(e.crossDomain||e.scriptAttrs)return{send:function(r,i){t=b(\""
+ ],
+ "text/plain": [
+ ":DynamicMap []\n",
+ " :Overlay\n",
+ " .RGB.I :RGB [date_time,traffic_volume] (R,G,B,A)\n",
+ " .RGB.II :RGB [date_time,traffic_volume] (R,G,B,A)\n",
+ " .RGB.III :RGB [date_time,traffic_volume] (R,G,B,A)"
+ ]
+ },
+ "metadata": {
+ "application/vnd.holoviews_exec.v0+json": {
+ "id": "1001"
+ }
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {},
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.holoviews_exec.v0+json": "",
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "
\n",
+ ""
+ ],
+ "text/plain": [
+ ":DynamicMap []\n",
+ " :Overlay\n",
+ " .RGB.I :RGB [index,log_pm2.5] (R,G,B,A)\n",
+ " .RGB.II :RGB [index,log_pm2.5] (R,G,B,A)\n",
+ " .RGB.III :RGB [index,log_pm2.5] (R,G,B,A)"
+ ]
+ },
+ "metadata": {
+ "application/vnd.holoviews_exec.v0+json": {
+ "id": "1458"
+ }
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {},
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.holoviews_exec.v0+json": "",
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "
\n",
+ ""
+ ],
+ "text/plain": [
+ ":DynamicMap []\n",
+ " :Overlay\n",
+ " .RGB.I :RGB [Time (s),R1 (MOhm)] (R,G,B,A)\n",
+ " .RGB.II :RGB [Time (s),R1 (MOhm)] (R,G,B,A)\n",
+ " .RGB.III :RGB [Time (s),R1 (MOhm)] (R,G,B,A)"
+ ]
+ },
+ "metadata": {
+ "application/vnd.holoviews_exec.v0+json": {
+ "id": "1915"
+ }
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {},
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.holoviews_exec.v0+json": "",
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "
\n",
+ ""
+ ],
+ "text/plain": [
+ ":DynamicMap []\n",
+ " :Overlay\n",
+ " .RGB.I :RGB [date,log_Appliances] (R,G,B,A)\n",
+ " .RGB.II :RGB [date,log_Appliances] (R,G,B,A)\n",
+ " .RGB.III :RGB [date,log_Appliances] (R,G,B,A)"
+ ]
+ },
+ "metadata": {
+ "application/vnd.holoviews_exec.v0+json": {
+ "id": "2372"
+ }
+ },
+ "output_type": "display_data"
+ },
+ {
+ "data": {},
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.holoviews_exec.v0+json": "",
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "
\n",
+ ""
+ ],
+ "text/plain": [
+ ":DynamicMap []\n",
+ " :Overlay\n",
+ " .RGB.I :RGB [TIME,SPD] (R,G,B,A)\n",
+ " .RGB.II :RGB [TIME,SPD] (R,G,B,A)\n",
+ " .RGB.III :RGB [TIME,SPD] (R,G,B,A)"
+ ]
+ },
+ "metadata": {
+ "application/vnd.holoviews_exec.v0+json": {
+ "id": "2829"
+ }
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# View train, test, val splits\n",
+ "l = hv.Layout()\n",
+ "for dataset in datasets:\n",
+ " d = dataset(datasets_root)\n",
+ " \n",
+ " p = dynspread(\n",
+ " datashade(hv.Scatter(d.df_train[d.columns_target[0]]),\n",
+ " cmap='red'))\n",
+ " p *= dynspread(\n",
+ " datashade(hv.Scatter(d.df_val[d.columns_target[0]]),\n",
+ " cmap='green'))\n",
+ " p *= dynspread(\n",
+ " datashade(hv.Scatter(d.df_test[d.columns_target[0]]),\n",
+ " cmap='blue'))\n",
+ " p = p.opts(title=f\"{dataset.__name__}, n={len(d)}, freq={d.df.index.freq.freqstr}\")\n",
+ " display(p)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Lightning\n",
+ "\n",
+ "We will use pytorch lightning to handle all the training scaffolding. We have a common pytorch lightning class that takes in the model and defines training steps and logging."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:03.014720Z",
+ "start_time": "2020-11-01T23:31:02.957669Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import pytorch_lightning as pl\n",
+ "\n",
+ "class PL_MODEL(pl.LightningModule):\n",
+ " def __init__(self, model, lr=3e-4, patience=None, weight_decay=0):\n",
+ " super().__init__()\n",
+ " self._model = model\n",
+ " self.lr = lr\n",
+ " self.patience = patience\n",
+ " self.weight_decay = weight_decay\n",
+ "\n",
+ " def forward(self, x_past, y_past, x_future, y_future=None):\n",
+ " \"\"\"Eval/Predict\"\"\"\n",
+ " y_dist, extra = self._model(x_past, y_past, x_future, y_future)\n",
+ " return y_dist, extra\n",
+ "\n",
+ " def training_step(self, batch, batch_idx, phase='train'):\n",
+ " x_past, y_past, x_future, y_future = batch\n",
+ " y_dist, extra = self.forward(*batch)\n",
+ " loss = -y_dist.log_prob(y_future).mean()\n",
+ " self.log_dict({f'loss/{phase}':loss})\n",
+ " if ('loss' in extra) and (phase=='train'):\n",
+ " # some models have a special loss\n",
+ " loss = extra['loss']\n",
+ " self.log_dict({f'model_loss/{phase}':loss})\n",
+ " return loss\n",
+ "\n",
+ " def validation_step(self, batch, batch_idx):\n",
+ " return self.training_step(batch, batch_idx, phase='val')\n",
+ " \n",
+ " def configure_optimizers(self):\n",
+ " optim = torch.optim.AdamW(self.parameters(), lr=self.lr, weight_decay=self.weight_decay)\n",
+ " scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(\n",
+ " optim,\n",
+ " patience=self.patience,\n",
+ " verbose=True,\n",
+ " min_lr=1e-7,\n",
+ " ) if self.patience else None\n",
+ " return {'optimizer': optim, 'lr_scheduler': scheduler, 'monitor': 'loss/val'}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:03.066664Z",
+ "start_time": "2020-11-01T23:31:03.018110Z"
+ },
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "from torch.utils.data import DataLoader\n",
+ "from pytorch_lightning.loggers import CSVLogger\n",
+ "from pytorch_lightning.callbacks.early_stopping import EarlyStopping"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Models"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:03.123216Z",
+ "start_time": "2020-11-01T23:31:03.070112Z"
+ },
+ "lines_to_end_of_cell_marker": 2,
+ "lines_to_next_cell": 0
+ },
+ "outputs": [],
+ "source": [
+ "from seq2seq_time.models.baseline import BaselineLast, BaselineMean\n",
+ "from seq2seq_time.models.lstm_seq2seq import LSTMSeq2Seq\n",
+ "from seq2seq_time.models.lstm import LSTM\n",
+ "from seq2seq_time.models.transformer import Transformer\n",
+ "from seq2seq_time.models.transformer_seq2seq import TransformerSeq2Seq\n",
+ "from seq2seq_time.models.neural_process import RANP\n",
+ "from seq2seq_time.models.transformer_process import TransformerProcess\n",
+ "from seq2seq_time.models.tcn import TCNSeq\n",
+ "from seq2seq_time.models.inceptiontime import InceptionTimeSeq\n",
+ "from seq2seq_time.models.xattention import CrossAttention"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:03.175536Z",
+ "start_time": "2020-11-01T23:31:03.126732Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import gc\n",
+ "\n",
+ "def free_mem():\n",
+ " gc.collect()\n",
+ " torch.cuda.empty_cache()\n",
+ " gc.collect()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:03.251181Z",
+ "start_time": "2020-11-01T23:31:03.180453Z"
+ },
+ "lines_to_next_cell": 0
+ },
+ "outputs": [],
+ "source": [
+ "hidden_size = 16\n",
+ "dropout=0.0\n",
+ "layers=6\n",
+ "nhead=2\n",
+ "\n",
+ "models = [\n",
+ " lambda xs, ys: BaselineLast(),\n",
+ " lambda xs, ys: BaselineMean(),\n",
+ " lambda xs, ys: Transformer(xs,\n",
+ " ys,\n",
+ " attention_dropout=dropout,\n",
+ " nhead=nhead*2,\n",
+ " nlayers=layers,\n",
+ " hidden_size=hidden_size),\n",
+ "\n",
+ " lambda xs, ys:TransformerProcess(xs,\n",
+ " ys, hidden_size=hidden_size, nhead=nhead,\n",
+ " latent_dim=hidden_size//2, dropout=dropout,\n",
+ " nlayers=layers),\n",
+ " lambda xs, ys:TCNSeq(xs, ys, hidden_size=hidden_size*2, nlayers=layers, dropout=dropout, kernel_size=2),\n",
+ " lambda xs, ys: RANP(xs,\n",
+ " ys, hidden_dim=hidden_size, dropout=dropout, \n",
+ " latent_dim=hidden_size//2, n_decoder_layers=layers),\n",
+ " lambda xs, ys: TransformerSeq2Seq(xs,\n",
+ " ys,\n",
+ " hidden_size=hidden_size,\n",
+ " nhead=nhead,\n",
+ " nlayers=layers,\n",
+ " attention_dropout=dropout\n",
+ " ),\n",
+ " lambda xs, ys: LSTM(xs,\n",
+ " ys,\n",
+ " hidden_size=hidden_size*2,\n",
+ " lstm_layers=layers,\n",
+ " lstm_dropout=dropout),\n",
+ "\n",
+ "\n",
+ " \n",
+ " lambda xs, ys: LSTMSeq2Seq(xs,\n",
+ " ys,\n",
+ " hidden_size=hidden_size,\n",
+ " lstm_layers=layers,\n",
+ " lstm_dropout=dropout),\n",
+ " lambda xs, ys: CrossAttention(xs,\n",
+ " ys,\n",
+ " hidden_size=hidden_size,),\n",
+ " lambda xs, ys: InceptionTimeSeq(xs,\n",
+ " ys,\n",
+ " layers=layers,\n",
+ " hidden_size=hidden_size,\n",
+ " bottleneck=hidden_size//4)\n",
+ "\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T07:30:40.569795Z",
+ "start_time": "2020-11-01T07:29:12.500374Z"
+ }
+ },
+ "source": [
+ "Lets summarize all models, and make sure they have a similar number of parameters"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:08.059894Z",
+ "start_time": "2020-11-01T23:31:03.258853Z"
+ },
+ "lines_to_next_cell": 0
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ " | Total params | Trainable params | Non-trainable params | Mult-Adds |
\n",
+ " \n",
+ " | BaselineLast | \n",
+ " 1.0 | \n",
+ " 1.0 | \n",
+ " 0.0 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | BaselineMean | \n",
+ " 1.0 | \n",
+ " 1.0 | \n",
+ " 0.0 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ " | Transformer | \n",
+ " 32.69k | \n",
+ " 32.69k | \n",
+ " 0.0 | \n",
+ " 31.216k | \n",
+ "
\n",
+ " \n",
+ " | TransformerProcess | \n",
+ " 72.978k | \n",
+ " 72.978k | \n",
+ " 0.0 | \n",
+ " 101.472k | \n",
+ "
\n",
+ " \n",
+ " | TCNSeq | \n",
+ " 24.546k | \n",
+ " 24.546k | \n",
+ " 0.0 | \n",
+ " 7.370816M | \n",
+ "
\n",
+ " \n",
+ " | RANP | \n",
+ " 20.09k | \n",
+ " 20.09k | \n",
+ " 0.0 | \n",
+ " 21.696k | \n",
+ "
\n",
+ " \n",
+ " | TransformerSeq2Seq | \n",
+ " 72.05k | \n",
+ " 72.05k | \n",
+ " 0.0 | \n",
+ " 68.624k | \n",
+ "
\n",
+ " \n",
+ " | LSTM | \n",
+ " 48.706k | \n",
+ " 48.706k | \n",
+ " 0.0 | \n",
+ " 47.168k | \n",
+ "
\n",
+ " \n",
+ " | LSTMSeq2Seq | \n",
+ " 26.082k | \n",
+ " 26.082k | \n",
+ " 0.0 | \n",
+ " 24.544k | \n",
+ "
\n",
+ " \n",
+ " | CrossAttention | \n",
+ " 43.474k | \n",
+ " 43.474k | \n",
+ " 0.0 | \n",
+ " 41.52k | \n",
+ "
\n",
+ " \n",
+ " | InceptionTimeSeq | \n",
+ " 52.054k | \n",
+ " 52.054k | \n",
+ " 0.0 | \n",
+ " 7.253632M | \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 17,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Summarize each models shape and weights\n",
+ "from seq2seq_time.torchsummaryX import summary\n",
+ "\n",
+ "# Get a batch\n",
+ "Dataset = datasets[0]\n",
+ "dataset = Dataset(datasets_root)\n",
+ "ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,\n",
+ " window_future=window_future)\n",
+ "dl_val = DataLoader(ds_val, batch_size=batch_size)\n",
+ "batch = next(iter(dl_val))\n",
+ "batch = [x.to(device).float() for x in batch]\n",
+ "x_past, y_past, x_future, y_future = batch\n",
+ "xs = x_past.shape[-1]\n",
+ "ys = y_future.shape[-1]\n",
+ "\n",
+ "# summary of each model\n",
+ "sizes=[]\n",
+ "for m_fn in models:\n",
+ " pt_model = m_fn(xs, ys).eval().to(device)\n",
+ " model_name = type(pt_model).__name__\n",
+ " with torch.no_grad():\n",
+ " df_summary, df_total = summary(pt_model, x_past, y_past, x_future, y_future, print_summary=False)\n",
+ " sizes.append(df_total.rename(columns={'Totals':model_name}))\n",
+ "df_model_sizes = pd.concat(sizes, 1).T\n",
+ "df_model_sizes.style.format(pd.io.formats.format.EngFormatter(use_eng_prefix=True))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T07:47:36.890566Z",
+ "start_time": "2020-11-01T07:46:21.174787Z"
+ },
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-10-23T23:36:11.052891Z",
+ "start_time": "2020-10-23T23:36:11.048874Z"
+ }
+ },
+ "source": [
+ "## Train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:08.139312Z",
+ "start_time": "2020-11-01T23:31:08.069097Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from collections import defaultdict\n",
+ "from seq2seq_time.metrics import rmse, smape\n",
+ "results = defaultdict(dict)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:31:08.197024Z",
+ "start_time": "2020-11-01T23:31:08.143633Z"
+ },
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "max_iters=20000"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.171Z"
+ },
+ "lines_to_next_cell": 0,
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "GPU available: True, used: True\n",
+ "TPU available: False, using: 0 TPU cores\n",
+ "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
+ "Using native 16bit precision.\n",
+ "\n",
+ " | Name | Type | Params\n",
+ "----------------------------------------\n",
+ "0 | _model | BaselineLast | 1 \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MetroInterstateTraffic BaselineLast\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=1.0, bar_style='info', layout=Layout…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "74b80a8841754bd8bc785825e041283c",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/wassname/anaconda/envs/seq2seq-time/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:45: UserWarning: Detected KeyboardInterrupt, attempting graceful shutdown...\n",
+ " warnings.warn(*args, **kwargs)\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='predict'), FloatProgress(value=0.0, max=61.0), HTML(value='')))"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ " | nll | MetroInterstateTraffic |
\n",
+ " \n",
+ " | BaselineLast | \n",
+ " 1.75 | \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "GPU available: True, used: True\n",
+ "TPU available: False, using: 0 TPU cores\n",
+ "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
+ "Using native 16bit precision.\n",
+ "\n",
+ " | Name | Type | Params\n",
+ "----------------------------------------\n",
+ "0 | _model | BaselineMean | 1 \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MetroInterstateTraffic BaselineMean\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=1.0, bar_style='info', layout=Layout…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "6c6d6e6d0927432cb8c18334a3b3f1da",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/home/wassname/anaconda/envs/seq2seq-time/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:45: UserWarning: Detected KeyboardInterrupt, attempting graceful shutdown...\n",
+ " warnings.warn(*args, **kwargs)\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='predict'), FloatProgress(value=0.0, max=61.0), HTML(value='')))"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ " | nll | MetroInterstateTraffic |
\n",
+ " \n",
+ " | BaselineLast | \n",
+ " 1.75 | \n",
+ "
\n",
+ " \n",
+ " | BaselineMean | \n",
+ " 1.43 | \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "GPU available: True, used: True\n",
+ "TPU available: False, using: 0 TPU cores\n",
+ "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
+ "Using native 16bit precision.\n",
+ "\n",
+ " | Name | Type | Params\n",
+ "---------------------------------------\n",
+ "0 | _model | Transformer | 32 K \n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "MetroInterstateTraffic Transformer\n"
+ ]
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validation sanity check'), FloatProgress(value=1.0, bar_style='info', layout=Layout…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "878b3f7040e8410f835553aa53aa9a45",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Training'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), max…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "HBox(children=(HTML(value='Validating'), FloatProgress(value=1.0, bar_style='info', layout=Layout(flex='2'), m…"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "for Dataset in datasets:\n",
+ " dataset_name = Dataset.__name__\n",
+ " dataset = Dataset(datasets_root)\n",
+ " ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,\n",
+ " window_future=window_future)\n",
+ "\n",
+ " # Init data\n",
+ " x_past, y_past, x_future, y_future = ds_train.get_rows(10)\n",
+ " xs = x_past.shape[-1]\n",
+ " ys = y_future.shape[-1]\n",
+ "\n",
+ " # Loaders\n",
+ " dl_train = DataLoader(ds_train,\n",
+ " batch_size=batch_size,\n",
+ " shuffle=True,\n",
+ " pin_memory=num_workers == 0,\n",
+ " num_workers=num_workers)\n",
+ " dl_val = DataLoader(ds_val,\n",
+ " shuffle=True,\n",
+ " batch_size=batch_size,\n",
+ " num_workers=num_workers)\n",
+ "\n",
+ " for m_fn in models:\n",
+ " try:\n",
+ " free_mem()\n",
+ " pt_model = m_fn(xs, ys)\n",
+ " model_name = type(pt_model).__name__\n",
+ " print(dataset_name, model_name)\n",
+ "\n",
+ " # Wrap in lightning\n",
+ " patience = 2\n",
+ " model = PL_MODEL(pt_model,\n",
+ " lr=1e-3, patience=patience,\n",
+ "# weight_decay=4e-5\n",
+ " ).to(device)\n",
+ "\n",
+ " # Trainer \n",
+ " trainer = pl.Trainer(\n",
+ " gpus=1,\n",
+ " min_epochs=2,\n",
+ " max_epochs=100,\n",
+ " amp_level='O1',\n",
+ " precision=16,\n",
+ " gradient_clip_val=20,\n",
+ " terminate_on_nan=True, \n",
+ " limit_train_batches=max_iters//batch_size,\n",
+ " limit_val_batches=max_iters//batch_size//5,\n",
+ " logger=CSVLogger(\"../outputs\", name=f'{timestamp}_{dataset_name}_{model_name}'),\n",
+ " callbacks=[\n",
+ " EarlyStopping(monitor='loss/val', patience=patience * 2),\n",
+ " ],\n",
+ " )\n",
+ "\n",
+ " # Train\n",
+ " trainer.fit(model, dl_train, dl_val)\n",
+ "\n",
+ " ds_preds = predict(model.to(device),\n",
+ " ds_test,\n",
+ " batch_size * 2,\n",
+ " device=device,\n",
+ " scaler=dataset.output_scaler)\n",
+ "\n",
+ "# print(dataset_name, model_name)\n",
+ "# print(f'mean_NLL {ds_preds.nll.mean().item():2.2f}')\n",
+ "# loss = ds_preds.nll.mean().item()\n",
+ "\n",
+ "# print(plot_hist(trainer))\n",
+ "# plot_performance(ds_preds)\n",
+ "\n",
+ " metrics = dict(\n",
+ " rmse=rmse(ds_preds.y_true, ds_preds.y_pred).item(), \n",
+ " smape=smape(ds_preds.y_true, ds_preds.y_pred).item(), \n",
+ " nll=ds_preds.nll.mean().item()\n",
+ " )\n",
+ " results[dataset_name][model_name] = metrics\n",
+ " display_results(results, 'nll')\n",
+ " \n",
+ " dset_to_nc(ds_preds, Path(trainer.logger.experiment.log_dir)/'ds_preds.nc')\n",
+ " model.cpu()\n",
+ " except Exception as e:\n",
+ " logging.exception('failed to run model')\n",
+ " \n",
+ "df_results = pd.concat({k:pd.DataFrame(v) for k,v in results.items()})\n",
+ "display(df_results)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T23:02:00.176044Z",
+ "start_time": "2020-11-01T23:02:00.119495Z"
+ }
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-10-27T22:06:50.235348Z",
+ "start_time": "2020-10-27T22:06:50.168117Z"
+ }
+ },
+ "source": [
+ "# Leaderboard"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.179Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "print(f'Negative Log-Likelihood (NLL).\\nover {window_future} steps')\n",
+ "df_results = pd.concat({k:pd.DataFrame(v) for k,v in results.items()})\n",
+ "display_results(results, 'nll')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Plots"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.185Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "\n",
+ "# Load saved preds\n",
+ "ds_predss = defaultdict(dict)\n",
+ "for Dataset in datasets:\n",
+ " dataset_name = Dataset.__name__\n",
+ " for m_fn in models:\n",
+ " pt_model = m_fn(xs, ys)\n",
+ " model_name = type(pt_model).__name__\n",
+ "\n",
+ " checkpoint_name = f\"{timestamp}_{dataset_name}_{model_name}\"\n",
+ " save_dir = Path(f\"../outputs\")/checkpoint_name\n",
+ " fs = sorted(save_dir.glob(\"**/ds_preds.nc\"))\n",
+ " if len(fs)>0:\n",
+ " ds_preds = xr.open_dataset(fs[-1])\n",
+ " ds_predss[dataset_name][model_name] = ds_preds"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.189Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "data_i = 100"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.192Z"
+ },
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# Plot mean of predictions\n",
+ "n = hv.Layout()\n",
+ "for dataset in ds_predss.keys():\n",
+ " d = next(iter(ds_predss[dataset].values())).isel(t_source=data_i)\n",
+ " p = hv_plot_true(d)\n",
+ " for model in results[dataset].keys():\n",
+ " ds_preds = ds_predss[dataset][model]\n",
+ " d = ds_preds.isel(t_source=data_i)\n",
+ " p *= hv_plot_pred(d).relabel(label=f\"{model}\")\n",
+ " n += p.opts(title=dataset, legend_position='top_left')\n",
+ "n.cols(1).opts(shared_axes=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.195Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "dataset='BejingPM25'\n",
+ "n = hv.Layout()\n",
+ "for i, model in enumerate(ds_predss[dataset].keys()):\n",
+ " ds_preds = ds_predss[dataset][model]\n",
+ " d = ds_preds.isel(t_source=data_i)\n",
+ " p = hv_plot_true(d)\n",
+ " p *= hv_plot_pred(d).relabel('pred')\n",
+ " p *= hv_plot_std(d)\n",
+ " n += p.opts(title=f'{dataset} {model}', legend_position='top_left')\n",
+ "n.cols(1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.199Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "plot_performance(ds_preds, full=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.202Z"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "def plot_at_i(data_i):\n",
+ " d = ds_preds.isel(t_source=data_i)\n",
+ " return hv_plot_prediction(d).relabel(label=f\"{model}\")\n",
+ "dmap = hv.DynamicMap(plot_at_i, kdims=['t_source'])\n",
+ "t = ds_preds.t_source.values\n",
+ "dmap = dmap.redim.values(t_source=range(len(t)))\n",
+ "dmap.opts(framewise=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.206Z"
+ },
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "# Plot series of predictions\n",
+ "t_ahead_i=6\n",
+ "d = ds_preds.isel(t_ahead=t_ahead_i)\n",
+ "p = datashade(hv.Scatter({\n",
+ " 'x': d.t_target,\n",
+ " 'y': d.y_true\n",
+ "}, label='true').opts(color='black'), cmap='black')\n",
+ "p *= datashade(hv.Curve({'x': d.t_target, 'y':d.y_pred}), cmap='blue')\n",
+ "p.opts(title=f'ahead by {d.freq} * {t_ahead_i}')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# DEBUG"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "start_time": "2020-11-01T23:30:50.217Z"
+ },
+ "lines_to_next_cell": 0
+ },
+ "outputs": [],
+ "source": [
+ "# # Summarize each models weights, and sanity check them\n",
+ "# dataset = datasets[0](datasets_root)\n",
+ "# ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,\n",
+ "# window_future=window_future)\n",
+ "# dl_val = DataLoader(ds_val, batch_size=batch_size)\n",
+ "# x_past, y_past, x_future, y_future = next(iter(dl_val))\n",
+ "# xs = x_past.shape[-1]\n",
+ "# ys = y_future.shape[-1]\n",
+ "\n",
+ "# for m_fn in models:\n",
+ " \n",
+ "# pt_model = m_fn(xs, ys)\n",
+ "\n",
+ "# # Wrap in lightning\n",
+ "# model = PL_MODEL(pt_model)\n",
+ "\n",
+ "# # Trainer \n",
+ "# free_mem()\n",
+ "# trainer = pl.Trainer(\n",
+ "# gpus=1,\n",
+ "# fast_dev_run=True,\n",
+ "# gradient_clip_val=4,\n",
+ "# progress_bar_refresh_rate=0,\n",
+ "# )\n",
+ "\n",
+ "# # Train\n",
+ "# trainer.fit(model, dl_val)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2020-11-01T07:30:49.786908Z",
+ "start_time": "2020-11-01T07:30:49.706024Z"
+ },
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "@webio": {
+ "lastCommId": null,
+ "lastKernelId": null
+ },
+ "jupytext": {
+ "encoding": "# -*- coding: utf-8 -*-",
+ "formats": "ipynb,py:light"
+ },
+ "kernelspec": {
+ "display_name": "seq2seq-time",
+ "language": "python",
+ "name": "seq2seq-time"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.8"
+ },
+ "toc": {
+ "base_numbering": 1,
+ "nav_menu": {},
+ "number_sections": true,
+ "sideBar": true,
+ "skip_h1_title": false,
+ "title_cell": "Table of Contents",
+ "title_sidebar": "Contents",
+ "toc_cell": false,
+ "toc_position": {
+ "height": "calc(100% - 180px)",
+ "left": "10px",
+ "top": "150px",
+ "width": "209.176px"
+ },
+ "toc_section_display": true,
+ "toc_window_display": true
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/05.2-mc-leaderboard.py b/notebooks/05.2-mc-leaderboard.py
new file mode 100644
index 0000000..c92d628
--- /dev/null
+++ b/notebooks/05.2-mc-leaderboard.py
@@ -0,0 +1,656 @@
+# -*- coding: utf-8 -*-
+# ---
+# jupyter:
+# jupytext:
+# formats: ipynb,py:light
+# text_representation:
+# extension: .py
+# format_name: light
+# format_version: '1.5'
+# jupytext_version: 1.6.0
+# kernelspec:
+# display_name: seq2seq-time
+# language: python
+# name: seq2seq-time
+# ---
+
+# # Sequence to Sequence Models for Timeseries Regression
+#
+#
+# In this notebook we are going to tackle a harder problem:
+# - predicting the future on a timeseries
+# - by outputing sequence of predictions
+# - with rough uncertainty (uncalibrated)
+# - using forecasted information (like weather report, week, or cycle of the moon)
+#
+# Not many papers benchmark movels for multivariate regression, much less seq prediction with uncertainty. So this notebook will try a range of models on a range of dataset.
+#
+# We do this using a sequence to sqequence interface
+#
+#
+#
+
+# - [ ] tensorboard / wandb
+# - [ ] show test train
+# - [ ] val
+# - [ ] don't overfit
+# - [ ] TCN
+# - [ ] make overlap between past and future
+
+# OPTIONAL: Load the "autoreload" extension so that code can change. But blacklist large modules
+# %load_ext autoreload
+# %autoreload 2
+# %aimport -pandas
+# %aimport -torch
+# %aimport -numpy
+# %aimport -matplotlib
+# %aimport -dask
+# %aimport -tqdm
+# %matplotlib inline
+
+# +
+# Imports
+import torch
+from torch import nn, optim
+from torch.nn import functional as F
+from torch.autograd import Variable
+import torch
+import torch.utils.data
+
+import xarray as xr
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+from pathlib import Path
+from tqdm.auto import tqdm
+
+import pytorch_lightning as pl
+# +
+import holoviews as hv
+from holoviews import opts
+from holoviews.operation.datashader import datashade, dynspread
+hv.extension('bokeh', inline=True)
+from seq2seq_time.visualization.hv_ggplot import ggplot_theme
+hv.renderer('bokeh').theme = ggplot_theme
+
+# holoview datashader timeseries options
+# %opts RGB [width=800 height=200 show_grid=True active_tools=["xwheel_zoom"] default_tools=["xpan","xwheel_zoom", "reset", "hover"] toolbar="right"]
+# %opts Curve [width=800 height=200 show_grid=True active_tools=["xwheel_zoom"] default_tools=["xpan","xwheel_zoom", "reset", "hover"] toolbar="right"]
+# %opts Scatter [width=800 height=200 show_grid=True active_tools=["xwheel_zoom"] default_tools=["xpan","xwheel_zoom", "reset", "hover"] toolbar="right"]
+# %opts Layout [width=800 height=200]
+# -
+
+
+from seq2seq_time.data.dataset import Seq2SeqDataSet, Seq2SeqDataSets
+from seq2seq_time.predict import predict, predict_multi
+from seq2seq_time.util import dset_to_nc
+
+import logging
+import warnings
+import seq2seq_time.silence
+warnings.simplefilter('once')
+warnings.simplefilter(action='ignore', category=FutureWarning)
+warnings.simplefilter(action='ignore', category=DeprecationWarning)
+warnings.filterwarnings('ignore', 'Consider increasing the value of the `num_workers` argument', UserWarning)
+warnings.filterwarnings('ignore', 'Your val_dataloader has `shuffle=True`', UserWarning)
+
+# ## Parameters
+
+# +
+device = "cuda" if torch.cuda.is_available() else "cpu"
+print(f'using {device}')
+
+timestamp = pd.Timestamp.now().strftime("%Y%m%d-%H%M%S")
+print(timestamp)
+window_past = 48*2
+window_future = 48
+batch_size = 64
+num_workers = 5
+datasets_root = Path('../data/processed/')
+window_past
+# -
+
+
+
+# ## Plot helpers
+
+# +
+def hv_plot_std(d: xr.Dataset):
+ """Plot predictions 2 standard deviations."""
+ xf = d.t_target
+ yp = d.y_pred
+ s = d.y_pred_std
+ return hv.Spread((xf, yp, s * 2),
+ label='2*std').opts(alpha=0.5, line_width=0)
+
+def hv_plot_pred(d: xr.Dataset):
+ """Plot prediction mean"""
+ xf = d.t_target
+ yp = d.y_pred
+ return hv.Curve({'x': xf, 'y': yp})
+
+def hv_plot_true(d: xr.Dataset):
+ """Plot true past and future data seperated by red line."""
+ # Plot true
+ x = np.concatenate([d.t_past, d.t_target])
+ yt = np.concatenate([d.y_past, d.y_true])
+ p = hv.Scatter({
+ 'x': x,
+ 'y': yt
+ }, label='true').opts(color='black')
+
+
+ # plot a red line for now
+ now=pd.Timestamp(d.t_source.squeeze().values)
+ p *= hv.VLine(now, label='now').opts(color='red', framewise=True)
+
+ p = p.opts(
+ ylabel=str(ds_preds.attrs['targets']),
+ xlabel=f'{now}'
+ )
+
+ return p
+
+def hv_plot_prediction(d: xr.Dataset) -> hv.Layout:
+ """Plot a prediction into the future, at a single point in time."""
+ p = hv_plot_true(d)
+ p *= hv_plot_pred(d)
+ p *= hv_plot_std(d)
+ return p
+
+
+# +
+def plot_performance(ds_preds, full=False):
+ """Multiple plots using xr_preds"""
+ p = hv_plot_prediction(ds_preds.isel(t_source=10))
+ display(p)
+
+ n = len(ds_preds.t_source)
+ d_ahead = ds_preds.mean(['t_source'])['nll'].groupby('t_ahead_hours').mean()
+ nll_vs_tahead = (hv.Curve(
+ (d_ahead.t_ahead_hours,
+ d_ahead)).redim(x='hours ahead',
+ y='nll').opts(
+ title=f'NLL vs time ahead (no. samples={n})'))
+ display(nll_vs_tahead)
+
+ # Make a plot of the NLL over time. Does this solution get worse with time?
+ if full:
+ d_source = ds_preds.mean(['t_ahead'])['nll'].groupby('t_source').mean()
+ nll_vs_time = (hv.Curve(d_source).opts(
+ title='Error vs time of prediction'))
+ display(nll_vs_time)
+
+ # A scatter plot is easy with xarray
+ if full:
+ tlim = (ds_preds.y_true.min().item(), ds_preds.y_true.max().item())
+ true_vs_pred = datashade(hv.Scatter(
+ (ds_preds.y_true,
+ ds_preds.y_pred))).redim(x='true', y='pred').opts(width=400,
+ height=400,
+ xlim=tlim,
+ ylim=tlim,
+ title='Scatter plot')
+ true_vs_pred = dynspread(true_vs_pred)
+ true_vs_pred
+ display(true_vs_pred)
+
+def plot_hist(trainer: pl.Trainer):
+ """If you used a CSVLogger you can load and plot history here"""
+ try:
+ df_hist = pd.read_csv(trainer.logger.experiment.metrics_file_path)
+ df_hist['epoch'] = df_hist['epoch'].ffill()
+
+ df_histe = df_hist.set_index('epoch').groupby('epoch').mean()
+ if len(df_histe)>1:
+ p = hv.Curve(df_histe, kdims=['epoch'], vdims=['loss/train']).relabel('train')
+ p *= hv.Curve(df_histe, kdims=['epoch'], vdims=['loss/val']).relabel('val')
+ display(p.opts(ylabel='loss'))
+ return df_histe
+ except Exception as e:
+ print(e)
+ pass
+
+
+# +
+# helpers to display our results as a dataframe
+
+def df_bold_min(data):
+ '''
+ highlight the maximum in a Series or DataFrame
+
+
+ Usage:
+ `df.style.apply(df_bold_min)`
+ '''
+ attr = 'font-weight: bold'
+ #remove % and cast to float
+ data = data.replace('%','', regex=True).astype(float)
+ if data.ndim == 1: # Series from .apply(axis=0) or axis=1
+ is_min = data == data.min()
+ return [attr if v else '' for v in is_min]
+ else: # from .apply(axis=None)
+ is_min = data == data.min().min()
+ return pd.DataFrame(np.where(is_min, attr, ''),
+ index=data.index, columns=data.columns)
+
+def format_results(results, metric=None):
+ df_results = pd.concat({k:pd.DataFrame(v) for k,v in results.items()}).T
+ if metric:
+ return df_results.xs(metric, axis=1, level=1).rename_axis(columns=metric)
+ return df_results
+
+def display_results(results, metric='nll', strformat="{:.2f}"):
+ df_results = format_results(results, metric=metric)
+
+ # display metric
+ display(df_results
+ .style.format(strformat)
+ .apply(df_bold_min)
+ )
+
+
+# -
+# ## Datasets
+#
+# From easy to hard, these dataset show different challenges, all of them with more than 20k datapoints and with a regression output. See the 00.01 notebook for more details, and the code for more information.
+#
+# Some such as MetroInterstateTraffic are easier, some are periodic such as BejingPM25, some are conditional on inputs such as GasSensor, and some are noisy and periodic like IMOSCurrentsVel
+
+from seq2seq_time.data.data import IMOSCurrentsVel, AppliancesEnergyPrediction, BejingPM25, GasSensor, MetroInterstateTraffic
+datasets = [MetroInterstateTraffic, BejingPM25, GasSensor, AppliancesEnergyPrediction, IMOSCurrentsVel]
+datasets
+
+
+
+# View train, test, val splits
+l = hv.Layout()
+for dataset in datasets:
+ d = dataset(datasets_root)
+
+ p = dynspread(
+ datashade(hv.Scatter(d.df_train[d.columns_target[0]]),
+ cmap='red'))
+ p *= dynspread(
+ datashade(hv.Scatter(d.df_val[d.columns_target[0]]),
+ cmap='green'))
+ p *= dynspread(
+ datashade(hv.Scatter(d.df_test[d.columns_target[0]]),
+ cmap='blue'))
+ p = p.opts(title=f"{dataset.__name__}, n={len(d)}, freq={d.df.index.freq.freqstr}")
+ display(p)
+
+# ## Lightning
+#
+# We will use pytorch lightning to handle all the training scaffolding. We have a common pytorch lightning class that takes in the model and defines training steps and logging.
+
+# +
+import pytorch_lightning as pl
+
+class PL_MODEL(pl.LightningModule):
+ def __init__(self, model, lr=3e-4, patience=None, weight_decay=0):
+ super().__init__()
+ self._model = model
+ self.lr = lr
+ self.patience = patience
+ self.weight_decay = weight_decay
+
+ def forward(self, x_past, y_past, x_future, y_future=None):
+ """Eval/Predict"""
+ y_dist, extra = self._model(x_past, y_past, x_future, y_future)
+ return y_dist, extra
+
+ def training_step(self, batch, batch_idx, phase='train'):
+ x_past, y_past, x_future, y_future = batch
+ y_dist, extra = self.forward(*batch)
+ loss = -y_dist.log_prob(y_future).mean()
+ self.log_dict({f'loss/{phase}':loss})
+ if ('loss' in extra) and (phase=='train'):
+ # some models have a special loss
+ loss = extra['loss']
+ self.log_dict({f'model_loss/{phase}':loss})
+ return loss
+
+ def validation_step(self, batch, batch_idx):
+ return self.training_step(batch, batch_idx, phase='val')
+
+ def configure_optimizers(self):
+ optim = torch.optim.AdamW(self.parameters(), lr=self.lr, weight_decay=self.weight_decay)
+ scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
+ optim,
+ patience=self.patience,
+ verbose=True,
+ min_lr=1e-7,
+ ) if self.patience else None
+ return {'optimizer': optim, 'lr_scheduler': scheduler, 'monitor': 'loss/val'}
+
+
+# -
+
+from torch.utils.data import DataLoader
+from pytorch_lightning.loggers import CSVLogger
+from pytorch_lightning.callbacks.early_stopping import EarlyStopping
+
+
+# ## Models
+
+from seq2seq_time.models.baseline import BaselineLast, BaselineMean
+from seq2seq_time.models.lstm_seq2seq import LSTMSeq2Seq
+from seq2seq_time.models.lstm import LSTM
+from seq2seq_time.models.transformer import Transformer
+from seq2seq_time.models.transformer_seq2seq import TransformerSeq2Seq
+from seq2seq_time.models.neural_process import RANP
+from seq2seq_time.models.transformer_process import TransformerProcess
+from seq2seq_time.models.tcn import TCNSeq
+from seq2seq_time.models.inceptiontime import InceptionTimeSeq
+from seq2seq_time.models.xattention import CrossAttention
+# +
+import gc
+
+def free_mem():
+ gc.collect()
+ torch.cuda.empty_cache()
+ gc.collect()
+
+
+# +
+hidden_size = 16
+dropout=0.0
+layers=6
+nhead=2
+
+models = [
+ lambda xs, ys: BaselineLast(),
+ lambda xs, ys: BaselineMean(),
+ lambda xs, ys: Transformer(xs,
+ ys,
+ attention_dropout=dropout,
+ nhead=nhead*2,
+ nlayers=layers,
+ hidden_size=hidden_size),
+
+ lambda xs, ys:TransformerProcess(xs,
+ ys, hidden_size=hidden_size, nhead=nhead,
+ latent_dim=hidden_size//2, dropout=dropout,
+ nlayers=layers),
+ lambda xs, ys:TCNSeq(xs, ys, hidden_size=hidden_size*2, nlayers=layers, dropout=dropout, kernel_size=2),
+ lambda xs, ys: RANP(xs,
+ ys, hidden_dim=hidden_size, dropout=dropout,
+ latent_dim=hidden_size//2, n_decoder_layers=layers),
+ lambda xs, ys: TransformerSeq2Seq(xs,
+ ys,
+ hidden_size=hidden_size,
+ nhead=nhead,
+ nlayers=layers,
+ attention_dropout=dropout
+ ),
+ lambda xs, ys: LSTM(xs,
+ ys,
+ hidden_size=hidden_size*2,
+ lstm_layers=layers,
+ lstm_dropout=dropout),
+
+
+
+ lambda xs, ys: LSTMSeq2Seq(xs,
+ ys,
+ hidden_size=hidden_size,
+ lstm_layers=layers,
+ lstm_dropout=dropout),
+ lambda xs, ys: CrossAttention(xs,
+ ys,
+ hidden_size=hidden_size,),
+ lambda xs, ys: InceptionTimeSeq(xs,
+ ys,
+ layers=layers,
+ hidden_size=hidden_size,
+ bottleneck=hidden_size//4)
+
+]
+# -
+# Lets summarize all models, and make sure they have a similar number of parameters
+
+# +
+# Summarize each models shape and weights
+from seq2seq_time.torchsummaryX import summary
+
+# Get a batch
+Dataset = datasets[0]
+dataset = Dataset(datasets_root)
+ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,
+ window_future=window_future)
+dl_val = DataLoader(ds_val, batch_size=batch_size)
+batch = next(iter(dl_val))
+batch = [x.to(device).float() for x in batch]
+x_past, y_past, x_future, y_future = batch
+xs = x_past.shape[-1]
+ys = y_future.shape[-1]
+
+# summary of each model
+sizes=[]
+for m_fn in models:
+ pt_model = m_fn(xs, ys).eval().to(device)
+ model_name = type(pt_model).__name__
+ with torch.no_grad():
+ df_summary, df_total = summary(pt_model, x_past, y_past, x_future, y_future, print_summary=False)
+ sizes.append(df_total.rename(columns={'Totals':model_name}))
+df_model_sizes = pd.concat(sizes, 1).T
+df_model_sizes.style.format(pd.io.formats.format.EngFormatter(use_eng_prefix=True))
+# -
+
+
+
+# ## Train
+
+from collections import defaultdict
+from seq2seq_time.metrics import rmse, smape
+results = defaultdict(dict)
+
+max_iters=20000
+
+
+# +
+for Dataset in datasets:
+ dataset_name = Dataset.__name__
+ dataset = Dataset(datasets_root)
+ ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,
+ window_future=window_future)
+
+ # Init data
+ x_past, y_past, x_future, y_future = ds_train.get_rows(10)
+ xs = x_past.shape[-1]
+ ys = y_future.shape[-1]
+
+ # Loaders
+ dl_train = DataLoader(ds_train,
+ batch_size=batch_size,
+ shuffle=True,
+ pin_memory=num_workers == 0,
+ num_workers=num_workers)
+ dl_val = DataLoader(ds_val,
+ shuffle=True,
+ batch_size=batch_size,
+ num_workers=num_workers)
+
+ for m_fn in models:
+ try:
+ free_mem()
+ pt_model = m_fn(xs, ys)
+ model_name = type(pt_model).__name__
+ print(dataset_name, model_name)
+
+ # Wrap in lightning
+ patience = 2
+ model = PL_MODEL(pt_model,
+ lr=1e-3, patience=patience,
+# weight_decay=4e-5
+ ).to(device)
+
+ # Trainer
+ trainer = pl.Trainer(
+ gpus=1,
+ min_epochs=2,
+ max_epochs=100,
+ amp_level='O1',
+ precision=16,
+ gradient_clip_val=20,
+ terminate_on_nan=True,
+ limit_train_batches=max_iters//batch_size,
+ limit_val_batches=max_iters//batch_size//5,
+ logger=CSVLogger("../outputs", name=f'{timestamp}_{dataset_name}_{model_name}'),
+ callbacks=[
+ EarlyStopping(monitor='loss/val', patience=patience * 2),
+ ],
+ )
+
+ # Train
+ trainer.fit(model, dl_train, dl_val)
+
+ ds_preds = predict(model.to(device),
+ ds_test,
+ batch_size * 2,
+ device=device,
+ scaler=dataset.output_scaler)
+
+# print(dataset_name, model_name)
+# print(f'mean_NLL {ds_preds.nll.mean().item():2.2f}')
+# loss = ds_preds.nll.mean().item()
+
+# print(plot_hist(trainer))
+# plot_performance(ds_preds)
+
+ metrics = dict(
+ rmse=rmse(ds_preds.y_true, ds_preds.y_pred).item(),
+ smape=smape(ds_preds.y_true, ds_preds.y_pred).item(),
+ nll=ds_preds.nll.mean().item()
+ )
+ results[dataset_name][model_name] = metrics
+ display_results(results, 'nll')
+
+ dset_to_nc(ds_preds, Path(trainer.logger.experiment.log_dir)/'ds_preds.nc')
+ model.cpu()
+ except Exception as e:
+ logging.exception('failed to run model')
+
+df_results = pd.concat({k:pd.DataFrame(v) for k,v in results.items()})
+display(df_results)
+# -
+
+
+# # Leaderboard
+
+print(f'Negative Log-Likelihood (NLL).\nover {window_future} steps')
+df_results = pd.concat({k:pd.DataFrame(v) for k,v in results.items()})
+display_results(results, 'nll')
+
+# # Plots
+
+# +
+
+# Load saved preds
+ds_predss = defaultdict(dict)
+for Dataset in datasets:
+ dataset_name = Dataset.__name__
+ for m_fn in models:
+ pt_model = m_fn(xs, ys)
+ model_name = type(pt_model).__name__
+
+ checkpoint_name = f"{timestamp}_{dataset_name}_{model_name}"
+ save_dir = Path(f"../outputs")/checkpoint_name
+ fs = sorted(save_dir.glob("**/ds_preds.nc"))
+ if len(fs)>0:
+ ds_preds = xr.open_dataset(fs[-1])
+ ds_predss[dataset_name][model_name] = ds_preds
+# -
+
+data_i = 100
+
+# Plot mean of predictions
+n = hv.Layout()
+for dataset in ds_predss.keys():
+ d = next(iter(ds_predss[dataset].values())).isel(t_source=data_i)
+ p = hv_plot_true(d)
+ for model in results[dataset].keys():
+ ds_preds = ds_predss[dataset][model]
+ d = ds_preds.isel(t_source=data_i)
+ p *= hv_plot_pred(d).relabel(label=f"{model}")
+ n += p.opts(title=dataset, legend_position='top_left')
+n.cols(1).opts(shared_axes=False)
+
+dataset='BejingPM25'
+n = hv.Layout()
+for i, model in enumerate(ds_predss[dataset].keys()):
+ ds_preds = ds_predss[dataset][model]
+ d = ds_preds.isel(t_source=data_i)
+ p = hv_plot_true(d)
+ p *= hv_plot_pred(d).relabel('pred')
+ p *= hv_plot_std(d)
+ n += p.opts(title=f'{dataset} {model}', legend_position='top_left')
+n.cols(1)
+
+plot_performance(ds_preds, full=True)
+
+
+def plot_at_i(data_i):
+ d = ds_preds.isel(t_source=data_i)
+ return hv_plot_prediction(d).relabel(label=f"{model}")
+dmap = hv.DynamicMap(plot_at_i, kdims=['t_source'])
+t = ds_preds.t_source.values
+dmap = dmap.redim.values(t_source=range(len(t)))
+dmap.opts(framewise=True)
+
+# Plot series of predictions
+t_ahead_i=6
+d = ds_preds.isel(t_ahead=t_ahead_i)
+p = datashade(hv.Scatter({
+ 'x': d.t_target,
+ 'y': d.y_true
+}, label='true').opts(color='black'), cmap='black')
+p *= datashade(hv.Curve({'x': d.t_target, 'y':d.y_pred}), cmap='blue')
+p.opts(title=f'ahead by {d.freq} * {t_ahead_i}')
+
+
+
+
+
+
+
+
+
+# # DEBUG
+
+# +
+# # Summarize each models weights, and sanity check them
+# dataset = datasets[0](datasets_root)
+# ds_train, ds_val, ds_test = dataset.to_datasets(window_past=window_past,
+# window_future=window_future)
+# dl_val = DataLoader(ds_val, batch_size=batch_size)
+# x_past, y_past, x_future, y_future = next(iter(dl_val))
+# xs = x_past.shape[-1]
+# ys = y_future.shape[-1]
+
+# for m_fn in models:
+
+# pt_model = m_fn(xs, ys)
+
+# # Wrap in lightning
+# model = PL_MODEL(pt_model)
+
+# # Trainer
+# free_mem()
+# trainer = pl.Trainer(
+# gpus=1,
+# fast_dev_run=True,
+# gradient_clip_val=4,
+# progress_bar_refresh_rate=0,
+# )
+
+# # Train
+# trainer.fit(model, dl_val)
+# -
+
+
+
+
+