diff --git a/tensorforce-PPO.ipynb b/tensorforce-PPO.ipynb new file mode 100644 index 0000000..5491dc3 --- /dev/null +++ b/tensorforce-PPO.ipynb @@ -0,0 +1,1077 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Uses tensorforce tensorforce-0.2.0" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.002254Z", + "start_time": "2017-10-15T01:42:13.297490Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# plotting\n", + "%matplotlib inline\n", + "from matplotlib import pyplot as plt\n", + "import seaborn as sns\n", + "plt.style.use('ggplot')\n", + "\n", + "# numeric\n", + "import numpy as np\n", + "from numpy import random\n", + "import pandas as pd\n", + "\n", + "# util\n", + "from collections import Counter\n", + "import pdb\n", + "import time\n", + "import tempfile\n", + "import itertools\n", + "from tqdm import tqdm_notebook as tqdm\n", + "import datetime\n", + "\n", + "# logging\n", + "import logging\n", + "logger = log = logging.getLogger(__name__)\n", + "# log.setLevel(logging.INFO)\n", + "logging.basicConfig()\n", + "log.info('%s logger started.', __name__)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.101075Z", + "start_time": "2017-10-15T01:42:14.003911Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "import gym\n", + "from gym import error, spaces, utils\n", + "from gym.utils import seeding" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.157669Z", + "start_time": "2017-10-15T01:42:14.102563Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "import os\n", + "os.sys.path.append(os.path.abspath('.'))\n", + "%reload_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.201990Z", + "start_time": "2017-10-15T01:42:14.158832Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'./outputs/tensorforce-PPO-prioritised/tensorforce-PPO-prioritised_20171012_04-42-55.model'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# params\n", + "window_length = 50\n", + "\n", + "save_path = './outputs/tensorforce-PPO-prioritised/tensorforce-PPO-prioritised_20171012_04-42-55.model'\n", + "save_path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Enviroment" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.245473Z", + "start_time": "2017-10-15T01:42:14.203055Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "from src.environments.portfolio import PortfolioEnv" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.311788Z", + "start_time": "2017-10-15T01:42:14.247810Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "from rl.memory import SequentialMemory, Memory\n", + "from collections import deque\n", + "\n", + "class EnvWrapper(PortfolioEnv):\n", + " \"\"\"Wraps env to normalise and reshape action.\"\"\"\n", + " def __init__(self, window_length=50, *args, **kwargs):\n", + " super().__init__(*args, **kwargs)\n", + " \n", + " def step(self, action):\n", + " # also it puts it in a list\n", + " if isinstance(action, list):\n", + " action = action[0]\n", + " \n", + " # we have to normalise for some reason softmax wont work\n", + " if isinstance(action, dict):\n", + " action = np.abs(list(action.values()))\n", + " action /= action.sum() \n", + " \n", + " return super().step(action) " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.618015Z", + "start_time": "2017-10-15T01:42:14.313685Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:gym.envs.registration:Making new env: CartPole-v0\n", + "[2017-10-15 09:42:14,602] Making new env: CartPole-v0\n", + "INFO:gym.envs.registration:Making new env: CartPole-v0\n", + "[2017-10-15 09:42:14,608] Making new env: CartPole-v0\n" + ] + } + ], + "source": [ + "df_train = pd.read_hdf('./data/poloniex_30m.hf',key='train')\n", + "env = EnvWrapper(\n", + " df=df_train,\n", + " steps=30, \n", + " scale=True, \n", + " augment=0.000,\n", + " trading_cost=0, # let just overfit first,\n", + " window_length = window_length,\n", + " \n", + ")\n", + "env.seed = 0 \n", + "\n", + "df_test = pd.read_hdf('./data/poloniex_30m.hf',key='test')\n", + "env_test = EnvWrapper(\n", + " df=df_test,\n", + " steps=30, \n", + " scale=True, \n", + " augment=0.00,\n", + " trading_cost=0, # let just overfit first\n", + " window_length=window_length,\n", + ")\n", + "env_test.seed = 0 \n", + "\n", + "from tensorforce.contrib.openai_gym import OpenAIGym\n", + "environment = OpenAIGym('CartPole-v0')\n", + "environment.gym = env\n", + "\n", + "environment_test = OpenAIGym('CartPole-v0')\n", + "environment_test.gym = env_test" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:14.690869Z", + "start_time": "2017-10-15T01:42:14.619668Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.0 False {'reward': 0.0, 'log_return': 0.0, 'portfolio_value': 1.0, 'return': 1.0038234525884082, 'rate_of_return': 0.0, 'cash_bias': 0.19920207637805584, 'cost': 0.0, 'market_value': 1.0015682040003333, 'date': 1421807400.0, 'steps': 2}\n", + "(5, 50, 3) (5, 50, 3)\n" + ] + } + ], + "source": [ + "# check shapes\n", + "obs1, reward, done, info=env.step(np.random.random(env.action_space.shape))\n", + "print(reward, done, info)\n", + "obs2 = env.reset()\n", + "print(obs1.shape,obs2.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "ExecuteTime": { + "end_time": "2017-07-16T04:41:21.116729Z", + "start_time": "2017-07-16T12:41:21.086620+08:00" + } + }, + "source": [ + "# Model\n", + "\n", + "Derived from https://github.com/reinforceio/tensorforce/blob/0d07fadec03f76537a2431e17c51cd759d53b5e9/tensorforce/core/networks/layers.py#L90" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:30.420676Z", + "start_time": "2017-10-15T01:42:14.692354Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "from tensorforce import Configuration\n", + "from tensorforce.agents import PPOAgent\n", + "from tensorforce.core.networks import layered_network_builder" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:30.769981Z", + "start_time": "2017-10-15T01:42:30.423107Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# layer helpers from:\n", + "# https://github.com/reinforceio/tensorforce/blob/0d07fadec03f76537a2431e17c51cd759d53b5e9/tensorforce/core/networks/layers.py#L90\n", + "import tensorflow as tf\n", + "from math import sqrt\n", + "from tensorforce import util\n", + "from tensorforce import TensorForceError\n", + "\n", + "def linear(x, size, bias=True, l2_regularization=0.0):\n", + " if util.rank(x) != 2:\n", + " raise TensorForceError('Invalid input rank for linear layer.')\n", + " with tf.variable_scope('linear'):\n", + " weights = tf.Variable(initial_value=tf.random_normal(shape=(x.get_shape()[1].value, size), stddev=sqrt(2.0 / (x.get_shape()[1].value + size))))\n", + " if l2_regularization > 0.0:\n", + " tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=weights))\n", + " x = tf.matmul(a=x, b=weights)\n", + " if bias:\n", + " bias = tf.Variable(initial_value=tf.zeros(shape=(size,)))\n", + " if l2_regularization > 0.0:\n", + " tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=bias))\n", + " x = tf.nn.bias_add(value=x, bias=bias)\n", + " return x\n", + "\n", + "def nonlinearity(x, name='relu'):\n", + " with tf.variable_scope('nonlinearity'):\n", + " if name == 'elu':\n", + " x = tf.nn.elu(features=x)\n", + " elif name == 'relu':\n", + " x = tf.nn.relu(features=x)\n", + " elif name == 'selu':\n", + " # https://arxiv.org/pdf/1706.02515.pdf\n", + " alpha = 1.6732632423543772848170429916717\n", + " scale = 1.0507009873554804934193349852946\n", + " negative = alpha * tf.nn.elu(features=x)\n", + " x = scale * tf.where(condition=(x >= 0.0), x=x, y=negative)\n", + " elif name == 'sigmoid':\n", + " x = tf.sigmoid(x=x)\n", + " elif name == 'softmax':\n", + " x = tf.nn.softmax(logits=x)\n", + " elif name == 'tanh':\n", + " x = tf.nn.tanh(x=x)\n", + " else:\n", + " raise TensorForceError('Invalid nonlinearity.')\n", + " return x\n", + "\n", + "def dense(x, size, bias=True, activation='relu', l2_regularization=0.0):\n", + " if util.rank(x) != 2:\n", + " raise TensorForceError('Invalid input rank for dense layer.')\n", + " with tf.variable_scope('dense'):\n", + " x = linear(x=x, size=size, bias=bias, l2_regularization=l2_regularization)\n", + " x = nonlinearity(x=x, name=activation)\n", + " return x\n", + "\n", + "def flatten(x):\n", + " with tf.variable_scope('flatten'):\n", + " x = tf.reshape(tensor=x, shape=(-1, util.prod(x.get_shape().as_list()[1:])))\n", + " return x\n", + "\n", + "def conv2d(x, size, window=(3,3), stride=(1,1), bias=False, activation='relu', l2_regularization=0.0, padding='SAME'):\n", + " if util.rank(x) != 4:\n", + " raise TensorForceError('Invalid input rank for conv2d layer.')\n", + " with tf.variable_scope('conv2d'):\n", + " filters = tf.Variable(initial_value=tf.random_normal(shape=(window[0], window[1], x.get_shape()[3].value, size), stddev=sqrt(2.0 / size)))\n", + " if l2_regularization > 0.0:\n", + " tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=filters))\n", + " x = tf.nn.conv2d(input=x, filter=filters, strides=(1, stride[0], stride[1], 1), padding=padding)\n", + " if bias:\n", + " bias = tf.Variable(initial_value=tf.zeros(shape=(size,)))\n", + " if l2_regularization > 0.0:\n", + " tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=bias))\n", + " x = tf.nn.bias_add(value=x, bias=bias)\n", + " x = nonlinearity(x=x, name=activation)\n", + " return x\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:31.635122Z", + "start_time": "2017-10-15T01:42:30.771243Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# build a network for a given input\n", + "def network_builder(inputs, summary_level):\n", + " if len(inputs) != 1:\n", + " raise TensorForceError('Layered network must have only one input.')\n", + " x = next(iter(inputs.values()))\n", + " \n", + " x = conv2d(x=x, size=2, window=(1,3), bias=True, activation='relu', l2_regularization=1e-8, padding='VALID')\n", + " x = conv2d(x=x, size=20, window=(1,window_length-2), bias=True, activation='relu', l2_regularization=1e-8, padding='VALID')\n", + " x = conv2d(x=x, size=1, window=(1,1), bias=True, activation='relu', l2_regularization=1e-8, padding='VALID')\n", + " x = flatten(x)\n", + " x = nonlinearity(x,name='softmax')\n", + " \n", + " return x\n", + "network=network_builder" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Agent" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:43:14.458362Z", + "start_time": "2017-10-15T01:43:14.384891Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "\n", + "# Check my PR is included\n", + "import tensorforce.core.memories\n", + "assert isinstance(runner.agent.memory,tensorforce.core.memories.PrioritizedReplay)\n", + "assert isinstance(runner.agent, tensorforce.agents.MemoryAgent)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:48:15.367117Z", + "start_time": "2017-10-15T01:48:15.294327Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'action0': {'continuous': True,\n", + " 'max_value': None,\n", + " 'min_value': None,\n", + " 'shape': ()},\n", + " 'action1': {'continuous': True,\n", + " 'max_value': None,\n", + " 'min_value': None,\n", + " 'shape': ()},\n", + " 'action2': {'continuous': True,\n", + " 'max_value': None,\n", + " 'min_value': None,\n", + " 'shape': ()},\n", + " 'action3': {'continuous': True,\n", + " 'max_value': None,\n", + " 'min_value': None,\n", + " 'shape': ()},\n", + " 'action4': {'continuous': True,\n", + " 'max_value': None,\n", + " 'min_value': None,\n", + " 'shape': ()},\n", + " 'action5': {'continuous': True,\n", + " 'max_value': None,\n", + " 'min_value': None,\n", + " 'shape': ()}}" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config.actions.as_dict()" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:51:06.768136Z", + "start_time": "2017-10-15T01:51:06.673474Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "({'action0': {'epsilon': 1,\n", + " 'epsilon_final': 0.005,\n", + " 'epsilon_timesteps': 100000.0,\n", + " 'start_after': 0,\n", + " 'type': 'epsilon_anneal'},\n", + " 'action1': {'epsilon': 1,\n", + " 'epsilon_final': 0.005,\n", + " 'epsilon_timesteps': 100000.0,\n", + " 'start_after': 0,\n", + " 'type': 'epsilon_anneal'},\n", + " 'action2': {'epsilon': 1,\n", + " 'epsilon_final': 0.005,\n", + " 'epsilon_timesteps': 100000.0,\n", + " 'start_after': 0,\n", + " 'type': 'epsilon_anneal'},\n", + " 'action3': {'epsilon': 1,\n", + " 'epsilon_final': 0.005,\n", + " 'epsilon_timesteps': 100000.0,\n", + " 'start_after': 0,\n", + " 'type': 'epsilon_anneal'},\n", + " 'action4': {'epsilon': 1,\n", + " 'epsilon_final': 0.005,\n", + " 'epsilon_timesteps': 100000.0,\n", + " 'start_after': 0,\n", + " 'type': 'epsilon_anneal'},\n", + " 'action5': {'epsilon': 1,\n", + " 'epsilon_final': 0.005,\n", + " 'epsilon_timesteps': 100000.0,\n", + " 'start_after': 0,\n", + " 'type': 'epsilon_anneal'}},)" + ] + }, + "execution_count": 43, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "exploration=dict(\n", + " type=\"epsilon_anneal\",\n", + " epsilon=1,\n", + " epsilon_final= 0.005,\n", + " epsilon_timesteps= 1e5,\n", + " start_after=0,\n", + ")\n", + "{'action' + str(n): exploration for n in range(env.action_space.shape[0])}," + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:53:21.251258Z", + "start_time": "2017-10-15T01:53:19.469753Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:tensorforce.agents.agent:Configuration values not accessed: first_update, memory_capacity, memory, update_frequency, repeat_update\n", + "[2017-10-15 09:53:21,245] Configuration values not accessed: first_update, memory_capacity, memory, update_frequency, repeat_update\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch_size=256\n", + "exploration=dict(\n", + " type=\"epsilon_anneal\",\n", + " epsilon=1,\n", + " epsilon_final= 0.005,\n", + " epsilon_timesteps= 1e5,\n", + " start_after=0,\n", + ")\n", + "config = Configuration( \n", + " # Each agent requires the following ``Configuration`` parameters:\n", + " # https://github.com/reinforceio/tensorforce/blob/master/tensorforce/agents/agent.py#L32\n", + " network=network,\n", + " states=dict(shape=tuple(env.observation_space.shape), type='float'),\n", + " actions={'action' + str(n): dict(continuous=True) for n in range(env.action_space.shape[0])},\n", + " preprocessing = None,# dict or list containing state preprocessing configuration.\n", + " exploration = {'action' + str(n): exploration for n in range(env.action_space.shape[0])}, # dict containing action exploration configuration.\n", + " \n", + " \n", + " # The `MemoryAgent` class additionally requires the following parameters:\n", + " first_update = batch_size*2, # integer indicating the number of steps to pass before the first update.\n", + " memory_capacity = 300000, # integer of maximum experiences to store. (takes 2s to sample with 100k)\n", + " memory = 'prioritized_replay', # string indicating memory type ('replay' or 'prioritized_replay').\n", + " update_frequency = int(batch_size/2), # integer indicating the number of steps between model updates.\n", + " repeat_update = 2, # integer indicating how often to repeat the model update.\n", + "\n", + " # Each model requires the following configuration parameters:\n", + " # https://github.com/reinforceio/tensorforce/blob/master/tensorforce/models/model.py#L33\n", + " discount = 0.97, # float of discount factor (gamma).\n", + " learning_rate = 1e-3, # float of learning rate (alpha). (3e-4 in paper 1e-3 (atari) and 3e-4 in baselines)\n", + " optimizer = 'adam', # string of optimizer to use (e.g. 'adam' in paper).\n", + " device = None, # string of tensorflow device name.\n", + "# tf_summary = log_dir, # string directory to write tensorflow summaries. Default None\n", + "# tf_summary_level = 1, # int indicating which tensorflow summaries to create.\n", + " tf_summary_interval = 1000, # int number of calls to get_action until writing tensorflow summaries on update.\n", + " log_level = 'info', # string containing log level (e.g. 'info').\n", + " distributed = False, # boolean indicating whether to use distributed tensorflow.\n", + " global_model = False, # global model.\n", + " session = None, # session to use. \n", + "\n", + " # A Policy Gradient Model expects the following additional configuration parameters:\n", + " # https://github.com/reinforceio/tensorforce/blob/master/tensorforce/models/policy_gradient_model.py#L35\n", + " # I edited my tensorflow install to have a flatten layer to make this work (my branch is in requirements.txt)\n", + " baseline=dict(\n", + " type=\"mlp\",\n", + " sizes=[128, 128],\n", + " epochs=1,\n", + " update_batch_size=128,\n", + " learning_rate=0.01\n", + " ), # string indicating the baseline value function (currently 'linear' or 'mlp').\n", + " gae_rewards= True, # boolean indicating whether to use GAE.\n", + " gae_lambda= 0.97, # float of the Generalized Advantage Estimation lambda.\n", + " normalize_rewards= False,# boolean indicating whether to normalize the advantage or not.\n", + " \n", + " # PPO Params \n", + " # https://github.com/reinforceio/tensorforce/blob/master/tensorforce/models/ppo_model.py\n", + " entropy_penalty=0.01, # 0 and 0.01 in baselines\n", + " loss_clipping=0.1, # Trust region clipping\n", + " epochs=4, # Number of training epochs for SGD, data is repeated this much 4 (atari),10 in baselines, 10 in paper\n", + " optimizer_batch_size=32, # Batch size for optimiser, should be small (e.g. 64 in paper)\n", + " random_sampling=True # Sampling strategy for minibatch replay memory\n", + ")\n", + "\n", + "# Create a Trust Region Policy Optimization agent\n", + "agent = PPOAgent(config=config)\n", + "agent" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:53:53.000937Z", + "start_time": "2017-10-15T01:53:52.906552Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:53:56.906721Z", + "start_time": "2017-10-15T01:53:56.825652Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "def sharpe(returns, freq=30, rfr=0):\n", + " \"\"\"Given a set of returns, calculates naive (rfr=0) sharpe (eq 28) \"\"\"\n", + " return (np.sqrt(freq) * np.mean(returns-rfr)) / np.std(returns - rfr)\n", + "\n", + "def MDD(returns):\n", + " \"\"\"Max drawdown.\"\"\"\n", + " peak = returns.max()\n", + " i = returns.argmax()\n", + " trough = returns[returns.argmax():].min()\n", + " return (trough-peak)/trough " + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:53:57.245985Z", + "start_time": "2017-10-15T01:53:57.163430Z" + }, + "code_folding": [], + "collapsed": true + }, + "outputs": [], + "source": [ + "# Callback function printing episode statistics\n", + "class EpisodeFinished(object):\n", + " \"\"\"Logger callback for tensorforce runner\"\"\"\n", + " \n", + " def __init__(self, log_intv):\n", + " self.log_intv = log_intv\n", + " self.portfolio_values = [] \n", + " self.mdds=[]\n", + " self.sharpes=[]\n", + " \n", + " def __call__(self, r):\n", + " if len(r.environment.gym.sim.infos):\n", + " self.portfolio_values.append(r.environment.gym.sim.infos[-1]['portfolio_value'])\n", + " \n", + " df = pd.DataFrame(r.environment.gym.sim.infos)\n", + " self.mdds.append(MDD(df.rate_of_return+1))\n", + " self.sharpes.append(sharpe(df.rate_of_return))\n", + " if r.episode % self.log_intv == 0:\n", + " print(\n", + " \"Finished episode {ep} after {ts} timesteps (reward: {reward: 2.4f} [{rewards_min: 2.4f}, {rewards_max: 2.4f}]) portfolio_value: {portfolio_value: 2.4f} [{portfolio_value_min: 2.4f}, {portfolio_value_max: 2.4f}] mdd={mdd: 2.2%} sharpe={sharpe: 2.2f}\".\n", + " format(\n", + " ep=r.episode,\n", + " ts=r.timestep,\n", + " reward=np.mean(r.episode_rewards[-self.log_intv:]),\n", + " rewards_min=np.min(r.episode_rewards[-self.log_intv:]),\n", + " rewards_max=np.max(r.episode_rewards[-self.log_intv:]),\n", + " portfolio_value=np.mean(self.portfolio_values[-self.log_intv:]),\n", + " portfolio_value_min=np.min(self.portfolio_values[-self.log_intv:]),\n", + " portfolio_value_max=np.max(self.portfolio_values[-self.log_intv:]),\n", + " mdd=np.mean(self.mdds[-self.log_intv:]),\n", + " sharpe=np.mean(self.sharpes[-self.log_intv:]),\n", + " )\n", + " )\n", + " return True" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:53:57.934576Z", + "start_time": "2017-10-15T01:53:57.833877Z" + }, + "code_folding": [], + "collapsed": true + }, + "outputs": [], + "source": [ + "# Callback EpisodeFinishedTQDM\n", + "from tqdm import tqdm_notebook\n", + "class EpisodeFinishedTQDM(EpisodeFinished):\n", + " \"\"\"Logger for tensorforce using tqdm_notebook for jupyter-notebook.\"\"\"\n", + " \n", + " def __init__(self, episodes, log_intv):\n", + " \"\"\"\n", + " log_intv - print the mean metrics every log_intv episodes\n", + " \"\"\"\n", + " super().__init__(log_intv=log_intv)\n", + " self.episodes = episodes\n", + " self.progbar = tqdm_notebook(desc='', \n", + " total=episodes, \n", + " leave=True, mininterval=5)\n", + " \n", + " def __call__(self, r):\n", + " super().__call__(r)\n", + " desc = \"reward: {reward: 2.4f} [{rewards_min: 2.4f}, {rewards_max: 2.4f}], portfolio_value: {portfolio_value: 2.4f} [{portfolio_value_min: 2.4f}, {portfolio_value_max: 2.4f}]\". format(\n", + " reward=np.mean(r.episode_rewards[-1:]),\n", + " rewards_min=np.min(r.episode_rewards[-1:]),\n", + " rewards_max=np.max(r.episode_rewards[-1:]),\n", + " portfolio_value=np.mean(self.portfolio_values[-1:]),\n", + " portfolio_value_min=np.min(self.portfolio_values[-1:]),\n", + " portfolio_value_max=np.max(self.portfolio_values[-1:])\n", + " )\n", + " self.progbar.desc = desc\n", + " self.progbar.update(1) # update\n", + " return True" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:53:59.814205Z", + "start_time": "2017-10-15T01:53:59.729623Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "from tensorforce.execution import Runner\n", + "runner = Runner(agent=agent, environment=environment, save_path=save_path, save_episodes=10000)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:54:00.406978Z", + "start_time": "2017-10-15T01:54:00.346179Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "#agent.load_model(save_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2017-10-15T01:54:00.897Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c02d07f406244008ab9da56c090f0355" + } + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "episodes=int(6e6/30)\n", + "runner.run(\n", + " episodes=episodes, max_timesteps=200, episode_finished=EpisodeFinishedTQDM(log_intv=1000, episodes=episodes))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:46.508299Z", + "start_time": "2017-10-15T01:42:13.091Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# save\n", + "agent.save_model(save_path)\n", + "save_path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-07-19T00:09:54.262405Z", + "start_time": "2017-07-19T08:09:54.226639+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:46.508945Z", + "start_time": "2017-10-15T01:42:13.093Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# one big test\n", + "df_test = pd.read_hdf('./data/poloniex_30m.hf',key='test')\n", + "steps=2400#len(df_test)-window_length-2\n", + "env_test = EnvWrapper(\n", + " df=df_test,\n", + " steps=steps, \n", + " scale=True, \n", + " augment=0.00,\n", + " trading_cost=0, # let just overfit first\n", + " window_length=window_length,\n", + ")\n", + "env_test.seed = 0 \n", + "environment_test = OpenAIGym('CartPole-v0')\n", + "environment_test.gym = env_test\n", + "\n", + "agent.load_model(save_path)\n", + "runner_test = Runner(agent=agent, environment=environment_test)\n", + "runner_test.run(\n", + "episodes=1, max_timesteps=steps, episode_finished=EpisodeFinished(10))\n", + "\n", + "df = pd.DataFrame(env_test.infos)\n", + "df.index=df['index']\n", + "\n", + "s=sharpe(df.rate_of_return+1)\n", + "mdd=MDD(df.rate_of_return+1)\n", + "apv=df.portfolio_value.iloc[-1]\n", + "print('APV (Accumulated portfolio value): \\t{: 2.6f}'.format(apv))\n", + "print('SR (Sharpe ratio): \\t{: 2.6f}'.format( s))\n", + "print('MDD (max drawdown): \\t{: 2.6%}'.format( mdd))\n", + "print('MMR (mean market returns): \\t{: 2.6f}'.format(df.mean_market_returns.cumprod().iloc[-1]))\n", + "print('')\n", + "\n", + "# show one run vs average market performance\n", + "plt.title('test MDD={}, Sharpe={}, APV={}'.format(mdd,s,apv))\n", + "df.portfolio_value.plot()\n", + "df.mean_market_returns.cumprod().plot(label='mean market performance')\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-07-19T00:48:39.193976Z", + "start_time": "2017-07-19T08:48:39.154752+08:00" + }, + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:46.509761Z", + "start_time": "2017-10-15T01:42:13.095Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "\n", + "\n", + "data=[]\n", + "for i in range(10):\n", + " agent.load_model(save_path)\n", + " df_test = pd.read_hdf('./data/poloniex_30m.hf',key='test')\n", + " \n", + " env_test = EnvWrapper(\n", + " df=df_test,\n", + " steps=1800, \n", + " scale=True, \n", + " augment=0.00,\n", + " trading_cost=0, # let just overfit first\n", + " window_length=window_length,\n", + " )\n", + " env_test.seed = 0 \n", + "\n", + "\n", + " environment_test = OpenAIGym('CartPole-v0')\n", + " environment_test.gym = env_test\n", + "\n", + " runner_test = Runner(agent=agent, environment=environment_test)\n", + " np.random.seed(i)\n", + " runner_test.run(\n", + " episodes=2, max_timesteps=32, episode_finished=EpisodeFinished(10))\n", + " df = pd.DataFrame(environment_test.gym.infos)\n", + "# df.index=df['index']\n", + " \n", + " s=sharpe(df.rate_of_return+1)\n", + " mdd=MDD(df.rate_of_return+1)\n", + " data.append(dict(sharpe=s,mdd=mdd))\n", + " print('APV (Accumulated portfolio value): \\t{: 2.6f}'.format(df.portfolio_value.iloc[-1]))\n", + " print('SR (Sharpe ratio): \\t{: 2.6f}'.format( s))\n", + " print('MDD (max drawdown): \\t{: 2.6%}'.format( mdd))\n", + " print('MMR (mean market returns): \\t{: 2.6f}'.format(df.mean_market_returns.cumprod().iloc[-1]))\n", + " print('')\n", + " df.portfolio_value.plot(label=str(i))\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:46.510600Z", + "start_time": "2017-10-15T01:42:13.096Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2017-10-15T01:42:46.511486Z", + "start_time": "2017-10-15T01:42:13.098Z" + }, + "collapsed": true + }, + "outputs": [], + "source": [ + "# one big test over train\n", + "# one big test\n", + "df_train = pd.read_hdf('./data/poloniex_30m.hf',key='train')\n", + "steps=len(df_train)-window_length-2\n", + "env = EnvWrapper(\n", + " df=df_train,\n", + " steps=steps, \n", + " scale=True, \n", + " augment=0.00,\n", + " trading_cost=0, # let just overfit first\n", + " window_length=window_length,\n", + ")\n", + "env.seed = 0 \n", + "environment = OpenAIGym('CartPole-v0')\n", + "environment.gym = env\n", + "\n", + "agent.load_model(save_path)\n", + "runner = Runner(agent=agent, environment=environment)\n", + "runner.run(\n", + "episodes=1, max_timesteps=steps, episode_finished=EpisodeFinished(10))\n", + "\n", + "df = pd.DataFrame(env.infos)\n", + "df.index=df['index']\n", + "\n", + "s=sharpe(df.rate_of_return+1)\n", + "mdd=MDD(df.rate_of_return+1)\n", + "data.append(dict(sharpe=s,mdd=mdd))\n", + "print('APV (Accumulated portfolio value): \\t{: 2.6f}'.format(df.portfolio_value.iloc[-1]))\n", + "print('SR (Sharpe ratio): \\t{: 2.6f}'.format( s))\n", + "print('MDD (max drawdown): \\t{: 2.6%}'.format( mdd))\n", + "print('')\n", + "\n", + "# show one run vs average market performance\n", + "plt.title('train')\n", + "df.portfolio_value.plot()\n", + "df.mean_market_returns.cumprod().plot(label='mean market performance')\n", + "plt.legend()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "jupyter3", + "language": "python", + "name": "jupyter3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}