rl-portfolio-management/tensorforce-VPG.ipynb

In [1]:

# plotting
%matplotlib inline
from matplotlib import pyplot as plt
import seaborn as sns
plt.style.use('ggplot')

# numeric
import numpy as np
from numpy import random
import pandas as pd

# util
from collections import Counter
import pdb
import time
import tempfile
import itertools
from tqdm import tqdm_notebook as tqdm
import datetime

# logging
import logging
logger = log = logging.getLogger(__name__)
# log.setLevel(logging.INFO)
logging.basicConfig()
log.info('%s logger started.', __name__)

In [2]:

import gym
from gym import error, spaces, utils
from gym.utils import seeding

In [3]:

import os
os.sys.path.append(os.path.abspath('.'))
%reload_ext autoreload
%autoreload 2

In [4]:

# params
window_length = 50

save_path = './outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model'
save_path

Out [4]:

'./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model'

Enviroment

In [5]:

from rl_portfolio_management.environments.portfolio import PortfolioEnv

In [6]:

from rl.memory import  SequentialMemory, Memory
from collections import deque

class EnvWrapper(PortfolioEnv):
    """Wraps env to normalise and reshape action."""
    def __init__(self, window_length=50, *args, **kwargs):
        super().__init__(*args, **kwargs)
        
    def step(self, action):
        # also it puts it in a list
        if isinstance(action, list):
            action = action[0]
        
        # we have to normalise for some reason softmax wont work
        if isinstance(action, dict):
            action = np.abs(list(action.values()))
            action /= action.sum()        
        
        return super().step(action) 

In [7]:

df_train = pd.read_hdf('./data/poloniex_30m.hf',key='train')
env = EnvWrapper(
    df=df_train,
    steps=30, 
    scale=True, 
    augment=0.000,
    trading_cost=0, # let just overfit first,
    window_length = window_length,
    
)
env.seed = 0   

df_test = pd.read_hdf('./data/poloniex_30m.hf',key='test')
env_test = EnvWrapper(
    df=df_test,
    steps=30, 
    scale=True, 
    augment=0.00,
    trading_cost=0, # let just overfit first
    window_length=window_length,
)
env_test.seed = 0  

from tensorforce.environments.openai_gym import OpenAIGym
environment = OpenAIGym('CartPole-v0')
environment.gym = env

environment_test = OpenAIGym('CartPole-v0')
environment_test.gym = env_test

INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-21 14:29:14,628] Making new env: CartPole-v0
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-21 14:29:14,638] Making new env: CartPole-v0

In [8]:

# check shapes
obs1, reward, done, info=env.step(np.random.random(env.action_space.shape))
print(reward, done, info)
obs2 = env.reset()
print(obs1.shape,obs2.shape)

0.0 False {'reward': 0.0, 'log_return': 0.0, 'portfolio_value': 1.0, 'rate_of_return': 0.0, 'cost': 0.0, 'steps': 2, 'cash_bias': 0.18594253920945863, 'mean_market_returns': 1.0024999885200685}
(5, 50, 3) (5, 50, 3)

Model

Derived from https://github.com/reinforceio/tensorforce/blob/0d07fadec03f76537a2431e17c51cd759d53b5e9/tensorforce/core/networks/layers.py#L90

In [9]:

from tensorforce import Configuration
from tensorforce.agents import VPGAgent
from tensorforce.core.networks import layered_network_builder

In [10]:

# layer helpers from:
# https://github.com/reinforceio/tensorforce/blob/0d07fadec03f76537a2431e17c51cd759d53b5e9/tensorforce/core/networks/layers.py#L90
import tensorflow as tf
from math import sqrt
from tensorforce import util
from tensorforce import TensorForceError

def linear(x, size, bias=True, l2_regularization=0.0):
    if util.rank(x) != 2:
        raise TensorForceError('Invalid input rank for linear layer.')
    with tf.variable_scope('linear'):
        weights = tf.Variable(initial_value=tf.random_normal(shape=(x.get_shape()[1].value, size), stddev=sqrt(2.0 / (x.get_shape()[1].value + size))))
        if l2_regularization > 0.0:
            tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=weights))
        x = tf.matmul(a=x, b=weights)
        if bias:
            bias = tf.Variable(initial_value=tf.zeros(shape=(size,)))
            if l2_regularization > 0.0:
                tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=bias))
            x = tf.nn.bias_add(value=x, bias=bias)
    return x

def nonlinearity(x, name='relu'):
    with tf.variable_scope('nonlinearity'):
        if name == 'elu':
            x = tf.nn.elu(features=x)
        elif name == 'relu':
            x = tf.nn.relu(features=x)
        elif name == 'selu':
            # https://arxiv.org/pdf/1706.02515.pdf
            alpha = 1.6732632423543772848170429916717
            scale = 1.0507009873554804934193349852946
            negative = alpha * tf.nn.elu(features=x)
            x = scale * tf.where(condition=(x >= 0.0), x=x, y=negative)
        elif name == 'sigmoid':
            x = tf.sigmoid(x=x)
        elif name == 'softmax':
            x = tf.nn.softmax(logits=x)
        elif name == 'tanh':
            x = tf.nn.tanh(x=x)
        else:
            raise TensorForceError('Invalid nonlinearity.')
    return x

def dense(x, size, bias=True, activation='relu', l2_regularization=0.0):
    if util.rank(x) != 2:
        raise TensorForceError('Invalid input rank for dense layer.')
    with tf.variable_scope('dense'):
        x = linear(x=x, size=size, bias=bias, l2_regularization=l2_regularization)
        x = nonlinearity(x=x, name=activation)
    return x

def flatten(x):
    with tf.variable_scope('flatten'):
        x = tf.reshape(tensor=x, shape=(-1, util.prod(x.get_shape().as_list()[1:])))
    return x

def conv2d(x, size, window=(3,3), stride=(1,1), bias=False, activation='relu', l2_regularization=0.0, padding='SAME'):
    if util.rank(x) != 4:
        raise TensorForceError('Invalid input rank for conv2d layer.')
    with tf.variable_scope('conv2d'):
        filters = tf.Variable(initial_value=tf.random_normal(shape=(window[0], window[1], x.get_shape()[3].value, size), stddev=sqrt(2.0 / size)))
        if l2_regularization > 0.0:
            tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=filters))
        x = tf.nn.conv2d(input=x, filter=filters, strides=(1, stride[0], stride[1], 1), padding=padding)
        if bias:
            bias = tf.Variable(initial_value=tf.zeros(shape=(size,)))
            if l2_regularization > 0.0:
                tf.losses.add_loss(l2_regularization * tf.nn.l2_loss(t=bias))
            x = tf.nn.bias_add(value=x, bias=bias)
        x = nonlinearity(x=x, name=activation)
    return x

In [11]:

# build a network for a given input
def network_builder(inputs):
    if len(inputs) != 1:
        raise TensorForceError('Layered network must have only one input.')
    x = next(iter(inputs.values()))
    
    x = conv2d(x=x, size=2, window=(1,3), bias=True, activation='relu', l2_regularization=1e-8, padding='VALID')
    x = conv2d(x=x, size=20, window=(1,window_length-2), bias=True, activation='relu', l2_regularization=1e-8, padding='VALID')
    x = conv2d(x=x, size=1, window=(1,1), bias=True, activation='relu', l2_regularization=1e-8, padding='VALID')
    x = flatten(x)
    x = dense(x, size=env.action_space.shape[0],activation='relu', l2_regularization=1e-8)
    x = nonlinearity(x,name='softmax')
    
    return x
network=network_builder

Agent

In [12]:

config = Configuration(   
    # Each model requires the following configuration parameters:
    discount=0,  # float of discount factor (gamma).
    learning_rate=3e-5,  # float of learning rate (alpha).
    optimizer='adam',  # string of optimizer to use (e.g. 'adam').
#     optimizer_args= ,  # list of arguments for optimizer.
#     optimizer_kwargs= ,  # dict of keyword arguments for optimizer.
#     device= ,  # string of tensorflow device name.
    tf_saver=True,  # boolean whether to save model parameters.
#     tf_summary= ,  # boolean indicating whether to use tensorflow summary file writer.
    log_level='info',  # string containing logleve (e.g. 'info').
    distributed=False ,  # boolean indicating whether to use distributed tensorflow.
#     global_model= ,  # global model.
#     session= ,  # session to use.
    
    # Each agent requires the following ``Configuration`` parameters:
    network=network,
    states=dict(shape=tuple(env.observation_space.shape), type='float'),
    actions={'action' + str(n): dict(continuous=True) for n in range(env.action_space.shape[0])},
    preprocessing = None,# dict or list containing state preprocessing configuration.
    exploration = dict(
        type='EpsilonDecay',
        kwargs=dict(epsilon=1, epsilon_final=0.01, epsilon_timesteps=1e4)
    ),

    # The `BatchAgent` class additionally requires the following parameters:
    batch_size = 50,# integer of the batch size.

    # A Policy Gradient Model expects the following additional configuration parameters:
    sample_actions= True,# boolean of whether to sample actions.
#     baseline='mlp' ,# string indicating the baseline value function (currently 'linear' or 'mlp').
#     baseline_args=dict(size=100, repeat_update=100) ,# list of arguments for the baseline value function.
    override_line_search=False,
    
#     baseline_kwargs= ,# dict of keyword arguments for the baseline value function.
    generalized_advantage_estimation= True ,# boolean indicating whether to use GAE.
    gae_lambda= 0.97,# float of the Generalized Advantage Estimation lambda.
    normalize_advantage= False,# boolean indicating whether to normalize the advantage or not.
)

# Create a Trust Region Policy Optimization agent
agent = VPGAgent(config=config)

Train

In [13]:

def sharpe(returns, freq=30, rfr=0):
    """Given a set of returns, calculates naive (rfr=0) sharpe (eq 28) """
    return (np.sqrt(freq) * np.mean(returns-rfr)) / np.std(returns - rfr)

def MDD(returns):
    """Max drawdown."""
    peak = returns.max()
    i = returns.argmax()
    trough = returns[returns.argmax():].min()
    return (trough-peak)/trough 

In [14]:

# Callback function printing episode statistics
class EpisodeFinished(object):
    """Logger callback for tensorforce runner"""
    
    def __init__(self, log_intv):
        self.log_intv = log_intv
        self.portfolio_values = [] 
        self.mdds=[]
        self.sharpes=[]
    
    def __call__(self, r):
        if len(r.environment.gym.sim.infos):
            self.portfolio_values.append(r.environment.gym.sim.infos[-1]['portfolio_value'])
            
            df = pd.DataFrame(r.environment.gym.sim.infos)
            self.mdds.append(MDD(df.rate_of_return+1))
            self.sharpes.append(sharpe(df.rate_of_return))
        if r.episode % self.log_intv == 0:
            print(
                "Finished episode {ep} after {ts} timesteps (reward: {reward: 2.4f} [{rewards_min: 2.4f}, {rewards_max: 2.4f}]) portfolio_value: {portfolio_value: 2.4f} [{portfolio_value_min: 2.4f}, {portfolio_value_max: 2.4f}] mdd={mdd: 2.2%} sharpe={sharpe: 2.2f}".
                format(
                    ep=r.episode,
                    ts=r.timestep,
                    reward=np.mean(r.episode_rewards[-self.log_intv:]),
                    rewards_min=np.min(r.episode_rewards[-self.log_intv:]),
                    rewards_max=np.max(r.episode_rewards[-self.log_intv:]),
                    portfolio_value=np.mean(self.portfolio_values[-self.log_intv:]),
                    portfolio_value_min=np.min(self.portfolio_values[-self.log_intv:]),
                    portfolio_value_max=np.max(self.portfolio_values[-self.log_intv:]),
                    mdd=np.mean(self.mdds[-self.log_intv:]),
                    sharpe=np.mean(self.sharpes[-self.log_intv:]),
                )
            )
        return True

In [15]:

# Callback EpisodeFinishedTQDM
from tqdm import tqdm_notebook
class EpisodeFinishedTQDM(EpisodeFinished):
    """Logger for tensorforce using tqdm_notebook for jupyter-notebook."""
    
    def __init__(self, episodes, log_intv):
        """
        log_intv - print the mean metrics every log_intv episodes
        """
        super().__init__(log_intv=log_intv)
        self.episodes = episodes
        self.progbar = tqdm_notebook(desc='', 
                                     total=episodes, 
                                     leave=True, mininterval=5)
    
    def __call__(self, r):
        super().__call__(r)
        desc = "reward: {reward: 2.4f} [{rewards_min: 2.4f}, {rewards_max: 2.4f}], portfolio_value: {portfolio_value: 2.4f} [{portfolio_value_min: 2.4f}, {portfolio_value_max: 2.4f}]".        format(
            reward=np.mean(r.episode_rewards[-1:]),
            rewards_min=np.min(r.episode_rewards[-1:]),
            rewards_max=np.max(r.episode_rewards[-1:]),
            portfolio_value=np.mean(self.portfolio_values[-1:]),
            portfolio_value_min=np.min(self.portfolio_values[-1:]),
            portfolio_value_max=np.max(self.portfolio_values[-1:])
        )
        self.progbar.desc = desc
        self.progbar.update(1)  # update
        return True

In [16]:

from tensorforce.execution import Runner
runner = Runner(agent=agent, environment=environment, save_path=save_path, save_episodes=10000)

In [17]:

agent.load_model(save_path)

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:06:02,111] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

In [18]:

episodes=int(2e6/30)
runner.run(
    episodes=episodes, max_timesteps=200, episode_finished=EpisodeFinishedTQDM(log_intv=1000, episodes=episodes))

[Data output - unsupported data type map[string]interface {} for mime type application/vnd.jupyter.widget-view+json]

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-18-53b610d06478> in <module>()
      1 episodes=int(2e6/30)
      2 runner.run(
----> 3     episodes=episodes, max_timesteps=200, episode_finished=EpisodeFinishedTQDM(log_intv=1000, episodes=episodes))

/media/isisilon/Data/linuxOpt/tensorforce/tensorforce/execution/runner.py in run(self, episodes, max_timesteps, episode_finished)
    123         self.episode = 1
    124         while True:
--> 125             state = self.environment.reset()
    126             self.agent.reset()
    127             episode_reward = 0

/media/isisilon/Data/linuxOpt/tensorforce/tensorforce/environments/openai_gym.py in reset(self)
     60 
     61     def reset(self):
---> 62         return self.gym.reset()
     63 
     64     def execute(self, action):

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/gym/core.py in reset(self)
    102             space.
    103         """
--> 104         return self._reset()
    105 
    106     def render(self, mode='human', close=False):

/media/isisilon/Data/My_Documents/Documents/eclipse-workspace/rl_keras_finance/rl-portfolio-management/src/environments/portfolio.py in _reset(self)
    250     def _reset(self):
    251         self.sim.reset()
--> 252         self.src.reset()
    253         self.infos = []
    254         action = self.sim.w0

/media/isisilon/Data/My_Documents/Documents/eclipse-workspace/rl_keras_finance/rl-portfolio-management/src/environments/portfolio.py in reset(self)
     66 
     67         # augment data to prevent overfitting
---> 68         data = data.apply(lambda x: random_shift(x, self.augment))
     69 
     70         self.data = data

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in apply(self, func, axis, broadcast, raw, reduce, args, **kwds)
   4260                         f, axis,
   4261                         reduce=reduce,
-> 4262                         ignore_failures=ignore_failures)
   4263             else:
   4264                 return self._apply_broadcast(f, axis)

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in _apply_standard(self, func, axis, ignore_failures, reduce)
   4373                 index = None
   4374 
-> 4375             result = self._constructor(data=results, index=index)
   4376             result.columns = res_index
   4377 

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in __init__(self, data, index, columns, dtype, copy)
    273                                  dtype=dtype, copy=copy)
    274         elif isinstance(data, dict):
--> 275             mgr = self._init_dict(data, index, columns, dtype=dtype)
    276         elif isinstance(data, ma.MaskedArray):
    277             import numpy.ma.mrecords as mrecords

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in _init_dict(self, data, index, columns, dtype)
    409             arrays = [data[k] for k in keys]
    410 
--> 411         return _arrays_to_mgr(arrays, data_names, index, columns, dtype=dtype)
    412 
    413     def _init_ndarray(self, values, index, columns, dtype=None, copy=False):

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in _arrays_to_mgr(arrays, arr_names, index, columns, dtype)
   5504     axes = [_ensure_index(columns), _ensure_index(index)]
   5505 
-> 5506     return create_block_manager_from_arrays(arrays, arr_names, axes)
   5507 
   5508 

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/internals.py in create_block_manager_from_arrays(arrays, names, axes)
   4307 
   4308     try:
-> 4309         blocks = form_blocks(arrays, names, axes)
   4310         mgr = BlockManager(blocks, axes)
   4311         mgr._consolidate_inplace()

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/internals.py in form_blocks(arrays, names, axes)
   4371     blocks = []
   4372     if len(float_items):
-> 4373         float_blocks = _multi_blockify(float_items)
   4374         blocks.extend(float_blocks)
   4375 

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/internals.py in _multi_blockify(tuples, dtype)
   4448     for dtype, tup_block in grouper:
   4449 
-> 4450         values, placement = _stack_arrays(list(tup_block), dtype)
   4451 
   4452         block = make_block(values, placement=placement)

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/internals.py in _stack_arrays(tuples, dtype)
   4493     stacked = np.empty(shape, dtype=dtype)
   4494     for i, arr in enumerate(arrays):
-> 4495         stacked[i] = _asarray_compat(arr)
   4496 
   4497     return stacked, placement

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/internals.py in _asarray_compat(x)
   4475     # fml
   4476     def _asarray_compat(x):
-> 4477         if isinstance(x, ABCSeries):
   4478             return x._values
   4479         else:

KeyboardInterrupt: 

In [19]:

# save
agent.save_model(save_path)
save_path

Out [19]:

'./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model'

Test

In [ ]:

In [16]:

# one big test
df_test = pd.read_hdf('./data/poloniex_30m.hf',key='test')
steps=2400#len(df_test)-window_length-2
env_test = EnvWrapper(
    df=df_test,
    steps=steps, 
    scale=True, 
    augment=0.00,
    trading_cost=0, # let just overfit first
    window_length=window_length,
)
env_test.seed = 0  
environment_test = OpenAIGym('CartPole-v0')
environment_test.gym = env_test

agent.load_model(save_path)
runner_test = Runner(agent=agent, environment=environment_test)
runner_test.run(
episodes=1, max_timesteps=steps, episode_finished=EpisodeFinished(10))

df = pd.DataFrame(env_test.infos)
df.index=df['index']

s=sharpe(df.rate_of_return+1)
mdd=MDD(df.rate_of_return+1)
apv=df.portfolio_value.iloc[-1]
print('APV (Accumulated portfolio value): \t{: 2.6f}'.format(apv))
print('SR (Sharpe ratio):                 \t{: 2.6f}'.format( s))
print('MDD (max drawdown):                \t{: 2.6%}'.format( mdd))
print('MMR (mean market returns):         \t{: 2.6f}'.format(df.mean_market_returns.cumprod().iloc[-1]))
print('')

# show one run vs average market performance
plt.title('test MDD={}, Sharpe={}, APV={}'.format(mdd,s,apv))
df.portfolio_value.plot()
df.mean_market_returns.cumprod().plot(label='mean market performance')
plt.legend()

INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-21 14:30:47,675] Making new env: CartPole-v0

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-21 14:30:47,680] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-16-61c933e5fe09> in <module>()
     15 
     16 agent.load_model(save_path)
---> 17 runner_test = Runner(agent=agent, environment=environment_test)
     18 runner_test.run(
     19 episodes=1, max_timesteps=steps, episode_finished=EpisodeFinished(10))

NameError: name 'Runner' is not defined

In [ ]:

In [27]:

data=[]
for i in range(10):
    agent.load_model(save_path)
    df_test = pd.read_hdf('./data/poloniex_30m.hf',key='test')
    
    env_test = EnvWrapper(
        df=df_test,
        steps=1800, 
        scale=True, 
        augment=0.00,
        trading_cost=0, # let just overfit first
        window_length=window_length,
    )
    env_test.seed = 0  


    environment_test = OpenAIGym('CartPole-v0')
    environment_test.gym = env_test

    runner_test = Runner(agent=agent, environment=environment_test)
    np.random.seed(i)
    runner_test.run(
    episodes=2, max_timesteps=32, episode_finished=EpisodeFinished(10))
    df = pd.DataFrame(environment_test.gym.infos)
#     df.index=df['index']
    
    s=sharpe(df.rate_of_return+1)
    mdd=MDD(df.rate_of_return+1)
    data.append(dict(sharpe=s,mdd=mdd))
    print('APV (Accumulated portfolio value): \t{: 2.6f}'.format(df.portfolio_value.iloc[-1]))
    print('SR (Sharpe ratio):                 \t{: 2.6f}'.format( s))
    print('MDD (max drawdown):                \t{: 2.6%}'.format( mdd))
    print('MMR (mean market returns):         \t{: 2.6f}'.format(df.mean_market_returns.cumprod().iloc[-1]))
    print('')
    df.portfolio_value.plot(label=str(i))
plt.legend()

Out [27]:

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:09,124] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:09,195] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 1.034579
SR (Sharpe ratio):                 	 322.898386
MDD (max drawdown):                	-7.676194%
MMR (mean market returns):         	 1.065456

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:09,803] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:09,873] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 0.962914
SR (Sharpe ratio):                 	 796.956403
MDD (max drawdown):                	-2.820202%
MMR (mean market returns):         	 0.979284

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:10,370] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:10,453] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 0.960063
SR (Sharpe ratio):                 	 692.779292
MDD (max drawdown):                	-2.389813%
MMR (mean market returns):         	 0.971125

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:11,027] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:11,106] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 1.030753
SR (Sharpe ratio):                 	 970.241430
MDD (max drawdown):                	-1.869627%
MMR (mean market returns):         	 1.050944

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:11,638] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:11,703] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 1.002456
SR (Sharpe ratio):                 	 863.774706
MDD (max drawdown):                	-1.959431%
MMR (mean market returns):         	 1.009633

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:12,236] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:12,305] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 0.943495
SR (Sharpe ratio):                 	 466.614285
MDD (max drawdown):                	-5.269238%
MMR (mean market returns):         	 0.968954

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:12,882] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:12,958] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 0.905622
SR (Sharpe ratio):                 	 551.573112
MDD (max drawdown):                	-3.440875%
MMR (mean market returns):         	 0.949998

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:13,478] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:13,557] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 1.020469
SR (Sharpe ratio):                 	 822.388480
MDD (max drawdown):                	-3.791417%
MMR (mean market returns):         	 1.016669

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:14,067] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:14,162] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 1.016870
SR (Sharpe ratio):                 	 637.664818
MDD (max drawdown):                	-3.573840%
MMR (mean market returns):         	 1.006212

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:13:14,642] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:13:14,716] Making new env: CartPole-v0

APV (Accumulated portfolio value): 	 1.061972
SR (Sharpe ratio):                 	 891.327791
MDD (max drawdown):                	-1.853956%
MMR (mean market returns):         	 1.058966

<matplotlib.legend.Legend at 0x7f0e2cdb2c18>

In [ ]:

data

In [ ]:

In [22]:

# one big test over train
# one big test
df_train = pd.read_hdf('./data/poloniex_30m.hf',key='train')
steps=len(df_train)-window_length-2
env = EnvWrapper(
    df=df_train,
    steps=steps, 
    scale=True, 
    augment=0.00,
    trading_cost=0, # let just overfit first
    window_length=window_length,
)
env.seed = 0  
environment = OpenAIGym('CartPole-v0')
environment.gym = env

agent.load_model(save_path)
runner = Runner(agent=agent, environment=environment)
runner.run(
episodes=1, max_timesteps=steps, episode_finished=EpisodeFinished(10))

df = pd.DataFrame(env.infos)
df.index=df['index']

s=sharpe(df.rate_of_return+1)
mdd=MDD(df.rate_of_return+1)
data.append(dict(sharpe=s,mdd=mdd))
print('APV (Accumulated portfolio value): \t{: 2.6f}'.format(df.portfolio_value.iloc[-1]))
print('SR (Sharpe ratio):                 \t{: 2.6f}'.format( s))
print('MDD (max drawdown):                \t{: 2.6%}'.format( mdd))
print('')

# show one run vs average market performance
plt.title('train')
df.portfolio_value.plot()
df.mean_market_returns.cumprod().plot(label='mean market performance')
plt.legend()

INFO:gym.envs.registration:Making new env: CartPole-v0
[2017-07-19 17:07:07,951] Making new env: CartPole-v0

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

INFO:tensorflow:Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model
[2017-07-19 17:07:07,954] Restoring parameters from ./outputs/tensorforce-VPG/tensorforce-VPG_20170717_04-42-55.model

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-22-7c12755d4551> in <module>()
     18 runner = Runner(agent=agent, environment=environment)
     19 runner.run(
---> 20 episodes=1, max_timesteps=steps, episode_finished=EpisodeFinished(10))
     21 
     22 df = pd.DataFrame(env.infos)

/media/isisilon/Data/linuxOpt/tensorforce/tensorforce/execution/runner.py in run(self, episodes, max_timesteps, episode_finished)
    139                             break
    140                 else:
--> 141                     state, reward, terminal = self.environment.execute(action=action)
    142 
    143                 self.agent.observe(reward=reward, terminal=terminal)

/media/isisilon/Data/linuxOpt/tensorforce/tensorforce/environments/openai_gym.py in execute(self, action)
     65         if isinstance(self.gym.action_space, gym.spaces.Box):
     66             action = [action]  # some gym environments expect a list (f.i. Pendulum-v0)
---> 67         state, reward, terminal, _ = self.gym.step(action)
     68         return state, reward, terminal
     69 

<ipython-input-6-368e3da9ea5a> in step(self, action)
     17             action /= action.sum()
     18 
---> 19         return super().step(action)

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/gym/core.py in step(self, action)
     94             info (dict): contains auxiliary diagnostic information (helpful for debugging, and sometimes learning)
     95         """
---> 96         return self._step(action)
     97 
     98     def reset(self):

/media/isisilon/Data/My_Documents/Documents/eclipse-workspace/rl_keras_finance/rl-portfolio-management/src/environments/portfolio.py in _step(self, action)
    226             np.sum(weights), 1.0, 3, err_msg='weights should sum to 1. action="%s"' % weights)
    227 
--> 228         observation, done1 = self.src._step()
    229 
    230         y1 = observation[:, -1, 0]  # relative price vector (open/close)

/media/isisilon/Data/My_Documents/Documents/eclipse-workspace/rl_keras_finance/rl-portfolio-management/src/environments/portfolio.py in _step(self)
     47         if self.scale:
     48             open = data_window.xs('open', axis=1, level='Price')
---> 49             data_window = data_window.divide(open.iloc[-1], level='Pair')
     50             data_window = data_window.drop('open', axis=1, level='Price')
     51 

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/ops.py in f(self, other, axis, level, fill_value)
   1234             return self._combine_frame(other, na_op, fill_value, level)
   1235         elif isinstance(other, ABCSeries):
-> 1236             return self._combine_series(other, na_op, fill_value, axis, level)
   1237         else:
   1238             if fill_value is not None:

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in _combine_series(self, other, func, fill_value, axis, level)
   3502             else:
   3503                 return self._combine_match_columns(other, func, level=level,
-> 3504                                                    fill_value=fill_value)
   3505         return self._combine_series_infer(other, func, level=level,
   3506                                           fill_value=fill_value)

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in _combine_match_columns(self, other, func, level, fill_value)
   3530     def _combine_match_columns(self, other, func, level=None, fill_value=None):
   3531         left, right = self.align(other, join='outer', axis=1, level=level,
-> 3532                                  copy=False)
   3533         if fill_value is not None:
   3534             raise NotImplementedError("fill_value %r not supported" %

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/frame.py in align(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis, broadcast_axis)
   2726                                             method=method, limit=limit,
   2727                                             fill_axis=fill_axis,
-> 2728                                             broadcast_axis=broadcast_axis)
   2729 
   2730     @Appender(_shared_docs['reindex'] % _shared_doc_kwargs)

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/generic.py in align(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis, broadcast_axis)
   4935                                       copy=copy, fill_value=fill_value,
   4936                                       method=method, limit=limit,
-> 4937                                       fill_axis=fill_axis)
   4938         else:  # pragma: no cover
   4939             raise TypeError('unsupported type: %s' % type(other))

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/generic.py in _align_series(self, other, join, axis, level, copy, fill_value, method, limit, fill_axis)
   5029                     join_index, lidx, ridx = self.columns.join(
   5030                         other.index, how=join, level=level,
-> 5031                         return_indexers=True)
   5032 
   5033                 if lidx is not None:

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/indexes/base.py in join(self, other, how, level, return_indexers, sort)
   2999         if level is not None and (self_is_mi or other_is_mi):
   3000             return self._join_level(other, level, how=how,
-> 3001                                     return_indexers=return_indexers)
   3002 
   3003         other = _ensure_index(other)

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/indexes/base.py in _join_level(self, other, level, how, return_indexers, keep_order)
   3255             right_indexer = algos.take_nd(right_lev_indexer,
   3256                                           join_index.labels[level],
-> 3257                                           allow_fill=False)
   3258         else:
   3259             right_indexer = join_index.labels[level]

/home/isisilon/.pyenv/versions/3.6.0/envs/jupyter3/lib/python3.6/site-packages/pandas/core/algorithms.py in take_nd(arr, indexer, axis, out, fill_value, mask_info, allow_fill)
   1455     # and the fill_value
   1456     if out is None:
-> 1457         out_shape = list(arr.shape)
   1458         out_shape[axis] = len(indexer)
   1459         out_shape = tuple(out_shape)

KeyboardInterrupt: 

In [ ]: