ddpg with a dynamics model

I'm trying a dynamics model to provide additional supervision. I'm using
this repo because it's performance tested on a competition and is in
pytorch. I'm intially testing with pendulum. Code is messy as it's a one
time experiment.

common/env_wrappers.py

@@ -1,7 +1,8 @@
 import numpy as np
 import gym
 from gym.spaces import Box
-from osim.env import RunEnv
+import sys
+# from osim.env import RunEnv
 
 from common.state_transform import StateVelCentr
 
@@ -59,12 +60,59 @@ class DdpgWrapper(gym.Wrapper):
         return observation
 
 
+# def create_env_old(args):
+#     env = RunEnv(visualize=False, max_obstacles=args.max_obstacles)
+#
+#     if hasattr(args, "baseline_wrapper") or hasattr(args, "ddpg_wrapper"):
+#         env = DdpgWrapper(env, args)
+#
+#     return env
+
+
+# class BasicTask:
+#     def __init__(self):
+#         self.normalized_state = True
+#
+#     def normalize_state(self, state):
+#         return state
+#
+#     def reset(self):
+#         state = self.env.reset()
+#         if self.normalized_state:
+#             return self.normalize_state(state)
+#         return state
+#
+#     def step(self, action):
+#         next_state, reward, done, info = self.env.step(action)
+#         if self.normalized_state:
+#             next_state = self.normalize_state(next_state)
+#         return next_state, np.sign(reward), done, info
+#
+#     def random_action(self):
+#         return self.env.action_space.sample()
+#
+#
+# class Pendulum(BasicTask):
+#     name = 'Pendulum-v0'
+#     success_threshold = -10
+#
+#     def __init__(self):
+#         BasicTask.__init__(self)
+#         self.env = gym.make(self.name)
+#         self.max_episode_steps = self.env._max_episode_steps
+#         self.env._max_episode_steps = sys.maxsize
+#         self.action_dim = self.env.action_space.shape[0]
+#         self.state_dim = self.env.observation_space.shape[0]
+#
+#     def step(self, action):
+#         action = np.clip(action, -2, 2)
+#         next_state, reward, done, info = self.env.step(action)
+#         return next_state, reward, done, info
+
+
 def create_env(args):
-    env = RunEnv(visualize=False, max_obstacles=args.max_obstacles)
-
-    if hasattr(args, "baseline_wrapper") or hasattr(args, "ddpg_wrapper"):
-        env = DdpgWrapper(env, args)
-
+    # env = Pendulum()
+    env = gym.make('Pendulum-v0')
     return env
 
 

ddpg/model.py

@@ -6,7 +6,7 @@ import time
 import torch.nn as nn
 from pprint import pprint
 
-from ddpg.nets import Actor, Critic
+from ddpg.nets import Actor, Critic, Base, ActorHead, CriticHead, DynamicsHead
 from common.torch_util import to_numpy, to_tensor, soft_update
 from common.misc_util import create_if_need, set_global_seeds
 from common.logger import Logger
@@ -15,33 +15,72 @@ from common.loss import create_loss, create_decay_fn
 from common.env_wrappers import create_env
 from common.random_process import create_random_process
 
-
 def create_model(args):
-    actor = Actor(
+    base = Base(
+        args.n_observation, args.n_action, args.actor_layers,
+        activation=args.actor_activation,
+        layer_norm=args.actor_layer_norm,
+        parameters_noise=args.actor_parameters_noise,
+        parameters_noise_factorised=args.actor_parameters_noise_factorised,
+        last_activation=nn.Tanh
+    )
+    actor = ActorHead(
+        base,
         args.n_observation, args.n_action, args.actor_layers,
         activation=args.actor_activation,
         layer_norm=args.actor_layer_norm,
         parameters_noise=args.actor_parameters_noise,
         parameters_noise_factorised=args.actor_parameters_noise_factorised,
         last_activation=nn.Tanh)
-    critic = Critic(
+    critic = CriticHead(
+        base,
+        args.n_observation, args.n_action, args.critic_layers,
+        activation=args.critic_activation,
+        layer_norm=args.critic_layer_norm,
+        parameters_noise=args.critic_parameters_noise,
+        parameters_noise_factorised=args.critic_parameters_noise_factorised)
+    dynamics = DynamicsHead(
+        base,
         args.n_observation, args.n_action, args.critic_layers,
         activation=args.critic_activation,
         layer_norm=args.critic_layer_norm,
         parameters_noise=args.critic_parameters_noise,
         parameters_noise_factorised=args.critic_parameters_noise_factorised)
 
+    pprint(base)
     pprint(actor)
     pprint(critic)
+    pprint(dynamics)
+    return actor, critic, dynamics
 
-    return actor, critic
+# def create_model_old(args):
+#     actor = Actor(
+#         args.n_observation, args.n_action, args.actor_layers,
+#         activation=args.actor_activation,
+#         layer_norm=args.actor_layer_norm,
+#         parameters_noise=args.actor_parameters_noise,
+#         parameters_noise_factorised=args.actor_parameters_noise_factorised,
+#         last_activation=nn.Tanh)
+#     critic = Critic(
+#         args.n_observation, args.n_action, args.critic_layers,
+#         activation=args.critic_activation,
+#         layer_norm=args.critic_layer_norm,
+#         parameters_noise=args.critic_parameters_noise,
+#         parameters_noise_factorised=args.critic_parameters_noise_factorised)
+#
+#     pprint(actor)
+#     pprint(critic)
+#
+#     return actor, critic
 
 
-def create_act_update_fns(actor, critic, target_actor, target_critic, args):
+def create_act_update_fns(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args):
     actor_optim = torch.optim.Adam(actor.parameters(), lr=args.actor_lr)
     critic_optim = torch.optim.Adam(critic.parameters(), lr=args.critic_lr)
+    dynamics_optim = torch.optim.Adam(dynamics.parameters(), lr=args.actor_lr)
 
     criterion = create_loss(args)
+    dynamics_criterion = nn.MSELoss()
 
     low_action_boundary = -1.
     high_action_boundary = 1.
@@ -56,7 +95,7 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args):
     def update_fn(
             observations, actions, rewards, next_observations, dones, weights,
             actor_lr=1e-4, critic_lr=1e-3):
-        nonlocal actor, critic, target_actor, target_critic, actor_optim, critic_optim
+        nonlocal actor, critic, dynamics, target_actor, target_critic, target_dynamics, actor_optim, critic_optim, dynamics_optim
 
         if hasattr(args, "flip_states"):
             observations_flip = args.flip_states(observations)
@@ -78,19 +117,41 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args):
         rewards = to_tensor(rewards)
         weights = to_tensor(weights, requires_grad=False)
 
-        next_v_values = target_critic(
+        # Dynamics update
+        next_observations_pred = dynamics(observations, actions)
+        dynamics_loss = criterion(
+            next_observations_pred,
+            to_tensor(next_obsno proervations),
+            weights=torch.stack([weights, weights, weights], 1)
+        )
+        dynamics.zero_grad()
+        dynamics_loss.backward()
+        torch.nn.utils.clip_grad_norm(dynamics.parameters(), args.grad_clip)
+        for param_group in actor_optim.param_groups:
+            param_group["lr"] = actor_lr  # TODO change to dynamics lr
+        dynamics_optim.step()
+
+        # Critic update
+        next_next_observations_pred = target_dynamics(
             to_tensor(next_observations, volatile=True),
             target_actor(to_tensor(next_observations, volatile=True)),
         )
+        next_v_values = target_critic(next_next_observations_pred)
         next_v_values.volatile = False
 
+        # next_v_values = target_critic(
+        #     to_tensor(next_observations, volatile=True),
+        #     target_actor(to_tensor(next_observations, volatile=True)),
+        # )
+        # next_v_values.volatile = False
+
         reward_predicted = dones * args.gamma * next_v_values
         td_target = rewards + reward_predicted
 
-        # Critic update
         critic.zero_grad()
 
-        v_values = critic(to_tensor(observations), to_tensor(actions))
+        # v_values = critic(to_tensor(observations), to_tensor(actions))
+        v_values = critic(dynamics(to_tensor(observations), to_tensor(actions)))
         value_loss = criterion(v_values, td_target, weights=weights)
         value_loss.backward()
 
@@ -104,8 +165,10 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args):
         actor.zero_grad()
 
         policy_loss = -critic(
-            to_tensor(observations),
-            actor(to_tensor(observations))
+            dynamics(
+                to_tensor(observations),
+                actor(to_tensor(observations))
+            )
         )
 
         policy_loss = torch.mean(policy_loss * weights)
@@ -127,8 +190,12 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args):
         }
 
         td_v_values = critic(
-            to_tensor(observations, volatile=True, requires_grad=False),
-            to_tensor(actions, volatile=True, requires_grad=False))
+            dynamics(
+                to_tensor(observations, volatile=True, requires_grad=False),
+                to_tensor(actions, volatile=True, requires_grad=False)
+            )
+        )
+
         td_error = td_target - td_v_values
 
         info = {
@@ -138,7 +205,7 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args):
         return metrics, info
 
     def save_fn(episode=None):
-        nonlocal actor, critic
+        nonlocal actor, critic, dynamics
         if episode is None:
             save_path = args.logdir
         else:
@@ -152,14 +219,14 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args):
     return act_fn, update_fn, save_fn
 
 
-def train_multi_thread(actor, critic, target_actor, target_critic, args, prepare_fn, best_reward):
+def train_multi_thread(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, prepare_fn, best_reward):
     workerseed = args.seed + 241 * args.thread
     set_global_seeds(workerseed)
 
     args.logdir = "{}/thread_{}".format(args.logdir, args.thread)
     create_if_need(args.logdir)
 
-    act_fn, update_fn, save_fn = prepare_fn(actor, critic, target_actor, target_critic, args)
+    act_fn, update_fn, save_fn = prepare_fn(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args)
     logger = Logger(args.logdir)
 
     buffer = create_buffer(args)
@@ -213,7 +280,7 @@ def train_multi_thread(actor, critic, target_actor, target_critic, args, prepare
             "epsilon": epsilon
         }
 
-        observation = env.reset(seed=seed, difficulty=args.difficulty)
+        observation = env.reset()#seed=seed, difficulty=args.difficulty)
         random_process.reset_states()
         done = False
 
@@ -288,7 +355,7 @@ def train_multi_thread(actor, critic, target_actor, target_critic, args, prepare
 
 
 def train_single_thread(
-        actor, critic, target_actor, target_critic, args, prepare_fn,
+        actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, prepare_fn,
         global_episode, global_update_step, episodes_queue):
     workerseed = args.seed + 241 * args.thread
     set_global_seeds(workerseed)
@@ -296,7 +363,7 @@ def train_single_thread(
     args.logdir = "{}/thread_{}".format(args.logdir, args.thread)
     create_if_need(args.logdir)
 
-    _, update_fn, save_fn = prepare_fn(actor, critic, target_actor, target_critic, args)
+    _, update_fn, save_fn = prepare_fn(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args)
 
     logger = Logger(args.logdir)
 
@@ -389,7 +456,7 @@ def train_single_thread(
 
 
 def play_single_thread(
-        actor, critic, target_actor, target_critic, args, prepare_fn,
+        actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, prepare_fn,
         global_episode, global_update_step, episodes_queue,
         best_reward):
     workerseed = args.seed + 241 * args.thread
@@ -398,7 +465,7 @@ def play_single_thread(
     args.logdir = "{}/thread_{}".format(args.logdir, args.thread)
     create_if_need(args.logdir)
 
-    act_fn, _, save_fn = prepare_fn(actor, critic, target_actor, target_critic, args)
+    act_fn, _, save_fn = prepare_fn(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args)
 
     logger = Logger(args.logdir)
     env = create_env(args)
@@ -430,7 +497,7 @@ def play_single_thread(
             "epsilon": epsilon
         }
 
-        observation = env.reset(seed=seed, difficulty=args.difficulty)
+        observation = env.reset()#seed=seed, difficulty=args.difficulty)
         random_process.reset_states()
         done = False
 

ddpg/nets.py

@@ -1,10 +1,19 @@
 import numpy as np
 import torch
 import torch.nn as nn
+from torch.autograd import Variable
 
 from common.nets import LinearNet
 from common.modules.NoisyLinear import NoisyLinear
 
+def to_torch_variable(x, dtype='float32'):
+    if isinstance(x, Variable):
+        return x
+    if not isinstance(x, torch.FloatTensor):
+        x = torch.from_numpy(np.asarray(x, dtype=dtype))
+    # if self.gpu:
+    #     x = x.cuda()
+    return Variable(x)
 
 def fanin_init(size, fanin=None):
     fanin = fanin or size[0]
@@ -48,7 +57,7 @@ class Actor(nn.Module):
                 layer.weight.data.uniform_(-init_w, init_w)
 
     def forward(self, observation):
-        x = observation
+        x = to_torch_variable(observation)
         x = self.feature_net.forward(x)
         x = self.policy_net.forward(x)
         return x
@@ -88,3 +97,111 @@ class Critic(nn.Module):
         x = self.feature_net.forward(x)
         x = self.value_net.forward(x)
         return x
+
+
+class Base(nn.Module):
+    def __init__(self, n_observation, n_action,
+                 layers, activation=torch.nn.ELU,
+                 layer_norm=False,
+                 parameters_noise=False, parameters_noise_factorised=False,
+                 last_activation=torch.nn.Tanh, init_w=3e-3):
+        super(Base, self).__init__()
+
+        if parameters_noise:
+            def linear_layer(x_in, x_out):
+                return NoisyLinear(x_in, x_out, factorised=parameters_noise_factorised)
+        else:
+            linear_layer = nn.Linear
+
+        self.feature_net = LinearNet(
+            layers=[n_observation] + layers,
+            activation=activation,
+            layer_norm=layer_norm,
+            linear_layer=linear_layer)
+        self.init_weights(init_w)
+
+    def init_weights(self, init_w):
+        for layer in self.feature_net.net:
+            if isinstance(layer, nn.Linear):
+                layer.weight.data = fanin_init(layer.weight.data.size())
+
+        for layer in self.feature_net.net:
+            if isinstance(layer, nn.Linear):
+                layer.weight.data.uniform_(-init_w, init_w)
+
+    def forward(self, observation):
+        x = to_torch_variable(observation)
+        x = self.feature_net.forward(x)
+        return x
+
+class CriticHead(nn.Module):
+    def __init__(self, base, n_observation, n_action,
+                 layers, activation=torch.nn.ELU,
+                 layer_norm=False,
+                 parameters_noise=False, parameters_noise_factorised=False,
+                 init_w=3e-3):
+        super(CriticHead, self).__init__()
+        self.base = base
+        self.value_net = nn.Linear(self.base.feature_net.output_shape, 1)
+        self.init_weights(init_w)
+
+    def init_weights(self, init_w):
+        self.value_net.weight.data.uniform_(-init_w, init_w)
+
+    def forward(self, observation):
+        x = self.base.forward(observation)
+        # x = torch.cat((x, action), dim=1)
+        x = self.value_net.forward(x)
+        return x
+
+
+
+class ActorHead(nn.Module):
+    def __init__(self, base, n_observation, n_action,
+                 layers, activation=torch.nn.ELU,
+                 layer_norm=False,
+                 parameters_noise=False, parameters_noise_factorised=False,
+                 last_activation=torch.nn.Tanh, init_w=3e-3):
+        super(ActorHead, self).__init__()
+        self.base = base
+
+        self.policy_net = LinearNet(
+            layers=[self.base.feature_net.output_shape, n_action],
+            activation=last_activation,
+            layer_norm=False
+        )
+        self.init_weights(init_w)
+
+    def init_weights(self, init_w):
+        for layer in self.policy_net.net:
+            if isinstance(layer, nn.Linear):
+                layer.weight.data.uniform_(-init_w, init_w)
+
+    def forward(self, observation):
+        x = observation
+        x = self.base.forward(x)
+        x = self.policy_net.forward(x)
+        return x
+
+
+
+class DynamicsHead(nn.Module):
+    def __init__(self, base, n_observation, n_action,
+                 layers, activation=torch.nn.ELU,
+                 layer_norm=False,
+                 parameters_noise=False, parameters_noise_factorised=False,
+                 init_w=3e-3):
+        super(DynamicsHead, self).__init__()
+        self.base = base
+        self.value_net = nn.Linear(self.base.feature_net.output_shape + n_action, n_observation)
+        self.init_weights(init_w)
+
+    def init_weights(self, init_w):
+        self.value_net.weight.data.uniform_(-init_w, init_w)
+
+    def forward(self, observation, action):
+        action = to_torch_variable(action)
+        x = self.base.forward(observation)
+        x = torch.cat((x, action), dim=1)
+        x = self.value_net.forward(x)
+        return x

ddpg/train.py

@@ -160,7 +160,7 @@ def train(args, model_fn, act_update_fns, multi_thread, train_single, play_singl
     args.actor_activation = activations[args.actor_activation]
     args.critic_activation = activations[args.critic_activation]
 
-    actor, critic = model_fn(args)
+    actor, critic, dynamics = model_fn(args)
 
     if args.restore_actor_from is not None:
         actor.load_state_dict(torch.load(args.restore_actor_from))
@@ -169,31 +169,42 @@ def train(args, model_fn, act_update_fns, multi_thread, train_single, play_singl
 
     actor.train()
     critic.train()
+    dynamics.train()
     actor.share_memory()
     critic.share_memory()
+    dynamics.share_memory()
 
     target_actor = copy.deepcopy(actor)
     target_critic = copy.deepcopy(critic)
+    target_dynamics = copy.deepcopy(dynamics)
 
     hard_update(target_actor, actor)
     hard_update(target_critic, critic)
+    hard_update(target_dynamics, dynamics)
 
     target_actor.train()
     target_critic.train()
+    target_dynamics.train()
     target_actor.share_memory()
     target_critic.share_memory()
+    target_dynamics.share_memory()
 
-    _, _, save_fn = act_update_fns(actor, critic, target_actor, target_critic, args)
+    _, _, save_fn = act_update_fns(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args)
 
     processes = []
     best_reward = Value("f", 0.0)
+
+    # debugging
+    args.thread = 1
+    multi_thread(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns, best_reward)
+
     try:
         if args.num_threads == args.num_train_threads:
             for rank in range(args.num_threads):
                 args.thread = rank
                 p = mp.Process(
                     target=multi_thread,
-                    args=(actor, critic, target_actor, target_critic, args, act_update_fns,
+                    args=(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns,
                           best_reward))
                 p.start()
                 processes.append(p)
@@ -206,12 +217,12 @@ def train(args, model_fn, act_update_fns, multi_thread, train_single, play_singl
                 if rank < args.num_train_threads:
                     p = mp.Process(
                         target=train_single,
-                        args=(actor, critic, target_actor, target_critic, args, act_update_fns,
+                        args=(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns,
                               global_episode, global_update_step, episodes_queue))
                 else:
                     p = mp.Process(
                         target=play_single,
-                        args=(actor, critic, target_actor, target_critic, args, act_update_fns,
+                        args=(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns,
                               global_episode, global_update_step, episodes_queue,
                               best_reward))
                 p.start()

trying_ddpg_with_implicit_dynamics.ipynb

@@ -0,0 +1,330 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-01-18T08:39:01.196406Z",
+     "start_time": "2018-01-18T08:39:01.193658Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "os.environ['CUDA_VISIBLE_DEVICES']=\"\"\n",
+    "os.environ[\"PYTHONPATH\"]='.'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-01-18T08:39:02.089265Z",
+     "start_time": "2018-01-18T08:39:01.197644Z"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Populating the interactive namespace from numpy and matplotlib\n"
+     ]
+    }
+   ],
+   "source": [
+    "%pylab --no-import-all inline\n",
+    "%reload_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "ExecuteTime": {
+     "end_time": "2018-01-18T08:39:02.125920Z",
+     "start_time": "2018-01-18T08:39:02.090922Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['ddpg/train.py',\n",
+       " '--logdir',\n",
+       " './logs_ddpg',\n",
+       " '--num-threads',\n",
+       " '1',\n",
+       " '--ddpg-wrapper',\n",
+       " '--skip-frames',\n",
+       " '5',\n",
+       " '--fail-reward',\n",
+       " '-0.2',\n",
+       " '--reward-scale',\n",
+       " '1',\n",
+       " '--flip-state-action',\n",
+       " '--actor-layers',\n",
+       " '64-64',\n",
+       " '--actor-layer-norm',\n",
+       " '--actor-parameters-noise',\n",
+       " '--actor-lr',\n",
+       " '0.001',\n",
+       " '--actor-lr-end',\n",
+       " '0.00001',\n",
+       " '--critic-layers',\n",
+       " '64-32',\n",
+       " '--critic-layer-norm',\n",
+       " '--critic-lr',\n",
+       " '0.002',\n",
+       " '--critic-lr-end',\n",
+       " '0.00001',\n",
+       " '--initial-epsilon',\n",
+       " '0.5',\n",
+       " '--final-epsilon',\n",
+       " '0.001',\n",
+       " '--tau',\n",
+       " '0.0001']"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "os.sys.argv=\"ddpg/train.py --logdir ./logs_ddpg --num-threads 1 --ddpg-wrapper --skip-frames 5 --fail-reward -0.2 --reward-scale 1 --flip-state-action --actor-layers 64-64 --actor-layer-norm --actor-parameters-noise --actor-lr 0.001 --actor-lr-end 0.00001 --critic-layers 64-32 --critic-layer-norm --critic-lr 0.002 --critic-lr-end 0.00001 --initial-epsilon 0.5 --final-epsilon 0.001 --tau 0.0001\".split(\" \")\n",
+    "os.sys.argv"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "start_time": "2018-01-18T08:39:00.919Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "from ddpg.train import *"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "start_time": "2018-01-18T08:39:00.924Z"
+    },
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "[2018-01-18 16:39:02,792] Making new env: Pendulum-v0\n",
+      "[2018-01-18 16:39:02,886] Making new env: Pendulum-v0\n",
+      "[2018-01-18 16:39:02,889] Making new env: Pendulum-v0\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Base (\n",
+      "  (feature_net): LinearNet (\n",
+      "    (net): Sequential (\n",
+      "      (linear_0): NoisyLinear (3 -> 64)\n",
+      "      (layer_norm_0): LayerNorm (\n",
+      "      )\n",
+      "      (act_0): ReLU ()\n",
+      "      (linear_1): NoisyLinear (64 -> 64)\n",
+      "      (layer_norm_1): LayerNorm (\n",
+      "      )\n",
+      "      (act_1): ReLU ()\n",
+      "    )\n",
+      "  )\n",
+      ")\n",
+      "ActorHead (\n",
+      "  (base): Base (\n",
+      "    (feature_net): LinearNet (\n",
+      "      (net): Sequential (\n",
+      "        (linear_0): NoisyLinear (3 -> 64)\n",
+      "        (layer_norm_0): LayerNorm (\n",
+      "        )\n",
+      "        (act_0): ReLU ()\n",
+      "        (linear_1): NoisyLinear (64 -> 64)\n",
+      "        (layer_norm_1): LayerNorm (\n",
+      "        )\n",
+      "        (act_1): ReLU ()\n",
+      "      )\n",
+      "    )\n",
+      "  )\n",
+      "  (policy_net): LinearNet (\n",
+      "    (net): Sequential (\n",
+      "      (linear_0): Linear (64 -> 1)\n",
+      "      (act_0): Tanh ()\n",
+      "    )\n",
+      "  )\n",
+      ")\n",
+      "CriticHead (\n",
+      "  (base): Base (\n",
+      "    (feature_net): LinearNet (\n",
+      "      (net): Sequential (\n",
+      "        (linear_0): NoisyLinear (3 -> 64)\n",
+      "        (layer_norm_0): LayerNorm (\n",
+      "        )\n",
+      "        (act_0): ReLU ()\n",
+      "        (linear_1): NoisyLinear (64 -> 64)\n",
+      "        (layer_norm_1): LayerNorm (\n",
+      "        )\n",
+      "        (act_1): ReLU ()\n",
+      "      )\n",
+      "    )\n",
+      "  )\n",
+      "  (value_net): Linear (64 -> 1)\n",
+      ")\n",
+      "DynamicsHead (\n",
+      "  (base): Base (\n",
+      "    (feature_net): LinearNet (\n",
+      "      (net): Sequential (\n",
+      "        (linear_0): NoisyLinear (3 -> 64)\n",
+      "        (layer_norm_0): LayerNorm (\n",
+      "        )\n",
+      "        (act_0): ReLU ()\n",
+      "        (linear_1): NoisyLinear (64 -> 64)\n",
+      "        (layer_norm_1): LayerNorm (\n",
+      "        )\n",
+      "        (act_1): ReLU ()\n",
+      "      )\n",
+      "    )\n",
+      "  )\n",
+      "  (value_net): Linear (65 -> 3)\n",
+      ")\n"
+     ]
+    }
+   ],
+   "source": [
+    "os.environ['OMP_NUM_THREADS'] = '1'\n",
+    "torch.set_num_threads(1)\n",
+    "args = parse_args()\n",
+    "train(args,\n",
+    "      create_model,\n",
+    "      create_act_update_fns,\n",
+    "      train_multi_thread,\n",
+    "      train_single_thread,\n",
+    "      play_single_thread)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "start_time": "2018-01-18T08:39:00.927Z"
+    }
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "ExecuteTime": {
+     "start_time": "2018-01-18T08:39:00.929Z"
+    }
+   },
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "jupyter3",
+   "language": "python",
+   "name": "jupyter3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  },
+  "toc": {
+   "colors": {
+    "hover_highlight": "#DAA520",
+    "navigate_num": "#000000",
+    "navigate_text": "#333333",
+    "running_highlight": "#FF0000",
+    "selected_highlight": "#FFD700",
+    "sidebar_border": "#EEEEEE",
+    "wrapper_background": "#FFFFFF"
+   },
+   "moveMenuLeft": true,
+   "nav_menu": {
+    "height": "12px",
+    "width": "252px"
+   },
+   "navigate_menu": true,
+   "number_sections": true,
+   "sideBar": true,
+   "threshold": 4,
+   "toc_cell": false,
+   "toc_section_display": "block",
+   "toc_window_display": false,
+   "widenNotebook": false
+  },
+  "varInspector": {
+   "cols": {
+    "lenName": 16,
+    "lenType": 16,
+    "lenVar": 40
+   },
+   "kernels_config": {
+    "python": {
+     "delete_cmd_postfix": "",
+     "delete_cmd_prefix": "del ",
+     "library": "var_list.py",
+     "varRefreshCmd": "print(var_dic_list())"
+    },
+    "r": {
+     "delete_cmd_postfix": ") ",
+     "delete_cmd_prefix": "rm(",
+     "library": "var_list.r",
+     "varRefreshCmd": "cat(var_dic_list()) "
+    }
+   },
+   "types_to_exclude": [
+    "module",
+    "function",
+    "builtin_function_or_method",
+    "instance",
+    "_Feature"
+   ],
+   "window_display": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}