From 0de68133cf1a23d661081d78104deb1900dd0aef Mon Sep 17 00:00:00 2001 From: wassname Date: Thu, 18 Jan 2018 16:41:20 +0800 Subject: [PATCH] ddpg with a dynamics model I'm trying a dynamics model to provide additional supervision. I'm using this repo because it's performance tested on a competition and is in pytorch. I'm intially testing with pendulum. Code is messy as it's a one time experiment. --- common/env_wrappers.py | 60 ++++- ddpg/model.py | 113 ++++++-- ddpg/nets.py | 119 +++++++- ddpg/train.py | 21 +- trying_ddpg_with_implicit_dynamics.ipynb | 330 +++++++++++++++++++++++ 5 files changed, 608 insertions(+), 35 deletions(-) create mode 100644 trying_ddpg_with_implicit_dynamics.ipynb diff --git a/common/env_wrappers.py b/common/env_wrappers.py index b323757..b75d09c 100644 --- a/common/env_wrappers.py +++ b/common/env_wrappers.py @@ -1,7 +1,8 @@ import numpy as np import gym from gym.spaces import Box -from osim.env import RunEnv +import sys +# from osim.env import RunEnv from common.state_transform import StateVelCentr @@ -59,12 +60,59 @@ class DdpgWrapper(gym.Wrapper): return observation +# def create_env_old(args): +# env = RunEnv(visualize=False, max_obstacles=args.max_obstacles) +# +# if hasattr(args, "baseline_wrapper") or hasattr(args, "ddpg_wrapper"): +# env = DdpgWrapper(env, args) +# +# return env + + +# class BasicTask: +# def __init__(self): +# self.normalized_state = True +# +# def normalize_state(self, state): +# return state +# +# def reset(self): +# state = self.env.reset() +# if self.normalized_state: +# return self.normalize_state(state) +# return state +# +# def step(self, action): +# next_state, reward, done, info = self.env.step(action) +# if self.normalized_state: +# next_state = self.normalize_state(next_state) +# return next_state, np.sign(reward), done, info +# +# def random_action(self): +# return self.env.action_space.sample() +# +# +# class Pendulum(BasicTask): +# name = 'Pendulum-v0' +# success_threshold = -10 +# +# def __init__(self): +# BasicTask.__init__(self) +# self.env = gym.make(self.name) +# self.max_episode_steps = self.env._max_episode_steps +# self.env._max_episode_steps = sys.maxsize +# self.action_dim = self.env.action_space.shape[0] +# self.state_dim = self.env.observation_space.shape[0] +# +# def step(self, action): +# action = np.clip(action, -2, 2) +# next_state, reward, done, info = self.env.step(action) +# return next_state, reward, done, info + + def create_env(args): - env = RunEnv(visualize=False, max_obstacles=args.max_obstacles) - - if hasattr(args, "baseline_wrapper") or hasattr(args, "ddpg_wrapper"): - env = DdpgWrapper(env, args) - + # env = Pendulum() + env = gym.make('Pendulum-v0') return env diff --git a/ddpg/model.py b/ddpg/model.py index d23110d..b7303fa 100644 --- a/ddpg/model.py +++ b/ddpg/model.py @@ -6,7 +6,7 @@ import time import torch.nn as nn from pprint import pprint -from ddpg.nets import Actor, Critic +from ddpg.nets import Actor, Critic, Base, ActorHead, CriticHead, DynamicsHead from common.torch_util import to_numpy, to_tensor, soft_update from common.misc_util import create_if_need, set_global_seeds from common.logger import Logger @@ -15,33 +15,72 @@ from common.loss import create_loss, create_decay_fn from common.env_wrappers import create_env from common.random_process import create_random_process - def create_model(args): - actor = Actor( + base = Base( + args.n_observation, args.n_action, args.actor_layers, + activation=args.actor_activation, + layer_norm=args.actor_layer_norm, + parameters_noise=args.actor_parameters_noise, + parameters_noise_factorised=args.actor_parameters_noise_factorised, + last_activation=nn.Tanh + ) + actor = ActorHead( + base, args.n_observation, args.n_action, args.actor_layers, activation=args.actor_activation, layer_norm=args.actor_layer_norm, parameters_noise=args.actor_parameters_noise, parameters_noise_factorised=args.actor_parameters_noise_factorised, last_activation=nn.Tanh) - critic = Critic( + critic = CriticHead( + base, + args.n_observation, args.n_action, args.critic_layers, + activation=args.critic_activation, + layer_norm=args.critic_layer_norm, + parameters_noise=args.critic_parameters_noise, + parameters_noise_factorised=args.critic_parameters_noise_factorised) + dynamics = DynamicsHead( + base, args.n_observation, args.n_action, args.critic_layers, activation=args.critic_activation, layer_norm=args.critic_layer_norm, parameters_noise=args.critic_parameters_noise, parameters_noise_factorised=args.critic_parameters_noise_factorised) + pprint(base) pprint(actor) pprint(critic) + pprint(dynamics) + return actor, critic, dynamics - return actor, critic +# def create_model_old(args): +# actor = Actor( +# args.n_observation, args.n_action, args.actor_layers, +# activation=args.actor_activation, +# layer_norm=args.actor_layer_norm, +# parameters_noise=args.actor_parameters_noise, +# parameters_noise_factorised=args.actor_parameters_noise_factorised, +# last_activation=nn.Tanh) +# critic = Critic( +# args.n_observation, args.n_action, args.critic_layers, +# activation=args.critic_activation, +# layer_norm=args.critic_layer_norm, +# parameters_noise=args.critic_parameters_noise, +# parameters_noise_factorised=args.critic_parameters_noise_factorised) +# +# pprint(actor) +# pprint(critic) +# +# return actor, critic -def create_act_update_fns(actor, critic, target_actor, target_critic, args): +def create_act_update_fns(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args): actor_optim = torch.optim.Adam(actor.parameters(), lr=args.actor_lr) critic_optim = torch.optim.Adam(critic.parameters(), lr=args.critic_lr) + dynamics_optim = torch.optim.Adam(dynamics.parameters(), lr=args.actor_lr) criterion = create_loss(args) + dynamics_criterion = nn.MSELoss() low_action_boundary = -1. high_action_boundary = 1. @@ -56,7 +95,7 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args): def update_fn( observations, actions, rewards, next_observations, dones, weights, actor_lr=1e-4, critic_lr=1e-3): - nonlocal actor, critic, target_actor, target_critic, actor_optim, critic_optim + nonlocal actor, critic, dynamics, target_actor, target_critic, target_dynamics, actor_optim, critic_optim, dynamics_optim if hasattr(args, "flip_states"): observations_flip = args.flip_states(observations) @@ -78,19 +117,41 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args): rewards = to_tensor(rewards) weights = to_tensor(weights, requires_grad=False) - next_v_values = target_critic( + # Dynamics update + next_observations_pred = dynamics(observations, actions) + dynamics_loss = criterion( + next_observations_pred, + to_tensor(next_obsno proervations), + weights=torch.stack([weights, weights, weights], 1) + ) + dynamics.zero_grad() + dynamics_loss.backward() + torch.nn.utils.clip_grad_norm(dynamics.parameters(), args.grad_clip) + for param_group in actor_optim.param_groups: + param_group["lr"] = actor_lr # TODO change to dynamics lr + dynamics_optim.step() + + # Critic update + next_next_observations_pred = target_dynamics( to_tensor(next_observations, volatile=True), target_actor(to_tensor(next_observations, volatile=True)), ) + next_v_values = target_critic(next_next_observations_pred) next_v_values.volatile = False + # next_v_values = target_critic( + # to_tensor(next_observations, volatile=True), + # target_actor(to_tensor(next_observations, volatile=True)), + # ) + # next_v_values.volatile = False + reward_predicted = dones * args.gamma * next_v_values td_target = rewards + reward_predicted - # Critic update critic.zero_grad() - v_values = critic(to_tensor(observations), to_tensor(actions)) + # v_values = critic(to_tensor(observations), to_tensor(actions)) + v_values = critic(dynamics(to_tensor(observations), to_tensor(actions))) value_loss = criterion(v_values, td_target, weights=weights) value_loss.backward() @@ -104,8 +165,10 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args): actor.zero_grad() policy_loss = -critic( - to_tensor(observations), - actor(to_tensor(observations)) + dynamics( + to_tensor(observations), + actor(to_tensor(observations)) + ) ) policy_loss = torch.mean(policy_loss * weights) @@ -127,8 +190,12 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args): } td_v_values = critic( - to_tensor(observations, volatile=True, requires_grad=False), - to_tensor(actions, volatile=True, requires_grad=False)) + dynamics( + to_tensor(observations, volatile=True, requires_grad=False), + to_tensor(actions, volatile=True, requires_grad=False) + ) + ) + td_error = td_target - td_v_values info = { @@ -138,7 +205,7 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args): return metrics, info def save_fn(episode=None): - nonlocal actor, critic + nonlocal actor, critic, dynamics if episode is None: save_path = args.logdir else: @@ -152,14 +219,14 @@ def create_act_update_fns(actor, critic, target_actor, target_critic, args): return act_fn, update_fn, save_fn -def train_multi_thread(actor, critic, target_actor, target_critic, args, prepare_fn, best_reward): +def train_multi_thread(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, prepare_fn, best_reward): workerseed = args.seed + 241 * args.thread set_global_seeds(workerseed) args.logdir = "{}/thread_{}".format(args.logdir, args.thread) create_if_need(args.logdir) - act_fn, update_fn, save_fn = prepare_fn(actor, critic, target_actor, target_critic, args) + act_fn, update_fn, save_fn = prepare_fn(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args) logger = Logger(args.logdir) buffer = create_buffer(args) @@ -213,7 +280,7 @@ def train_multi_thread(actor, critic, target_actor, target_critic, args, prepare "epsilon": epsilon } - observation = env.reset(seed=seed, difficulty=args.difficulty) + observation = env.reset()#seed=seed, difficulty=args.difficulty) random_process.reset_states() done = False @@ -288,7 +355,7 @@ def train_multi_thread(actor, critic, target_actor, target_critic, args, prepare def train_single_thread( - actor, critic, target_actor, target_critic, args, prepare_fn, + actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, prepare_fn, global_episode, global_update_step, episodes_queue): workerseed = args.seed + 241 * args.thread set_global_seeds(workerseed) @@ -296,7 +363,7 @@ def train_single_thread( args.logdir = "{}/thread_{}".format(args.logdir, args.thread) create_if_need(args.logdir) - _, update_fn, save_fn = prepare_fn(actor, critic, target_actor, target_critic, args) + _, update_fn, save_fn = prepare_fn(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args) logger = Logger(args.logdir) @@ -389,7 +456,7 @@ def train_single_thread( def play_single_thread( - actor, critic, target_actor, target_critic, args, prepare_fn, + actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, prepare_fn, global_episode, global_update_step, episodes_queue, best_reward): workerseed = args.seed + 241 * args.thread @@ -398,7 +465,7 @@ def play_single_thread( args.logdir = "{}/thread_{}".format(args.logdir, args.thread) create_if_need(args.logdir) - act_fn, _, save_fn = prepare_fn(actor, critic, target_actor, target_critic, args) + act_fn, _, save_fn = prepare_fn(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args) logger = Logger(args.logdir) env = create_env(args) @@ -430,7 +497,7 @@ def play_single_thread( "epsilon": epsilon } - observation = env.reset(seed=seed, difficulty=args.difficulty) + observation = env.reset()#seed=seed, difficulty=args.difficulty) random_process.reset_states() done = False diff --git a/ddpg/nets.py b/ddpg/nets.py index 06b2807..fd2efc2 100644 --- a/ddpg/nets.py +++ b/ddpg/nets.py @@ -1,10 +1,19 @@ import numpy as np import torch import torch.nn as nn +from torch.autograd import Variable from common.nets import LinearNet from common.modules.NoisyLinear import NoisyLinear +def to_torch_variable(x, dtype='float32'): + if isinstance(x, Variable): + return x + if not isinstance(x, torch.FloatTensor): + x = torch.from_numpy(np.asarray(x, dtype=dtype)) + # if self.gpu: + # x = x.cuda() + return Variable(x) def fanin_init(size, fanin=None): fanin = fanin or size[0] @@ -48,7 +57,7 @@ class Actor(nn.Module): layer.weight.data.uniform_(-init_w, init_w) def forward(self, observation): - x = observation + x = to_torch_variable(observation) x = self.feature_net.forward(x) x = self.policy_net.forward(x) return x @@ -88,3 +97,111 @@ class Critic(nn.Module): x = self.feature_net.forward(x) x = self.value_net.forward(x) return x + + +class Base(nn.Module): + def __init__(self, n_observation, n_action, + layers, activation=torch.nn.ELU, + layer_norm=False, + parameters_noise=False, parameters_noise_factorised=False, + last_activation=torch.nn.Tanh, init_w=3e-3): + super(Base, self).__init__() + + if parameters_noise: + def linear_layer(x_in, x_out): + return NoisyLinear(x_in, x_out, factorised=parameters_noise_factorised) + else: + linear_layer = nn.Linear + + self.feature_net = LinearNet( + layers=[n_observation] + layers, + activation=activation, + layer_norm=layer_norm, + linear_layer=linear_layer) + self.init_weights(init_w) + + def init_weights(self, init_w): + for layer in self.feature_net.net: + if isinstance(layer, nn.Linear): + layer.weight.data = fanin_init(layer.weight.data.size()) + + for layer in self.feature_net.net: + if isinstance(layer, nn.Linear): + layer.weight.data.uniform_(-init_w, init_w) + + def forward(self, observation): + x = to_torch_variable(observation) + x = self.feature_net.forward(x) + return x + +class CriticHead(nn.Module): + def __init__(self, base, n_observation, n_action, + layers, activation=torch.nn.ELU, + layer_norm=False, + parameters_noise=False, parameters_noise_factorised=False, + init_w=3e-3): + super(CriticHead, self).__init__() + self.base = base + self.value_net = nn.Linear(self.base.feature_net.output_shape, 1) + self.init_weights(init_w) + + def init_weights(self, init_w): + self.value_net.weight.data.uniform_(-init_w, init_w) + + def forward(self, observation): + x = self.base.forward(observation) + # x = torch.cat((x, action), dim=1) + x = self.value_net.forward(x) + return x + + + +class ActorHead(nn.Module): + def __init__(self, base, n_observation, n_action, + layers, activation=torch.nn.ELU, + layer_norm=False, + parameters_noise=False, parameters_noise_factorised=False, + last_activation=torch.nn.Tanh, init_w=3e-3): + super(ActorHead, self).__init__() + self.base = base + + self.policy_net = LinearNet( + layers=[self.base.feature_net.output_shape, n_action], + activation=last_activation, + layer_norm=False + ) + self.init_weights(init_w) + + def init_weights(self, init_w): + for layer in self.policy_net.net: + if isinstance(layer, nn.Linear): + layer.weight.data.uniform_(-init_w, init_w) + + def forward(self, observation): + x = observation + x = self.base.forward(x) + x = self.policy_net.forward(x) + return x + + + +class DynamicsHead(nn.Module): + def __init__(self, base, n_observation, n_action, + layers, activation=torch.nn.ELU, + layer_norm=False, + parameters_noise=False, parameters_noise_factorised=False, + init_w=3e-3): + super(DynamicsHead, self).__init__() + self.base = base + self.value_net = nn.Linear(self.base.feature_net.output_shape + n_action, n_observation) + self.init_weights(init_w) + + def init_weights(self, init_w): + self.value_net.weight.data.uniform_(-init_w, init_w) + + def forward(self, observation, action): + action = to_torch_variable(action) + x = self.base.forward(observation) + x = torch.cat((x, action), dim=1) + x = self.value_net.forward(x) + return x diff --git a/ddpg/train.py b/ddpg/train.py index 91a5281..6e9af18 100644 --- a/ddpg/train.py +++ b/ddpg/train.py @@ -160,7 +160,7 @@ def train(args, model_fn, act_update_fns, multi_thread, train_single, play_singl args.actor_activation = activations[args.actor_activation] args.critic_activation = activations[args.critic_activation] - actor, critic = model_fn(args) + actor, critic, dynamics = model_fn(args) if args.restore_actor_from is not None: actor.load_state_dict(torch.load(args.restore_actor_from)) @@ -169,31 +169,42 @@ def train(args, model_fn, act_update_fns, multi_thread, train_single, play_singl actor.train() critic.train() + dynamics.train() actor.share_memory() critic.share_memory() + dynamics.share_memory() target_actor = copy.deepcopy(actor) target_critic = copy.deepcopy(critic) + target_dynamics = copy.deepcopy(dynamics) hard_update(target_actor, actor) hard_update(target_critic, critic) + hard_update(target_dynamics, dynamics) target_actor.train() target_critic.train() + target_dynamics.train() target_actor.share_memory() target_critic.share_memory() + target_dynamics.share_memory() - _, _, save_fn = act_update_fns(actor, critic, target_actor, target_critic, args) + _, _, save_fn = act_update_fns(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args) processes = [] best_reward = Value("f", 0.0) + + # debugging + args.thread = 1 + multi_thread(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns, best_reward) + try: if args.num_threads == args.num_train_threads: for rank in range(args.num_threads): args.thread = rank p = mp.Process( target=multi_thread, - args=(actor, critic, target_actor, target_critic, args, act_update_fns, + args=(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns, best_reward)) p.start() processes.append(p) @@ -206,12 +217,12 @@ def train(args, model_fn, act_update_fns, multi_thread, train_single, play_singl if rank < args.num_train_threads: p = mp.Process( target=train_single, - args=(actor, critic, target_actor, target_critic, args, act_update_fns, + args=(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns, global_episode, global_update_step, episodes_queue)) else: p = mp.Process( target=play_single, - args=(actor, critic, target_actor, target_critic, args, act_update_fns, + args=(actor, critic, dynamics, target_actor, target_critic, target_dynamics, args, act_update_fns, global_episode, global_update_step, episodes_queue, best_reward)) p.start() diff --git a/trying_ddpg_with_implicit_dynamics.ipynb b/trying_ddpg_with_implicit_dynamics.ipynb new file mode 100644 index 0000000..9f49fd7 --- /dev/null +++ b/trying_ddpg_with_implicit_dynamics.ipynb @@ -0,0 +1,330 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2018-01-18T08:39:01.196406Z", + "start_time": "2018-01-18T08:39:01.193658Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "os.environ['CUDA_VISIBLE_DEVICES']=\"\"\n", + "os.environ[\"PYTHONPATH\"]='.'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2018-01-18T08:39:02.089265Z", + "start_time": "2018-01-18T08:39:01.197644Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Populating the interactive namespace from numpy and matplotlib\n" + ] + } + ], + "source": [ + "%pylab --no-import-all inline\n", + "%reload_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2018-01-18T08:39:02.125920Z", + "start_time": "2018-01-18T08:39:02.090922Z" + }, + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['ddpg/train.py',\n", + " '--logdir',\n", + " './logs_ddpg',\n", + " '--num-threads',\n", + " '1',\n", + " '--ddpg-wrapper',\n", + " '--skip-frames',\n", + " '5',\n", + " '--fail-reward',\n", + " '-0.2',\n", + " '--reward-scale',\n", + " '1',\n", + " '--flip-state-action',\n", + " '--actor-layers',\n", + " '64-64',\n", + " '--actor-layer-norm',\n", + " '--actor-parameters-noise',\n", + " '--actor-lr',\n", + " '0.001',\n", + " '--actor-lr-end',\n", + " '0.00001',\n", + " '--critic-layers',\n", + " '64-32',\n", + " '--critic-layer-norm',\n", + " '--critic-lr',\n", + " '0.002',\n", + " '--critic-lr-end',\n", + " '0.00001',\n", + " '--initial-epsilon',\n", + " '0.5',\n", + " '--final-epsilon',\n", + " '0.001',\n", + " '--tau',\n", + " '0.0001']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "os.sys.argv=\"ddpg/train.py --logdir ./logs_ddpg --num-threads 1 --ddpg-wrapper --skip-frames 5 --fail-reward -0.2 --reward-scale 1 --flip-state-action --actor-layers 64-64 --actor-layer-norm --actor-parameters-noise --actor-lr 0.001 --actor-lr-end 0.00001 --critic-layers 64-32 --critic-layer-norm --critic-lr 0.002 --critic-lr-end 0.00001 --initial-epsilon 0.5 --final-epsilon 0.001 --tau 0.0001\".split(\" \")\n", + "os.sys.argv" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2018-01-18T08:39:00.919Z" + } + }, + "outputs": [], + "source": [ + "from ddpg.train import *" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2018-01-18T08:39:00.924Z" + }, + "scrolled": true + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2018-01-18 16:39:02,792] Making new env: Pendulum-v0\n", + "[2018-01-18 16:39:02,886] Making new env: Pendulum-v0\n", + "[2018-01-18 16:39:02,889] Making new env: Pendulum-v0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Base (\n", + " (feature_net): LinearNet (\n", + " (net): Sequential (\n", + " (linear_0): NoisyLinear (3 -> 64)\n", + " (layer_norm_0): LayerNorm (\n", + " )\n", + " (act_0): ReLU ()\n", + " (linear_1): NoisyLinear (64 -> 64)\n", + " (layer_norm_1): LayerNorm (\n", + " )\n", + " (act_1): ReLU ()\n", + " )\n", + " )\n", + ")\n", + "ActorHead (\n", + " (base): Base (\n", + " (feature_net): LinearNet (\n", + " (net): Sequential (\n", + " (linear_0): NoisyLinear (3 -> 64)\n", + " (layer_norm_0): LayerNorm (\n", + " )\n", + " (act_0): ReLU ()\n", + " (linear_1): NoisyLinear (64 -> 64)\n", + " (layer_norm_1): LayerNorm (\n", + " )\n", + " (act_1): ReLU ()\n", + " )\n", + " )\n", + " )\n", + " (policy_net): LinearNet (\n", + " (net): Sequential (\n", + " (linear_0): Linear (64 -> 1)\n", + " (act_0): Tanh ()\n", + " )\n", + " )\n", + ")\n", + "CriticHead (\n", + " (base): Base (\n", + " (feature_net): LinearNet (\n", + " (net): Sequential (\n", + " (linear_0): NoisyLinear (3 -> 64)\n", + " (layer_norm_0): LayerNorm (\n", + " )\n", + " (act_0): ReLU ()\n", + " (linear_1): NoisyLinear (64 -> 64)\n", + " (layer_norm_1): LayerNorm (\n", + " )\n", + " (act_1): ReLU ()\n", + " )\n", + " )\n", + " )\n", + " (value_net): Linear (64 -> 1)\n", + ")\n", + "DynamicsHead (\n", + " (base): Base (\n", + " (feature_net): LinearNet (\n", + " (net): Sequential (\n", + " (linear_0): NoisyLinear (3 -> 64)\n", + " (layer_norm_0): LayerNorm (\n", + " )\n", + " (act_0): ReLU ()\n", + " (linear_1): NoisyLinear (64 -> 64)\n", + " (layer_norm_1): LayerNorm (\n", + " )\n", + " (act_1): ReLU ()\n", + " )\n", + " )\n", + " )\n", + " (value_net): Linear (65 -> 3)\n", + ")\n" + ] + } + ], + "source": [ + "os.environ['OMP_NUM_THREADS'] = '1'\n", + "torch.set_num_threads(1)\n", + "args = parse_args()\n", + "train(args,\n", + " create_model,\n", + " create_act_update_fns,\n", + " train_multi_thread,\n", + " train_single_thread,\n", + " play_single_thread)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2018-01-18T08:39:00.927Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "start_time": "2018-01-18T08:39:00.929Z" + } + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "jupyter3", + "language": "python", + "name": "jupyter3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.0" + }, + "toc": { + "colors": { + "hover_highlight": "#DAA520", + "navigate_num": "#000000", + "navigate_text": "#333333", + "running_highlight": "#FF0000", + "selected_highlight": "#FFD700", + "sidebar_border": "#EEEEEE", + "wrapper_background": "#FFFFFF" + }, + "moveMenuLeft": true, + "nav_menu": { + "height": "12px", + "width": "252px" + }, + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 4, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false, + "widenNotebook": false + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}