wassname/DeepRL
1
0
Fork 0
mirror of https://github.com/wassname/DeepRL.git synced 2026-09-10 11:40:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite state/reward normalizer

Browse Source
This commit is contained in:
Shangtong Zhang
2018-04-06 11:36:44 -06:00
parent 4c6481d5be
commit 8dfe6ff7c8
23 changed files with 125 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files
utils/config.py
+4 -7
View File
@@ -3,6 +3,7 @@
# Permission given to modify the code as long as you keep this #
# declaration at the top #
#######################################################################
from .normalizer import *
class Config:
def __init__(self):
@@ -22,22 +23,18 @@ class Config:
self.exploration_steps = 0
self.logger = None
self.history_length = 1
self.test_interval = 0
self.test_repetitions = 50
self.double_q = False
self.tag = 'vanilla'
self.num_workers = 1
self.worker = None
self.update_interval = 1
self.gradient_clip = 40
self.gradient_clip = 0.5
self.entropy_weight = 0.01
self.use_gae = False
self.gae_tau = 1.0
self.noise_decay_interval = 0
self.target_network_mix = 0.001
self.action_shift_fn = lambda a: a
self.reward_shift_fn = lambda r: r
self.reward_weight = 1
self.state_normalizer = RescaleNormalizer()
self.reward_normalizer = RescaleNormalizer()
self.hybrid_reward = False
self.episode_limit = 0
self.min_memory_size = 200
utils/misc.py
+1 -24
View File
@@ -16,7 +16,6 @@ def run_episodes(agent):
ep = 0
rewards = []
steps = []
avg_test_rewards = []
agent_type = agent.__class__.__name__
while True:
ep += 1
@@ -38,25 +37,8 @@ def run_episodes(agent):
if config.max_steps and agent.total_steps > config.max_steps:
break
if config.test_interval and ep % config.test_interval == 0:
config.logger.info('Testing...')
agent.save('data/%s-%s-model-%s.bin' % (agent_type, config.tag, agent.task.name))
test_rewards = []
for _ in range(config.test_repetitions):
test_rewards.append(agent.episode(True)[0])
avg_reward = np.mean(test_rewards)
avg_test_rewards.append(avg_reward)
config.logger.info('Avg reward %f(%f)' % (
avg_reward, np.std(test_rewards) / np.sqrt(config.test_repetitions)))
with open('data/%s-%s-all-stats-%s.bin' % (agent_type, config.tag, agent.task.name), 'wb') as f:
pickle.dump({'rewards': rewards,
'steps': steps,
'test_rewards': avg_test_rewards}, f)
if avg_reward > config.success_threshold:
break
agent.close()
return steps, rewards, avg_test_rewards
return steps, rewards
def run_iterations(agent):
config = agent.config
@@ -79,11 +61,6 @@ def run_iterations(agent):
pickle.dump({'rewards': rewards,
'steps': steps}, f)
agent.save('data/%s-%s-model-%s.bin' % (agent_name, config.tag, agent.task.name))
if config.test_interval and iteration % config.test_interval == 0:
test_rewards, test_steps = agent.evaluate()
config.logger.info('total steps %d, test reward %f, test steps %d' % (
agent.total_steps, test_rewards, test_steps
))
iteration += 1
if config.max_steps and agent.total_steps >= config.max_steps:
agent.close()
utils/normalizer.py
+20 -76
View File
@@ -6,16 +6,22 @@
import torch
import numpy as np
class Normalizer:
def __init__(self, x_size):
class RunningStatsNormalizer:
def __init__(self):
self.needs_reset = True
def reset(self, x_size):
self.m = np.zeros(x_size)
self.v = np.zeros(x_size)
self.n = 1.0
self.n = 0.0
self.needs_reset = False
def __call__(self, x):
if np.isscalar(x) or len(x.shape) == 1:
if self.needs_reset: self.reset(1)
return self.nomalize_single(x)
elif len(x.shape) == 2:
if self.needs_reset: self.reset(x.shape[1])
new_x = np.zeros(x.shape)
for i in range(x.shape[0]):
new_x[i] = self.nomalize_single(x[i])
@@ -38,79 +44,17 @@ class Normalizer:
x = np.asscalar(x)
return x
class StaticNormalizer:
def __init__(self, o_size):
self.offline_stats = SharedStats(o_size)
self.online_stats = SharedStats(o_size)
class RescaleNormalizer:
def __init__(self, coef=1.0):
self.coef = coef
def __call__(self, o_):
if np.isscalar(o_):
o = torch.FloatTensor([o_])
else:
o = torch.FloatTensor(o_)
self.online_stats.feed(o)
if self.offline_stats.n[0] == 0:
return o_
std = (self.offline_stats.v + 1e-6) ** .5
o = (o - self.offline_stats.m) / std
o = o.numpy()
if np.isscalar(o_):
o = np.asscalar(o)
else:
o = o.reshape(o_.shape)
return o
def state_dict(self):
return self.offline_stats.state_dict()
def __call__(self, x):
return self.coef * x
def load_state_dict(self, saved):
self.offline_stats.load_state_dict(saved)
class ImageNormalizer(RescaleNormalizer):
def __init__(self):
RescaleNormalizer.__init__(self, 1.0 / 255)
class SharedStats:
def __init__(self, o_size):
self.m = torch.zeros(o_size)
self.v = torch.zeros(o_size)
self.n = torch.zeros(1)
self.m.share_memory_()
self.v.share_memory_()
self.n.share_memory_()
def feed(self, o):
n = self.n[0]
new_m = self.m * (n / (n + 1)) + o / (n + 1)
self.v.copy_(self.v * (n / (n + 1)) + (o - self.m) * (o - new_m) / (n + 1))
self.m.copy_(new_m)
self.n.add_(1)
def zero(self):
self.m.zero_()
self.v.zero_()
self.n.zero_()
def load(self, stats):
self.m.copy_(stats.m)
self.v.copy_(stats.v)
self.n.copy_(stats.n)
def merge(self, B):
A = self
n_A = self.n[0]
n_B = B.n[0]
n = n_A + n_B
delta = B.m - A.m
m = A.m + delta * n_B / n
v = A.v * n_A + B.v * n_B + delta * delta * n_A * n_B / n
v /= n
self.m.copy_(m)
self.v.copy_(v)
self.n.add_(B.n)
def state_dict(self):
return {'m': self.m.numpy(),
'v': self.v.numpy(),
'n': self.n.numpy()}
def load_state_dict(self, saved):
self.m = torch.FloatTensor(saved['m'])
self.v = torch.FloatTensor(saved['v'])
self.n = torch.FloatTensor(saved['n'])
class SignNormalizer:
def __call__(self, x):
return np.sign(x)