diff --git a/network/continuous_action_network.py b/network/continuous_action_network.py index d900094..f434996 100644 --- a/network/continuous_action_network.py +++ b/network/continuous_action_network.py @@ -5,6 +5,7 @@ ####################################################################### from .base_network import * +import math class DeterministicActorNet(nn.Module, BasicNet): def __init__(self, @@ -147,18 +148,35 @@ class GaussianActorNet(nn.Module, BasicNet): else: std = F.softplus(self.action_std(phi)) + 1e-5 log_std = std.log() + + # clip for numeric stability https://github.com/reinforceio/tensorforce/blob/master/tensorforce/core/distributions/gaussian.py#L53 + log_eps = log(util.epsilon) # epsilon < 1.0, hence negative + log_std.clamp(log_eps,-log_eps) + return mean, std, log_std def predict(self, x): return self.forward(x) def log_density(self, x, mean, log_std, std): - var = std.pow(2) - log_density = -(x - mean).pow(2) / (2 * var + 1e-5) - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x)) - log_std + # x is action + # https://github.com/reinforceio/tensorforce/blob/master/tensorforce/core/distributions/gaussian.py#L85 + # same as tensorforce but max instead of + eps + sq_stddev = torch.max(std.pow(2), 1e-5) + sq_mean_distance = (x - mean).pow(2) + sq_stddev = tf.maximum(x=var, y=util.epsilon) + log_density = - 0.5 * sq_mean_distance / sq_stddev \ + - 0.5 * torch.log(2 * Variable(torch.FloatTensor([np.pi])).expand_as(x))\ + - log_std return log_density.sum(1) def entropy(self, std): - return 0.5 * (1 + (2 * std.pow(2) * np.pi + 1e-5).log()).sum(1).mean() + # 8.18 http://www.biopsychology.org/norwich/isp/chap8.pdf + sq_stddev = torch.max(std.pow(2), 1e-5) + # return 0.5 * (1 + (2 * sq_stddev * np.pi).log()).sum(1).mean() + return 0.5 * (2 * sq_stddev * np.pi * np.e).log().sum(1).mean() + # return std.log() + 0.5 * log(2.0 * Variable(torch.FloatTensor([math.pi*math.e]))) + # return 0.5 * (2.0 * sq_stddev * Variable(torch.FloatTensor([math.pi*math.e]))).log() class GaussianCriticNet(nn.Module, BasicNet): def __init__(self, state_dim, gpu=False):