mirror of
https://github.com/wassname/DeepRL.git
synced 2026-10-29 12:40:58 +08:00
Finalize async methods
This commit is contained in:
Shangtong Zhang committed
1 parent
9c73abfbff
commit
8ea108e4bb
6 files changed
+76
-111
No files matched your search
+33
-81
@@ -18,9 +18,7 @@ class BasicNet:
|
||||
self.gpu = gpu and torch.cuda.is_available()
|
||||
self.LSTM = LSTM
|
||||
if self.gpu:
|
||||
print 'Transferring network to GPU...'
|
||||
self.cuda()
|
||||
print 'Network transferred.'
|
||||
|
||||
def to_torch_variable(self, x, dtype='float32'):
|
||||
if not isinstance(x, torch.FloatTensor):
|
||||
@@ -74,9 +72,9 @@ class DuelingNet(BasicNet):
|
||||
# Starting of several network instances
|
||||
|
||||
# Network for CartPole with value based methods
|
||||
class FullyConnectedNet(nn.Module, VanillaNet):
|
||||
class FCNet(nn.Module, VanillaNet):
|
||||
def __init__(self, dims, optimizer_fn=None, gpu=True):
|
||||
super(FullyConnectedNet, self).__init__()
|
||||
super(FCNet, self).__init__()
|
||||
self.fc1 = nn.Linear(dims[0], dims[1])
|
||||
self.fc2 = nn.Linear(dims[1], dims[2])
|
||||
self.fc3 = nn.Linear(dims[2], dims[3])
|
||||
@@ -92,9 +90,9 @@ class FullyConnectedNet(nn.Module, VanillaNet):
|
||||
return y
|
||||
|
||||
# Network for CartPole with dueling architecture
|
||||
class DuelingFullyConnectedNet(nn.Module, DuelingNet):
|
||||
class DuelingFCNet(nn.Module, DuelingNet):
|
||||
def __init__(self, dims, optimizer_fn=None, gpu=True):
|
||||
super(DuelingFullyConnectedNet, self).__init__()
|
||||
super(DuelingFCNet, self).__init__()
|
||||
self.fc1 = nn.Linear(dims[0], dims[1])
|
||||
self.fc2 = nn.Linear(dims[1], dims[2])
|
||||
self.fc_value = nn.Linear(dims[2], 1)
|
||||
@@ -109,10 +107,26 @@ class DuelingFullyConnectedNet(nn.Module, DuelingNet):
|
||||
phi = F.relu(self.fc2(y))
|
||||
return phi
|
||||
|
||||
# Network for CartPole with actor critic
|
||||
class ActorCriticFCNet(nn.Module, ActorCriticNet):
|
||||
def __init__(self,
|
||||
dims):
|
||||
super(ActorCriticFCNet, self).__init__()
|
||||
self.layer1 = nn.Linear(dims[0], dims[1])
|
||||
self.fc_actor = nn.Linear(dims[1], dims[2])
|
||||
self.fc_critic = nn.Linear(dims[1], 1)
|
||||
BasicNet.__init__(self, None, False)
|
||||
|
||||
def forward(self, x, update_LSTM=True):
|
||||
x = self.to_torch_variable(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
phi = self.layer1(x)
|
||||
return phi
|
||||
|
||||
# Network for pixel Atari game with value based methods
|
||||
class ConvNet(nn.Module, VanillaNet):
|
||||
class NatureConvNet(nn.Module, VanillaNet):
|
||||
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
|
||||
super(ConvNet, self).__init__()
|
||||
super(NatureConvNet, self).__init__()
|
||||
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
|
||||
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
|
||||
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
|
||||
@@ -130,28 +144,10 @@ class ConvNet(nn.Module, VanillaNet):
|
||||
y = F.relu(self.fc4(y))
|
||||
return self.fc5(y)
|
||||
|
||||
class NipsConvNet(nn.Module, VanillaNet):
|
||||
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
|
||||
super(NipsConvNet, self).__init__()
|
||||
self.conv1 = nn.Conv2d(in_channels, 16, kernel_size=8, stride=4)
|
||||
self.conv2 = nn.Conv2d(16, 32, kernel_size=4, stride=2)
|
||||
self.fc3 = nn.Linear(9 * 9 * 32, 256)
|
||||
self.fc4 = nn.Linear(256, n_actions)
|
||||
self.criterion = nn.MSELoss()
|
||||
BasicNet.__init__(self, optimizer_fn, gpu)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.to_torch_variable(x)
|
||||
y = F.relu(self.conv1(x))
|
||||
y = F.relu(self.conv2(y))
|
||||
y = y.view(y.size(0), -1)
|
||||
y = F.relu(self.fc3(y))
|
||||
return self.fc4(y)
|
||||
|
||||
# Network for pixel Atari game with dueling architecture
|
||||
class DuelingConvNet(nn.Module, DuelingNet):
|
||||
class DuelingNatureConvNet(nn.Module, DuelingNet):
|
||||
def __init__(self, in_channels, n_actions, optimizer_fn=None, gpu=True):
|
||||
super(DuelingConvNet, self).__init__()
|
||||
super(DuelingNatureConvNet, self).__init__()
|
||||
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
|
||||
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
|
||||
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
|
||||
@@ -170,45 +166,17 @@ class DuelingConvNet(nn.Module, DuelingNet):
|
||||
phi = F.relu(self.fc4(y))
|
||||
return phi
|
||||
|
||||
# Network for CartPole with actor critic
|
||||
class FCActorCriticNet(nn.Module, ActorCriticNet):
|
||||
def __init__(self,
|
||||
dims,
|
||||
LSTM=False):
|
||||
super(FCActorCriticNet, self).__init__()
|
||||
if LSTM:
|
||||
self.layer1 = nn.LSTMCell(dims[0], dims[1])
|
||||
else:
|
||||
self.layer1 = nn.Linear(dims[0], dims[1])
|
||||
self.fc_actor = nn.Linear(dims[1], dims[2])
|
||||
self.fc_critic = nn.Linear(dims[1], 1)
|
||||
BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM)
|
||||
if LSTM:
|
||||
self.h = self.to_torch_variable(np.zeros((1, dims[1])))
|
||||
self.c = self.to_torch_variable(np.zeros((1, dims[1])))
|
||||
|
||||
def forward(self, x, update_LSTM=True):
|
||||
x = self.to_torch_variable(x)
|
||||
x = x.view(x.size(0), -1)
|
||||
if self.LSTM:
|
||||
h, c = self.layer1(x, (self.h, self.c))
|
||||
if update_LSTM:
|
||||
self.h = h
|
||||
self.c = c
|
||||
phi = h
|
||||
else:
|
||||
phi = self.layer1(x)
|
||||
return phi
|
||||
|
||||
# Network for pixel Atari game with actor critic
|
||||
class ConvActorCriticNet(nn.Module, ActorCriticNet):
|
||||
class ActorCriticNatureConvNet(nn.Module, ActorCriticNet):
|
||||
def __init__(self,
|
||||
in_channels,
|
||||
n_actions,
|
||||
xentropy_weight=0.01,
|
||||
grad_threshold=40,
|
||||
gpu=True):
|
||||
super(ConvActorCriticNet, self).__init__()
|
||||
super(ActorCriticNatureConvNet, self).__init__()
|
||||
self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)
|
||||
self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)
|
||||
self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)
|
||||
@@ -227,12 +195,12 @@ class ConvActorCriticNet(nn.Module, ActorCriticNet):
|
||||
y = y.view(y.size(0), -1)
|
||||
return F.elu(self.fc4(y))
|
||||
|
||||
class OpenAIConvActorCriticNet(nn.Module, ActorCriticNet):
|
||||
class OpenAIActorCriticConvNet(nn.Module, ActorCriticNet):
|
||||
def __init__(self,
|
||||
in_channels,
|
||||
n_actions,
|
||||
LSTM=False):
|
||||
super(OpenAIConvActorCriticNet, self).__init__()
|
||||
super(OpenAIActorCriticConvNet, self).__init__()
|
||||
self.conv1 = nn.Conv2d(in_channels, 32, 3, stride=2, padding=1)
|
||||
self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
|
||||
self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
|
||||
@@ -273,27 +241,18 @@ class OpenAIConvActorCriticNet(nn.Module, ActorCriticNet):
|
||||
class OpenAIConvNet(nn.Module, VanillaNet):
|
||||
def __init__(self,
|
||||
in_channels,
|
||||
n_actions,
|
||||
LSTM=False):
|
||||
n_actions):
|
||||
super(OpenAIConvNet, self).__init__()
|
||||
self.conv1 = nn.Conv2d(in_channels, 32, 3, stride=2, padding=1)
|
||||
self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
|
||||
self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
|
||||
self.conv4 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
|
||||
|
||||
self.LSTM = LSTM
|
||||
hidden_units = 256
|
||||
|
||||
if LSTM:
|
||||
self.layer5 = nn.LSTMCell(32 * 3 * 3, hidden_units)
|
||||
else:
|
||||
self.layer5 = nn.Linear(32 * 3 * 3, hidden_units)
|
||||
|
||||
self.layer5 = nn.Linear(32 * 3 * 3, hidden_units)
|
||||
self.fc6 = nn.Linear(hidden_units, n_actions)
|
||||
BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=LSTM)
|
||||
if LSTM:
|
||||
self.h = self.to_torch_variable(np.zeros((1, hidden_units)))
|
||||
self.c = self.to_torch_variable(np.zeros((1, hidden_units)))
|
||||
|
||||
BasicNet.__init__(self, optimizer_fn=None, gpu=False, LSTM=False)
|
||||
|
||||
def forward(self, x, update_LSTM=True):
|
||||
x = self.to_torch_variable(x)
|
||||
@@ -302,12 +261,5 @@ class OpenAIConvNet(nn.Module, VanillaNet):
|
||||
y = F.elu(self.conv3(y))
|
||||
y = F.elu(self.conv4(y))
|
||||
y = y.view(y.size(0), -1)
|
||||
if self.LSTM:
|
||||
h, c = self.layer5(y, (self.h, self.c))
|
||||
if update_LSTM:
|
||||
self.h = h
|
||||
self.c = c
|
||||
phi = h
|
||||
else:
|
||||
phi = F.elu(self.layer5(y))
|
||||
phi = F.elu(self.layer5(y))
|
||||
return self.fc6(phi)
|
||||
Reference in new issue
Block a user