mirror of
https://github.com/wassname/Deep-reinforcement-learning-with-pytorch.git
synced 2026-09-09 11:13:45 +08:00
816 lines
49 KiB
Python
816 lines
49 KiB
Python
# coding:utf-8
|
||
|
||
import sys,numpy,os,scipy.signal,threading,time,random,copy
|
||
import tensorflow as tf
|
||
sys.path.append('../../pythonModules')
|
||
import wgenv,common,wgfeature,wgsdata
|
||
from resnet_model import ResNet
|
||
## HELPER FUNCTIONS
|
||
|
||
# Copies one set of variables to another.
|
||
# Used to set worker network parameters to those of global network.
|
||
def update_target_graph(from_scope,to_scope):
|
||
from_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, from_scope)
|
||
to_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, to_scope)
|
||
op_holder = []
|
||
for from_var,to_var in zip(from_vars,to_vars):
|
||
op_holder.append(to_var.assign(from_var))
|
||
return op_holder
|
||
|
||
def filterMetadata(env,BattleMode):
|
||
'''sgr修改，去除部分加载算子，在简单情况下验证算法
|
||
battlemode = 66
|
||
'''
|
||
try:
|
||
|
||
l_rbops,l_bbops, l_pbops= env.obj_main.dic_metadata['l_rbops'],env.obj_main.dic_metadata['l_bbops'],env.obj_main.dic_metadata['l_pbops']
|
||
if BattleMode == '6v6':
|
||
return l_rbops, l_bbops, l_pbops
|
||
elif BattleMode == '1v0':
|
||
l_f_rbops = []
|
||
l_f_bbops = []
|
||
for bop in l_bbops:
|
||
if bop.ObjTypeX == 0:
|
||
l_f_bbops.append(bop)
|
||
break
|
||
l_f_pbops = []
|
||
return l_f_rbops,l_f_bbops,l_f_pbops
|
||
elif BattleMode == '1v1':
|
||
l_f_rbops = []
|
||
l_f_bbops = []
|
||
for bop in l_bbops:
|
||
if bop.ObjTypeX == 0:
|
||
l_f_bbops.append(bop)
|
||
break
|
||
for bop in l_rbops:
|
||
if bop.ObjTypeX == 0:
|
||
l_f_rbops.append(bop)
|
||
break
|
||
l_f_pbops = []
|
||
l_rbops, l_bbops, l_pbops = l_f_rbops, l_f_bbops, l_f_pbops
|
||
except Exception as e:
|
||
print('error in wgsdata.py -> filterMetadata() ' + str(e))
|
||
raise
|
||
|
||
def process_observation(obj_main):
|
||
wgfeature.calUpdateFeature(obj_main)
|
||
np_mapFeature = obj_main.dic_metadata['np_mapFeature']
|
||
np_mapFeature = numpy.swapaxes(np_mapFeature,0, 1)
|
||
np_mapFeature = numpy.swapaxes(np_mapFeature, 1, 2)
|
||
np_bopFeature = obj_main.dic_metadata['np_bopFeature']
|
||
return np_mapFeature[numpy.newaxis,:,:,:],np_bopFeature[numpy.newaxis,:]
|
||
|
||
def normalized_columns_initializer(std=1.0):
|
||
def _initializer(shape, dtype=None, partition_info=None):
|
||
out = numpy.random.randn(*shape).astype(numpy.float32)
|
||
out *= std / numpy.sqrt(numpy.square(out).sum(axis=0, keepdims=True))
|
||
return tf.constant(out)
|
||
return _initializer
|
||
|
||
# Discounting function used to calculate discounted returns.
|
||
def discount(x, gamma):
|
||
return scipy.signal.lfilter([1], [1, -gamma], x[::-1], axis=0)[::-1]
|
||
|
||
#
|
||
def calScore(obj_main,flag_rlaicolor):
|
||
try:
|
||
l_values = wgsdata.getSdataValue(obj_main.dic_metadata)
|
||
l_scores = [l_values[0],120-l_values[2],l_values[1]] if flag_rlaicolor == 0 else [l_values[2],120-l_values[0],l_values[3]]
|
||
return l_scores
|
||
except Exception as e:
|
||
print('error in test_rl.py -> calScore() : ' + str(e))
|
||
raise
|
||
|
||
|
||
class AC_Network():
|
||
def __init__(self, scope, trainer, trainer_critic, nonspatial_size,map_size,dict_actionsepc):
|
||
'''modify0 2018年11月07日 sgr 修改网络节点的name属性，共享节点.name =scope/public...
|
||
，policy = scope/policy... value = scope/value...为了反向传播时只更新特定层的梯度
|
||
modify1 2018年11月07日 sgr 增加dic_hparas['UpdateGradsMethod'] in ['Integration','Divide'] 指示整体训练policy和value网，
|
||
还是分开训。分训的时候value 网只更新全连接层参数
|
||
'''
|
||
try:
|
||
with tf.variable_scope(scope):
|
||
# get size of features from action_spec and observation_spec
|
||
assert dic_hparas['NetType'] in ['AtariNet','ResNet']
|
||
with tf.variable_scope('public'): # name 'scope/public/...'
|
||
if dic_hparas['NetType'] == 'AtariNet':
|
||
self.inputs_nonspatial = tf.placeholder(shape=[None, nonspatial_size], dtype=tf.float32)
|
||
self.inputs_spatial_screen = tf.placeholder(shape=[None, map_size[0], map_size[1], map_size[2]],dtype=tf.float32)
|
||
# Architecture here follows Atari-net Agent described in [1] Section 4.3
|
||
self.nonspatial_dense = tf.layers.dense(inputs=self.inputs_nonspatial,units=32,activation=tf.tanh)
|
||
self.screen_conv1 = tf.layers.conv2d(inputs=self.inputs_spatial_screen,filters=16,kernel_size=[8,8],strides=[4,4],padding='valid',activation=tf.nn.relu)
|
||
self.screen_conv2 = tf.layers.conv2d(inputs=self.screen_conv1,filters=32,kernel_size=[4,4],strides=[2,2],padding='valid',activation=tf.nn.relu)
|
||
|
||
# According to [1]: "The results are concatenated and sent through a linear layer with a ReLU activation."
|
||
screen_output_length = 1
|
||
for dim in self.screen_conv2.get_shape().as_list()[1:]:
|
||
screen_output_length *= dim
|
||
|
||
self.latent_vector = tf.layers.dense(
|
||
inputs=tf.concat([self.nonspatial_dense, tf.reshape(self.screen_conv2,shape=[-1,screen_output_length])], axis=1),units=256,activation=tf.nn.relu)
|
||
elif dic_hparas['NetType'] == 'ResNet':
|
||
self.inputs_nonspatial = tf.placeholder(shape=[None, nonspatial_size], dtype=tf.float32)
|
||
self.inputs_spatial_screen = tf.placeholder(shape=[None, map_size[0], map_size[1], map_size[2]],dtype=tf.float32)
|
||
# Architecture here follows ResNet
|
||
batch_size = self.inputs_spatial_screen.get_shape().as_list()[0]
|
||
self.resnet = ResNet(self.inputs_spatial_screen,batch_size)
|
||
self.latent_vector = self.resnet._build_model()
|
||
|
||
### 下面是HQ加的网络
|
||
self.policy_dense_1 = {}
|
||
self.policy_dense_2 = {}
|
||
self.value_dense_1 = None
|
||
self.value_dense_2 = None
|
||
self.policy_arg = {}
|
||
self.value = None
|
||
|
||
else:
|
||
common.echosentence_color('dic_hparas.NetType error.')
|
||
|
||
# Output layers for policy and value estimations
|
||
# 1 policy network for base actions
|
||
# 16 policy networks for arguments
|
||
# - All modeled independently
|
||
# - Spatial arguments have the x and y values modeled independently as well
|
||
# 1 value network
|
||
with tf.variable_scope('policy'): # name = 'scope/policy/...'
|
||
|
||
for key,value in dict_actionsepc.items():
|
||
self.policy_dense_1[key] = tf.layers.dense(inputs=self.latent_vector,units=128,activation=tf.nn.tanh)
|
||
self.policy_dense_2[key] = tf.layers.dense(inputs=self.policy_dense_1[key], units=64,
|
||
activation=tf.nn.tanh)
|
||
self.policy_arg[key] = tf.layers.dense(inputs=self.policy_dense_2[key], units=value,
|
||
activation=tf.nn.softmax,
|
||
kernel_initializer=normalized_columns_initializer(0.01))
|
||
|
||
with tf.variable_scope('value'): # name = 'scope/value/...'
|
||
self.value_dense_1 = tf.layers.dense(inputs=self.latent_vector, units=128,
|
||
activation=tf.nn.tanh)
|
||
self.value_dense_2 = tf.layers.dense(inputs=self.value_dense_1, units=64,
|
||
activation=tf.nn.tanh)
|
||
self.value = tf.layers.dense(inputs=self.value_dense_2,units=1,kernel_initializer=normalized_columns_initializer(1.0))
|
||
|
||
# Only the worker network need ops for loss functions and gradient updating.
|
||
if scope != 'global':
|
||
self.actions_arg = dict()
|
||
self.actions_onehot_arg = dict()
|
||
for key, value in dict_actionsepc.items():
|
||
self.actions_arg[key] = tf.placeholder(shape=[None],dtype=tf.int32)
|
||
self.actions_onehot_arg[key] = tf.one_hot(self.actions_arg[key],value,dtype=tf.float32)
|
||
|
||
self.target_v = tf.placeholder(shape=[None],dtype=tf.float32)
|
||
self.advantages = tf.placeholder(shape=[None],dtype=tf.float32)
|
||
|
||
|
||
self.responsible_outputs_arg = dict()
|
||
|
||
for key, value in dict_actionsepc.items():
|
||
self.responsible_outputs_arg[key] = tf.reduce_sum(self.policy_arg[key] * self.actions_onehot_arg[key], [1])
|
||
# tf_c1 = tf.assign(self.responsible_outputs_arg[key],1)
|
||
# #shape = tf.shape(self.responsible_outputs_arg[key])
|
||
# shape = self.screen_conv2.get_shape()
|
||
# tf_c1= tf.constant(1,shape=self.screen_conv2.get_shape())
|
||
# self.responsible_outputs_arg[key] = tf.where(tf.equal(self.responsible_outputs_arg[key],0),tf_c1,self.responsible_outputs_arg[key])
|
||
|
||
# Loss functions
|
||
self.value_loss = 0.5 * tf.reduce_sum(tf.square(self.target_v - tf.reshape(self.value,[-1])))
|
||
|
||
self.entropy_arg = dict()
|
||
for key, value in dict_actionsepc.items():
|
||
self.entropy_arg[key] = -tf.reduce_sum(self.policy_arg[key] * tf.log(tf.clip_by_value(self.policy_arg[key],1e-20,1.)))
|
||
|
||
self.entropy = 0
|
||
for key, value in dict_actionsepc.items():
|
||
self.entropy += self.entropy_arg[key]
|
||
|
||
self.policy_loss_arg = dict()
|
||
for key, value in dict_actionsepc.items():
|
||
self.policy_loss_arg[key] = - tf.reduce_sum(tf.log(tf.clip_by_value(self.responsible_outputs_arg[key]+0.05, 1e-20, 1.05)) * self.advantages)
|
||
|
||
self.policy_loss = 0
|
||
for key, value in dict_actionsepc.items():
|
||
self.policy_loss += self.policy_loss_arg[key]
|
||
|
||
# Get gradients from local network using local losses
|
||
self.local_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope)
|
||
self.var_norms = tf.global_norm(self.local_vars)
|
||
|
||
if 'UpdateGradsMethod' not in dic_hparas.keys() or dic_hparas['UpdateGradsMethod'] not in ['Integration','Divide']:
|
||
print('dic_hparas key error : UpdateGradsMethod')
|
||
if dic_hparas['UpdateGradsMethod'] == 'Integration':
|
||
self.loss = 0.5 * self.value_loss + self.policy_loss - self.entropy * 0.01
|
||
# self.gradients - gradients of loss wrt local_vars
|
||
self.gradients = tf.gradients(self.loss,self.local_vars)
|
||
|
||
self.grads,self.grad_norms = tf.clip_by_global_norm(self.gradients,0.5) # old=40 2018年09月10日修改，梯度太大
|
||
|
||
# Apply local gradients to global network
|
||
self.global_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, 'global')
|
||
self.apply_grads = trainer.apply_gradients(zip(self.grads,self.global_vars))
|
||
elif dic_hparas['UpdateGradsMethod'] == 'Divide':
|
||
# policy
|
||
self.loss_policy = self.policy_loss - self.entropy * 0.01
|
||
self.local_policy_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope+'/public') + tf.get_collection(
|
||
tf.GraphKeys.TRAINABLE_VARIABLES,scope+'/policy')
|
||
self.gradients_policy = tf.gradients(self.loss_policy,self.local_policy_vars)
|
||
self.grads_policy,self.grad_norms_policy = tf.clip_by_global_norm(self.gradients_policy,0.5)
|
||
self.global_vars_policy = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,'global'+'/public') + tf.get_collection(
|
||
tf.GraphKeys.TRAINABLE_VARIABLES,'global'+'/policy')
|
||
self.apply_grads_policy = trainer.apply_gradients(zip(self.grads_policy,self.global_vars_policy))
|
||
#value
|
||
self.loss_value = self.value_loss
|
||
self.local_value_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,scope+'/value')
|
||
self.gradients_value = tf.gradients(self.loss_value,self.local_value_vars)
|
||
self.grads_value,self.grad_norms_value = tf.clip_by_global_norm(self.gradients_value,0.5)
|
||
self.global_vars_value = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,'global'+'/value')
|
||
self.apply_grads_value = trainer_critic.apply_gradients(zip(self.grads_value,self.global_vars_value))
|
||
except Exception as e:
|
||
common.echosentence_color('error in AC_NetWork -> init: {}'.format(str(e)) )
|
||
raise
|
||
|
||
|
||
## WORKER AGENT
|
||
class Worker():
|
||
def __init__(self, name, trainer, trainer_critic, model_path,log_path,roomrd_path, global_episodes,flag_rlaicolor,dic_mainparas,
|
||
flag_testai = False,testai_freq = 500,testai_runnums = 20,step_length = 100,history_policy_len = 5):
|
||
self.name = "worker_" + str(name)
|
||
self.number = name
|
||
self.model_path = model_path
|
||
self.roomrd_path = roomrd_path
|
||
self.flag_rlaicolor = flag_rlaicolor
|
||
self.trainer = trainer
|
||
self.trainer_critic = trainer_critic
|
||
self.global_episodes = global_episodes
|
||
self.increment = self.global_episodes.assign_add(1)
|
||
self.episode_rewards = []
|
||
self.episode_lengths = []
|
||
self.episode_mean_values = []
|
||
self.episode_winnums = []
|
||
self.l_actkeys = ['remain_scores','defeat_scores','city_scores']
|
||
self.dict_episode_scores = {'remain_scores':[],'defeat_scores':[],'city_scores':[]} #分数
|
||
self.dict_episode_actnums = {} #动作数
|
||
|
||
self.summary_writer = tf.summary.FileWriter(log_path) # 修改，由外部传入
|
||
self.flag_testai = flag_testai # 是否测试ai
|
||
self.testai_freq = testai_freq # 测试ai的频率
|
||
self.testai_runnums = testai_runnums # 测试50场的平均胜率
|
||
self.history_policy_len = history_policy_len #指训练价值网时所使用的历史策略网长度
|
||
self.step_length = step_length
|
||
self.episode_test_winrates = [] #测试的平均胜率
|
||
self.episode_test_winscores = [] #测试的平均净胜分
|
||
self.prob_dec_fn = lambda x : 1.0/(0.0009*x+1.1)
|
||
print('Initializing environment #{}...'.format(self.number))
|
||
self.env = wgenv.ENV(dic_mainparas = dic_mainparas,flag_rlaicolor=flag_rlaicolor)
|
||
filterMetadata(self.env,dic_hparas['BattleMode'])
|
||
# 初始化保存动作执行变量
|
||
self.num_whats = self.env.getWhatNums()
|
||
for i in range(self.num_whats):
|
||
self.dict_episode_actnums[i] = []
|
||
print('initlize static map feature...')
|
||
wgfeature.cfunRegister(self.env.obj_main)
|
||
wgfeature.calOneTimeFeature(self.env.obj_main)
|
||
wgfeature.calUpdateFeature(self.env.obj_main)
|
||
print('initlize static map feature done...')
|
||
# Create the local copy of the network and the tensorflow op to copy global paramters to local network
|
||
self.nonspatial_size = self.env.obj_main.dic_metadata['np_bopFeature'].shape[0]
|
||
tmp_map_size = self.env.obj_main.dic_metadata['np_mapFeature'].shape
|
||
self.map_size = (tmp_map_size[1],tmp_map_size[2],tmp_map_size[0])
|
||
self.l_action_keys = ['who','what','how']
|
||
dict_actionsepc ={self.l_action_keys[0]:self.env.getWhoNums(),self.l_action_keys[1]:self.env.getWhatNums(),self.l_action_keys[2]:self.env.getHowNums()}
|
||
self.local_AC = AC_Network(self.name, trainer, trainer_critic, self.nonspatial_size,self.map_size,dict_actionsepc)
|
||
self.update_local_ops = update_target_graph('global', self.name)
|
||
|
||
def train(self, list_rollout, sess, gamma, bootstrap_value,updateParas = None):
|
||
'''modify: 2018年11月09日 宋国瑞 添加updateParas 取值:['Policy','Value','None','Both'] None或Both表示两者都更新'''
|
||
try:
|
||
common.echosentence_color('--' * 20, 'red')
|
||
common.echosentence_color('Start Train {}'.format(updateParas))
|
||
|
||
for index in range(len(list_rollout)):
|
||
rollout = numpy.array(list_rollout[index])
|
||
|
||
s_np_mapFeature,s_np_bopFeature = rollout[:, 0],rollout[:,1]
|
||
np_l_chooseArg, rewards = rollout[:,2],rollout[:,3]
|
||
s1_np_mapFeature, s1_np_bopFeature = rollout[:,4],rollout[:,5]
|
||
episode_end, values = rollout[:,6],rollout[:,7]
|
||
|
||
actions_arg_stack = dict()
|
||
for l_chooseArg in np_l_chooseArg:
|
||
for i in range(len(self.l_action_keys)):
|
||
key = self.l_action_keys[i]
|
||
if key not in actions_arg_stack:
|
||
actions_arg_stack[key] = []
|
||
actions_arg_stack[key].append(l_chooseArg[i])
|
||
|
||
# Here we take the rewards and values from the rollout, and use them to calculate the advantage and discounted returns
|
||
# The advantage function uses generalized advantage estimation from [2]
|
||
self.rewards_plus = numpy.asarray(rewards.tolist() + [bootstrap_value])
|
||
discounted_rewards = discount(self.rewards_plus, gamma)[:-1]
|
||
self.value_plus = numpy.asarray(values.tolist() + [bootstrap_value])
|
||
advantages = rewards + gamma * self.value_plus[1:] - self.value_plus[:-1]
|
||
advantages = discount(advantages, gamma)
|
||
|
||
# Update the global network using gradients from loss
|
||
# Generate network statistics to periodically save
|
||
if index == 0:
|
||
feed_dict = {self.local_AC.target_v: discounted_rewards,
|
||
self.local_AC.inputs_spatial_screen: numpy.stack(s_np_mapFeature).reshape(-1, self.map_size[0],self.map_size[1],self.map_size[2]),
|
||
self.local_AC.inputs_nonspatial: numpy.stack(s_np_bopFeature).reshape(-1, self.nonspatial_size),
|
||
self.local_AC.advantages: advantages}
|
||
|
||
for key,value in actions_arg_stack.items():
|
||
feed_dict[self.local_AC.actions_arg[key]] = value
|
||
else:
|
||
feed_dict[self.local_AC.target_v] = numpy.concatenate((feed_dict[self.local_AC.target_v],discounted_rewards),axis = 0)
|
||
np_mapFeature = numpy.stack(s_np_mapFeature).reshape(-1, self.map_size[0],self.map_size[1],self.map_size[2])
|
||
np_bopFeature = numpy.stack(s_np_bopFeature).reshape(-1, self.nonspatial_size)
|
||
feed_dict[self.local_AC.inputs_spatial_screen] = numpy.concatenate((feed_dict[self.local_AC.inputs_spatial_screen], np_mapFeature), axis=0)
|
||
feed_dict[self.local_AC.inputs_nonspatial] = numpy.concatenate((feed_dict[self.local_AC.inputs_nonspatial], np_bopFeature), axis=0)
|
||
feed_dict[self.local_AC.advantages] = numpy.concatenate((feed_dict[self.local_AC.advantages], advantages), axis=0)
|
||
for key,value in actions_arg_stack.items():
|
||
feed_dict[self.local_AC.actions_arg[key]] += value
|
||
if dic_hparas['UpdateGradsMethod'] == 'Integration':
|
||
v_l, p_l, e_l, g_n, v_n, _ = sess.run([self.local_AC.value_loss,
|
||
self.local_AC.policy_loss,
|
||
self.local_AC.entropy,
|
||
self.local_AC.grad_norms,
|
||
self.local_AC.var_norms,
|
||
self.local_AC.apply_grads],
|
||
feed_dict=feed_dict)
|
||
elif dic_hparas['UpdateGradsMethod'] == 'Divide':
|
||
if updateParas == 'Policy':
|
||
v_l, p_l, e_l, g_n, v_n, _, = sess.run([self.local_AC.value_loss,
|
||
self.local_AC.policy_loss,
|
||
self.local_AC.entropy,
|
||
self.local_AC.grad_norms_policy,
|
||
self.local_AC.var_norms,
|
||
self.local_AC.apply_grads_policy,
|
||
],feed_dict=feed_dict)
|
||
elif updateParas == 'Value':
|
||
v_l, p_l, e_l, g_n, v_n, _, = sess.run([self.local_AC.value_loss,
|
||
self.local_AC.policy_loss,
|
||
self.local_AC.entropy,
|
||
self.local_AC.grad_norms_policy,
|
||
self.local_AC.var_norms,
|
||
self.local_AC.apply_grads_value,
|
||
],feed_dict=feed_dict)
|
||
else:
|
||
v_l, p_l, e_l, g_n, v_n, _, _, = sess.run([self.local_AC.value_loss,
|
||
self.local_AC.policy_loss,
|
||
self.local_AC.entropy,
|
||
self.local_AC.grad_norms_policy,
|
||
self.local_AC.var_norms,
|
||
self.local_AC.apply_grads_policy,
|
||
self.local_AC.apply_grads_value,
|
||
], feed_dict=feed_dict)
|
||
|
||
common.echosentence_color('--' * 20, 'red')
|
||
return v_l / rollout.shape[0] / rollout.shape[1], p_l / rollout.shape[0] / rollout.shape[1], \
|
||
e_l / rollout.shape[0] / rollout.shape[1], g_n, v_n
|
||
except Exception as e:
|
||
common.echosentence_color('error in test_rl.py->train():{}'.format(str(e)))
|
||
self.env.__del__()
|
||
raise
|
||
except KeyboardInterrupt as k:
|
||
common.echosentence_color('error in test_rl.py->train():{}'.format(str(k)))
|
||
self.env.__del__()
|
||
raise
|
||
|
||
def getwincolor(self,dic_metadata):
|
||
'''打印态势信息'''
|
||
try:
|
||
list_values = wgsdata.getSdataValue(dic_metadata)
|
||
# cout_left = [0] * 2
|
||
# for cur_bop in self.dic_metadata['l_rbops'] + self.dic_metadata['l_bbops'] + self.dic_metadata['l_pbops']:
|
||
# if cur_bop.ObjBlood > 0:
|
||
# cout_left[int(cur_bop.GameColor)] += 1
|
||
# print '\t 红方剩余算子数量：%d. 剩余兵力分值：%d, 夺控得分：%d' % (cout_left[0], list_values[0], list_values[1])
|
||
# print '\t 蓝方剩余算子数量：%d. 剩余兵力分值：%d, 夺控得分：%d' % (cout_left[1], list_values[2], list_values[3])
|
||
return 0 if list_values[0] + list_values[1] - list_values[2] > \
|
||
list_values[2] + list_values[3] - list_values[0] \
|
||
else 1
|
||
except Exception as e:
|
||
common.echosentence_color('wgsdata > showSDDat():{}'.format(str(e)))
|
||
raise
|
||
|
||
def work(self, max_episode_length, gamma, sess, coord, saver):
|
||
try:
|
||
episode_count = sess.run(self.global_episodes)
|
||
total_steps = 0
|
||
print ("Starting worker " + str(self.number))
|
||
with sess.as_default(), sess.graph.as_default():
|
||
t_start = time.time()
|
||
winnums = 0
|
||
list3_episode_buffer = []
|
||
while True: #not coord.should_stop():
|
||
# Download copy of parameters from global network
|
||
# prob = self.prob_dec_fn(episode_count) #用规则ai的概率
|
||
prob = 0
|
||
sess.run(self.update_local_ops)
|
||
episode_buffer = []
|
||
episode_values = []
|
||
# episode_frames = []
|
||
episode_reward = 0
|
||
episode_step_count = 0
|
||
episode_end = False
|
||
|
||
# 测试ai性能
|
||
if self.flag_testai and episode_count % self.testai_freq == 0 and episode_count >= 0:
|
||
winrate,winscore = self.test_ai(sess,self.testai_runnums)
|
||
self.episode_test_winrates.append(winrate)
|
||
self.episode_test_winscores.append(winscore)
|
||
common.echosentence_color('====================','blue')
|
||
common.echosentence_color('测试{}场的胜率为：{}% ；净胜分为 {}'.format(self.testai_runnums,winrate*100,winscore))
|
||
common.echosentence_color('====================', 'yellow')
|
||
summary = tf.Summary()
|
||
summary.value.add(tag='test/WinRate', simple_value=float(winrate))
|
||
summary.value.add(tag='test/WinScore', simple_value=float(winscore))
|
||
self.summary_writer.add_summary(summary, episode_count)
|
||
self.summary_writer.flush()
|
||
|
||
flag_wincolor = 0
|
||
# Start new episode
|
||
flag_goon = self.env.reset()
|
||
episode_end = not flag_goon
|
||
np_mapFeature,np_bopFeature = process_observation(self.env.obj_main) #resize np_mapFeature to [1,2,0]
|
||
s_np_mapFeature,s_np_bopFeature = np_mapFeature,np_bopFeature
|
||
for i in range(self.num_whats):
|
||
self.dict_episode_actnums[i].append(0)
|
||
|
||
while not episode_end:
|
||
flag_baseai = False
|
||
if(random.random() < prob):
|
||
flag_thisvalid,flag_goon,reward,list2_chooseArg,list_npMapFea,list_npBopFea = self.env.stepFollowBaseAI() # 依概率选择按照规则AI走,不参与训练
|
||
if flag_thisvalid: #生成有效动作
|
||
flag_baseai = True
|
||
else: #否则按照rl-ai选择动作
|
||
flag_baseai = False
|
||
policy_args, v = sess.run([self.local_AC.policy_arg, self.local_AC.value],
|
||
feed_dict={self.local_AC.inputs_spatial_screen: np_mapFeature,
|
||
self.local_AC.inputs_nonspatial: np_bopFeature})
|
||
l_nppolicy = []
|
||
for policy_key in self.l_action_keys:
|
||
l_nppolicy.append(policy_args[policy_key][0, :])
|
||
|
||
flag_goon, reward, l_chooseArg = self.env.step(l_nppolicy[0], l_nppolicy[1],l_nppolicy[2])
|
||
else:
|
||
policy_args, v = sess.run([self.local_AC.policy_arg, self.local_AC.value],
|
||
feed_dict={self.local_AC.inputs_spatial_screen: np_mapFeature,
|
||
self.local_AC.inputs_nonspatial: np_bopFeature})
|
||
# nonspatial_dense,screen_conv1,screen_conv2,latent_vector= sess.run([self.local_AC.nonspatial_dense, self.local_AC.screen_conv1,self.local_AC.screen_conv2,self.local_AC.latent_vector],
|
||
# feed_dict={self.local_AC.inputs_spatial_screen: np_mapFeature,
|
||
# self.local_AC.inputs_nonspatial: np_bopFeature})
|
||
#
|
||
# local_vars,global_vars = sess.run([self.local_AC.local_vars,self.local_AC.global_vars])
|
||
|
||
l_nppolicy = []
|
||
for policy_key in self.l_action_keys:
|
||
l_nppolicy.append(policy_args[policy_key][0, :])
|
||
|
||
flag_goon, reward, l_chooseArg = self.env.step(l_nppolicy[0], l_nppolicy[1], l_nppolicy[2])
|
||
|
||
episode_end = not flag_goon
|
||
np_mapFeature, np_bopFeature = process_observation(self.env.obj_main) # resize np_mapFeature to [1,2,0]
|
||
|
||
if episode_end: #游戏结束，根据游戏胜负修正最后的reward
|
||
flag_wincolor = self.getwincolor(self.env.obj_main.dic_metadata)
|
||
if flag_rlaicolor == flag_wincolor:
|
||
reward += 20 # [0,200]
|
||
|
||
s1_np_mapFeature, s1_np_bopFeature = np_mapFeature, np_bopFeature
|
||
# if not episode_end:
|
||
# s1_np_mapFeature, s1_np_bopFeature = np_mapFeature, np_bopFeature
|
||
# else:
|
||
# s1_np_mapFeature, s1_np_bopFeature = s_np_mapFeature,s_np_bopFeature
|
||
|
||
# Append latest state to buffer
|
||
if not flag_baseai:
|
||
if l_chooseArg != None:
|
||
self.dict_episode_actnums[l_chooseArg[1]][-1] += 1
|
||
episode_buffer.append([s_np_mapFeature,s_np_bopFeature, l_chooseArg, reward, s1_np_mapFeature,s1_np_bopFeature, episode_end, v[0, 0]])
|
||
episode_values.append(v[0, 0])
|
||
else:
|
||
assert (len(list_npBopFea) == len(list2_chooseArg))
|
||
list_npMapFea = [s_np_mapFeature] + list_npMapFea # 修复bug 动作执行之前的状态应该是s_np_mapFeature
|
||
list_npBopFea = [s_np_bopFeature] + list_npBopFea
|
||
for l_i in range(len(list2_chooseArg)):
|
||
l_chooseArg,s_np_mapFeature,s_np_bopFeature = list2_chooseArg[l_i],list_npMapFea[l_i],list_npBopFea[l_i]
|
||
s1_np_mapFeature,s1_np_bopFeature = list_npMapFea[l_i+1],list_npBopFea[l_i+1]
|
||
v = sess.run(self.local_AC.value,
|
||
feed_dict={self.local_AC.inputs_spatial_screen: s_np_mapFeature,
|
||
self.local_AC.inputs_nonspatial: s_np_bopFeature})
|
||
self.dict_episode_actnums[l_chooseArg[1]][-1] += 1
|
||
episode_buffer.append([s_np_mapFeature, s_np_bopFeature, l_chooseArg, reward, s1_np_mapFeature,s1_np_bopFeature, episode_end, v[0, 0]])
|
||
episode_values.append(v[0, 0])
|
||
|
||
episode_reward = reward
|
||
s_np_mapFeature, s_np_bopFeature = s1_np_mapFeature, s1_np_bopFeature
|
||
total_steps += 1
|
||
episode_step_count += 1
|
||
|
||
# If the episode hasn't ended, but the experience buffer is full, then we make an update step using that experience rollout
|
||
if len(episode_buffer) == self.step_length and not episode_end and episode_step_count != max_episode_length - 1:
|
||
# Since we don't know what the true final return is, we "bootstrap" from our current value estimation
|
||
v1 = sess.run(self.local_AC.value,feed_dict={self.local_AC.inputs_spatial_screen: np_mapFeature,
|
||
self.local_AC.inputs_nonspatial: np_bopFeature})
|
||
|
||
v_l, p_l, e_l, g_n, v_n = self.train(episode_buffer, sess, gamma, v1)
|
||
episode_buffer = []
|
||
sess.run(self.update_local_ops)
|
||
if episode_end:
|
||
break
|
||
|
||
if flag_wincolor == flag_rlaicolor:
|
||
winnums += 1
|
||
self.episode_rewards.append(episode_reward)
|
||
self.episode_lengths.append(episode_step_count)
|
||
self.episode_mean_values.append(numpy.mean(episode_values))
|
||
self.episode_winnums.append(winnums)
|
||
#计算分数
|
||
l_scores = calScore(self.env.obj_main,flag_rlaicolor)
|
||
for i in range(len(self.l_actkeys)):
|
||
self.dict_episode_scores[self.l_actkeys[i]].append(l_scores[i])
|
||
|
||
episode_count += 1
|
||
|
||
global _max_score, _running_avg_score, _episodes, _steps
|
||
if _max_score < episode_reward:
|
||
_max_score = episode_reward
|
||
_running_avg_score = (2.0 / 101) * (episode_reward - _running_avg_score) + _running_avg_score
|
||
_episodes[self.number] = episode_count
|
||
_steps[self.number] = total_steps
|
||
|
||
t_now = time.time()
|
||
print("{} Step #{} Episode #{} Reward: {}".format(self.name, total_steps, episode_count,
|
||
episode_reward))
|
||
print("Total Steps: {}\tTotal Episodes: {}\tTotal time: {}\tMax Score: {}\tAvg Score: {}".format(numpy.sum(_steps),
|
||
numpy.sum(_episodes),
|
||
t_now-t_start,
|
||
_max_score,
|
||
_running_avg_score))
|
||
print(self.l_actkeys[0] + ' : ' + str(l_scores[0]) + ' '+self.l_actkeys[1] + ' : ' + str(l_scores[1]) + ' '
|
||
+self.l_actkeys[2] + ' : ' + str(l_scores[2]) + ' ')
|
||
common.echosentence_color('=='*10,'blue')
|
||
# Update the network using the episode buffer at the end of the episode
|
||
|
||
if len(episode_buffer) != 0:
|
||
if len(list3_episode_buffer) == self.history_policy_len: # 保证buffer中最多有history_policy_len个策略生成的数据
|
||
list3_episode_buffer = list3_episode_buffer[1:]
|
||
list3_episode_buffer.append(episode_buffer)
|
||
# v1 = sess.run(self.local_AC.value,feed_dict={self.local_AC.inputs_spatial_screen: np_mapFeature,
|
||
# self.local_AC.inputs_nonspatial: np_bopFeature})
|
||
if dic_hparas['UpdateGradsMethod'] == 'Integration':
|
||
v_l, p_l, e_l, g_n, v_n = self.train(episode_buffer, sess, gamma, 0,
|
||
updateParas='Both') # 用v1替换0; 2018年09月11日重新替换为0
|
||
else:
|
||
v_l, p_l, e_l, g_n, v_n = self.train([episode_buffer], sess, gamma, 0,updateParas = 'Policy') # 用v1替换0; 2018年09月11日重新替换为0
|
||
v_l, p_l, e_l, g_n, v_n = self.train(list3_episode_buffer, sess, gamma, 0,
|
||
updateParas='Value') # 用v1替换0; 2018年09月11日重新替换为0
|
||
if episode_count % 5 == 0 and episode_count != 0:
|
||
if episode_count % 250 == 0 and self.name == 'worker_0':
|
||
saver.save(sess, self.model_path + '/model-' + str(0) + '.cptk')
|
||
print ("Saved Model")
|
||
|
||
mean_reward = numpy.mean(self.episode_rewards[-5:])
|
||
mean_length = numpy.mean(self.episode_lengths[-5:])
|
||
mean_value = numpy.mean(self.episode_mean_values[-5:])
|
||
mean_winnums = numpy.mean(self.episode_winnums[-5:])
|
||
summary = tf.Summary()
|
||
summary.value.add(tag='Perf/Reward', simple_value=float(mean_reward))
|
||
summary.value.add(tag='Perf/Length', simple_value=float(mean_length))
|
||
summary.value.add(tag='Perf/Value', simple_value=float(mean_value))
|
||
summary.value.add(tag='Perf/winums', simple_value=float(mean_winnums))
|
||
summary.value.add(tag='Losses/Value Loss', simple_value=float(v_l))
|
||
summary.value.add(tag='Losses/Policy Loss', simple_value=float(p_l))
|
||
summary.value.add(tag='Losses/Entropy', simple_value=float(e_l))
|
||
summary.value.add(tag='Losses/Grad Norm', simple_value=float(g_n))
|
||
summary.value.add(tag='Losses/Var Norm', simple_value=float(v_n))
|
||
for i in range(self.num_whats):
|
||
summary.value.add(tag='ActNums/'+str(i),simple_value=float(self.dict_episode_actnums[i][-1]))
|
||
for actkey in self.l_actkeys:
|
||
summary.value.add(tag='Scores/' + actkey, simple_value=float(self.dict_episode_scores[actkey][-1]))
|
||
self.summary_writer.add_summary(summary, episode_count)
|
||
|
||
self.summary_writer.flush()
|
||
if self.name == 'worker_0':
|
||
sess.run(self.increment)
|
||
except Exception as e:
|
||
common.echosentence_color('error in test_rl.py->work():{}'.format(str(e)))
|
||
self.env.__del__()
|
||
raise
|
||
except KeyboardInterrupt as k:
|
||
common.echosentence_color('error in test_rl.py->work():{}'.format(str(k)))
|
||
self.env.__del__()
|
||
raise
|
||
|
||
def test_ai(self,sess,test_nums):
|
||
try:
|
||
winrate = 0
|
||
winscore = 0
|
||
for gamenum in range(test_nums):
|
||
flag_goon = self.env.reset()
|
||
episode_end = not flag_goon
|
||
np_mapFeature, np_bopFeature = process_observation(self.env.obj_main) # resize np_mapFeature to [1,2,0]
|
||
while not episode_end:
|
||
policy_args, v = sess.run([self.local_AC.policy_arg, self.local_AC.value],
|
||
feed_dict={self.local_AC.inputs_spatial_screen: np_mapFeature,
|
||
self.local_AC.inputs_nonspatial: np_bopFeature})
|
||
l_nppolicy = []
|
||
for policy_key in self.l_action_keys:
|
||
l_nppolicy.append(policy_args[policy_key][0, :])
|
||
|
||
flag_goon, reward, l_chooseArg = self.env.step(l_nppolicy[0], l_nppolicy[1], l_nppolicy[2])
|
||
|
||
episode_end = not flag_goon
|
||
np_mapFeature, np_bopFeature = process_observation(self.env.obj_main) # resize np_mapFeature to [1,2,0]
|
||
|
||
list_values = wgsdata.getSdataValue(self.env.obj_main.dic_metadata)
|
||
red_score = 2*list_values[0] + list_values[1] - 2* list_values[2] - list_values[3]
|
||
myscore = red_score if self.flag_rlaicolor == 0 else -red_score
|
||
winrate = winrate + 1 if myscore > 0 else winrate
|
||
winscore += myscore
|
||
wincolor = 'RED' if red_score > 0 else 'BLUE'
|
||
if self.env.obj_main.obj_pk.dic_paras['flag_qd_rm'] and random.random() < 0.1:
|
||
str_savefile = self.roomrd_path + 'BAK_Trainnums-{}_Testnums-{}-WinColor={}.xls'.format(int(_episodes[self.number])
|
||
,gamenum,wincolor)
|
||
wgsdata.record(self.env.obj_main,str_savefile)
|
||
winrate /= test_nums
|
||
winscore /= test_nums
|
||
return winrate,winscore
|
||
except Exception as e:
|
||
common.echosentence_color('error in test_rl.py->Worker->test_ai():{}'.format(str(e)))
|
||
self.env.__del__()
|
||
raise
|
||
except KeyboardInterrupt as k:
|
||
common.echosentence_color('error in test_rl.py->Worker->test_ai():{}'.format(str(k)))
|
||
self.env.__del__()
|
||
raise
|
||
|
||
def defmainparas():
|
||
try:
|
||
num_xd, num_plays, num_objcutility = 0, 1, 1
|
||
strategy_id_r, strategy_id_b = 0, 1
|
||
import AI_QD, wgdensestranet
|
||
dic2_rolloutaiparas = {
|
||
'red': {'type_ai': AI_QD.AI_QD_BASE,
|
||
'type_stra': 'rule-base',
|
||
'type_stranet': wgdensestranet.StraDenseNet,
|
||
},
|
||
'blue': {'type_ai': AI_QD.AI_QD_STRA,
|
||
'type_stra': 'random',
|
||
'type_stranet': wgdensestranet.StraDenseNet,
|
||
},
|
||
}
|
||
dic_mainparas = {'str_wgrootdir': '../../',
|
||
'str_global_flag': 'QD',
|
||
'flag_show': True,
|
||
'num_plays': num_plays,
|
||
'num_objcutility': num_objcutility,
|
||
'num_xd': num_xd,
|
||
'strategy_ids': (strategy_id_r,strategy_id_b),
|
||
'flag_action_cache': False,
|
||
'flag_qd_rm': True,
|
||
'flag_cache': False,
|
||
'flag_gpu': False,
|
||
'flag_afm': True, # 两个AI都为BASE时，flag_afm=False; 否则为True
|
||
'flag_dllnum': 0,
|
||
'cuda_id': 0,
|
||
'flag_savestate': False,
|
||
'dic2_aiparas': {
|
||
'flag_color4acai': 0,
|
||
'red': {'type_ai': AI_QD.AI_QD_DEMO,
|
||
'type_stra': 'rule-base',
|
||
# type of stratree of nodes, how to select next path, [rule-base, random, net]
|
||
'type_stranet': wgdensestranet.StraDenseNet,
|
||
'dic2_rolloutaiparas': dic2_rolloutaiparas,
|
||
'flag_candidateactions': 'rule-base'
|
||
# [rule-base, stra] how to get candidate actions
|
||
},
|
||
'blue': {'type_ai': AI_QD.AI_QD_HA,
|
||
'type_stra': 'net',
|
||
'type_stranet': wgdensestranet.StraDenseNet,
|
||
'dic2_rolloutaiparas': dic2_rolloutaiparas,
|
||
'flag_candidateactions': 'stra'
|
||
},
|
||
},
|
||
'rewrad_type': 'nowscore-base', # ['nowsocoe-base','action-base']
|
||
}
|
||
return dic_mainparas
|
||
except Exception as e:
|
||
print('error in defmainparas() : '+str(e))
|
||
raise
|
||
|
||
dic_hparas = {
|
||
'BattleMode' : '6v6',
|
||
'NetType': 'ResNet', #['AtariNet','ResNet']
|
||
'UpdateGradsMethod':'Divide' #['Integration','Divide']
|
||
}
|
||
global flag_ignoreEnemyReward
|
||
flag_ignoreEnemyReward = True
|
||
if __name__ == '__main__':
|
||
try:
|
||
'''注： 需要修改参数说明
|
||
修改rl颜色： flag_rlaicolor
|
||
defmainparas()中的dic_mainparas中dic2_paras的'red'和'blue'互换
|
||
修改算子数： BattleMode (1v0,1v1,6v6)
|
||
是否加载已训练模型： load_model
|
||
注意：每次更换实验条件之后，记得修改exp_version,免得覆盖之前的游戏版本
|
||
'''
|
||
load_model = False
|
||
flag_rlaicolor = 1
|
||
exp_version = 'v10'
|
||
num_workers = 1 # 多线程数 multiprocessing.cpu_count()
|
||
flag_testai = True
|
||
testai_freq = 500 # 测试ai的频率
|
||
testai_runnums = 50 # 测试50场的平均胜率
|
||
step_length = 200 # 推演多少步进行一次训练
|
||
max_episode_length = 300
|
||
gamma = 0.99 # Discount rate for advantage estimation and reward discounting
|
||
history_policy_len = 5 # 指训练价值网时所使用的历史策略网长度
|
||
str_ai = 'red' if flag_rlaicolor == 0 else 'blue'
|
||
save_path = '../../../a3c-explog/'+ dic_hparas['BattleMode']+'_'+str_ai+'_'+exp_version+'/'
|
||
model_path = save_path + 'model/'
|
||
log_path = save_path + 'train_0/'
|
||
roomrd_path = save_path + 'roomrecord/'
|
||
dic_mainparas = defmainparas()
|
||
|
||
tf.reset_default_graph()
|
||
if not os.path.exists(model_path):
|
||
load_model = False
|
||
os.makedirs(model_path)
|
||
os.makedirs(roomrd_path)
|
||
|
||
common.echosentence_color('BattleMode = ' + dic_hparas['BattleMode'] ,'yellow')
|
||
print('Initializing temporary environment to retrive action_spec...')
|
||
env = wgenv.ENV(dic_mainparas = dic_mainparas,flag_rlaicolor=flag_rlaicolor)
|
||
filterMetadata(env,dic_hparas['BattleMode'])
|
||
wgfeature.cfunRegister(env.obj_main)
|
||
|
||
wgfeature.calOneTimeFeature(env.obj_main)
|
||
wgfeature.calUpdateFeature(env.obj_main)
|
||
|
||
nonspatial_size = env.obj_main.dic_metadata['np_bopFeature'].shape[0]
|
||
tmp_map_size = env.obj_main.dic_metadata['np_mapFeature'].shape
|
||
|
||
map_size = (tmp_map_size[1], tmp_map_size[2], tmp_map_size[0])
|
||
# if dic_hparas['NetType'] == 'ResNet':
|
||
# map_size = (tmp_map_size[1], tmp_map_size[1], tmp_map_size[0])
|
||
l_action_keys = ['who', 'what', 'how']
|
||
dict_actionsepc = {l_action_keys[0]: env.getWhoNums(), l_action_keys[1]: env.getWhatNums(),l_action_keys[2]: env.getHowNums()}
|
||
print('Initializing temporary environment to retrive action_spec done...')
|
||
|
||
with tf.device("/cpu:0"):
|
||
global_episodes = tf.Variable(0, dtype=tf.int32, name='global_episodes', trainable=False)
|
||
trainer = tf.train.AdamOptimizer(learning_rate=1e-4)
|
||
trainer_critic = tf.train.AdamOptimizer(learning_rate=3e-4)
|
||
master_network = AC_Network('global', None, None, nonspatial_size,map_size,dict_actionsepc) # Generate global network
|
||
global _max_score, _running_avg_score, _steps, _episodes
|
||
_max_score = -200
|
||
_running_avg_score = 0
|
||
_steps = numpy.zeros(num_workers)
|
||
_episodes = numpy.zeros(num_workers)
|
||
workers = []
|
||
# Create worker classes
|
||
for i in range(num_workers):
|
||
tmp_dic_mainparas = copy.deepcopy(dic_mainparas)
|
||
dic_mainparas['flag_dllnum'] = i
|
||
workers.append(Worker(i, trainer, trainer_critic, model_path, log_path, roomrd_path,global_episodes, flag_rlaicolor,dic_mainparas,
|
||
flag_testai=flag_testai,testai_freq=testai_freq,testai_runnums= testai_runnums,step_length=step_length
|
||
,history_policy_len = history_policy_len))
|
||
saver = tf.train.Saver(max_to_keep=5)
|
||
|
||
config = tf.ConfigProto()
|
||
config.gpu_options.allow_growth = True
|
||
with tf.Session(config = config) as sess:
|
||
coord = tf.train.Coordinator()
|
||
if load_model == True:
|
||
print ('Loading Model...')
|
||
ckpt = tf.train.get_checkpoint_state(model_path)
|
||
saver.restore(sess, ckpt.model_checkpoint_path)
|
||
else:
|
||
sess.run(tf.global_variables_initializer())
|
||
|
||
# This is where the asynchronous magic happens
|
||
# Start the "work" process for each worker in a separate thread
|
||
# while(True):
|
||
# for worker in workers:
|
||
# worker.work(max_episode_length, gamma, sess, None, saver)
|
||
worker_threads = []
|
||
for worker in workers:
|
||
worker_work = lambda: worker.work(max_episode_length, gamma, sess, coord, saver)
|
||
t = threading.Thread(target=(worker_work))
|
||
t.start()
|
||
time.sleep(0.001)
|
||
worker_threads.append(t)
|
||
coord.join(worker_threads)
|
||
except Exception as e:
|
||
common.echosentence_color('error in test_rl.py-> main() : {}'.format(str(e)))
|
||
env.__del__()
|
||
raise
|
||
except KeyboardInterrupt as k:
|
||
common.echosentence_color('error in test_rl.py-> main() : {}'.format(str(k)))
|
||
env.__del__()
|
||
raise
|
||
#
|