mirror of
https://github.com/wassname/DeepRL.git
synced 2026-08-21 11:09:46 +08:00
129 lines
4.2 KiB
Python
129 lines
4.2 KiB
Python
#######################################################################
|
|
# Copyright (C) 2017 Shangtong Zhang(zhangshangtong.cpp@gmail.com) #
|
|
# Permission given to modify the code as long as you keep this #
|
|
# declaration at the top #
|
|
#######################################################################
|
|
|
|
import numpy as np
|
|
import pickle
|
|
import os
|
|
import datetime
|
|
import torch
|
|
try:
|
|
# python >= 3.5
|
|
from pathlib import Path
|
|
except:
|
|
# python == 2.7
|
|
from pathlib2 import Path
|
|
|
|
def random_seed():
|
|
np.random.seed()
|
|
torch.manual_seed(np.random.randint(int(1e6)))
|
|
|
|
def run_episodes(agent):
|
|
random_seed()
|
|
config = agent.config
|
|
window_size = 100
|
|
ep = 0
|
|
rewards = []
|
|
steps = []
|
|
avg_test_rewards = []
|
|
agent_type = agent.__class__.__name__
|
|
while True:
|
|
ep += 1
|
|
reward, step = agent.episode()
|
|
rewards.append(reward)
|
|
steps.append(step)
|
|
avg_reward = np.mean(rewards[-window_size:])
|
|
config.logger.info('episode %d, reward %f, avg reward %f, total steps %d, episode step %d' % (
|
|
ep, reward, avg_reward, agent.total_steps, step))
|
|
|
|
if config.save_interval and ep % config.save_interval == 0:
|
|
with open('data/%s-%s-online-stats-%s.bin' % (
|
|
agent_type, config.tag, agent.task.name), 'wb') as f:
|
|
pickle.dump([steps, rewards], f)
|
|
agent.save('data/%s-%s-model-%s.bin' % (agent_type, config.tag, agent.task.name))
|
|
|
|
if config.episode_limit and ep > config.episode_limit:
|
|
break
|
|
|
|
if config.max_steps and agent.total_steps > config.max_steps:
|
|
break
|
|
|
|
agent.close()
|
|
return steps, rewards, avg_test_rewards
|
|
|
|
def run_iterations(agent):
|
|
random_seed()
|
|
config = agent.config
|
|
agent_name = agent.__class__.__name__
|
|
iteration = 0
|
|
steps = []
|
|
rewards = []
|
|
while True:
|
|
agent.iteration()
|
|
steps.append(agent.total_steps)
|
|
rewards.append(np.mean(agent.last_episode_rewards))
|
|
if iteration % config.iteration_log_interval == 0:
|
|
config.logger.info('total steps %d, mean/max/min reward %f/%f/%f' % (
|
|
agent.total_steps, np.mean(agent.last_episode_rewards),
|
|
np.max(agent.last_episode_rewards),
|
|
np.min(agent.last_episode_rewards)
|
|
))
|
|
if iteration % (config.iteration_log_interval * 100) == 0:
|
|
with open('data/%s-%s-online-stats-%s.bin' % (agent_name, config.tag, agent.task.name), 'wb') as f:
|
|
pickle.dump({'rewards': rewards,
|
|
'steps': steps}, f)
|
|
agent.save('data/%s-%s-model-%s.bin' % (agent_name, config.tag, agent.task.name))
|
|
iteration += 1
|
|
if config.max_steps and agent.total_steps >= config.max_steps:
|
|
agent.close()
|
|
break
|
|
|
|
return steps, rewards
|
|
|
|
def get_time_str():
|
|
return datetime.datetime.now().strftime("%y%m%d-%H%M%S")
|
|
|
|
def get_default_log_dir(name):
|
|
return './log/%s-%s' % (name, get_time_str())
|
|
|
|
def sync_grad(target_network, src_network):
|
|
for param, src_param in zip(target_network.parameters(), src_network.parameters()):
|
|
param._grad = src_param.grad.clone()
|
|
|
|
def mkdir(path):
|
|
Path(path).mkdir(parents=True, exist_ok=True)
|
|
|
|
def set_one_thread():
|
|
os.environ['OMP_NUM_THREADS'] = '1'
|
|
os.environ['MKL_NUM_THREADS'] = '1'
|
|
torch.set_num_threads(1)
|
|
|
|
class Batcher:
|
|
def __init__(self, batch_size, data):
|
|
self.batch_size = batch_size
|
|
self.data = data
|
|
self.num_entries = len(data[0])
|
|
self.reset()
|
|
|
|
def reset(self):
|
|
self.batch_start = 0
|
|
self.batch_end = self.batch_start + self.batch_size
|
|
|
|
def end(self):
|
|
return self.batch_start >= self.num_entries
|
|
|
|
def next_batch(self):
|
|
batch = []
|
|
for d in self.data:
|
|
batch.append(d[self.batch_start: self.batch_end])
|
|
self.batch_start = self.batch_end
|
|
self.batch_end = min(self.batch_start + self.batch_size, self.num_entries)
|
|
return batch
|
|
|
|
def shuffle(self):
|
|
indices = np.arange(self.num_entries)
|
|
np.random.shuffle(indices)
|
|
self.data = [d[indices] for d in self.data]
|