wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-13 13:02:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use flake8-comprehensions (#1976)

Browse Source 
* Add flake8 to Travis

* Add flake8-comprehensions

[flake8 plugin](https://github.com/adamchainz/flake8-comprehensions) that
checks for useless constructions.

* Use generators instead of lists where appropriate

A lot of the builtins can take in generators instead of lists.

This commit applies `flake8-comprehensions` to find them.

* Fix lint error

* Fix some string formatting

The rest can be fixed in another PR

* Fix compound literals syntax

This should probably be merged after #1963.

* dict() -> {}

* Use dict literal syntax

dict(...) -> {...}

* Rewrite nested dicts

* Fix hanging indent

* Add missing import

* Add missing quote

* fmt

* Add missing whitespace

* rm duplicate pip install

This is already installed in another file.

* Fix indent

* move `merge_dicts` into utils

* Bring up to date with `master`

* Add automatic syntax upgrade

* rm pyupgrade

In case users want to still use it on their own, the upgrade-syn.sh script was
left in the `.travis` dir.
This commit is contained in:
Alok Singh
2018-05-20 16:15:06 -07:00
committed by Philipp Moritz
parent 99ae74e1d2
commit f795173b51
37 changed files with 329 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/rllib/ddpg/apex.py
+25 -22
View File
@@ -3,29 +3,32 @@ from __future__ import division
from __future__ import print_function
from ray.rllib.ddpg.ddpg import DDPGAgent, DEFAULT_CONFIG as DDPG_CONFIG
from ray.utils import merge_dicts
APEX_DDPG_DEFAULT_CONFIG = dict(DDPG_CONFIG,
**dict(
optimizer_class="ApexOptimizer",
optimizer_config=dict(
DDPG_CONFIG["optimizer_config"],
**dict(
max_weight_sync_delay=400,
num_replay_buffer_shards=4,
debug=False,
)),
n_step=3,
num_workers=32,
buffer_size=2000000,
learning_starts=50000,
train_batch_size=512,
sample_batch_size=50,
max_weight_sync_delay=400,
target_network_update_freq=500000,
timesteps_per_iteration=25000,
per_worker_exploration=True,
worker_side_prioritization=True,
))
APEX_DDPG_DEFAULT_CONFIG = merge_dicts(
DDPG_CONFIG,
{
'optimizer_class': 'ApexOptimizer',
'optimizer_config':
merge_dicts(
DDPG_CONFIG['optimizer_config'], {
'max_weight_sync_delay': 400,
'num_replay_buffer_shards': 4,
'debug': False
}),
'n_step': 3,
'num_workers': 32,
'buffer_size': 2000000,
'learning_starts': 50000,
'train_batch_size': 512,
'sample_batch_size': 50,
'max_weight_sync_delay': 400,
'target_network_update_freq': 500000,
'timesteps_per_iteration': 25000,
'per_worker_exploration': True,
'worker_side_prioritization': True,
},
)
class ApexDDPGAgent(DDPGAgent):
python/ray/rllib/ddpg/ddpg.py
+41 -40
View File
@@ -20,89 +20,89 @@ OPTIMIZER_SHARED_CONFIGS = [
"train_batch_size", "learning_starts", "clip_rewards"
]
DEFAULT_CONFIG = dict(
DEFAULT_CONFIG = {
# === Model ===
# Hidden layer sizes of the policy networks
actor_hiddens=[64, 64],
'actor_hiddens': [64, 64],
# Hidden layer sizes of the policy networks
critic_hiddens=[64, 64],
'critic_hiddens': [64, 64],
# N-step Q learning
n_step=1,
'n_step': 1,
# Config options to pass to the model constructor
model={},
'model': {},
# Discount factor for the MDP
gamma=0.99,
'gamma': 0.99,
# Arguments to pass to the env creator
env_config={},
'env_config': {},
# === Exploration ===
# Max num timesteps for annealing schedules. Exploration is annealed from
# 1.0 to exploration_fraction over this number of timesteps scaled by
# exploration_fraction
schedule_max_timesteps=100000,
'schedule_max_timesteps': 100000,
# Number of env steps to optimize for before returning
timesteps_per_iteration=1000,
'timesteps_per_iteration': 1000,
# Fraction of entire training period over which the exploration rate is
# annealed
exploration_fraction=0.1,
'exploration_fraction': 0.1,
# Final value of random action probability
exploration_final_eps=0.02,
'exploration_final_eps': 0.02,
# OU-noise scale
noise_scale=0.1,
'noise_scale': 0.1,
# theta
exploration_theta=0.15,
'exploration_theta': 0.15,
# sigma
exploration_sigma=0.2,
'exploration_sigma': 0.2,
# Update the target network every `target_network_update_freq` steps.
target_network_update_freq=0,
'target_network_update_freq': 0,
# Update the target by \tau * policy + (1-\tau) * target_policy
tau=0.002,
'tau': 0.002,
# Whether to start with random actions instead of noops.
random_starts=True,
'random_starts': True,
# === Replay buffer ===
# Size of the replay buffer. Note that if async_updates is set, then
# each worker will have a replay buffer of this size.
buffer_size=50000,
'buffer_size': 50000,
# If True prioritized replay buffer will be used.
prioritized_replay=True,
'prioritized_replay': True,
# Alpha parameter for prioritized replay buffer.
prioritized_replay_alpha=0.6,
'prioritized_replay_alpha': 0.6,
# Beta parameter for sampling from prioritized replay buffer.
prioritized_replay_beta=0.4,
'prioritized_replay_beta': 0.4,
# Epsilon to add to the TD errors when updating priorities.
prioritized_replay_eps=1e-6,
'prioritized_replay_eps': 1e-6,
# Whether to clip rewards to [-1, 1] prior to adding to the replay buffer.
clip_rewards=True,
'clip_rewards': True,
# === Optimization ===
# Learning rate for adam optimizer
actor_lr=1e-4,
critic_lr=1e-3,
'actor_lr': 1e-4,
'critic_lr': 1e-3,
# If True, use huber loss instead of squared loss for critic network
# Conventionally, no need to clip gradients if using a huber loss
use_huber=False,
'use_huber': False,
# Threshold of a huber loss
huber_threshold=1.0,
'huber_threshold': 1.0,
# Weights for L2 regularization
l2_reg=1e-6,
'l2_reg': 1e-6,
# If not None, clip gradients during optimization at this value
grad_norm_clipping=None,
'grad_norm_clipping': None,
# How many steps of the model to sample before learning starts.
learning_starts=1500,
'learning_starts': 1500,
# Update the replay buffer with this many samples at once. Note that this
# setting applies per-worker if num_workers > 1.
sample_batch_size=1,
'sample_batch_size': 1,
# Size of a batched sampled from replay buffer for training. Note that
# if async_updates is set, then each worker returns gradients for a
# batch of this size.
train_batch_size=256,
'train_batch_size': 256,
# Smooth the current average reward over this many previous episodes.
smoothing_num_episodes=100,
'smoothing_num_episodes': 100,
# === Tensorflow ===
# Arguments to pass to tensorflow
tf_session_args={
'tf_session_args': {
"device_count": {
"CPU": 2
},
@@ -119,17 +119,18 @@ DEFAULT_CONFIG = dict(
# Number of workers for collecting samples with. This only makes sense
# to increase if your environment is particularly slow to sample, or if
# you're using the Async or Ape-X optimizers.
num_workers=0,
'num_workers': 0,
# Whether to allocate GPUs for workers (if > 0).
num_gpus_per_worker=0,
'num_gpus_per_worker': 0,
# Optimizer class to use.
optimizer_class="LocalSyncReplayOptimizer",
'optimizer_class': "LocalSyncReplayOptimizer",
# Config to pass to the optimizer.
optimizer_config=dict(),
'optimizer_config': {},
# Whether to use a distribution of epsilons across workers for exploration.
per_worker_exploration=False,
'per_worker_exploration': False,
# Whether to compute priorities on workers.
worker_side_prioritization=False)
'worker_side_prioritization': False
}
class DDPGAgent(Agent):