[rllib] Modularize Torch and TF policy graphs (#2294)

* wip

* cls

* re

* wip

* wip

* a3c working

* torch support

* pg works

* lint

* rm v2

* consumer id

* clean up pg

* clean up more

* fix python 2.7

* tf session management

* docs

* dqn wip

* fix compile

* dqn

* apex runs

* up

* impotrs

* ddpg

* quotes

* fix tests

* fix last r

* fix tests

* lint

* pass checkpoint restore

* kwar

* nits

* policy graph

* fix yapf

* com

* class

* pyt

* vectorization

* update

* test cpe

* unit test

* fix ddpg2

* changes

* wip

* args

* faster test

* common

* fix

* add alg option

* batch mode and policy serving

* multi serving test

* todo

* wip

* serving test

* doc async env

* num envs

* comments

* thread

* remove init hook

* update

* fix ppo

* comments1

* fix

* updates

* add jenkins tests

* fix

* fix pytorch

* fix

* fixes

* fix a3c policy

* fix squeeze

* fix trunc on apex

* fix squeezing for real

* update

* remove horizon test for now

* multiagent wip

* update

* fix race condition

* fix ma

* t

* doc

* st

* wip

* example

* wip

* working

* cartpole

* wip

* batch wip

* fix bug

* make other_batches None default

* working

* debug

* nit

* warn

* comments

* fix ppo

* fix obs filter

* update

* wip

* tf

* update

* fix

* cleanup

* cleanup

* spacing

* model

* fix

* dqn

* fix ddpg

* doc

* keep names

* update

* fix

* com

* docs

* clarify model outputs

* Update torch_policy_graph.py

* fix obs filter

* pass thru worker index

* fix

* rename

* vlad torch comments

* fix log action

* debug name

* fix lstm

* remove unused ddpg net

* remove conv net

* revert lstm

* cast

* clean up

* fix lstm check

* move to end

* fix sphinx

* fix cmd

* remove bad doc

* clarify

* copy

* async sa

* fix

test/jenkins_tests/run_multi_node_tests.sh

@@ -11,51 +11,6 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 DOCKER_SHA=$($ROOT_DIR/../../build-docker.sh --output-sha --no-cache)
 echo "Using Docker image" $DOCKER_SHA
 
-python $ROOT_DIR/multi_node_docker_test.py \
-    --docker-image=$DOCKER_SHA \
-    --num-nodes=5 \
-    --num-redis-shards=10 \
-    --test-script=/ray/test/jenkins_tests/multi_node_tests/test_0.py
-
-python $ROOT_DIR/multi_node_docker_test.py \
-    --docker-image=$DOCKER_SHA \
-    --num-nodes=5 \
-    --num-redis-shards=5 \
-    --num-gpus=0,1,2,3,4 \
-    --num-drivers=7 \
-    --driver-locations=0,1,0,1,2,3,4 \
-    --test-script=/ray/test/jenkins_tests/multi_node_tests/remove_driver_test.py
-
-python $ROOT_DIR/multi_node_docker_test.py \
-    --docker-image=$DOCKER_SHA \
-    --num-nodes=5 \
-    --num-redis-shards=2 \
-    --num-gpus=0,0,5,6,50 \
-    --num-drivers=100 \
-    --test-script=/ray/test/jenkins_tests/multi_node_tests/many_drivers_test.py
-
-python $ROOT_DIR/multi_node_docker_test.py \
-    --docker-image=$DOCKER_SHA \
-    --num-nodes=1 \
-    --mem-size=60G \
-    --shm-size=60G \
-    --test-script=/ray/test/jenkins_tests/multi_node_tests/large_memory_test.py
-
-# Test that the example applications run.
-
-# docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
-#     python /ray/examples/lbfgs/driver.py
-
-# docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
-#     python /ray/examples/rl_pong/driver.py \
-#     --iterations=3
-
-# docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
-#     python /ray/examples/hyperopt/hyperopt_simple.py
-
-# docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
-#     python /ray/examples/hyperopt/hyperopt_adaptive.py
-
 docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
     --env PongDeterministic-v0 \
@@ -96,7 +51,6 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     --env CartPole-v0 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
-    --config '{"use_lstm": false}'
 
 docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
@@ -151,14 +105,14 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     --env PongDeterministic-v4 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
-    --config '{"num_workers": 2, "use_lstm": false, "use_pytorch": true, "model": {"grayscale": true, "zero_mean": false, "dim": 80, "channel_major": true}}'
+    --config '{"num_workers": 2, "use_pytorch": true, "model": {"use_lstm": false, "grayscale": true, "zero_mean": false, "dim": 80, "channel_major": true}}'
 
 docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
     --env CartPole-v1 \
     --run A3C \
     --stop '{"training_iteration": 2}' \
-    --config '{"num_workers": 2, "use_lstm": false, "use_pytorch": true}'
+    --config '{"num_workers": 2, "use_pytorch": true}'
 
 docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/train.py \
@@ -260,3 +214,33 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
 
 docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
     python /ray/python/ray/rllib/examples/multiagent_cartpole.py
+
+python $ROOT_DIR/multi_node_docker_test.py \
+    --docker-image=$DOCKER_SHA \
+    --num-nodes=5 \
+    --num-redis-shards=10 \
+    --test-script=/ray/test/jenkins_tests/multi_node_tests/test_0.py
+
+python $ROOT_DIR/multi_node_docker_test.py \
+    --docker-image=$DOCKER_SHA \
+    --num-nodes=5 \
+    --num-redis-shards=5 \
+    --num-gpus=0,1,2,3,4 \
+    --num-drivers=7 \
+    --driver-locations=0,1,0,1,2,3,4 \
+    --test-script=/ray/test/jenkins_tests/multi_node_tests/remove_driver_test.py
+
+python $ROOT_DIR/multi_node_docker_test.py \
+    --docker-image=$DOCKER_SHA \
+    --num-nodes=5 \
+    --num-redis-shards=2 \
+    --num-gpus=0,0,5,6,50 \
+    --num-drivers=100 \
+    --test-script=/ray/test/jenkins_tests/multi_node_tests/many_drivers_test.py
+
+python $ROOT_DIR/multi_node_docker_test.py \
+    --docker-image=$DOCKER_SHA \
+    --num-nodes=1 \
+    --mem-size=60G \
+    --shm-size=60G \
+    --test-script=/ray/test/jenkins_tests/multi_node_tests/large_memory_test.py