diff --git a/README.md b/README.md index 85dffa1..1b8928b 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,26 @@ -# SAC+CPC implementaiton in PyTorch - -# +# CURL: Contrastive Unsupervised Representation Learning for Sample-Efficient Reinforcement Learning ## Instructions -To train an SAC+CPC agent on the `cheetah run` task from image-based observations run: +To train a CURL agent on the `cartpole swingup` task from image-based observations run `bash script/run.sh` from the root of this directory. The `run.sh` file contains the following command, which you can modify to try different environments / hyperparamters. ``` -CUDA_VISIBLE_DEVICES=4 python train_cpc.py \ - --dmc2gym \ - --domain_name reacher \ - --task_name easy \ +CUDA_VISIBLE_DEVICES=0 python train.py \ + --domain_name cartpole \ + --task_name swingup \ --encoder_type pixel \ - --decoder_type identity \ - --action_repeat 4 --frame_stack 1 \ - --save_tb --save_video --num_train_steps 1000000 \ - --work_dir ./tmp/dmc/reacher_easy_cpc \ - --agent sac_cpc \ - --seed 2 --critic_lr 1e-3 --actor_lr 1e-3 --eval_freq 10000 - -CUDA_VISIBLE_DEVICES=7 python train.py \ - --domain_name walker \ - --task_name walk --dmc2gym \ - --encoder_type pixel \ - --decoder_type identity \ - --action_repeat 4 \ - --save_tb --pre_transform_image_size 84 --image_size 84 \ - --work_dir ./tmp/icml/vanilla_sac/ML0107walker_vanilla_sac \ - --agent sac_ae --frame_stack 3 \ + --action_repeat 8 \ + --save_tb --pre_transform_image_size 100 --image_size 84 \ + --work_dir ./tmp \ + --agent curl_sac --frame_stack 3 \ --seed -1 --critic_lr 1e-3 --actor_lr 1e-3 --eval_freq 10000 --batch_size 128 --num_train_steps 1000000 ``` -Try - reducing log std actor max from 2->1 or increase 2->3 +This will store data related to the run in the specified `working_dir`. To enable model or video saving, use the `--save_model` or `--save_video` flags. For all available flags, inspect `train.py`. To visualize progress with tensorboard run: -This will produce 'log' folder, where all the outputs are going to be stored including train/eval logs, tensorboard blobs, and evaluation episode videos. One can attacha tensorboard to monitor training by running: ``` -tensorboard --logdir log +tensorboard --logdir log --port 6006 ``` -and opening up tensorboad in your browser. + +and go to `localhost:6006` in your browser. If you're running headlessly, try port forwarding with ssh. The console output is also available in a form: ``` @@ -49,9 +34,9 @@ S - total number of environment steps D - duration in seconds to train 1 episode R - episode reward BR - average reward of sampled batch -ALOSS - average loss of actor -CLOSS - average loss of critic -RLOSS - average reconstruction loss (only if is trained from pixels and decoder) +A_LOSS - average loss of actor +CR_LOSS - average loss of critic +CU_LOSS - average loss of the CURL encoder ``` while an evaluation entry: ``` diff --git a/curl_sac.py b/curl_sac.py index 99c0f2d..aa26aee 100644 --- a/curl_sac.py +++ b/curl_sac.py @@ -442,7 +442,7 @@ class CurlSacAgent(object): self.encoder_optimizer.step() self.cpc_optimizer.step() if step % self.log_interval == 0: - L.log('train_cpc/cpc_loss', loss, step) + L.log('train_curl/curl_loss', loss, step) def update(self, replay_buffer, L, step): diff --git a/logger.py b/logger.py index 8e31fd4..e35d6a0 100644 --- a/logger.py +++ b/logger.py @@ -13,8 +13,8 @@ FORMAT_CONFIG = { 'train': [ ('episode', 'E', 'int'), ('step', 'S', 'int'), ('duration', 'D', 'time'), ('episode_reward', 'R', 'float'), - ('batch_reward', 'BR', 'float'), ('actor_loss', 'ALOSS', 'float'), - ('critic_loss', 'CLOSS', 'float'), ('ae_loss', 'RLOSS', 'float') + ('batch_reward', 'BR', 'float'), ('actor_loss', 'A_LOSS', 'float'), + ('critic_loss', 'CR_LOSS', 'float'), ('curl_loss', 'CU_LOSS', 'float') ], 'eval': [('step', 'S', 'int'), ('episode_reward', 'ER', 'float')] } diff --git a/scripts/long_experiments.sh b/scripts/long_experiments.sh deleted file mode 100755 index e69de29..0000000 diff --git a/scripts/run.sh b/scripts/run.sh index 90016db..d44d7d2 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -1,66 +1,9 @@ - -# curl cheetah, crop 76 > 64, grayscale + random crop, deep stack -# batch size = 256 instead of 128. maybe 256 makes 64x64 work. And, try using 512 with 64x64. -# then try adam LR (smaller) - 3e-4... You can try 2e-4 and 5e-4. -# try using the stochastic policy for eval. (you can do later.. for now the important thing is to run ablations.) -# try bigger frame stack, maybe 8. -# parser.add_argument('--critic_tau', default=0.01, type=float) # try 0.05 or 0.1 -# run 1: batch 256, first try with 84 -# run 2: batch 512, first try with 84, then try 64, then try their encoder -# run 3: batch 256, first try with 84 -# run 4: batch 512, first try with 84, then try 64, then try their encoder -# run 5: 2e-4 lr for all -# run 6: 5e-4 for all -# run 7: critic higher tau, 0.05 - -# try stochastic critic eval -CUDA_VISIBLE_DEVICES=1 python train_cpc.py \ - --domain_name cheetah \ - --task_name run --dmc2gym \ +CUDA_VISIBLE_DEVICES=5 python train.py \ + --domain_name cartpole \ + --task_name swingup \ --encoder_type pixel \ - --decoder_type identity \ - --action_repeat 4 --batch_size 256 \ - --save_tb --work_dir ./tmp/icml/feb2cheetah/curl_cheetah_b256_84 \ - --agent sac_cpc --frame_stack 3 --pre_transform_image_size 100 --image_size 84 \ - --seed 23 --critic_lr 1e-3 --actor_lr 1e-3 --eval_freq 10000 --batch_size 128 --num_train_steps 3000000 & - -CUDA_VISIBLE_DEVICES=2 python train_cpc.py \ - --domain_name cheetah \ - --task_name run --dmc2gym \ - --encoder_type pixel \ - --decoder_type identity \ - --action_repeat 4 \ - --save_tb --work_dir ./tmp/icml/feb2cheetah/curl_cheetah_b512_84 \ - --agent sac_cpc --frame_stack 3 --pre_transform_image_size 100 --image_size 84 \ - --seed 23 --critic_lr 1e-3 --actor_lr 1e-3 --eval_freq 20000 --batch_size 512 --num_train_steps 3000000 & - -CUDA_VISIBLE_DEVICES=3 python train.py \ - --domain_name cheetah \ - --task_name run --dmc2gym \ - --encoder_type pixel \ - --decoder_type identity \ - --action_repeat 4 --batch_size 256 \ - --save_tb --work_dir ./tmp/icml/feb2cheetah/rad_cheetah_b256_84 \ - --agent sac_ae --frame_stack 3 --pre_transform_image_size 100 --image_size 84 \ - --seed 23 --critic_lr 1e-3 --actor_lr 1e-3 --eval_freq 20000 --batch_size 128 --num_train_steps 3000000 & - -CUDA_VISIBLE_DEVICES=4 python train.py \ - --domain_name cheetah \ - --task_name run --dmc2gym \ - --encoder_type pixel \ - --decoder_type identity \ - --action_repeat 4 \ - --save_tb --work_dir ./tmp/icml/feb2cheetah/rad_cheetah_b512_84 \ - --agent sac_ae --frame_stack 3 --pre_transform_image_size 100 --image_size 84 \ - --seed 23 --critic_lr 1e-3 --actor_lr 1e-3 --eval_freq 20000 --batch_size 512 --num_train_steps 3000000 & - -CUDA_VISIBLE_DEVICES=7 python train_cpc.py \ - --domain_name cheetah \ - --task_name run --dmc2gym \ - --encoder_type pixel \ - --decoder_type identity \ - --action_repeat 4 \ - --save_tb --work_dir ./tmp/icml/feb2cheetah/curl_cheetah_b256_84_lr3e4 \ - --agent sac_cpc --frame_stack 3 --pre_transform_image_size 100 --image_size 84 \ - --seed 23 --encoder_lr 3e-4 --critic_lr 3e-4 --actor_lr 3e-4 \ - --eval_freq 20000 --batch_size 256 --num_train_steps 3000000 \ No newline at end of file + --action_repeat 8 \ + --save_tb --pre_transform_image_size 100 --image_size 84 \ + --work_dir ./tmp/cartpole \ + --agent curl_sac --frame_stack 3 \ + --seed -1 --critic_lr 1e-3 --actor_lr 1e-3 --eval_freq 10000 --batch_size 128 --num_train_steps 1000000 \ No newline at end of file diff --git a/scripts/short_experiments.sh b/scripts/short_experiments.sh deleted file mode 100755 index e69de29..0000000