From 25f5491ac7e9bbbc59413766f8d1cb2d2252d416 Mon Sep 17 00:00:00 2001
From: William Falcon <falc@fb.com>
Date: Sun, 21 Jul 2019 08:32:17 -0400
Subject: [PATCH] updated docs

---
 docs/Trainer/Distributed training.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/Trainer/Distributed training.md b/docs/Trainer/Distributed training.md
index 486dd6c6..8c8e2fe0 100644
--- a/docs/Trainer/Distributed training.md	
+++ b/docs/Trainer/Distributed training.md	
@@ -80,6 +80,20 @@ cluster = SlurmCluster(
     log_path='/some/path/to/save',
 )
 
+# OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT
+# which interface your nodes use for communication
+cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
+
+# see output of the NCCL connection process
+# NCCL is how the nodes talk to each other
+cluster.add_command('export NCCL_DEBUG=INFO')
+
+# setting a master port here is a good idea.
+cluster.add_command(f'export MASTER_PORT={PORT}')
+
+# good to load the latest NCCL version
+cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])
+
 # configure cluster
 cluster.per_experiment_nb_nodes = 12 
 cluster.per_experiment_nb_gpus = 8