#!/bin/bash
#
# Auto-generated by test-tube (https://github.com/williamFalcon/test-tube)
#################

# set a job name
#SBATCH --job-name=lightning_test
#################

# a file for job output, you can check job progress
#SBATCH --output=/slurm_output_%j.out
#################

# a file for errors
#SBATCH --error=/slurm_output_%j.err
#################

# time needed for job
#SBATCH --time=01:00:00
#################

# gpus per node
#SBATCH --gres=gpu:8
#################

# cpus per job
#SBATCH --cpus-per-task=10
#################

# number of requested nodes
#SBATCH --nodes=2
#################

# memory per node (0 means all)
#SBATCH --mem=0
#################

# slurm will send a signal this far out before it kills the job
#SBATCH --signal=USR1@300
#################

# comment
#SBATCH --comment=lightning_demo
#################

# 1 task per gpu
#SBATCH --ntasks-per-node=8
#################

source activate YourEnv

# debugging flags (optional)
export NCCL_DEBUG=INFO
export PYTHONFAULTHANDLER=1

# on your cluster you might need these:
# set the network interface
export NCCL_SOCKET_IFNAME=^docker0,lo

# might need the latest cuda
module load NCCL/2.4.7-1-cuda.10.0

# random port between 12k and 20k
export MASTER_PORT=$((12000 + RANDOM % 20000))$

srun python multi_node_own_slurm_script.py