Retry and exception for hang on memory store full (#5143)

python/ray/ray_constants.py

@@ -17,6 +17,12 @@ ID_SIZE = 20
 # The default maximum number of bytes to allocate to the object store unless
 # overridden by the user.
 DEFAULT_OBJECT_STORE_MAX_MEMORY_BYTES = 20 * 10**9
+# The default number of retries to call `put` when the object store is full.
+DEFAULT_PUT_OBJECT_RETRIES = 5
+# The default seconds for delay between calls to retry `put` when
+# the object store is full. This delay is exponentially doubled up to
+# DEFAULT_PUT_OBJECT_RETRIES times.
+DEFAULT_PUT_OBJECT_DELAY = 1
 # The smallest cap on the memory used by the object store that we allow.
 OBJECT_STORE_MINIMUM_MEMORY_BYTES = 10**7
 # The default maximum number of bytes that the non-primary Redis shards are