diff --git a/rllib/utils/torch_ops.py b/rllib/utils/torch_ops.py
index 9b9c52c37..53c26dc4d 100644
--- a/rllib/utils/torch_ops.py
+++ b/rllib/utils/torch_ops.py
@@ -1,6 +1,7 @@
 from gym.spaces import Discrete, MultiDiscrete
 import numpy as np
 import tree
+import warnings
 
 from ray.rllib.models.repeated_values import RepeatedValues
 from ray.rllib.utils.framework import try_import_torch
@@ -62,7 +63,13 @@ def convert_to_torch_tensor(x, device=None):
             return RepeatedValues(
                 tree.map_structure(mapping, item.values), item.lengths,
                 item.max_len)
-        tensor = torch.from_numpy(np.asarray(item))
+        # Non-writable numpy-arrays will cause PyTorch warning.
+        if isinstance(item, np.ndarray) and item.flags.writeable is False:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                tensor = torch.from_numpy(item)
+        else:
+            tensor = torch.from_numpy(np.asarray(item))
         # Floatify all float64 tensors.
         if tensor.dtype == torch.double:
             tensor = tensor.float()