[Tune] Parametrize Cloud Syncing Frequency (#8771)

python/ray/tune/tests/test_cluster.py

@@ -18,7 +18,7 @@ from ray.tune.error import TuneError
 from ray.tune.ray_trial_executor import RayTrialExecutor
 from ray.tune.resources import Resources
 from ray.tune.suggest import BasicVariantGenerator
-from ray.tune.syncer import Syncer
+from ray.tune.syncer import CloudSyncer
 from ray.tune.trainable import TrainableUtil
 from ray.tune.trial import Trial
 from ray.tune.trial_runner import TrialRunner
@@ -521,7 +521,7 @@ def test_cluster_down_full(start_connected_cluster, tmpdir, trainable_id):
 
     mock_get_client = "ray.tune.trial_runner.get_cloud_syncer"
     with patch(mock_get_client) as mock_get_cloud_syncer:
-        mock_syncer = Syncer(local_dir, upload_dir, mock_storage_client())
+        mock_syncer = CloudSyncer(local_dir, upload_dir, mock_storage_client())
         mock_get_cloud_syncer.return_value = mock_syncer
 
         tune.run_experiments(all_experiments, raise_on_failed_trial=False)

python/ray/tune/tests/test_sync.py

@@ -3,6 +3,7 @@ import os
 import shutil
 import sys
 import tempfile
+import time
 import unittest
 from unittest.mock import patch
 
@@ -151,6 +152,38 @@ class TestSyncFunctionality(unittest.TestCase):
         shutil.rmtree(tmpdir)
         shutil.rmtree(tmpdir2)
 
+    @patch("ray.tune.sync_client.S3_PREFIX", "test")
+    def testCloudSyncPeriod(self):
+        """Tests that changing CLOUD_SYNC_PERIOD affects syncing frequency."""
+        tmpdir = tempfile.mkdtemp()
+
+        def trainable(config):
+            for i in range(10):
+                time.sleep(1)
+                tune.report(score=i)
+
+        mock = unittest.mock.Mock()
+
+        def counter(local, remote):
+            mock()
+
+        tune.syncer.CLOUD_SYNC_PERIOD = 1
+        [trial] = tune.run(
+            trainable,
+            name="foo",
+            max_failures=0,
+            local_dir=tmpdir,
+            upload_dir="test",
+            sync_to_cloud=counter,
+            stop={
+                "training_iteration": 10
+            },
+            global_checkpoint_period=0.5,
+        ).trials
+
+        self.assertEqual(mock.call_count, 12)
+        shutil.rmtree(tmpdir)
+
     def testClusterSyncFunction(self):
         def sync_func_driver(source, target):
             assert ":" in source, "Source {} not a remote path.".format(source)