[tune] Test background syncer serialization (#28699)

Syncer (and thus Trial and Trial Runner) serialization fails because of threading objects that are not correctly unset on getstate.

Signed-off-by: Kai Fricke <[email protected]>

python/ray/tune/syncer.py

@@ -444,6 +444,11 @@ def retry(self):
  self._sync_process = _BackgroundProcess(cmd)
  self._sync_process.start(**kwargs)
 
+ def __getstate__(self):
+ state = self.__dict__.copy()
+ state["_sync_process"] = None
+ return state
+
 
 class _DefaultSyncer(_BackgroundSyncer):
  """Default syncer between local storage and remote URI."""

python/ray/tune/tests/test_syncer.py

@@ -9,6 +9,7 @@
 from freezegun import freeze_time
 
 import ray
+import ray.cloudpickle as pickle
 from ray import tune
 from ray.tune import TuneError
 from ray.tune.syncer import Syncer, _DefaultSyncer, _validate_upload_dir
@@ -473,6 +474,19 @@ def test_trainable_syncer_custom_command(ray_start_2_cpus, temp_data_dirs):
  assert_file(False, tmp_target, os.path.join(checkpoint_dir, "checkpoint.data"))
 
 
+def test_syncer_serialize(temp_data_dirs):
+ """Check that syncing up and down works"""
+ tmp_source, tmp_target = temp_data_dirs
+
+ syncer = _DefaultSyncer()
+
+ syncer.sync_up(
+ local_dir=tmp_source, remote_dir="memory:https:///test/test_syncer_sync_up_down"
+ )
+
+ pickle.dumps(syncer)
+
+
 if __name__ == "__main__":
  import sys