[RLlib] Replace remaining mentions of "trainer" by "algorithm". (ray-…

…project#36557)

rllib/BUILD

@@ -3703,7 +3703,7 @@ py_test(
 )
 
 # Taking out this test for now: Mixed torch- and tf- policies within the same
-# Trainer never really worked.
+# Algorothm never really worked.
 # py_test(
 # name = "examples/multi_agent_two_trainers_mixed_torch_tf",
 # main = "examples/multi_agent_two_trainers.py",

rllib/algorithms/a3c/tests/test_a3c.py

@@ -64,14 +64,14 @@ def test_a3c_entropy_coeff_schedule(self):
  min_time_s_per_iteration=0, min_sample_timesteps_per_iteration=20
  )
 
- def _step_n_times(trainer, n: int):
- """Step trainer n times.
+ def _step_n_times(algo, n: int):
+ """Step Algorithm n times.
 
  Returns:
  learning rate at the end of the execution.
  """
  for _ in range(n):
- results = trainer.train()
+ results = algo.train()
  return results["info"][LEARNER_INFO][DEFAULT_POLICY_ID][LEARNER_STATS_KEY][
  "entropy_coeff"
  ]

rllib/algorithms/algorithm.py

@@ -362,7 +362,9 @@ def __init__(
  # Last resort: Create core AlgorithmConfig from merged dicts.
  if isinstance(default_config, dict):
  config = AlgorithmConfig.from_dict(
- config_dict=self.merge_trainer_configs(default_config, config, True)
+ config_dict=self.merge_algorithm_configs(
+ default_config, config, True
+ )
  )
  # Default config is an AlgorithmConfig -> update its properties
  # from the given config dict.
@@ -569,17 +571,17 @@ def setup(self, config: AlgorithmConfig) -> None:
  )
  self.config.off_policy_estimation_methods = ope_dict
 
- # Deprecated way of implementing Trainer sub-classes (or "templates"
+ # Deprecated way of implementing Algorithm sub-classes (or "templates"
  # via the `build_trainer` utility function).
  # Instead, sub-classes should override the Trainable's `setup()`
  # method and call super().setup() from within that override at some
  # point.
- # Old design: Override `Trainer._init`.
+ # Old design: Override `Algorithm._init`.
  _init = False
  try:
  self._init(self.config, self.env_creator)
  _init = True
- # New design: Override `Trainable.setup()` (as indented by tune.Trainable)
+ # New design: Override `Algorithm.setup()` (as indented by tune.Trainable)
  # and do or don't call `super().setup()` from within your override.
  # By default, `super().setup()` will create both worker sets:
  # "rollout workers" for collecting samples for training and - if
@@ -743,7 +745,7 @@ def setup(self, config: AlgorithmConfig) -> None:
  # Run `on_algorithm_init` callback after initialization is done.
  self.callbacks.on_algorithm_init(algorithm=self)
 
- # TODO: Deprecated: In your sub-classes of Trainer, override `setup()`
+ # TODO: Deprecated: In your sub-classes of Algorithm, override `setup()`
  # directly and call super().setup() from within it if you would like the
  # default setup behavior plus some own setup logic.
  # If you don't need the env/workers/config/etc.. setup for you by super,
@@ -767,13 +769,13 @@ def get_default_policy_class(
 
  @override(Trainable)
  def step(self) -> ResultDict:
- """Implements the main `Trainer.train()` logic.
+ """Implements the main `Algorithm.train()` logic.
 
  Takes n attempts to perform a single training step. Thereby
  catches RayErrors resulting from worker failures. After n attempts,
  fails gracefully.
 
- Override this method in your Trainer sub-classes if you would like to
+ Override this method in your Algorithm sub-classes if you would like to
  handle worker failures yourself.
  Otherwise, override only `training_step()` to implement the core
  algorithm logic.
@@ -815,7 +817,7 @@ def step(self) -> ResultDict:
  if not evaluate_this_iter and self.config.always_attach_evaluation_results:
  assert isinstance(
  self.evaluation_metrics, dict
- ), "Trainer.evaluate() needs to return a dict."
+ ), "Algorithm.evaluate() needs to return a dict."
  results.update(self.evaluation_metrics)
 
  if hasattr(self, "workers") and isinstance(self.workers, WorkerSet):
@@ -865,9 +867,6 @@ def evaluate(
  ) -> dict:
  """Evaluates current policy under `evaluation_config` settings.
 
- Note that this default implementation does not do anything beyond
- merging evaluation_config with the normal trainer config.
-
  Args:
  duration_fn: An optional callable taking the already run
  num episodes as only arg and returning the number of
@@ -914,7 +913,7 @@ def evaluate(
  ):
  raise ValueError(
  "Cannot evaluate w/o an evaluation worker set in "
- "the Trainer or w/o an env on the local worker!\n"
+ "the Algorithm or w/o an env on the local worker!\n"
  "Try one of the following:\n1) Set "
  "`evaluation_interval` >= 0 to force creating a "
  "separate evaluation worker set.\n2) Set "
@@ -1105,7 +1104,7 @@ def duration_fn(num_units_done):
  metrics["off_policy_estimator"][name] = avg_estimate
 
  # Evaluation does not run for every step.
- # Save evaluation metrics on trainer, so it can be attached to
+ # Save evaluation metrics on Algorithm, so it can be attached to
  # subsequent step results as latest evaluation result.
  self.evaluation_metrics = {"evaluation": metrics}
 
@@ -1298,7 +1297,7 @@ def remote_fn(worker):
  metrics["off_policy_estimator"][name] = estimates
 
  # Evaluation does not run for every step.
- # Save evaluation metrics on trainer, so it can be attached to
+ # Save evaluation metrics on Algorithm, so it can be attached to
  # subsequent step results as latest evaluation result.
  self.evaluation_metrics = {"evaluation": metrics}
 
@@ -1360,7 +1359,7 @@ def training_step(self) -> ResultDict:
  """Default single iteration logic of an algorithm.
 
  - Collect on-policy samples (SampleBatches) in parallel using the
- Trainer's RolloutWorkers (@ray.remote).
+ Algorithm's RolloutWorkers (@ray.remote).
  - Concatenate collected SampleBatches into one train batch.
  - Note that we may have more than one policy in the multi-agent case:
  Call the different policies' `learn_on_batch` (simple optimizer) OR
@@ -1431,10 +1430,10 @@ def training_step(self) -> ResultDict:
  @staticmethod
  def execution_plan(workers, config, **kwargs):
  raise NotImplementedError(
- "It is not longer recommended to use Trainer's `execution_plan` method/API."
+ "It is no longer supported to use the `Algorithm.execution_plan()` API!"
  " Set `_disable_execution_plan_api=True` in your config and override the "
- "`Trainer.training_step()` method with your algo's custom "
- "execution logic."
+ "`Algorithm.training_step()` method with your algo's custom "
+ "execution logic instead."
  )
 
  @PublicAPI
@@ -1454,9 +1453,6 @@ def compute_single_action(
  episode: Optional[Episode] = None,
  unsquash_action: Optional[bool] = None,
  clip_action: Optional[bool] = None,
- # Deprecated args.
- unsquash_actions=DEPRECATED_VALUE,
- clip_actions=DEPRECATED_VALUE,
  # Kwargs placeholder for future compatibility.
  **kwargs,
  ) -> Union[
@@ -1506,24 +1502,9 @@ def compute_single_action(
  or we have an RNN-based Policy.
 
  Raises:
- KeyError: If the `policy_id` cannot be found in this Trainer's
- local worker.
+ KeyError: If the `policy_id` cannot be found in this Algorithm's local
+ worker.
  """
- if clip_actions != DEPRECATED_VALUE:
- deprecation_warning(
- old="Trainer.compute_single_action(`clip_actions`=...)",
- new="Trainer.compute_single_action(`clip_action`=...)",
- error=True,
- )
- clip_action = clip_actions
- if unsquash_actions != DEPRECATED_VALUE:
- deprecation_warning(
- old="Trainer.compute_single_action(`unsquash_actions`=...)",
- new="Trainer.compute_single_action(`unsquash_action`=...)",
- error=True,
- )
- unsquash_action = unsquash_actions
-
  # `unsquash_action` is None: Use value of config['normalize_actions'].
  if unsquash_action is None:
  unsquash_action = self.config.normalize_actions
@@ -1535,7 +1516,7 @@ def compute_single_action(
  # are all None.
  err_msg = (
  "Provide either `input_dict` OR [`observation`, ...] as "
- "args to Trainer.compute_single_action!"
+ "args to `Algorithm.compute_single_action()`!"
  )
  if input_dict is not None:
  assert (
@@ -1549,12 +1530,12 @@ def compute_single_action(
  assert observation is not None, err_msg
 
  # Get the policy to compute the action for (in the multi-agent case,
- # Trainer may hold >1 policies).
+ # Algorithm may hold >1 policies).
  policy = self.get_policy(policy_id)
  if policy is None:
  raise KeyError(
  f"PolicyID '{policy_id}' not found in PolicyMap of the "
- f"Trainer's local worker!"
+ f"Algorithm's local worker!"
  )
  local_worker = self.workers.local_worker()
 
@@ -1657,8 +1638,6 @@ def compute_actions(
  episodes: Optional[List[Episode]] = None,
  unsquash_actions: Optional[bool] = None,
  clip_actions: Optional[bool] = None,
- # Deprecated.
- normalize_actions=None,
  **kwargs,
  ):
  """Computes an action for the specified policy on the local Worker.
@@ -1700,14 +1679,6 @@ def compute_actions(
  the full output of policy.compute_actions_from_input_dict() if
  full_fetch=True or we have an RNN-based Policy.
  """
- if normalize_actions is not None:
- deprecation_warning(
- old="Trainer.compute_actions(`normalize_actions`=...)",
- new="Trainer.compute_actions(`unsquash_actions`=...)",
- error=True,
- )
- unsquash_actions = normalize_actions
-
  # `unsquash_actions` is None: Use value of config['normalize_actions'].
  if unsquash_actions is None:
  unsquash_actions = self.config.normalize_actions
@@ -1834,8 +1805,6 @@ def add_policy(
  ] = None,
  evaluation_workers: bool = True,
  module_spec: Optional[SingleAgentRLModuleSpec] = None,
- # Deprecated.
- workers: Optional[List[Union[RolloutWorker, ActorHandle]]] = DEPRECATED_VALUE,
  ) -> Optional[Policy]:
  """Adds a new policy to this Algorithm.
 
@@ -1873,27 +1842,13 @@ def add_policy(
  module_spec: In the new RLModule API we need to pass in the module_spec for
  the new module that is supposed to be added. Knowing the policy spec is
  not sufficient.
- workers: A list of RolloutWorker/ActorHandles (remote
- RolloutWorkers) to add this policy to. If defined, will only
- add the given policy to these workers.
-
 
  Returns:
  The newly added policy (the copy that got added to the local
  worker). If `workers` was provided, None is returned.
  """
  validate_policy_id(policy_id, error=True)
 
- if workers is not DEPRECATED_VALUE:
- deprecation_warning(
- old="Algorithm.add_policy(.., workers=..)",
- help=(
- "The `workers` argument to `Algorithm.add_policy()` is deprecated! "
- "Please do not use it anymore."
- ),
- error=True,
- )
-
  self.workers.add_policy(
  policy_id,
  policy_cls,
@@ -2016,7 +1971,6 @@ def export_policy_model(
  def export_policy_checkpoint(
  self,
  export_dir: str,
- filename_prefix=DEPRECATED_VALUE, # deprecated arg, do not use anymore
  policy_id: PolicyID = DEFAULT_POLICY_ID,
  ) -> None:
  """Exports Policy checkpoint to a local directory and returns an AIR Checkpoint.
@@ -2039,14 +1993,6 @@ def export_policy_checkpoint(
  >>> algo.train() # doctest: +SKIP
  >>> algo.export_policy_checkpoint("/tmp/export_dir") # doctest: +SKIP
  """
- # `filename_prefix` should not longer be used as new Policy checkpoints
- # contain more than one file with a fixed filename structure.
- if filename_prefix != DEPRECATED_VALUE:
- deprecation_warning(
- old="Algorithm.export_policy_checkpoint(filename_prefix=...)",
- error=True,
- )
-
  policy = self.get_policy(policy_id)
  if policy is None:
  raise KeyError(f"Policy with ID {policy_id} not found in Algorithm!")
@@ -2173,7 +2119,8 @@ def load_checkpoint(self, checkpoint: str) -> None:
  def log_result(self, result: ResultDict) -> None:
  # Log after the callback is invoked, so that the user has a chance
  # to mutate the result.
- # TODO: Remove `trainer` arg at some point to fully deprecate the old signature.
+ # TODO: Remove `algorithm` arg at some point to fully deprecate the old
+ # signature.
  self.callbacks.on_train_result(algorithm=self, result=result)
  # Then log according to Trainable's logging logic.
  Trainable.log_result(self, result)
@@ -2477,7 +2424,7 @@ def get_auto_filled_metrics(
  return auto_filled
 
  @classmethod
- def merge_trainer_configs(
+ def merge_algorithm_configs(
  cls,
  config1: AlgorithmConfigDict,
  config2: PartialAlgorithmConfigDict,
@@ -2754,7 +2701,7 @@ def _checkpoint_info_to_algorithm_state(
  if isinstance(default_config, AlgorithmConfig):
  new_config = default_config.update_from_dict(state["config"])
  else:
- new_config = Algorithm.merge_trainer_configs(
+ new_config = Algorithm.merge_algorithm_configs(
  default_config, state["config"]
  )
 
@@ -3146,21 +3093,8 @@ def _record_usage(self, config):
  alg = "USER_DEFINED"
  record_extra_usage_tag(TagKey.RLLIB_ALGORITHM, alg)
 
- @Deprecated(new="Algorithm.compute_single_action()", error=True)
- def compute_action(self, *args, **kwargs):
- return self.compute_single_action(*args, **kwargs)
-
- @Deprecated(new="construct WorkerSet(...) instance directly", error=True)
- def _make_workers(self, *args, **kwargs):
- pass
-
- @Deprecated(new="AlgorithmConfig.validate()", error=False)
- def validate_config(self, config):
- pass
-
- @staticmethod
  @Deprecated(new="AlgorithmConfig.validate()", error=True)
- def _validate_config(config, trainer_or_none):
+ def validate_config(self, config):
  pass
 
 

rllib/algorithms/algorithm_config.py

@@ -127,7 +127,7 @@ class AlgorithmConfig(_Config):
  ... .resources(num_gpus=0)
  ... .rollouts(num_rollout_workers=4)
  ... .callbacks(MemoryTrackingCallbacks)
- >>> # A config object can be used to construct the respective Trainer.
+ >>> # A config object can be used to construct the respective Algorithm.
  >>> rllib_algo = config.build() # doctest: +SKIP
 
  Example:
@@ -139,7 +139,7 @@ class AlgorithmConfig(_Config):
  >>> # Use `to_dict()` method to get the legacy plain python config dict
  >>> # for usage with `tune.Tuner().fit()`.
  >>> tune.Tuner( # doctest: +SKIP
- ... "[registered trainer class]", param_space=config.to_dict()
+ ... "[registered Algorithm class]", param_space=config.to_dict()
  ... ).fit()
  """
 
@@ -234,7 +234,7 @@ def overrides(cls, **kwargs):
  def __init__(self, algo_class=None):
  # Define all settings and their default values.
 
- # Define the default RLlib Trainer class that this AlgorithmConfig will be
+ # Define the default RLlib Algorithm class that this AlgorithmConfig will be
  # applied to.
  self.algo_class = algo_class
 
@@ -1125,7 +1125,7 @@ def resources(
  `num_gpus_per_learner_worker` accordingly (e.g. 4 GPUs total, and model
  needs 2 GPUs: `num_learner_workers = 2` and
  `num_gpus_per_learner_worker = 2`)
- num_cpus_per_learner_worker: Number of CPUs allocated per trainer worker.
+ num_cpus_per_learner_worker: Number of CPUs allocated per Learner worker.
  Only necessary for custom processing pipeline inside each Learner
  requiring multiple CPU cores. Ignored if `num_learner_workers = 0`.
  num_gpus_per_learner_worker: Number of GPUs allocated per worker. If
@@ -3095,7 +3095,7 @@ def get_default_learner_class(self) -> Union[Type["Learner"], str]:
 
  Returns:
  The Learner class to use for this algorithm either as a class type or as
- a string (e.g. ray.rllib.core.learner.testing.torch.BCTrainer).
+ a string (e.g. ray.rllib.core.learner.testing.torch.BC).
  """
  raise NotImplementedError