[Tune/CI] Fix Hyperopt notebook example (ray-project#26469)

Fixes failing hyperopt notebook in CI (as found in ray-project#26410). The cause was a mismatch between keys in points to evaluate and the search space - now, an informative exception will be raised.

Signed-off-by: Antoni Baum <[email protected]>

doc/source/tune/examples/BUILD

@@ -21,7 +21,6 @@ py_test_run_all_notebooks(
  "tune-xgboost.ipynb",
  "nyc_taxi_basic_processing.ipynb", # REGRESSION
  "ocr_example.ipynb", # REGRESSION
- "hyperopt_example.ipynb", # REGRESSION
  "sigopt_example.ipynb", # REGRESSION
  "tune-sklearn.ipynb", # REGRESSION
  ],
@@ -36,4 +35,4 @@ py_test_run_all_notebooks(
  exclude = [],
  data = ["//doc/source/tune/examples:tune_examples"],
  tags = ["exclusive", "team:ml", "gpu"],
-)
+)

doc/source/tune/examples/hyperopt_example.ipynb

@@ -36,7 +36,10 @@
  "metadata": {
  "tags": [
  "remove-cell"
- ]
+ ],
+ "vscode": {
+ "languageId": "python"
+ }
  },
  "outputs": [],
  "source": [
@@ -61,7 +64,10 @@
  "metadata": {
  "tags": [
  "hide-input"
- ]
+ ],
+ "vscode": {
+ "languageId": "python"
+ }
  },
  "outputs": [],
  "source": [
@@ -90,7 +96,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "12d4efc8",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "def evaluate(step, width, height):\n",
@@ -111,7 +121,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "c9818009",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "def objective(config):\n",
@@ -127,7 +141,10 @@
  "metadata": {
  "tags": [
  "remove-cell"
- ]
+ ],
+ "vscode": {
+ "languageId": "python"
+ }
  },
  "outputs": [],
  "source": [
@@ -150,13 +167,16 @@
  "execution_count": null,
  "id": "d4615bed",
  "metadata": {
- "lines_to_next_cell": 0
+ "lines_to_next_cell": 0,
+ "vscode": {
+ "languageId": "python"
+ }
  },
  "outputs": [],
  "source": [
  "initial_params = [\n",
- " {\"width\": 1, \"height\": 2},\n",
- " {\"width\": 4, \"height\": 2},\n",
+ " {\"width\": 1, \"height\": 2, \"activation\": \"relu\"},\n",
+ " {\"width\": 4, \"height\": 2, \"activation\": \"tanh\"},\n",
  "]\n",
  "algo = HyperOptSearch(points_to_evaluate=initial_params)\n",
  "algo = ConcurrencyLimiter(algo, max_concurrent=4)"
@@ -175,7 +195,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "2dbb2be0",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "num_samples = 1000"
@@ -188,7 +212,10 @@
  "metadata": {
  "tags": [
  "remove-cell"
- ]
+ ],
+ "vscode": {
+ "languageId": "python"
+ }
  },
  "outputs": [],
  "source": [
@@ -209,14 +236,18 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "65189946",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "search_config = {\n",
  " \"steps\": 100,\n",
  " \"width\": tune.uniform(0, 20),\n",
  " \"height\": tune.uniform(-100, 100),\n",
- " \"activation\": tune.choice([\"relu, tanh\"])\n",
+ " \"activation\": tune.choice([\"relu\", \"tanh\"])\n",
  "}"
  ]
  },
@@ -232,7 +263,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "9a99a3a7",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "analysis = tune.run(\n",
@@ -258,7 +293,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "7036798c",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "print(\"Best hyperparameters found were: \", analysis.best_config)"
@@ -278,7 +317,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "2f7b5449",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "def evaluation_fn(step, width, height, mult=1):\n",
@@ -289,7 +332,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "4b83b81c",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "def objective_two(config):\n",
@@ -307,7 +354,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "75cea99e",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "conditional_space = {\n",
@@ -336,7 +387,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "ea2c71a6",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "algo = HyperOptSearch(space=conditional_space, metric=\"mean_loss\", mode=\"min\")\n",
@@ -355,7 +410,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "14111e9e",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "analysis = tune.run(\n",
@@ -379,7 +438,11 @@
  "cell_type": "code",
  "execution_count": null,
  "id": "03c3fc49",
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
  "outputs": [],
  "source": [
  "print(\"Best hyperparameters found were: \", analysis.best_config)"
@@ -392,7 +455,10 @@
  "metadata": {
  "tags": [
  "remove-cell"
- ]
+ ],
+ "vscode": {
+ "languageId": "python"
+ }
  },
  "outputs": [],
  "source": [

python/ray/tune/search/hyperopt/hyperopt_search.py

@@ -30,8 +30,10 @@
  hyperopt_logger = logging.getLogger("hyperopt")
  hyperopt_logger.setLevel(logging.WARNING)
  import hyperopt as hpo
+ from hyperopt.pyll import Apply
 except ImportError:
  hpo = None
+ Apply = None
 
 from ray.tune.error import TuneError
 
@@ -209,7 +211,8 @@ def _lookup(config_dict, space_dict, key):
  _lookup(config_dict[key], space_dict[key], k)
  else:
  if (
- isinstance(space_dict[key], hpo.base.pyll.Apply)
+ key in space_dict
+ and isinstance(space_dict[key], hpo.base.pyll.Apply)
  and space_dict[key].name == "switch"
  ):
  if len(space_dict[key].pos_args) > 0:
@@ -277,9 +280,11 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
  )
 
  if self._points_to_evaluate > 0:
+ using_point_to_evaluate = True
  new_trial = self._hpopt_trials.trials[self._points_to_evaluate - 1]
  self._points_to_evaluate -= 1
  else:
+ using_point_to_evaluate = False
  new_ids = self._hpopt_trials.new_trial_ids(1)
  self._hpopt_trials.refresh()
 
@@ -307,11 +312,29 @@ def suggest(self, trial_id: str) -> Optional[Dict]:
  self.domain.expr, ctrl, hpo.base.Ctrl, memo
  )
 
- suggested_config = hpo.pyll.rec_eval(
- self.domain.expr,
- memo=memo,
- print_node_on_error=self.domain.rec_eval_print_node_on_error,
- )
+ try:
+ suggested_config = hpo.pyll.rec_eval(
+ self.domain.expr,
+ memo=memo,
+ print_node_on_error=self.domain.rec_eval_print_node_on_error,
+ )
+ except (AssertionError, TypeError) as e:
+ if using_point_to_evaluate and (
+ isinstance(e, AssertionError) or "GarbageCollected" in str(e)
+ ):
+ raise ValueError(
+ "HyperOpt encountered a GarbageCollected switch argument. "
+ "Usually this is caused by a config in "
+ "`points_to_evaluate` "
+ "missing a key present in `space`. Ensure that "
+ "`points_to_evaluate` contains "
+ "all non-constant keys from `space`.\n"
+ "Config from `points_to_evaluate`: "
+ f"{config}\n"
+ "HyperOpt search space: "
+ f"{self._space}"
+ ) from e
+ raise e
  return copy.deepcopy(suggested_config)
 
  def on_trial_result(self, trial_id: str, result: Dict) -> None:

python/ray/tune/tests/test_sample.py

@@ -1684,6 +1684,24 @@ def testPointsToEvaluateHyperOpt(self):
 
  from ray.tune.search.hyperopt import HyperOptSearch
 
+ # See if we catch hyperopt errors caused by points to evaluate missing
+ # keys found in space
+ points_to_evaluate_missing_one = [
+ {k: v.sample() for k, v in list(config.items())[:-1]}
+ ]
+ print(f"Points to evaluate: {points_to_evaluate_missing_one}")
+ searcher = HyperOptSearch(points_to_evaluate=points_to_evaluate_missing_one)
+
+ with self.assertRaises(ValueError):
+ tune.run(
+ _mock_objective,
+ config=config,
+ metric="metric",
+ mode="max",
+ search_alg=searcher,
+ num_samples=5,
+ )
+
  return self._testPointsToEvaluate(HyperOptSearch, config)
 
  def testPointsToEvaluateHyperOptNested(self):