[data] Fix html reprs to work with ipywidgets v8 (ray-project#30039)

python/ray/data/dataset.py

@@ -103,6 +103,7 @@
 from ray.util.annotations import DeveloperAPI, PublicAPI
 from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
 from ray.widgets import Template
+from ray.widgets.util import ensure_notebook_deps
 
 if sys.version_info >= (3, 8):
  from typing import Literal
@@ -4022,31 +4023,26 @@ def _aggregate_result(self, result: Union[Tuple, TableRow]) -> U:
  else:
  return result
 
+ @ensure_notebook_deps(
+ ["ipywidgets", "8"],
+ )
  def _ipython_display_(self):
- try:
- from ipywidgets import HTML, VBox, Layout
- except ImportError:
- logger.warn(
- "'ipywidgets' isn't installed. Run `pip install ipywidgets` to "
- "enable notebook widgets."
- )
- return None
-
+ from ipywidgets import HTML, VBox, Layout
  from IPython.display import display
 
  title = HTML(f"<h2>{self.__class__.__name__}</h2>")
- display(VBox([title, self._tab_repr_()], layout=Layout(width="100%")))
+ tab = self._tab_repr_()
 
+ if tab:
+ display(VBox([title, tab], layout=Layout(width="100%")))
+
+ @ensure_notebook_deps(
+ ["tabulate", None],
+ ["ipywidgets", "8"],
+ )
  def _tab_repr_(self):
- try:
- from tabulate import tabulate
- from ipywidgets import Tab, HTML
- except ImportError:
- logger.info(
- "For rich Dataset reprs in notebooks, run "
- "`pip install tabulate ipywidgets`."
- )
- return ""
+ from tabulate import tabulate
+ from ipywidgets import Tab, HTML
 
  metadata = {
  "num_blocks": self._plan.initial_num_blocks(),
@@ -4077,27 +4073,22 @@ def _tab_repr_(self):
  max_height="300px",
  )
 
- tab = Tab()
  children = []
-
- tab.set_title(0, "Metadata")
  children.append(
- Template("scrollableTable.html.j2").render(
- table=tabulate(
- tabular_data=metadata.items(),
- tablefmt="html",
- showindex=False,
- headers=["Field", "Value"],
- ),
- max_height="300px",
+ HTML(
+ Template("scrollableTable.html.j2").render(
+ table=tabulate(
+ tabular_data=metadata.items(),
+ tablefmt="html",
+ showindex=False,
+ headers=["Field", "Value"],
+ ),
+ max_height="300px",
+ )
  )
  )
-
- tab.set_title(1, "Schema")
- children.append(schema_repr)
-
- tab.children = [HTML(child) for child in children]
- return tab
+ children.append(HTML(schema_repr))
+ return Tab(children, titles=["Metadata", "Schema"])
 
  def __repr__(self) -> str:
  schema = self.schema()

python/ray/train/data_parallel_trainer.py

@@ -21,6 +21,7 @@
 from ray.train.trainer import BaseTrainer, GenDataset
 from ray.util.annotations import DeveloperAPI
 from ray.widgets import Template
+from ray.widgets.util import ensure_notebook_deps
 
 if TYPE_CHECKING:
  from ray.data.preprocessor import Preprocessor
@@ -385,54 +386,38 @@ def get_dataset_config(self) -> Dict[str, DatasetConfig]:
  """
  return self._dataset_config.copy()
 
+ @ensure_notebook_deps(
+ ["tabulate", None],
+ ["ipywidgets", "8"],
+ )
  def _ipython_display_(self):
- try:
- from ipywidgets import HTML, VBox, Tab, Layout
- except ImportError:
- logger.warn(
- "'ipywidgets' isn't installed. Run `pip install ipywidgets` to "
- "enable notebook widgets."
- )
- return None
-
+ from ipywidgets import HTML, VBox, Tab, Layout
  from IPython.display import display
 
  title = HTML(f"<h2>{self.__class__.__name__}</h2>")
 
- tab = Tab()
- children = []
-
- tab.set_title(0, "Datasets")
- children.append(self._datasets_repr_() if self.datasets else None)
-
- tab.set_title(1, "Dataset Config")
- children.append(
- HTML(self._dataset_config_repr_html_()) if self._dataset_config else None
- )
-
- tab.set_title(2, "Train Loop Config")
- children.append(
+ children = [
+ self._datasets_repr_() if self.datasets else None,
+ HTML(self._dataset_config_repr_html_()) if self._dataset_config else None,
  HTML(self._train_loop_config_repr_html_())
  if self._train_loop_config
- else None
- )
-
- tab.set_title(3, "Scaling Config")
- children.append(
- HTML(self.scaling_config._repr_html_()) if self.scaling_config else None
+ else None,
+ HTML(self.scaling_config._repr_html_()) if self.scaling_config else None,
+ HTML(self.run_config._repr_html_()) if self.run_config else None,
+ HTML(self._backend_config._repr_html_()) if self._backend_config else None,
+ ]
+
+ tab = Tab(
+ children,
+ titles=[
+ "Datasets",
+ "Dataset Config",
+ "Train Loop Config",
+ "Scaling Config",
+ "Run Config",
+ "Backend Config",
+ ],
  )
-
- tab.set_title(4, "Run Config")
- children.append(
- HTML(self.run_config._repr_html_()) if self.run_config else None
- )
-
- tab.set_title(5, "Backend Config")
- children.append(
- HTML(self._backend_config._repr_html_()) if self._backend_config else None
- )
-
- tab.children = children
  display(VBox([title, tab], layout=Layout(width="100%")))
 
  def _train_loop_config_repr_html_(self) -> str:
@@ -471,26 +456,23 @@ def _dataset_config_repr_html_(self) -> str:
 
  return Template("rendered_html_common.html.j2").render(content=content)
 
+ @ensure_notebook_deps(["ipywidgets", "8"])
  def _datasets_repr_(self) -> str:
- try:
- from ipywidgets import HTML, VBox, Layout
- except ImportError:
- logger.warn(
- "'ipywidgets' isn't installed. Run `pip install ipywidgets` to "
- "enable notebook widgets."
- )
- return None
+ from ipywidgets import HTML, VBox, Layout
+
  content = []
  if self.datasets:
  for name, config in self.datasets.items():
- content.append(
- HTML(
- Template("title_data.html.j2").render(
- title=f"Dataset - <code>{name}</code>", data=None
+ tab = config._tab_repr_()
+ if tab:
+ content.append(
+ HTML(
+ Template("title_data.html.j2").render(
+ title=f"Dataset - <code>{name}</code>", data=None
+ )
  )
  )
- )
- content.append(config._tab_repr_())
+ content.append(config._tab_repr_())
 
  return VBox(content, layout=Layout(width="100%"))
 

python/ray/widgets/util.py

@@ -1,8 +1,22 @@
-from typing import Any, Optional
+import importlib
+import logging
+import textwrap
+from functools import wraps
+from typing import Any, Callable, Iterable, Optional, TypeVar, Union
 
 from ray.util.annotations import DeveloperAPI
 from ray.widgets import Template
 
+try:
+ from packaging.version import Version
+except ImportError:
+ from distutils.version import LooseVersion as Version
+
+
+logger = logging.getLogger(__name__)
+
+F = TypeVar("F", bound=Callable[..., Any])
+
 
 @DeveloperAPI
 def make_table_html_repr(
@@ -59,3 +73,93 @@ def make_table_html_repr(
  content = table
 
  return content
+
+
+@DeveloperAPI
+def ensure_notebook_deps(
+ *deps: Iterable[Union[str, Optional[str]]],
+ missing_message: Optional[str] = None,
+ outdated_message: Optional[str] = None,
+) -> Callable[[F], F]:
+ """Generate a decorator which checks for soft dependencies.
+
+ This decorator is meant to wrap _ipython_display_. If the dependency is not found,
+ or a version is specified here and the version of the package is older than the
+ specified version, the wrapped function is not executed and None is returned. If
+ the dependency is missing or the version is old, a log message is displayed.
+
+ Args:
+ *deps: Iterable of (dependency name, min version (optional))
+ missing_message: Message to log if missing package is found
+ outdated_message: Message to log if outdated package is found
+
+ Returns:
+ Wrapped function. Guaranteed to be safe to import soft dependencies specified
+ above.
+ """
+
+ def wrapper(func: F) -> F:
+ @wraps(func)
+ def wrapped(*args, **kwargs):
+ if _has_missing(*deps, message=missing_message) or _has_outdated(
+ *deps, message=outdated_message
+ ):
+ return None
+ return func(*args, **kwargs)
+
+ return wrapped
+
+ return wrapper
+
+
+def _has_missing(
+ *deps: Iterable[Union[str, Optional[str]]], message: Optional[str] = None
+):
+ missing = []
+ for (lib, _) in deps:
+ try:
+ importlib.import_module(lib)
+ except ImportError:
+ missing.append(lib)
+
+ if missing:
+ if not message:
+ message = f"Run `pip install {' '.join(missing)}` for rich notebook output."
+
+ # stacklevel=3: First level is this function, then ensure_notebook_deps, then
+ # the actual function affected.
+ logger.warning(f"Missing packages: {missing}. {message}", stacklevel=3)
+
+ return missing
+
+
+def _has_outdated(
+ *deps: Iterable[Union[str, Optional[str]]], message: Optional[str] = None
+):
+ outdated = []
+ for (lib, version) in deps:
+ try:
+ module = importlib.import_module(lib)
+ if version and Version(module.__version__) < Version(version):
+ outdated.append([lib, version, module.__version__])
+ except ImportError:
+ pass
+
+ if outdated:
+ outdated_strs = []
+ install_args = []
+ for lib, version, installed in outdated:
+ outdated_strs.append(f"{lib}=={installed} found, needs {lib}>={version}")
+ install_args.append(f"{lib}>={version}")
+
+ outdated_str = textwrap.indent("\n".join(outdated_strs), " ")
+ install_str = " ".join(install_args)
+
+ if not message:
+ message = f"Run `pip install -U {install_str}` for rich notebook output."
+
+ # stacklevel=3: First level is this function, then ensure_notebook_deps, then
+ # the actual function affected.
+ logger.warning(f"Outdated packages:\n{outdated_str}\n{message}", stacklevel=3)
+
+ return outdated