From 349cbdb761f086fb63bf8422d6afef37ebb6b865 Mon Sep 17 00:00:00 2001 From: Peyton Murray Date: Wed, 9 Nov 2022 10:10:02 -0800 Subject: [PATCH] [data] Fix html reprs to work with ipywidgets v8 (#30039) --- python/ray/data/dataset.py | 63 ++++++------- python/ray/train/data_parallel_trainer.py | 90 ++++++++---------- python/ray/widgets/util.py | 106 +++++++++++++++++++++- 3 files changed, 168 insertions(+), 91 deletions(-) diff --git a/python/ray/data/dataset.py b/python/ray/data/dataset.py index 12ce4788eff66..2f73206e6b666 100644 --- a/python/ray/data/dataset.py +++ b/python/ray/data/dataset.py @@ -103,6 +103,7 @@ from ray.util.annotations import DeveloperAPI, PublicAPI from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy from ray.widgets import Template +from ray.widgets.util import ensure_notebook_deps if sys.version_info >= (3, 8): from typing import Literal @@ -4022,31 +4023,26 @@ def _aggregate_result(self, result: Union[Tuple, TableRow]) -> U: else: return result + @ensure_notebook_deps( + ["ipywidgets", "8"], + ) def _ipython_display_(self): - try: - from ipywidgets import HTML, VBox, Layout - except ImportError: - logger.warn( - "'ipywidgets' isn't installed. Run `pip install ipywidgets` to " - "enable notebook widgets." - ) - return None - + from ipywidgets import HTML, VBox, Layout from IPython.display import display title = HTML(f"

{self.__class__.__name__}

") - display(VBox([title, self._tab_repr_()], layout=Layout(width="100%"))) + tab = self._tab_repr_() + if tab: + display(VBox([title, tab], layout=Layout(width="100%"))) + + @ensure_notebook_deps( + ["tabulate", None], + ["ipywidgets", "8"], + ) def _tab_repr_(self): - try: - from tabulate import tabulate - from ipywidgets import Tab, HTML - except ImportError: - logger.info( - "For rich Dataset reprs in notebooks, run " - "`pip install tabulate ipywidgets`." - ) - return "" + from tabulate import tabulate + from ipywidgets import Tab, HTML metadata = { "num_blocks": self._plan.initial_num_blocks(), @@ -4077,27 +4073,22 @@ def _tab_repr_(self): max_height="300px", ) - tab = Tab() children = [] - - tab.set_title(0, "Metadata") children.append( - Template("scrollableTable.html.j2").render( - table=tabulate( - tabular_data=metadata.items(), - tablefmt="html", - showindex=False, - headers=["Field", "Value"], - ), - max_height="300px", + HTML( + Template("scrollableTable.html.j2").render( + table=tabulate( + tabular_data=metadata.items(), + tablefmt="html", + showindex=False, + headers=["Field", "Value"], + ), + max_height="300px", + ) ) ) - - tab.set_title(1, "Schema") - children.append(schema_repr) - - tab.children = [HTML(child) for child in children] - return tab + children.append(HTML(schema_repr)) + return Tab(children, titles=["Metadata", "Schema"]) def __repr__(self) -> str: schema = self.schema() diff --git a/python/ray/train/data_parallel_trainer.py b/python/ray/train/data_parallel_trainer.py index 79397eb2359c8..03b42fe768279 100644 --- a/python/ray/train/data_parallel_trainer.py +++ b/python/ray/train/data_parallel_trainer.py @@ -21,6 +21,7 @@ from ray.train.trainer import BaseTrainer, GenDataset from ray.util.annotations import DeveloperAPI from ray.widgets import Template +from ray.widgets.util import ensure_notebook_deps if TYPE_CHECKING: from ray.data.preprocessor import Preprocessor @@ -385,54 +386,38 @@ def get_dataset_config(self) -> Dict[str, DatasetConfig]: """ return self._dataset_config.copy() + @ensure_notebook_deps( + ["tabulate", None], + ["ipywidgets", "8"], + ) def _ipython_display_(self): - try: - from ipywidgets import HTML, VBox, Tab, Layout - except ImportError: - logger.warn( - "'ipywidgets' isn't installed. Run `pip install ipywidgets` to " - "enable notebook widgets." - ) - return None - + from ipywidgets import HTML, VBox, Tab, Layout from IPython.display import display title = HTML(f"

{self.__class__.__name__}

") - tab = Tab() - children = [] - - tab.set_title(0, "Datasets") - children.append(self._datasets_repr_() if self.datasets else None) - - tab.set_title(1, "Dataset Config") - children.append( - HTML(self._dataset_config_repr_html_()) if self._dataset_config else None - ) - - tab.set_title(2, "Train Loop Config") - children.append( + children = [ + self._datasets_repr_() if self.datasets else None, + HTML(self._dataset_config_repr_html_()) if self._dataset_config else None, HTML(self._train_loop_config_repr_html_()) if self._train_loop_config - else None - ) - - tab.set_title(3, "Scaling Config") - children.append( - HTML(self.scaling_config._repr_html_()) if self.scaling_config else None + else None, + HTML(self.scaling_config._repr_html_()) if self.scaling_config else None, + HTML(self.run_config._repr_html_()) if self.run_config else None, + HTML(self._backend_config._repr_html_()) if self._backend_config else None, + ] + + tab = Tab( + children, + titles=[ + "Datasets", + "Dataset Config", + "Train Loop Config", + "Scaling Config", + "Run Config", + "Backend Config", + ], ) - - tab.set_title(4, "Run Config") - children.append( - HTML(self.run_config._repr_html_()) if self.run_config else None - ) - - tab.set_title(5, "Backend Config") - children.append( - HTML(self._backend_config._repr_html_()) if self._backend_config else None - ) - - tab.children = children display(VBox([title, tab], layout=Layout(width="100%"))) def _train_loop_config_repr_html_(self) -> str: @@ -471,26 +456,23 @@ def _dataset_config_repr_html_(self) -> str: return Template("rendered_html_common.html.j2").render(content=content) + @ensure_notebook_deps(["ipywidgets", "8"]) def _datasets_repr_(self) -> str: - try: - from ipywidgets import HTML, VBox, Layout - except ImportError: - logger.warn( - "'ipywidgets' isn't installed. Run `pip install ipywidgets` to " - "enable notebook widgets." - ) - return None + from ipywidgets import HTML, VBox, Layout + content = [] if self.datasets: for name, config in self.datasets.items(): - content.append( - HTML( - Template("title_data.html.j2").render( - title=f"Dataset - {name}", data=None + tab = config._tab_repr_() + if tab: + content.append( + HTML( + Template("title_data.html.j2").render( + title=f"Dataset - {name}", data=None + ) ) ) - ) - content.append(config._tab_repr_()) + content.append(config._tab_repr_()) return VBox(content, layout=Layout(width="100%")) diff --git a/python/ray/widgets/util.py b/python/ray/widgets/util.py index b2d260a29f5d4..8e16810e41eec 100644 --- a/python/ray/widgets/util.py +++ b/python/ray/widgets/util.py @@ -1,8 +1,22 @@ -from typing import Any, Optional +import importlib +import logging +import textwrap +from functools import wraps +from typing import Any, Callable, Iterable, Optional, TypeVar, Union from ray.util.annotations import DeveloperAPI from ray.widgets import Template +try: + from packaging.version import Version +except ImportError: + from distutils.version import LooseVersion as Version + + +logger = logging.getLogger(__name__) + +F = TypeVar("F", bound=Callable[..., Any]) + @DeveloperAPI def make_table_html_repr( @@ -59,3 +73,93 @@ def make_table_html_repr( content = table return content + + +@DeveloperAPI +def ensure_notebook_deps( + *deps: Iterable[Union[str, Optional[str]]], + missing_message: Optional[str] = None, + outdated_message: Optional[str] = None, +) -> Callable[[F], F]: + """Generate a decorator which checks for soft dependencies. + + This decorator is meant to wrap _ipython_display_. If the dependency is not found, + or a version is specified here and the version of the package is older than the + specified version, the wrapped function is not executed and None is returned. If + the dependency is missing or the version is old, a log message is displayed. + + Args: + *deps: Iterable of (dependency name, min version (optional)) + missing_message: Message to log if missing package is found + outdated_message: Message to log if outdated package is found + + Returns: + Wrapped function. Guaranteed to be safe to import soft dependencies specified + above. + """ + + def wrapper(func: F) -> F: + @wraps(func) + def wrapped(*args, **kwargs): + if _has_missing(*deps, message=missing_message) or _has_outdated( + *deps, message=outdated_message + ): + return None + return func(*args, **kwargs) + + return wrapped + + return wrapper + + +def _has_missing( + *deps: Iterable[Union[str, Optional[str]]], message: Optional[str] = None +): + missing = [] + for (lib, _) in deps: + try: + importlib.import_module(lib) + except ImportError: + missing.append(lib) + + if missing: + if not message: + message = f"Run `pip install {' '.join(missing)}` for rich notebook output." + + # stacklevel=3: First level is this function, then ensure_notebook_deps, then + # the actual function affected. + logger.warning(f"Missing packages: {missing}. {message}", stacklevel=3) + + return missing + + +def _has_outdated( + *deps: Iterable[Union[str, Optional[str]]], message: Optional[str] = None +): + outdated = [] + for (lib, version) in deps: + try: + module = importlib.import_module(lib) + if version and Version(module.__version__) < Version(version): + outdated.append([lib, version, module.__version__]) + except ImportError: + pass + + if outdated: + outdated_strs = [] + install_args = [] + for lib, version, installed in outdated: + outdated_strs.append(f"{lib}=={installed} found, needs {lib}>={version}") + install_args.append(f"{lib}>={version}") + + outdated_str = textwrap.indent("\n".join(outdated_strs), " ") + install_str = " ".join(install_args) + + if not message: + message = f"Run `pip install -U {install_str}` for rich notebook output." + + # stacklevel=3: First level is this function, then ensure_notebook_deps, then + # the actual function affected. + logger.warning(f"Outdated packages:\n{outdated_str}\n{message}", stacklevel=3) + + return outdated