From 752bd06b16465f2791e256b1abfa49d5e6759f5b Mon Sep 17 00:00:00 2001 From: angelinalg <122562471+angelinalg@users.noreply.github.com> Date: Tue, 23 May 2023 17:07:51 -0700 Subject: [PATCH 1/5] fixing broken references, links, note Signed-off-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- doc/source/_toc.yml | 1 - doc/source/ray-air/getting-started.rst | 2 +- doc/source/ray-observability/key-concepts.rst | 9 +- .../user-guides/add-app-metrics.rst | 2 +- .../user-guides/configure-logging.rst | 2 +- .../user-guides/debug-apps/debug-failures.rst | 16 +- .../user-guides/debug-apps/debug-memory.rst | 10 +- .../user-guides/debug-apps/index.md | 2 +- .../debug-apps/optimize-performance.rst | 2 + .../ray-observability/user-guides/index.md | 2 +- doc/source/ray-observability/warnings.txt | 141 ++++++++++++++++++ python/ray/_private/log.py | 2 +- 12 files changed, 166 insertions(+), 25 deletions(-) create mode 100644 doc/source/ray-observability/warnings.txt diff --git a/doc/source/_toc.yml b/doc/source/_toc.yml index 41a73f095f70e..0cab0e91e7f98 100644 --- a/doc/source/_toc.yml +++ b/doc/source/_toc.yml @@ -407,7 +407,6 @@ parts: - file: ray-observability/user-guides/debug-apps/debug-failures - file: ray-observability/user-guides/debug-apps/optimize-performance - file: ray-observability/user-guides/debug-apps/ray-debugging - - file: ray-observability/user-guides/debug-apps/ray-core-profiling - file: ray-observability/user-guides/cli-sdk - file: ray-observability/user-guides/configure-logging - file: ray-observability/user-guides/add-app-metrics diff --git a/doc/source/ray-air/getting-started.rst b/doc/source/ray-air/getting-started.rst index 9bcdd66f18eae..dfdfad46f5e07 100644 --- a/doc/source/ray-air/getting-started.rst +++ b/doc/source/ray-air/getting-started.rst @@ -216,4 +216,4 @@ Next Steps - :ref:`air-examples-ref` - :ref:`API reference ` - :ref:`Technical whitepaper ` -- To check how your application is doing, you can use the :ref:`Ray dashboard`. +- To check how your application is doing, you can use the :ref:`Ray dashboard`. diff --git a/doc/source/ray-observability/key-concepts.rst b/doc/source/ray-observability/key-concepts.rst index 1b48c12c9501a..fac8ac07f2ca2 100644 --- a/doc/source/ray-observability/key-concepts.rst +++ b/doc/source/ray-observability/key-concepts.rst @@ -93,9 +93,9 @@ See :ref:`Ray Debugger ` for more details. Profiling --------- -Ray is compatible with Python profiling tools such as ``CProfile``. It also supports its built-in profiling tool such as :ref:```ray timeline`` `. +Ray is compatible with Python profiling tools such as ``CProfile``. It also supports its built-in profiling tool such as :ref:`ray timeline `. -See :ref:`Profiling ` for more details. +See :ref:`Profiling ` for more details. Tracing ------- @@ -171,8 +171,9 @@ Logging directory structure By default, Ray logs are stored in a ``/tmp/ray/session_*/logs`` directory. -..{note}: -The default temp directory is ``/tmp/ray`` (for Linux and MacOS). To change the temp directory, specify it when you call ``ray start`` or ``ray.init()``. +.. note:: + + The default temp directory is ``/tmp/ray`` (for Linux and MacOS). To change the temp directory, specify it when you call ``ray start`` or ``ray.init()``. A new Ray instance creates a new session ID to the temp directory. The latest session ID is symlinked to ``/tmp/ray/session_latest``. diff --git a/doc/source/ray-observability/user-guides/add-app-metrics.rst b/doc/source/ray-observability/user-guides/add-app-metrics.rst index b82946ed1544c..c742a7fba2873 100644 --- a/doc/source/ray-observability/user-guides/add-app-metrics.rst +++ b/doc/source/ray-observability/user-guides/add-app-metrics.rst @@ -8,7 +8,7 @@ There are currently three metrics supported: Counter, Gauge, and Histogram. These metrics correspond to the same `Prometheus metric types `_. Below is a simple example of an actor that exports metrics using these APIs: -.. literalinclude:: doc_code/metrics_example.py +.. literalinclude:: ../doc_code/metrics_example.py :language: python While the script is running, the metrics are exported to ``localhost:8080`` (this is the endpoint that Prometheus would be configured to scrape). diff --git a/doc/source/ray-observability/user-guides/configure-logging.rst b/doc/source/ray-observability/user-guides/configure-logging.rst index a7f3ae7ef0607..e57f0424649bc 100644 --- a/doc/source/ray-observability/user-guides/configure-logging.rst +++ b/doc/source/ray-observability/user-guides/configure-logging.rst @@ -106,7 +106,7 @@ By default Ray prints Actor logs prefixes in light blue: Users may instead activate multi-color prefixes by setting the environment variable ``RAY_COLOR_PREFIX=1``. This will index into an array of colors modulo the PID of each process. -.. image:: ./images/coloring-actor-log-prefixes.png +.. image:: ../images/coloring-actor-log-prefixes.png :align: center Distributed progress bars (tqdm) diff --git a/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst b/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst index 206adb88853e6..e4e51ab4f362b 100644 --- a/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst +++ b/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst @@ -96,7 +96,7 @@ it will raise an exception with one of the following error messages (which indic Also, you can use the `dmesg `_ CLI command to verify the processes are killed by the Linux out-of-memory killer. -.. image:: ../images/dmsg.png +.. image:: ../../images/dmsg.png :align: center If the worker is killed by Ray's memory monitor, they are automatically retried (see the :ref:`link ` for the detail). @@ -130,10 +130,10 @@ Ray memory monitor also periodically prints the aggregated out-of-memory killer Ray Dashboard's :ref:`metrics page ` and :ref:`event page ` also provides the out-of-memory killer-specific events and metrics. -.. image:: ../images/oom-metrics.png +.. image:: ../../images/oom-metrics.png :align: center -.. image:: ../images/oom-events.png +.. image:: ../../images/oom-events.png :align: center .. _troubleshooting-out-of-memory-task-actor-mem-usage: @@ -150,7 +150,7 @@ The memory usage from the per component graph uses RSS - SHR. See the below for Alternatively, you can also use the CLI command `htop `_. -.. image:: ../images/htop.png +.. image:: ../../images/htop.png :align: center See the ``allocate_memory`` row. See two columns, RSS and SHR. @@ -173,12 +173,12 @@ Head Node Out-of-Memory Error First, check the head node memory usage from the metrics page. Find the head node address from the cluster page. -.. image:: ../images/head-node-addr.png +.. image:: ../../images/head-node-addr.png :align: center And then check the memory usage from the head node from the node memory usage view inside the Dashboard :ref:`metrics view `. -.. image:: ../images/metrics-node-view.png +.. image:: ../../images/metrics-node-view.png :align: center Ray head node has more memory-demanding system components such as GCS or the dashboard. @@ -201,10 +201,10 @@ You can verify it by looking at the :ref:`per task and actor memory usage graph First, see the memory usage of a ``allocate_memory`` task. It is total 18GB. At the same time, you can verify 15 concurrent tasks running. -.. image:: ../images/component-memory.png +.. image:: ../../images/component-memory.png :align: center -.. image:: ../images/tasks-graph.png +.. image:: ../../images/tasks-graph.png :align: center It means each task uses about 18GB / 15 == 1.2 GB. To reduce the parallelism, diff --git a/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst b/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst index 10d10620cb6bf..89784fc0254f9 100644 --- a/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst +++ b/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst @@ -1,5 +1,3 @@ -.. _ray-core-profiling: - .. _ray-core-mem-profiling: Debugging Memory Issues @@ -22,7 +20,7 @@ This will allow you to download profiling files from other nodes. .. tab-item:: Actors - .. literalinclude:: ../doc_code/memray_profiling.py + .. literalinclude:: ../../doc_code/memray_profiling.py :language: python :start-after: __memray_profiling_start__ :end-before: __memray_profiling_end__ @@ -31,19 +29,19 @@ This will allow you to download profiling files from other nodes. Note that tasks have a shorter lifetime, so there could be lots of memory profiling files. - .. literalinclude:: ../doc_code/memray_profiling.py + .. literalinclude:: ../../doc_code/memray_profiling.py :language: python :start-after: __memray_profiling_task_start__ :end-before: __memray_profiling_task_end__ Once the task or actor runs, go to the :ref:`Logs View ` of the dashboard. Find and click the log file name. -.. image:: ../images/memory-profiling-files.png +.. image:: ../../images/memory-profiling-files.png :align: center Click the download button. -.. image:: ../images/download-memory-profiling-files.png +.. image:: ../../images/download-memory-profiling-files.png :align: center Now, you have the memory profiling file. Running diff --git a/doc/source/ray-observability/user-guides/debug-apps/index.md b/doc/source/ray-observability/user-guides/debug-apps/index.md index 53e9ee3e962e0..24c8c7ce4e277 100644 --- a/doc/source/ray-observability/user-guides/debug-apps/index.md +++ b/doc/source/ray-observability/user-guides/debug-apps/index.md @@ -1,4 +1,4 @@ -(observability-user-guides)= +(observability-debug-apps)= # Troubleshooting Applications diff --git a/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst b/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst index cad5e6f312ac1..a829333f2dc9a 100644 --- a/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst +++ b/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst @@ -119,6 +119,8 @@ not have root permissions, the dashboard will prompt with instructions on how to Alternatively, you can start Ray with passwordless sudo / root permissions. +.. _dashboard-cprofile: + Profiling Using Python's CProfile --------------------------------- diff --git a/doc/source/ray-observability/user-guides/index.md b/doc/source/ray-observability/user-guides/index.md index 9b81d03a55915..ce5ca14e0215c 100644 --- a/doc/source/ray-observability/user-guides/index.md +++ b/doc/source/ray-observability/user-guides/index.md @@ -6,7 +6,7 @@ These guides help you monitor and debug your Ray applications and clusters. The guides include: * {ref}`observability-general-troubleshoot` -* {ref}`observability-user-guides` +* {ref}`observability-debug-apps` * {ref}`observability-programmatic` * {ref}`configure-logging` * {ref}`application-level-metrics` diff --git a/doc/source/ray-observability/warnings.txt b/doc/source/ray-observability/warnings.txt new file mode 100644 index 0000000000000..f95fa9b6f5c1b --- /dev/null +++ b/doc/source/ray-observability/warnings.txt @@ -0,0 +1,141 @@ +/Users/angelina/src/ray/doc/source/cluster/vms/user-guides/index.md:: WARNING: toctree contains reference to nonexisting document 'cluster/vms/user-guides/logging' [etoc.ref] +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_actor:1: WARNING: duplicate object description of ray.util.state.get_actor, other instance in ray-observability/reference/doc/ray.util.state.get_actor, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_log:1: WARNING: duplicate object description of ray.util.state.get_log, other instance in ray-observability/reference/doc/ray.util.state.get_log, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_node:1: WARNING: duplicate object description of ray.util.state.get_node, other instance in ray-observability/reference/doc/ray.util.state.get_node, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_objects:1: WARNING: duplicate object description of ray.util.state.get_objects, other instance in ray-observability/reference/doc/ray.util.state.get_objects, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_placement_group:1: WARNING: duplicate object description of ray.util.state.get_placement_group, other instance in ray-observability/reference/doc/ray.util.state.get_placement_group, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_task:1: WARNING: duplicate object description of ray.util.state.get_task, other instance in ray-observability/reference/doc/ray.util.state.get_task, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_worker:1: WARNING: duplicate object description of ray.util.state.get_worker, other instance in ray-observability/reference/doc/ray.util.state.get_worker, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_actors:1: WARNING: duplicate object description of ray.util.state.list_actors, other instance in ray-observability/reference/doc/ray.util.state.list_actors, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_jobs:1: WARNING: duplicate object description of ray.util.state.list_jobs, other instance in ray-observability/reference/doc/ray.util.state.list_jobs, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_logs:1: WARNING: duplicate object description of ray.util.state.list_logs, other instance in ray-observability/reference/doc/ray.util.state.list_logs, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_nodes:1: WARNING: duplicate object description of ray.util.state.list_nodes, other instance in ray-observability/reference/doc/ray.util.state.list_nodes, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_objects:1: WARNING: duplicate object description of ray.util.state.list_objects, other instance in ray-observability/reference/doc/ray.util.state.list_objects, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_placement_groups:1: WARNING: duplicate object description of ray.util.state.list_placement_groups, other instance in ray-observability/reference/doc/ray.util.state.list_placement_groups, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_runtime_envs:1: WARNING: duplicate object description of ray.util.state.list_runtime_envs, other instance in ray-observability/reference/doc/ray.util.state.list_runtime_envs, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_tasks:1: WARNING: duplicate object description of ray.util.state.list_tasks, other instance in ray-observability/reference/doc/ray.util.state.list_tasks, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_workers:1: WARNING: duplicate object description of ray.util.state.list_workers, other instance in ray-observability/reference/doc/ray.util.state.list_workers, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_actors:1: WARNING: duplicate object description of ray.util.state.summarize_actors, other instance in ray-observability/reference/doc/ray.util.state.summarize_actors, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_objects:1: WARNING: duplicate object description of ray.util.state.summarize_objects, other instance in ray-observability/reference/doc/ray.util.state.summarize_objects, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_tasks:1: WARNING: duplicate object description of ray.util.state.summarize_tasks, other instance in ray-observability/reference/doc/ray.util.state.summarize_tasks, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_actor:1: WARNING: duplicate object description of ray.util.state.get_actor, other instance in ray-observability/api/state/doc/ray.util.state.get_actor, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_log:1: WARNING: duplicate object description of ray.util.state.get_log, other instance in ray-observability/api/state/doc/ray.util.state.get_log, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_node:1: WARNING: duplicate object description of ray.util.state.get_node, other instance in ray-observability/api/state/doc/ray.util.state.get_node, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_objects:1: WARNING: duplicate object description of ray.util.state.get_objects, other instance in ray-observability/api/state/doc/ray.util.state.get_objects, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_placement_group:1: WARNING: duplicate object description of ray.util.state.get_placement_group, other instance in ray-observability/api/state/doc/ray.util.state.get_placement_group, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_task:1: WARNING: duplicate object description of ray.util.state.get_task, other instance in ray-observability/api/state/doc/ray.util.state.get_task, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_worker:1: WARNING: duplicate object description of ray.util.state.get_worker, other instance in ray-observability/api/state/doc/ray.util.state.get_worker, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_actors:1: WARNING: duplicate object description of ray.util.state.list_actors, other instance in ray-observability/api/state/doc/ray.util.state.list_actors, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_jobs:1: WARNING: duplicate object description of ray.util.state.list_jobs, other instance in ray-observability/api/state/doc/ray.util.state.list_jobs, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_logs:1: WARNING: duplicate object description of ray.util.state.list_logs, other instance in ray-observability/api/state/doc/ray.util.state.list_logs, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_nodes:1: WARNING: duplicate object description of ray.util.state.list_nodes, other instance in ray-observability/api/state/doc/ray.util.state.list_nodes, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_objects:1: WARNING: duplicate object description of ray.util.state.list_objects, other instance in ray-observability/api/state/doc/ray.util.state.list_objects, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_placement_groups:1: WARNING: duplicate object description of ray.util.state.list_placement_groups, other instance in ray-observability/api/state/doc/ray.util.state.list_placement_groups, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_runtime_envs:1: WARNING: duplicate object description of ray.util.state.list_runtime_envs, other instance in ray-observability/api/state/doc/ray.util.state.list_runtime_envs, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_tasks:1: WARNING: duplicate object description of ray.util.state.list_tasks, other instance in ray-observability/api/state/doc/ray.util.state.list_tasks, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_workers:1: WARNING: duplicate object description of ray.util.state.list_workers, other instance in ray-observability/api/state/doc/ray.util.state.list_workers, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_actors:1: WARNING: duplicate object description of ray.util.state.summarize_actors, other instance in ray-observability/api/state/doc/ray.util.state.summarize_actors, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_objects:1: WARNING: duplicate object description of ray.util.state.summarize_objects, other instance in ray-observability/api/state/doc/ray.util.state.summarize_objects, use :noindex: for one of them +/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_tasks:1: WARNING: duplicate object description of ray.util.state.summarize_tasks, other instance in ray-observability/api/state/doc/ray.util.state.summarize_tasks, use :noindex: for one of them +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/add-app-metrics.rst:11: WARNING: Include file '/Users/angelina/src/ray/doc/source/ray-observability/user-guides/doc_code/metrics_example.py' not found or reading it failed +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/configure-logging.rst:: WARNING: image file not readable: ray-observability/user-guides/images/coloring-actor-log-prefixes.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/dmsg.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/oom-metrics.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/oom-events.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/htop.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/head-node-addr.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/metrics-node-view.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/component-memory.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/tasks-graph.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:25: WARNING: Include file '/Users/angelina/src/ray/doc/source/ray-observability/user-guides/doc_code/memray_profiling.py' not found or reading it failed +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:34: WARNING: Include file '/Users/angelina/src/ray/doc/source/ray-observability/user-guides/doc_code/memray_profiling.py' not found or reading it failed +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:: WARNING: image file not readable: ray-observability/user-guides/images/memory-profiling-files.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:: WARNING: image file not readable: ray-observability/user-guides/images/download-memory-profiling-files.png +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/index.md:: WARNING: toctree contains reference to nonexisting document 'ray-observability/user-guides/debug-apps/ray-core-profiling' [etoc.ref] +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/index.md:3: WARNING: duplicate label observability-user-guides, other instance in /Users/angelina/src/ray/doc/source/ray-observability/user-guides/index.md +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/index.md:3: WARNING: duplicate label observability-user-guides, other instance in /Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/index.md +/Users/angelina/src/ray/doc/source/rllib/rllib-catalogs.rst:152: WARNING: Include file '/Users/angelina/src/ray/rllib/algorithms/ppo/ppo_base_rl_module.py' not found or reading it failed +looking for now-outdated files... none found +pickling environment... done +checking consistency... /Users/angelina/src/ray/doc/source/cluster/running-applications/job-submission/doc/ray.job_submission.JobSubmissionClient.__init__.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ActorState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ActorSummaries.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ActorSummaryPerClass.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.JobState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.NodeState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ObjectState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ObjectSummaries.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ObjectSummaryPerKey.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.PlacementGroupState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.RuntimeEnvState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.StateSummary.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.TaskState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.TaskSummaries.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.TaskSummaryPerFuncOrClassName.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.WorkerState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.exception.RayStateApiException.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_actor.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_log.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_node.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_objects.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_placement_group.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_task.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_worker.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_actors.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_jobs.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_logs.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_nodes.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_objects.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_placement_groups.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_runtime_envs.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_tasks.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_workers.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.summarize_actors.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.summarize_objects.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.summarize_tasks.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_actor.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_log.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_node.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_objects.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_placement_group.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_task.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_worker.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_actors.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_jobs.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_logs.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_nodes.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_objects.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_placement_groups.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_runtime_envs.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_tasks.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_workers.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.summarize_actors.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.summarize_objects.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.summarize_tasks.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ActorState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ActorSummaries.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ActorSummaryPerClass.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.JobState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.NodeState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ObjectState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ObjectSummaries.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ObjectSummaryPerKey.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.PlacementGroupState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.RuntimeEnvState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.StateSummary.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.TaskState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.TaskSummaries.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.TaskSummaryPerFuncOrClassName.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.WorkerState.rst: WARNING: document isn't included in any toctree +/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.exception.RayStateApiException.rst: WARNING: document isn't included in any toctree +done +preparing documents... done +^C/Users/angelina/src/ray/doc/source/ray-air/getting-started.rst:219: WARNING: undefined label: robservability-getting-started +/Users/angelina/src/ray/doc/source/ray-observability/getting-started.rst:304: WARNING: undefined label: logging-directory-structure +/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:69: WARNING: undefined label: ray-metrics +/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:120: WARNING: undefined label: ray-logging +/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:129: WARNING: undefined label: logging-directory-structure +/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:138: WARNING: undefined label: logging-directory-structure +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/configure-logging.rst:43: WARNING: undefined label: customizing worker loggers +/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst:86: WARNING: undefined label: ray-dashboard + diff --git a/python/ray/_private/log.py b/python/ray/_private/log.py index ea19fbae7926b..034dd6e536f06 100644 --- a/python/ray/_private/log.py +++ b/python/ray/_private/log.py @@ -72,7 +72,7 @@ def emit(self, record: logging.LogRecord): import ray if ( - hasattr(ray, "_private") + hasattr(ray, "_private") and hasattr(ray._private, "worker") and ray._private.worker.global_worker.mode == ray._private.worker.WORKER_MODE ): From bdfbe0a5f1be4895b6057bd0a7d332f251fd80aa Mon Sep 17 00:00:00 2001 From: angelinalg <122562471+angelinalg@users.noreply.github.com> Date: Tue, 23 May 2023 19:46:19 -0700 Subject: [PATCH 2/5] revert mistaken change Signed-off-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- python/ray/_private/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ray/_private/log.py b/python/ray/_private/log.py index 034dd6e536f06..ea19fbae7926b 100644 --- a/python/ray/_private/log.py +++ b/python/ray/_private/log.py @@ -72,7 +72,7 @@ def emit(self, record: logging.LogRecord): import ray if ( - hasattr(ray, "_private") and hasattr(ray._private, "worker") + hasattr(ray, "_private") and ray._private.worker.global_worker.mode == ray._private.worker.WORKER_MODE ): From fdd5b7185dad154fb82b715822d6f794e2f5c1ea Mon Sep 17 00:00:00 2001 From: angelinalg <122562471+angelinalg@users.noreply.github.com> Date: Tue, 23 May 2023 21:15:11 -0700 Subject: [PATCH 3/5] remove file that was accidentally committed Signed-off-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- doc/source/ray-observability/warnings.txt | 141 ---------------------- 1 file changed, 141 deletions(-) delete mode 100644 doc/source/ray-observability/warnings.txt diff --git a/doc/source/ray-observability/warnings.txt b/doc/source/ray-observability/warnings.txt deleted file mode 100644 index f95fa9b6f5c1b..0000000000000 --- a/doc/source/ray-observability/warnings.txt +++ /dev/null @@ -1,141 +0,0 @@ -/Users/angelina/src/ray/doc/source/cluster/vms/user-guides/index.md:: WARNING: toctree contains reference to nonexisting document 'cluster/vms/user-guides/logging' [etoc.ref] -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_actor:1: WARNING: duplicate object description of ray.util.state.get_actor, other instance in ray-observability/reference/doc/ray.util.state.get_actor, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_log:1: WARNING: duplicate object description of ray.util.state.get_log, other instance in ray-observability/reference/doc/ray.util.state.get_log, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_node:1: WARNING: duplicate object description of ray.util.state.get_node, other instance in ray-observability/reference/doc/ray.util.state.get_node, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_objects:1: WARNING: duplicate object description of ray.util.state.get_objects, other instance in ray-observability/reference/doc/ray.util.state.get_objects, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_placement_group:1: WARNING: duplicate object description of ray.util.state.get_placement_group, other instance in ray-observability/reference/doc/ray.util.state.get_placement_group, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_task:1: WARNING: duplicate object description of ray.util.state.get_task, other instance in ray-observability/reference/doc/ray.util.state.get_task, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_worker:1: WARNING: duplicate object description of ray.util.state.get_worker, other instance in ray-observability/reference/doc/ray.util.state.get_worker, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_actors:1: WARNING: duplicate object description of ray.util.state.list_actors, other instance in ray-observability/reference/doc/ray.util.state.list_actors, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_jobs:1: WARNING: duplicate object description of ray.util.state.list_jobs, other instance in ray-observability/reference/doc/ray.util.state.list_jobs, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_logs:1: WARNING: duplicate object description of ray.util.state.list_logs, other instance in ray-observability/reference/doc/ray.util.state.list_logs, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_nodes:1: WARNING: duplicate object description of ray.util.state.list_nodes, other instance in ray-observability/reference/doc/ray.util.state.list_nodes, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_objects:1: WARNING: duplicate object description of ray.util.state.list_objects, other instance in ray-observability/reference/doc/ray.util.state.list_objects, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_placement_groups:1: WARNING: duplicate object description of ray.util.state.list_placement_groups, other instance in ray-observability/reference/doc/ray.util.state.list_placement_groups, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_runtime_envs:1: WARNING: duplicate object description of ray.util.state.list_runtime_envs, other instance in ray-observability/reference/doc/ray.util.state.list_runtime_envs, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_tasks:1: WARNING: duplicate object description of ray.util.state.list_tasks, other instance in ray-observability/reference/doc/ray.util.state.list_tasks, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_workers:1: WARNING: duplicate object description of ray.util.state.list_workers, other instance in ray-observability/reference/doc/ray.util.state.list_workers, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_actors:1: WARNING: duplicate object description of ray.util.state.summarize_actors, other instance in ray-observability/reference/doc/ray.util.state.summarize_actors, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_objects:1: WARNING: duplicate object description of ray.util.state.summarize_objects, other instance in ray-observability/reference/doc/ray.util.state.summarize_objects, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_tasks:1: WARNING: duplicate object description of ray.util.state.summarize_tasks, other instance in ray-observability/reference/doc/ray.util.state.summarize_tasks, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_actor:1: WARNING: duplicate object description of ray.util.state.get_actor, other instance in ray-observability/api/state/doc/ray.util.state.get_actor, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_log:1: WARNING: duplicate object description of ray.util.state.get_log, other instance in ray-observability/api/state/doc/ray.util.state.get_log, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_node:1: WARNING: duplicate object description of ray.util.state.get_node, other instance in ray-observability/api/state/doc/ray.util.state.get_node, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_objects:1: WARNING: duplicate object description of ray.util.state.get_objects, other instance in ray-observability/api/state/doc/ray.util.state.get_objects, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_placement_group:1: WARNING: duplicate object description of ray.util.state.get_placement_group, other instance in ray-observability/api/state/doc/ray.util.state.get_placement_group, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_task:1: WARNING: duplicate object description of ray.util.state.get_task, other instance in ray-observability/api/state/doc/ray.util.state.get_task, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.get_worker:1: WARNING: duplicate object description of ray.util.state.get_worker, other instance in ray-observability/api/state/doc/ray.util.state.get_worker, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_actors:1: WARNING: duplicate object description of ray.util.state.list_actors, other instance in ray-observability/api/state/doc/ray.util.state.list_actors, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_jobs:1: WARNING: duplicate object description of ray.util.state.list_jobs, other instance in ray-observability/api/state/doc/ray.util.state.list_jobs, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_logs:1: WARNING: duplicate object description of ray.util.state.list_logs, other instance in ray-observability/api/state/doc/ray.util.state.list_logs, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_nodes:1: WARNING: duplicate object description of ray.util.state.list_nodes, other instance in ray-observability/api/state/doc/ray.util.state.list_nodes, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_objects:1: WARNING: duplicate object description of ray.util.state.list_objects, other instance in ray-observability/api/state/doc/ray.util.state.list_objects, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_placement_groups:1: WARNING: duplicate object description of ray.util.state.list_placement_groups, other instance in ray-observability/api/state/doc/ray.util.state.list_placement_groups, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_runtime_envs:1: WARNING: duplicate object description of ray.util.state.list_runtime_envs, other instance in ray-observability/api/state/doc/ray.util.state.list_runtime_envs, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_tasks:1: WARNING: duplicate object description of ray.util.state.list_tasks, other instance in ray-observability/api/state/doc/ray.util.state.list_tasks, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.list_workers:1: WARNING: duplicate object description of ray.util.state.list_workers, other instance in ray-observability/api/state/doc/ray.util.state.list_workers, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_actors:1: WARNING: duplicate object description of ray.util.state.summarize_actors, other instance in ray-observability/api/state/doc/ray.util.state.summarize_actors, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_objects:1: WARNING: duplicate object description of ray.util.state.summarize_objects, other instance in ray-observability/api/state/doc/ray.util.state.summarize_objects, use :noindex: for one of them -/Users/angelina/src/ray/python/ray/util/state/api.py:docstring of ray.util.state.api.summarize_tasks:1: WARNING: duplicate object description of ray.util.state.summarize_tasks, other instance in ray-observability/api/state/doc/ray.util.state.summarize_tasks, use :noindex: for one of them -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/add-app-metrics.rst:11: WARNING: Include file '/Users/angelina/src/ray/doc/source/ray-observability/user-guides/doc_code/metrics_example.py' not found or reading it failed -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/configure-logging.rst:: WARNING: image file not readable: ray-observability/user-guides/images/coloring-actor-log-prefixes.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/dmsg.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/oom-metrics.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/oom-events.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/htop.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/head-node-addr.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/metrics-node-view.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/component-memory.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-failures.rst:: WARNING: image file not readable: ray-observability/user-guides/images/tasks-graph.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:25: WARNING: Include file '/Users/angelina/src/ray/doc/source/ray-observability/user-guides/doc_code/memray_profiling.py' not found or reading it failed -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:34: WARNING: Include file '/Users/angelina/src/ray/doc/source/ray-observability/user-guides/doc_code/memray_profiling.py' not found or reading it failed -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:: WARNING: image file not readable: ray-observability/user-guides/images/memory-profiling-files.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/debug-memory.rst:: WARNING: image file not readable: ray-observability/user-guides/images/download-memory-profiling-files.png -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/index.md:: WARNING: toctree contains reference to nonexisting document 'ray-observability/user-guides/debug-apps/ray-core-profiling' [etoc.ref] -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/index.md:3: WARNING: duplicate label observability-user-guides, other instance in /Users/angelina/src/ray/doc/source/ray-observability/user-guides/index.md -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/index.md:3: WARNING: duplicate label observability-user-guides, other instance in /Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/index.md -/Users/angelina/src/ray/doc/source/rllib/rllib-catalogs.rst:152: WARNING: Include file '/Users/angelina/src/ray/rllib/algorithms/ppo/ppo_base_rl_module.py' not found or reading it failed -looking for now-outdated files... none found -pickling environment... done -checking consistency... /Users/angelina/src/ray/doc/source/cluster/running-applications/job-submission/doc/ray.job_submission.JobSubmissionClient.__init__.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ActorState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ActorSummaries.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ActorSummaryPerClass.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.JobState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.NodeState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ObjectState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ObjectSummaries.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.ObjectSummaryPerKey.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.PlacementGroupState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.RuntimeEnvState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.StateSummary.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.TaskState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.TaskSummaries.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.TaskSummaryPerFuncOrClassName.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.common.WorkerState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.exception.RayStateApiException.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_actor.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_log.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_node.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_objects.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_placement_group.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_task.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.get_worker.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_actors.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_jobs.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_logs.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_nodes.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_objects.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_placement_groups.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_runtime_envs.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_tasks.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.list_workers.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.summarize_actors.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.summarize_objects.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/api/state/doc/ray.util.state.summarize_tasks.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_actor.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_log.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_node.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_objects.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_placement_group.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_task.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.get_worker.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_actors.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_jobs.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_logs.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_nodes.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_objects.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_placement_groups.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_runtime_envs.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_tasks.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.list_workers.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.summarize_actors.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.summarize_objects.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.api.summarize_tasks.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ActorState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ActorSummaries.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ActorSummaryPerClass.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.JobState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.NodeState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ObjectState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ObjectSummaries.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.ObjectSummaryPerKey.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.PlacementGroupState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.RuntimeEnvState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.StateSummary.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.TaskState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.TaskSummaries.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.TaskSummaryPerFuncOrClassName.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.common.WorkerState.rst: WARNING: document isn't included in any toctree -/Users/angelina/src/ray/doc/source/ray-observability/reference/state/doc/ray.experimental.state.exception.RayStateApiException.rst: WARNING: document isn't included in any toctree -done -preparing documents... done -^C/Users/angelina/src/ray/doc/source/ray-air/getting-started.rst:219: WARNING: undefined label: robservability-getting-started -/Users/angelina/src/ray/doc/source/ray-observability/getting-started.rst:304: WARNING: undefined label: logging-directory-structure -/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:69: WARNING: undefined label: ray-metrics -/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:120: WARNING: undefined label: ray-logging -/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:129: WARNING: undefined label: logging-directory-structure -/Users/angelina/src/ray/doc/source/ray-observability/key-concepts.rst:138: WARNING: undefined label: logging-directory-structure -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/configure-logging.rst:43: WARNING: undefined label: customizing worker loggers -/Users/angelina/src/ray/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst:86: WARNING: undefined label: ray-dashboard - From 5b1fae402ab6ff113a2b2a1b955ef54c07d686a1 Mon Sep 17 00:00:00 2001 From: angelinalg <122562471+angelinalg@users.noreply.github.com> Date: Tue, 23 May 2023 21:35:58 -0700 Subject: [PATCH 4/5] fix references Signed-off-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- doc/source/ray-observability/key-concepts.rst | 4 +++- .../user-guides/debug-apps/optimize-performance.rst | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/source/ray-observability/key-concepts.rst b/doc/source/ray-observability/key-concepts.rst index fac8ac07f2ca2..0f856b40420c5 100644 --- a/doc/source/ray-observability/key-concepts.rst +++ b/doc/source/ray-observability/key-concepts.rst @@ -66,7 +66,7 @@ internal stats (e.g., number of actors in the cluster, number of worker failures and custom metrics (e.g., metrics defined by users). All stats can be exported as time series data (to Prometheus by default) and used to monitor the cluster over time. -See :ref:`Ray Metrics ` for more details. +See :ref:`Ray Metrics ` for more details. Exceptions ---------- @@ -166,6 +166,8 @@ Actor log messages look like the following by default. (MyActor pid=480956) actor log message +.. _logging-directory-structure: + Logging directory structure ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst b/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst index a829333f2dc9a..86d91b8f93a09 100644 --- a/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst +++ b/doc/source/ray-observability/user-guides/debug-apps/optimize-performance.rst @@ -83,7 +83,7 @@ Then open `chrome://tracing`_ in the Chrome web browser, and load Python CPU Profiling in the Dashboard ------------------------------------- -The :ref:`ray-dashboard` lets you profile Ray worker processes by clicking on the "Stack Trace" or "CPU Flame Graph" +The :ref:`Ray dashboard ` lets you profile Ray worker processes by clicking on the "Stack Trace" or "CPU Flame Graph" actions for active workers, actors, and jobs. .. image:: /images/profile.png From 0d8a0df972d07c56656952679e4f8c5dbbe47bd4 Mon Sep 17 00:00:00 2001 From: angelinalg <122562471+angelinalg@users.noreply.github.com> Date: Wed, 24 May 2023 13:42:07 -0700 Subject: [PATCH 5/5] move customizing worker loggers from k8s to configure logging Signed-off-by: angelinalg <122562471+angelinalg@users.noreply.github.com> --- .../cluster/kubernetes/user-guides/logging.md | 42 ---------------- .../user-guides/configure-logging.rst | 50 ++++++++++++++++++- 2 files changed, 48 insertions(+), 44 deletions(-) diff --git a/doc/source/cluster/kubernetes/user-guides/logging.md b/doc/source/cluster/kubernetes/user-guides/logging.md index 7a349e1ee2693..2a5687ade5739 100644 --- a/doc/source/cluster/kubernetes/user-guides/logging.md +++ b/doc/source/cluster/kubernetes/user-guides/logging.md @@ -144,48 +144,6 @@ kubectl logs raycluster-complete-logs-head-xxxxx -c fluentbit [KubDoc]: https://kubernetes.io/docs/concepts/cluster-administration/logging/ [ConfigLink]: https://raw.githubusercontent.com/ray-project/ray/releases/2.4.0/doc/source/cluster/kubernetes/configs/ray-cluster.log.yaml -## Customizing Worker Loggers - -When using Ray, all tasks and actors are executed remotely in Ray's worker processes. - -:::{note} -To stream logs to a driver, they should be flushed to stdout and stderr. -::: - -```python -import ray -import logging -# Initiate a driver. -ray.init() - -@ray.remote -class Actor: - def __init__(self): - # Basic config automatically configures logs to - # be streamed to stdout and stderr. - # Set the severity to INFO so that info logs are printed to stdout. - logging.basicConfig(level=logging.INFO) - - def log(self, msg): - logger = logging.getLogger(__name__) - logger.info(msg) - -actor = Actor.remote() -ray.get(actor.log.remote("A log message for an actor.")) - -@ray.remote -def f(msg): - logging.basicConfig(level=logging.INFO) - logger = logging.getLogger(__name__) - logger.info(msg) - -ray.get(f.remote("A log message for a task.")) -``` - -```bash -(Actor pid=179641) INFO:__main__:A log message for an actor. -(f pid=177572) INFO:__main__:A log message for a task. -``` ## Using structured logging The metadata of tasks or actors may be obtained by Ray's :ref:`runtime_context APIs `. diff --git a/doc/source/ray-observability/user-guides/configure-logging.rst b/doc/source/ray-observability/user-guides/configure-logging.rst index e57f0424649bc..2cc4efa63499a 100644 --- a/doc/source/ray-observability/user-guides/configure-logging.rst +++ b/doc/source/ray-observability/user-guides/configure-logging.rst @@ -6,7 +6,7 @@ Configuring Logging This guide helps you modify the default configuration of Ray's logging system. -Internal Ray Logging Configuration +Internal Ray logging configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ When ``import ray`` is executed, Ray's logger is initialized, generating a sensible configuration given in ``python/ray/_private/log.py``. The default logging level is ``logging.INFO``. @@ -40,7 +40,7 @@ Similarly, to modify the logging configuration for any Ray subcomponent, specify # Here's how to add an aditional file handler for ray tune: ray_tune_logger.addHandler(logging.FileHandler("extra_ray_tune_log.log")) -For more information about logging in workers, see :ref:`Customizing worker loggers`. +For more information about logging in workers, see :ref:`Customizing worker loggers `. Disabling logging to the driver ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -129,3 +129,49 @@ Limitations: By default, the builtin print will also be patched to use `ray.experimental.tqdm_ray.safe_print` when `tqdm_ray` is used. This avoids progress bar corruption on driver print statements. To disable this, set `RAY_TQDM_PATCH_PRINT=0`. + +.. _customize-worker-loggers: + +Customizing worker loggers +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When using Ray, all tasks and actors are executed remotely in Ray's worker processes. + +.. note:: + + To stream logs to a driver, they should be flushed to stdout and stderr. + +.. code-block:: python + + import ray + import logging + # Initiate a driver. + ray.init() + + @ray.remote + class Actor: + def __init__(self): + # Basic config automatically configures logs to + # be streamed to stdout and stderr. + # Set the severity to INFO so that info logs are printed to stdout. + logging.basicConfig(level=logging.INFO) + + def log(self, msg): + logger = logging.getLogger(__name__) + logger.info(msg) + + actor = Actor.remote() + ray.get(actor.log.remote("A log message for an actor.")) + + @ray.remote + def f(msg): + logging.basicConfig(level=logging.INFO) + logger = logging.getLogger(__name__) + logger.info(msg) + + ray.get(f.remote("A log message for a task.")) + +.. code-block:: bash + + (Actor pid=179641) INFO:__main__:A log message for an actor. + (f pid=177572) INFO:__main__:A log message for a task. \ No newline at end of file