[Dashboard] Turn on new dashboard by default pt 2 (ray-project#11510)

.gitignore

@@ -1,4 +1,5 @@
 # The build output should clearly not be checked in
+*test-output.xml
 /bazel-*
 /python/ray/core
 /python/ray/pickle5_files/
@@ -11,7 +12,7 @@
 /thirdparty/pkg/
 /build/java
 .jar
-
+/dashboard/client/build
 # Files generated by flatc should be ignored
 /src/ray/gcs/format/*_generated.h
 /src/ray/object_manager/format/*_generated.h

.travis.yml

@@ -448,9 +448,6 @@ script:
  # ray serve tests
  - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci $(./scripts/bazel_export_options) --test_tag_filters=-jenkins_only python/ray/serve/...; fi
 
- # ray dashboard tests
- - if [ "$RAY_CI_DASHBOARD_AFFECTED" == "1" ]; then ./ci/keep_alive bazel test python/ray/dashboard/...; fi
-
  # ray new dashboard tests
  - if [ "$RAY_CI_DASHBOARD_AFFECTED" == "1" ]; then ./ci/keep_alive bazel test python/ray/new_dashboard/...; fi
 

BUILD.bazel

@@ -1786,8 +1786,7 @@ filegroup(
  "python/ray/core/generated/__init__.py",
  "python/ray/core/generated/ray/__init__.py",
  "python/ray/core/generated/ray/protocol/__init__.py",
- "python/ray/dashboard/*.py",
- "python/ray/dashboard/metrics_exporter/*.py",
+ "python/ray/new_dashboard/**/*.py",
  "python/ray/experimental/*.py",
  "python/ray/util/*.py",
  "python/ray/internal/*.py",

ci/travis/ci.sh

@@ -159,7 +159,6 @@ test_python() {
  -python/ray/tests:test_resource_demand_scheduler
  -python/ray/tests:test_stress # timeout
  -python/ray/tests:test_stress_sharded # timeout
- -python/ray/tests:test_webui
  )
  fi
  if [ 0 -lt "${#args[@]}" ]; then # Any targets to test?
@@ -208,7 +207,7 @@ build_dashboard_front_end() {
  { echo "WARNING: Skipping dashboard due to NPM incompatibilities with Windows"; } 2> /dev/null
  else
  (
- cd ray/dashboard/client
+ cd ray/new_dashboard/client
  set +x # suppress set -x since it'll get very noisy here
  . "${HOME}/.nvm/nvm.sh"
  nvm use --silent node
@@ -334,7 +333,7 @@ lint_bazel() {
 
 lint_web() {
  (
- cd "${WORKSPACE_DIR}"/python/ray/dashboard/client
+ cd "${WORKSPACE_DIR}"/python/ray/new_dashboard/client
  set +x # suppress set -x since it'll get very noisy here
  . "${HOME}/.nvm/nvm.sh"
  install_npm_project

ci/travis/format.sh

@@ -39,12 +39,6 @@ check_command_exist yapf
 check_command_exist flake8
 check_command_exist mypy
 
-ver=$(yapf --version)
-if ! echo "$ver" | grep -q 0.23.0; then
- echo "Wrong YAPF version installed: 0.23.0 is required, not $ver. $YAPF_DOWNLOAD_COMMAND_MSG"
- exit 1
-fi
-
 # this stops git rev-parse from failing if we run this from the .git directory
 builtin cd "$(dirname "${BASH_SOURCE:-$0}")"
 

ci/travis/test-wheels.sh

@@ -22,7 +22,8 @@ if [ -z "${BUILD_DIR}" ]; then
 fi
 TEST_DIR="${BUILD_DIR}/python/ray/tests"
 TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py")
-UI_TEST_SCRIPT="${BUILD_DIR}/python/ray/tests/test_webui.py"
+DASHBOARD_TEST_SCRIPT="${BUILD_DIR}/python/ray/tests/test_dashboard.py"
+
 
 function retry {
  local n=1
@@ -77,9 +78,7 @@ if [[ "$platform" == "linux" ]]; then
  for SCRIPT in "${TEST_SCRIPTS[@]}"; do
  retry "$PYTHON_EXE" "$SCRIPT"
  done
-
- # Run the UI test to make sure that the packaged UI works.
- retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
+ retry "$PYTHON_EXE" "$DASHBOARD_TEST_SCRIPT"
  done
 
  # Check that the other wheels are present.
@@ -118,12 +117,6 @@ elif [[ "$platform" == "macosx" ]]; then
  for SCRIPT in "${TEST_SCRIPTS[@]}"; do
  retry "$PYTHON_EXE" "$SCRIPT"
  done
-
- if (( $(echo "$PY_MM >= 3.0" | bc) )); then
- # Run the UI test to make sure that the packaged UI works.
- retry "$PYTHON_EXE" "$UI_TEST_SCRIPT"
- fi
-
  done
 elif [ "${platform}" = windows ]; then
  echo "WARNING: Wheel testing not yet implemented for Windows."

dashboard/agent.py

@@ -3,6 +3,7 @@
 import logging
 import logging.handlers
 import os
+import platform
 import sys
 import socket
 import json
@@ -38,6 +39,7 @@
 class DashboardAgent(object):
  def __init__(self,
  redis_address,
+ dashboard_agent_port,
  redis_password=None,
  temp_dir=None,
  log_dir=None,
@@ -51,6 +53,7 @@ def __init__(self,
  self.redis_password = redis_password
  self.temp_dir = temp_dir
  self.log_dir = log_dir
+ self.dashboard_agent_port = dashboard_agent_port
  self.metrics_export_port = metrics_export_port
  self.node_manager_port = node_manager_port
  self.object_store_name = object_store_name
@@ -59,7 +62,8 @@ def __init__(self,
  assert self.node_id, "Empty node id (RAY_NODE_ID)."
  self.ip = ray._private.services.get_node_ip_address()
  self.server = aiogrpc.server(options=(("grpc.so_reuseport", 0), ))
- self.grpc_port = self.server.add_insecure_port("[::]:0")
+ self.grpc_port = self.server.add_insecure_port(
+ f"[::]:{self.dashboard_agent_port}")
  logger.info("Dashboard agent grpc address: %s:%s", self.ip,
  self.grpc_port)
  self.aioredis_client = None
@@ -186,6 +190,11 @@ async def _check_parent():
  required=True,
  type=int,
  help="The port to expose metrics through Prometheus.")
+ parser.add_argument(
+ "--dashboard-agent-port",
+ required=True,
+ type=int,
+ help="The port on which the dashboard agent will receive GRPCs.")
  parser.add_argument(
  "--node-manager-port",
  required=True,
@@ -247,35 +256,23 @@ async def _check_parent():
  format(dashboard_consts.LOGGING_ROTATE_BACKUP_COUNT))
  parser.add_argument(
  "--log-dir",
- required=False,
+ required=True,
  type=str,
  default=None,
  help="Specify the path of log directory.")
  parser.add_argument(
  "--temp-dir",
- required=False,
+ required=True,
  type=str,
  default=None,
  help="Specify the path of the temporary directory use by Ray process.")
 
  args = parser.parse_args()
  try:
- if args.temp_dir:
- temp_dir = "/" + args.temp_dir.strip("/")
- else:
- temp_dir = "/tmp/ray"
- os.makedirs(temp_dir, exist_ok=True)
-
- if args.log_dir:
- log_dir = args.log_dir
- else:
- log_dir = os.path.join(temp_dir, "session_latest/logs")
- os.makedirs(log_dir, exist_ok=True)
-
  if args.logging_filename:
  logging_handlers = [
  logging.handlers.RotatingFileHandler(
- os.path.join(log_dir, args.logging_filename),
+ os.path.join(args.log_dir, args.logging_filename),
  maxBytes=args.logging_rotate_bytes,
  backupCount=args.logging_rotate_backup_count)
  ]
@@ -288,9 +285,10 @@ async def _check_parent():
 
  agent = DashboardAgent(
  args.redis_address,
+ args.dashboard_agent_port,
  redis_password=args.redis_password,
- temp_dir=temp_dir,
- log_dir=log_dir,
+ temp_dir=args.temp_dir,
+ log_dir=args.log_dir,
  metrics_export_port=args.metrics_export_port,
  node_manager_port=args.node_manager_port,
  object_store_name=args.object_store_name,
@@ -304,7 +302,7 @@ async def _check_parent():
  args.redis_address, password=args.redis_password)
  traceback_str = ray.utils.format_error_message(traceback.format_exc())
  message = ("The agent on node {} failed with the following "
- "error:\n{}".format(os.uname()[1], traceback_str))
+ "error:\n{}".format(platform.uname()[1], traceback_str))
  ray.utils.push_error_to_driver_through_redis(
  redis_client, ray_constants.DASHBOARD_AGENT_DIED_ERROR, message)
  raise e

dashboard/client/src/api.ts

@@ -60,7 +60,7 @@ type ProcessStats = {
  iowait: number;
  };
  cpuPercent: number;
-}
+};
 
 export type Worker = {
  pid: number;

dashboard/client/src/pages/dashboard/logical-view/Actor.tsx

@@ -195,9 +195,7 @@ const Actor: React.FC<ActorProps> = ({ actor }) => {
  </span>
  )}
  </Typography>
- <ActorDetailsPane
- actor={actor}
- />
+ <ActorDetailsPane actor={actor} />
  {isFullActorInfo(actor) && (
  <React.Fragment>
  {actorCustomDisplay.length > 0 && (