[wheel] exclude 3rd-party __pycache__ and tests dir (#46405)

the pycache and tests dirs are not useful, non deterministic, and just
making the wheel larger.

Signed-off-by: Lonnie Liu <[email protected]>

python/setup.py

@@ -599,9 +599,14 @@ def build(build_python, build_java, build_cpp):
  )
 
 
-def walk_directory(directory):
+def _walk_thirdparty_dir(directory):
  file_list = []
  for root, dirs, filenames in os.walk(directory):
+ # Exclude generated bytecode cache directories and tests directories
+ # from vendored packages.
+ for exclude_dir in ["__pycache__", "tests"]:
+ if exclude_dir in dirs:
+ dirs.remove(exclude_dir)
  for name in filenames:
  file_list.append(os.path.join(root, name))
  return file_list
@@ -611,7 +616,7 @@ def copy_file(target_dir, filename, rootdir):
  # TODO(rkn): This feels very brittle. It may not handle all cases. See
  # https://github.com/apache/arrow/blob/master/python/setup.py for an
  # example.
- # File names can be absolute paths, e.g. from walk_directory().
+ # File names can be absolute paths, e.g. from _walk_thirdparty_dir().
  source = os.path.relpath(filename, rootdir)
  destination = os.path.join(target_dir, source)
  # Create the target directory if it doesn't already exist.
@@ -650,12 +655,14 @@ def pip_run(build_ext):
  setup_spec.files_to_include += ray_files
 
  thirdparty_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
- setup_spec.files_to_include += walk_directory(thirdparty_dir)
+ setup_spec.files_to_include += _walk_thirdparty_dir(thirdparty_dir)
 
  runtime_env_agent_thirdparty_dir = os.path.join(
  ROOT_DIR, RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR
  )
- setup_spec.files_to_include += walk_directory(runtime_env_agent_thirdparty_dir)
+ setup_spec.files_to_include += _walk_thirdparty_dir(
+ runtime_env_agent_thirdparty_dir
+ )
 
  # Copy over the autogenerated protobuf Python bindings.
  for directory in generated_python_directories: