Skip to content

Commit

Permalink
End loop if header is found & Add max attempts limit
Browse files Browse the repository at this point in the history
  • Loading branch information
xinzhengzhang committed Mar 18, 2023
1 parent 4244747 commit 0befc09
Showing 1 changed file with 29 additions and 8 deletions.
37 changes: 29 additions & 8 deletions refresh.template.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
import time
import types
import typing # MIN_PY=3.9: Switch e.g. typing.List[str] -> list[str]
import threading
import itertools


@enum.unique
Expand Down Expand Up @@ -182,6 +184,7 @@ def _get_cached_adjusted_modified_time(path: str):
# Roughly 1 year into the future. This is safely below bazel's 10 year margin, but large enough that no sane normal file should be past this.
BAZEL_INTERNAL_SOURCE_CUTOFF = time.time() + 60*60*24*365

BAZEL_INTERNAL_MAX_HEADER_SEARCH_COUNT = 500

def _get_headers_gcc(compile_args: typing.List[str], source_path: str, action_key: str):
"""Gets the headers used by a particular compile command that uses gcc arguments formatting (including clang.)
Expand Down Expand Up @@ -766,22 +769,28 @@ def _all_platform_patch(compile_args: typing.List[str]):
return compile_args


def _get_cpp_command_for_files(compile_action):
def _get_cpp_command_for_files(args):
"""Reformat compile_action into a compile command clangd can understand.
Undo Bazel-isms and figures out which files clangd should apply the command to.
"""
(compile_action, event, should_stop_lambda) = args
if event.is_set():
return set(), set(), []

# Patch command by platform
compile_action.arguments = _all_platform_patch(compile_action.arguments)
compile_action.arguments = _apple_platform_patch(compile_action.arguments)
# Android and Linux and grailbio LLVM toolchains: Fine as is; no special patching needed.

source_files, header_files = _get_files(compile_action)

if not event.is_set() and should_stop_lambda(source_files, header_files):
event.set()
return source_files, header_files, compile_action.arguments


def _convert_compile_commands(aquery_output):
def _convert_compile_commands(aquery_output, should_stop_lambda):
"""Converts from Bazel's aquery format to de-Bazeled compile_commands.json entries.
Input: jsonproto output from aquery, pre-filtered to (Objective-)C(++) compile actions for a given build.
Expand All @@ -805,8 +814,8 @@ def _convert_compile_commands(aquery_output):
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(32, (os.cpu_count() or 1) + 4) # Backport. Default in MIN_PY=3.8. See "using very large resources implicitly on many-core machines" in https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
) as threadpool:
outputs = threadpool.map(_get_cpp_command_for_files, aquery_output.actions)

event = threading.Event()
outputs = threadpool.map(_get_cpp_command_for_files, map(lambda action: (action, event, should_stop_lambda), aquery_output.actions))
# Yield as compile_commands.json entries
header_files_already_written = set()
for source_files, header_files, compile_command_args in outputs:
Expand All @@ -833,7 +842,19 @@ def _convert_compile_commands(aquery_output):

def _get_commands(target: str, flags: str):
"""Return compile_commands.json entries for a given target and flags, gracefully tolerating errors."""
def _get_commands(target_statment):
lock = threading.RLock()
counter = itertools.count()
def _should_stop(headers, file_path):
if file_path:
with lock:
tried_count = next(counter)
if tried_count >= BAZEL_INTERNAL_MAX_HEADER_SEARCH_COUNT:
log_warning(f""">>> Bazel lists no applicable compile commands for {file_path} in {target} under {tried_count} Attempt.""")
return True
return any(header.endswith(file_path) for header in headers)
return False

def _get_commands(target_statment, file_path):
aquery_args = [
'bazel',
'aquery',
Expand Down Expand Up @@ -899,7 +920,7 @@ def _get_commands(target_statment):
Continuing gracefully...""")
return []

return _convert_compile_commands(parsed_aquery_output)
return _convert_compile_commands(parsed_aquery_output, lambda _, headers: _should_stop(headers, file_path))

# Log clear completion messages
log_info(f">>> Analyzing commands used in {target}")
Expand Down Expand Up @@ -949,7 +970,7 @@ def _get_commands(target_statment):
])

for target_statment in target_statment_canidates:
compile_commands.extend( _get_commands(target_statment))
compile_commands.extend( _get_commands(target_statment, file_path))
if any(command['file'].endswith(file_path) for command in reversed(compile_commands)):
found = True
break
Expand All @@ -960,7 +981,7 @@ def _get_commands(target_statment):
if {exclude_external_sources}:
# For efficiency, have bazel filter out external targets (and therefore actions) before they even get turned into actions or serialized and sent to us. Note: this is a different mechanism than is used for excluding just external headers.
target_statment = f"filter('^(//|@//)',{target_statment})"
compile_commands.extend(_get_commands(target_statment))
compile_commands.extend(_get_commands(target_statment, None))
if len(compile_commands) == 0:
log_warning(f""">>> Bazel lists no applicable compile commands for {target}
If this is a header-only library, please instead specify a test or binary target that compiles it (search "header-only" in README.md).
Expand Down

0 comments on commit 0befc09

Please sign in to comment.