From 8f001b0c387f6dd166d753ab85a51797f7bdbf0d Mon Sep 17 00:00:00 2001 From: elpekenin Date: Tue, 28 Nov 2023 15:45:35 +0100 Subject: [PATCH 1/5] Refactor, not cleaned up --- lib/python/qmk/cli/find.py | 4 +- lib/python/qmk/search.py | 205 ++++++++++++++++++++++++++----------- 2 files changed, 145 insertions(+), 64 deletions(-) diff --git a/lib/python/qmk/cli/find.py b/lib/python/qmk/cli/find.py index 55a053009248..8f3a29c90ce4 100644 --- a/lib/python/qmk/cli/find.py +++ b/lib/python/qmk/cli/find.py @@ -1,7 +1,7 @@ """Command to search through all keyboards and keymaps for a given search criteria. """ from milc import cli -from qmk.search import search_keymap_targets +from qmk.search import filter_help, search_keymap_targets @cli.argument( @@ -11,7 +11,7 @@ action='append', default=[], help= # noqa: `format-python` and `pytest` don't agree here. - "Filter the list of keyboards based on their info.json data. Accepts the formats key=value, function(key), or function(key,value), eg. 'features.rgblight=true'. Valid functions are 'absent', 'contains', 'exists' and 'length'. May be passed multiple times; all filters need to match. Value may include wildcards such as '*' and '?'." # noqa: `format-python` and `pytest` don't agree here. + f"Filter the list of keyboards based on their info.json data. Accepts the formats key=value, function(key), or function(key,value), eg. 'features.rgblight=true'. Valid functions are {filter_help()}. May be passed multiple times; all filters need to match. Value may include wildcards such as '*' and '?'." # noqa: `format-python` and `pytest` don't agree here. ) @cli.argument('-p', '--print', arg_only=True, action='append', default=[], help="For each matched target, print the value of the supplied info.json key. May be passed multiple times.") @cli.argument('-km', '--keymap', type=str, default='default', help="The keymap name to build. Default is 'default'.") diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index 1140abe69dfa..66b1ed9b2d22 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -5,7 +5,7 @@ import fnmatch import logging import re -from typing import List, Tuple +from typing import Callable, List, Optional, Tuple from dotty_dict import dotty from milc import cli @@ -15,6 +15,92 @@ from qmk.keymap import list_keymaps, locate_keymap from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget +TargetInfo = Tuple[str, str, dict] + + +# by using a class for filters, we dont need to worry about capturing values +# see details +class FilterFunction: + """Base class for filters. + It provides: + - __init__: capture key and value + + Each subclass should provide: + - func_name: how it will be specified on CLI + >>> qmk find -f ... + - apply: function that actually applies the filter + ie: return whether the input kb/km satisfies the condition + """ + + key: str + value: Optional[str] + + func_name: str + apply: Callable[[TargetInfo], bool] + help: str + + def __init__(self, key, value): + self.key = key + self.value = value + + +class Exists(FilterFunction): + func_name = "exists" + + def apply(self, target_info: TargetInfo) -> bool: + _kb, _km, info = target_info + return self.key in info + + +class Absent(FilterFunction): + func_name = "absent" + + def apply(self, target_info: TargetInfo) -> bool: + _kb, _km, info = target_info + return self.key not in info + + +class Length(FilterFunction): + func_name = "length" + + def apply(self, target_info: TargetInfo) -> bool: + _kb, _km, info = target_info + return ( + self.key in info + and len(info[self.key]) == int(self.value) + ) + +class Contains(FilterFunction): + func_name = "contains" + + def apply(self, target_info: TargetInfo) -> bool: + _kb, _km, info = target_info + return ( + self.key in info + and self.value in info[self.key] + ) + + +def _get_filter(func_name: str, key: str, value: str) -> Optional[FilterFunction]: + """Initialize a filter subclass based on regex findings, + returning its filtering method. + + Or None if no there's no filter with the name provided. + """ + + for subclass in FilterFunction.__subclasses__(): + if func_name == subclass.func_name: + return subclass(key, value).apply + + return None + + +def filter_help() -> str: + names = [f"'{f.func_name}'" for f in FilterFunction.__subclasses__()] + return ( + ", ".join(names[:-1]) + + f" and {names[-1]}" + ) def _set_log_level(level): cli.acquire_lock() @@ -48,11 +134,12 @@ def _keymap_exists(keyboard, keymap): return keyboard if locate_keymap(keyboard, keymap) is not None else None -def _load_keymap_info(kb_km): +def _load_keymap_info(target: Tuple[str, str]) -> TargetInfo: """Returns a tuple of (keyboard, keymap, info.json) for the given keyboard/keymap combination. """ + kb, km = target with ignore_logging(): - return (kb_km[0], kb_km[1], keymap_json(kb_km[0], kb_km[1])) + return (kb, km, keymap_json(kb, km)) def expand_make_targets(targets: List[str]) -> List[Tuple[str, str]]: @@ -113,68 +200,62 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str Optionally includes the values of the queried info.json keys. """ if len(filters) == 0: - targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list] - else: - cli.log.info('Parsing data for all matching keyboard/keymap combinations...') - valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] - - function_re = re.compile(r'^(?P[a-zA-Z]+)\((?P[a-zA-Z0-9_\.]+)(,\s*(?P[^#]+))?\)$') - equals_re = re.compile(r'^(?P[a-zA-Z0-9_\.]+)\s*=\s*(?P[^#]+)$') - - for filter_expr in filters: - function_match = function_re.match(filter_expr) - equals_match = equals_re.match(filter_expr) - - if function_match is not None: - func_name = function_match.group('function').lower() - key = function_match.group('key') - value = function_match.group('value') - - if value is not None: - if func_name == 'length': - valid_keymaps = filter(lambda e, key=key, value=value: key in e[2] and len(e[2].get(key)) == int(value), valid_keymaps) - elif func_name == 'contains': - valid_keymaps = filter(lambda e, key=key, value=value: key in e[2] and value in e[2].get(key), valid_keymaps) - else: - cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}') - continue - - cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}, {{fg_cyan}}{value}{{fg_reset}})...') - else: - if func_name == 'exists': - valid_keymaps = filter(lambda e, key=key: key in e[2], valid_keymaps) - elif func_name == 'absent': - valid_keymaps = filter(lambda e, key=key: key not in e[2], valid_keymaps) - else: - cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}') - continue - - cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}})...') - - elif equals_match is not None: - key = equals_match.group('key') - value = equals_match.group('value') - cli.log.info(f'Filtering on condition: {{fg_cyan}}{key}{{fg_reset}} == {{fg_cyan}}{value}{{fg_reset}}...') - - def _make_filter(k, v): - expr = fnmatch.translate(v) - rule = re.compile(f'^{expr}$', re.IGNORECASE) - - def f(e): - lhs = e[2].get(k) - lhs = str(False if lhs is None else lhs) - return rule.search(lhs) is not None - - return f - - valid_keymaps = filter(_make_filter(key, value), valid_keymaps) - else: - cli.log.warning(f'Unrecognized filter expression: {filter_expr}') + return [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list] + + cli.log.info('Parsing data for all matching keyboard/keymap combinations...') + valid_keymaps: List[TargetInfo] = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] + + function_re = re.compile(r'^(?P[a-zA-Z]+)\((?P[a-zA-Z0-9_\.]+)(,\s*(?P[^#]+))?\)$') + equals_re = re.compile(r'^(?P[a-zA-Z0-9_\.]+)\s*=\s*(?P[^#]+)$') + + for filter_expr in filters: + function_match = function_re.match(filter_expr) + equals_match = equals_re.match(filter_expr) + + if function_match is not None: + func_name = function_match.group('function').lower() + key = function_match.group('key') + value = function_match.group('value') + + filter_ = _get_filter(func_name, key, value) + if filter_ is None: + cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}') continue + valid_keymaps = filter(filter_.apply, valid_keymaps) + + value_str = ( + f", {{fg_cyan}}{value}{{fg_reset}})" + if value is not None + else "" + ) + cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}{value_str}...') + + elif equals_match is not None: + key = equals_match.group('key') + value = equals_match.group('value') + cli.log.info(f'Filtering on condition: {{fg_cyan}}{key}{{fg_reset}} == {{fg_cyan}}{value}{{fg_reset}}...') + + def _make_filter(k: str, v: TargetInfo): + expr = fnmatch.translate(v) + rule = re.compile(f'^{expr}$', re.IGNORECASE) - targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps] + def f(e): + lhs = e[2].get(k) + lhs = str(False if lhs is None else lhs) + return rule.search(lhs) is not None + + return f + + valid_keymaps = filter(_make_filter(key, value), valid_keymaps) + + else: + cli.log.warning(f'Unrecognized filter expression: {filter_expr}') - return targets + # return filtered targets + return [ + KeyboardKeymapBuildTarget(keyboard=kb, keymap=km, json=json) + for (kb, km, json) in valid_keymaps + ] def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]: From fd03188c60b40d1e2eb2ffd6f2192ab17cfc5506 Mon Sep 17 00:00:00 2001 From: elpekenin Date: Tue, 28 Nov 2023 16:38:17 +0100 Subject: [PATCH 2/5] Lint --- lib/python/qmk/search.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index 7ae46a4ea741..4ada8d61a582 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -64,20 +64,14 @@ class Length(FilterFunction): def apply(self, target_info: TargetInfo) -> bool: _kb, _km, info = target_info - return ( - self.key in info - and len(info[self.key]) == int(self.value) - ) + return (self.key in info and len(info[self.key]) == int(self.value)) class Contains(FilterFunction): func_name = "contains" def apply(self, target_info: TargetInfo) -> bool: _kb, _km, info = target_info - return ( - self.key in info - and self.value in info[self.key] - ) + return (self.key in info and self.value in info[self.key]) def _get_filter_class(func_name: str, key: str, value: str) -> Optional[FilterFunction]: @@ -94,10 +88,7 @@ def _get_filter_class(func_name: str, key: str, value: str) -> Optional[FilterFu def filter_help() -> str: names = [f"'{f.func_name}'" for f in FilterFunction.__subclasses__()] - return ( - ", ".join(names[:-1]) - + f" and {names[-1]}" - ) + return (", ".join(names[:-1]) + f" and {names[-1]}") def _set_log_level(level): cli.acquire_lock() @@ -229,11 +220,7 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str continue valid_keymaps = filter(filter_class.apply, valid_keymaps) - value_str = ( - f", {{fg_cyan}}{value}{{fg_reset}})" - if value is not None - else "" - ) + value_str = (f", {{fg_cyan}}{value}{{fg_reset}})" if value is not None else "") cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}{value_str}...') elif equals_match is not None: @@ -257,11 +244,7 @@ def f(e): else: cli.log.warning(f'Unrecognized filter expression: {filter_expr}') - # return filtered targets - return [ - KeyboardKeymapBuildTarget(keyboard=kb, keymap=km, json=json) - for (kb, km, json) in valid_keymaps - ] + return [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km, json=json) for (kb, km, json) in valid_keymaps] def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]: From 37fac6b5c1bae7bff686069a796f6f0fc6d2e05b Mon Sep 17 00:00:00 2001 From: elpekenin Date: Tue, 28 Nov 2023 22:59:42 +0100 Subject: [PATCH 3/5] Fix missing bits --- lib/python/qmk/search.py | 78 +++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index 4ada8d61a582..2e95a2a8979e 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -6,7 +6,7 @@ import logging import re from typing import Callable, List, Optional, Tuple -from dotty_dict import dotty +from dotty_dict import dotty, Dotty from milc import cli from qmk.util import parallel_map @@ -197,54 +197,58 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str """ if len(filters) == 0: cli.log.info('Preparing target list...') - return list(set(parallel_map(_construct_build_target_kb_km, target_list))) - - cli.log.info('Parsing data for all matching keyboard/keymap combinations...') - valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] + targets = list(set(parallel_map(_construct_build_target_kb_km, target_list))) + else: + cli.log.info('Parsing data for all matching keyboard/keymap combinations...') + valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] - function_re = re.compile(r'^(?P[a-zA-Z]+)\((?P[a-zA-Z0-9_\.]+)(,\s*(?P[^#]+))?\)$') - equals_re = re.compile(r'^(?P[a-zA-Z0-9_\.]+)\s*=\s*(?P[^#]+)$') + function_re = re.compile(r'^(?P[a-zA-Z]+)\((?P[a-zA-Z0-9_\.]+)(,\s*(?P[^#]+))?\)$') + equals_re = re.compile(r'^(?P[a-zA-Z0-9_\.]+)\s*=\s*(?P[^#]+)$') - for filter_expr in filters: - function_match = function_re.match(filter_expr) - equals_match = equals_re.match(filter_expr) + for filter_expr in filters: + function_match = function_re.match(filter_expr) + equals_match = equals_re.match(filter_expr) - if function_match is not None: - func_name = function_match.group('function').lower() - key = function_match.group('key') - value = function_match.group('value') + if function_match is not None: + func_name = function_match.group('function').lower() + key = function_match.group('key') + value = function_match.group('value') - filter_class = _get_filter_class(func_name, key, value) - if filter_class is None: - cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}') - continue - valid_keymaps = filter(filter_class.apply, valid_keymaps) + filter_class = _get_filter_class(func_name, key, value) + if filter_class is None: + cli.log.warning(f'Unrecognized filter expression: {function_match.group(0)}') + continue + valid_keymaps = filter(filter_class.apply, valid_keymaps) - value_str = (f", {{fg_cyan}}{value}{{fg_reset}})" if value is not None else "") - cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}{value_str}...') + value_str = (f", {{fg_cyan}}{value}{{fg_reset}})" if value is not None else "") + cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}{value_str}...') - elif equals_match is not None: - key = equals_match.group('key') - value = equals_match.group('value') - cli.log.info(f'Filtering on condition: {{fg_cyan}}{key}{{fg_reset}} == {{fg_cyan}}{value}{{fg_reset}}...') + elif equals_match is not None: + key = equals_match.group('key') + value = equals_match.group('value') + cli.log.info(f'Filtering on condition: {{fg_cyan}}{key}{{fg_reset}} == {{fg_cyan}}{value}{{fg_reset}}...') - def _make_filter(k: str, v: TargetInfo): - expr = fnmatch.translate(v) - rule = re.compile(f'^{expr}$', re.IGNORECASE) + def _make_filter(k, v): + expr = fnmatch.translate(v) + rule = re.compile(f'^{expr}$', re.IGNORECASE) - def f(e): - lhs = e[2].get(k) - lhs = str(False if lhs is None else lhs) - return rule.search(lhs) is not None + def f(e): + lhs = e[2].get(k) + lhs = str(False if lhs is None else lhs) + return rule.search(lhs) is not None - return f + return f - valid_keymaps = filter(_make_filter(key, value), valid_keymaps) + valid_keymaps = filter(_make_filter(key, value), valid_keymaps) + else: + cli.log.warning(f'Unrecognized filter expression: {filter_expr}') + continue - else: - cli.log.warning(f'Unrecognized filter expression: {filter_expr}') + cli.log.info('Preparing target list...') + valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps] # need to convert dotty_dict back to dict because it doesn't survive parallelisation + targets = list(set(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps)))) - return [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km, json=json) for (kb, km, json) in valid_keymaps] + return targets def search_keymap_targets(targets: List[Tuple[str, str]] = [('all', 'default')], filters: List[str] = []) -> List[BuildTarget]: From 8007af1ba2488a00d994fcb21735f4d37760aeb6 Mon Sep 17 00:00:00 2001 From: elpekenin Date: Tue, 28 Nov 2023 23:02:53 +0100 Subject: [PATCH 4/5] Remove extra parens from linting changes --- lib/python/qmk/search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index 2e95a2a8979e..2e7d3617dd23 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -88,7 +88,7 @@ def _get_filter_class(func_name: str, key: str, value: str) -> Optional[FilterFu def filter_help() -> str: names = [f"'{f.func_name}'" for f in FilterFunction.__subclasses__()] - return (", ".join(names[:-1]) + f" and {names[-1]}") + return ", ".join(names[:-1]) + f" and {names[-1]}" def _set_log_level(level): cli.acquire_lock() @@ -220,7 +220,7 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str continue valid_keymaps = filter(filter_class.apply, valid_keymaps) - value_str = (f", {{fg_cyan}}{value}{{fg_reset}})" if value is not None else "") + value_str = f", {{fg_cyan}}{value}{{fg_reset}})" if value is not None else "" cli.log.info(f'Filtering on condition: {{fg_green}}{func_name}{{fg_reset}}({{fg_cyan}}{key}{{fg_reset}}{value_str}...') elif equals_match is not None: From 28169fff63d2689f0e99731a80b864998aaf101e Mon Sep 17 00:00:00 2001 From: zvecr Date: Fri, 16 Feb 2024 13:49:02 +0000 Subject: [PATCH 5/5] lint --- lib/python/qmk/search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py index 2e7d3617dd23..33550a3db274 100644 --- a/lib/python/qmk/search.py +++ b/lib/python/qmk/search.py @@ -66,6 +66,7 @@ def apply(self, target_info: TargetInfo) -> bool: _kb, _km, info = target_info return (self.key in info and len(info[self.key]) == int(self.value)) + class Contains(FilterFunction): func_name = "contains" @@ -90,6 +91,7 @@ def filter_help() -> str: names = [f"'{f.func_name}'" for f in FilterFunction.__subclasses__()] return ", ".join(names[:-1]) + f" and {names[-1]}" + def _set_log_level(level): cli.acquire_lock() old = cli.log_level