From d1860c14a4dfd848c830b28fbc1cbea4ae4fe294 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 15 Jul 2023 15:42:28 -0400 Subject: [PATCH 1/6] compare types as singletons --- pyupgrade/_plugins/legacy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyupgrade/_plugins/legacy.py b/pyupgrade/_plugins/legacy.py index 19a11372..937ca451 100644 --- a/pyupgrade/_plugins/legacy.py +++ b/pyupgrade/_plugins/legacy.py @@ -64,7 +64,7 @@ def _targets_same(target: ast.AST, yield_value: ast.AST) -> bool: # ignore `ast.Load` / `ast.Store` if _all_isinstance((t1, t2), ast.expr_context): continue - elif type(t1) != type(t2): + elif type(t1) is not type(t2): return False elif not _fields_same(t1, t2): return False From 6bef40c1a90e6a49eff20c8de6abc0418ef4b294 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 30 Jul 2023 19:34:51 -0400 Subject: [PATCH 2/6] fix some fstring edge cases for python 3.12 --- pyupgrade/_main.py | 10 +- pyupgrade/_plugins/default_encoding.py | 4 +- pyupgrade/_plugins/format_locals.py | 7 +- pyupgrade/_plugins/imports.py | 4 +- pyupgrade/_plugins/io_open.py | 4 +- pyupgrade/_plugins/legacy.py | 4 +- pyupgrade/_plugins/lru_cache.py | 7 +- pyupgrade/_plugins/mock.py | 4 +- pyupgrade/_plugins/native_literals.py | 4 +- pyupgrade/_plugins/open_mode.py | 4 +- pyupgrade/_plugins/oserror_aliases.py | 4 +- pyupgrade/_plugins/set_literals.py | 16 +-- pyupgrade/_plugins/shlex_join.py | 8 +- pyupgrade/_plugins/six_calls.py | 4 +- pyupgrade/_plugins/six_metaclasses.py | 8 +- pyupgrade/_plugins/subprocess_run.py | 6 +- pyupgrade/_plugins/type_of_primitive.py | 4 +- pyupgrade/_plugins/typing_pep604.py | 14 +-- pyupgrade/_plugins/typing_pep646_unpack.py | 4 +- pyupgrade/_token_helpers.py | 110 ++++++++++----------- setup.cfg | 2 +- tests/features/capture_output_test.py | 28 ++++++ tests/features/default_encoding_test.py | 5 + tests/features/extra_parens_test.py | 10 ++ tests/features/set_literals_test.py | 10 ++ tests/features/six_test.py | 10 ++ tests/features/typing_pep604_test.py | 36 +++++++ tests/features/yield_from_test.py | 30 ++++++ 28 files changed, 242 insertions(+), 119 deletions(-) diff --git a/pyupgrade/_main.py b/pyupgrade/_main.py index 4292174e..ee762b4b 100644 --- a/pyupgrade/_main.py +++ b/pyupgrade/_main.py @@ -25,8 +25,8 @@ from pyupgrade._string_helpers import is_codec from pyupgrade._string_helpers import parse_format from pyupgrade._string_helpers import unparse_parsed_string -from pyupgrade._token_helpers import CLOSING -from pyupgrade._token_helpers import OPENING +from pyupgrade._token_helpers import is_close +from pyupgrade._token_helpers import is_open from pyupgrade._token_helpers import remove_brace @@ -161,9 +161,9 @@ def _fix_extraneous_parens(tokens: list[Token], i: int) -> None: # found comma or yield at depth 1: this is a tuple / coroutine if depth == 1 and tokens[i].src in {',', 'yield'}: return - elif tokens[i].src in OPENING: + elif is_open(tokens[i]): depth += 1 - elif tokens[i].src in CLOSING: + elif is_close(tokens[i]): depth -= 1 end = i @@ -284,7 +284,7 @@ def _fix_tokens(contents_text: str) -> str: for i, token in reversed_enumerate(tokens): if token.name == 'STRING': tokens[i] = _fix_escape_sequences(_remove_u_prefix(tokens[i])) - elif token.src == '(': + elif token.matches(name='OP', src='('): _fix_extraneous_parens(tokens, i) elif token.src == 'format' and i > 0 and tokens[i - 1].src == '.': _fix_format_literal(tokens, i - 2) diff --git a/pyupgrade/_plugins/default_encoding.py b/pyupgrade/_plugins/default_encoding.py index 22cd023b..ab24b9ca 100644 --- a/pyupgrade/_plugins/default_encoding.py +++ b/pyupgrade/_plugins/default_encoding.py @@ -13,11 +13,11 @@ from pyupgrade._data import TokenFunc from pyupgrade._string_helpers import is_codec from pyupgrade._token_helpers import find_closing_bracket -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op def _fix_default_encoding(i: int, tokens: list[Token]) -> None: - i = find_open_paren(tokens, i + 1) + i = find_op(tokens, i + 1, '(') j = find_closing_bracket(tokens, i) del tokens[i + 1:j] diff --git a/pyupgrade/_plugins/format_locals.py b/pyupgrade/_plugins/format_locals.py index 501d4f33..cc63c442 100644 --- a/pyupgrade/_plugins/format_locals.py +++ b/pyupgrade/_plugins/format_locals.py @@ -12,13 +12,12 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import find_closing_bracket -from pyupgrade._token_helpers import find_open_paren -from pyupgrade._token_helpers import find_token +from pyupgrade._token_helpers import find_op def _fix(i: int, tokens: list[Token]) -> None: - dot_pos = find_token(tokens, i, '.') - open_pos = find_open_paren(tokens, dot_pos) + dot_pos = find_op(tokens, i, '.') + open_pos = find_op(tokens, dot_pos, '(') close_pos = find_closing_bracket(tokens, open_pos) for string_idx in rfind_string_parts(tokens, dot_pos - 1): tok = tokens[string_idx] diff --git a/pyupgrade/_plugins/imports.py b/pyupgrade/_plugins/imports.py index 56b54a87..1b379589 100644 --- a/pyupgrade/_plugins/imports.py +++ b/pyupgrade/_plugins/imports.py @@ -16,7 +16,7 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import find_end -from pyupgrade._token_helpers import find_token +from pyupgrade._token_helpers import find_name from pyupgrade._token_helpers import has_space_before from pyupgrade._token_helpers import indented_amount @@ -292,7 +292,7 @@ def parse(cls, i: int, tokens: list[Token]) -> FromImport: j += 1 mod_start = j - import_token = find_token(tokens, j, 'import') + import_token = find_name(tokens, j, 'import') j = import_token - 1 while tokens[j].name != 'NAME': j -= 1 diff --git a/pyupgrade/_plugins/io_open.py b/pyupgrade/_plugins/io_open.py index 571a8bf1..72fe4493 100644 --- a/pyupgrade/_plugins/io_open.py +++ b/pyupgrade/_plugins/io_open.py @@ -10,11 +10,11 @@ from pyupgrade._data import register from pyupgrade._data import State from pyupgrade._data import TokenFunc -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op def _replace_io_open(i: int, tokens: list[Token]) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') tokens[i:j] = [tokens[i]._replace(name='NAME', src='open')] diff --git a/pyupgrade/_plugins/legacy.py b/pyupgrade/_plugins/legacy.py index 937ca451..c042a05c 100644 --- a/pyupgrade/_plugins/legacy.py +++ b/pyupgrade/_plugins/legacy.py @@ -19,13 +19,13 @@ from pyupgrade._token_helpers import Block from pyupgrade._token_helpers import find_and_replace_call from pyupgrade._token_helpers import find_block_start -from pyupgrade._token_helpers import find_token +from pyupgrade._token_helpers import find_name FUNC_TYPES = (ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef) def _fix_yield(i: int, tokens: list[Token]) -> None: - in_token = find_token(tokens, i, 'in') + in_token = find_name(tokens, i, 'in') colon = find_block_start(tokens, i) block = Block.find(tokens, i, trim_end=True) container = tokens_to_src(tokens[in_token + 1:colon]).strip() diff --git a/pyupgrade/_plugins/lru_cache.py b/pyupgrade/_plugins/lru_cache.py index 8f40391e..6d690ecb 100644 --- a/pyupgrade/_plugins/lru_cache.py +++ b/pyupgrade/_plugins/lru_cache.py @@ -13,13 +13,12 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import find_and_replace_call -from pyupgrade._token_helpers import find_open_paren -from pyupgrade._token_helpers import find_token +from pyupgrade._token_helpers import find_op def _remove_call(i: int, tokens: list[Token]) -> None: - i = find_open_paren(tokens, i) - j = find_token(tokens, i, ')') + i = find_op(tokens, i, '(') + j = find_op(tokens, i, ')') del tokens[i:j + 1] diff --git a/pyupgrade/_plugins/mock.py b/pyupgrade/_plugins/mock.py index 9b41fc9d..0e1c391b 100644 --- a/pyupgrade/_plugins/mock.py +++ b/pyupgrade/_plugins/mock.py @@ -10,11 +10,11 @@ from pyupgrade._data import register from pyupgrade._data import State from pyupgrade._data import TokenFunc -from pyupgrade._token_helpers import find_token +from pyupgrade._token_helpers import find_name def _fix_mock_mock(i: int, tokens: list[Token]) -> None: - j = find_token(tokens, i + 1, 'mock') + j = find_name(tokens, i + 1, 'mock') del tokens[i + 1:j + 1] diff --git a/pyupgrade/_plugins/native_literals.py b/pyupgrade/_plugins/native_literals.py index 865c788a..ba535d64 100644 --- a/pyupgrade/_plugins/native_literals.py +++ b/pyupgrade/_plugins/native_literals.py @@ -13,7 +13,7 @@ from pyupgrade._data import register from pyupgrade._data import State from pyupgrade._data import TokenFunc -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import parse_call_args from pyupgrade._token_helpers import replace_call @@ -21,7 +21,7 @@ def _fix_literal(i: int, tokens: list[Token], *, empty: str) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') func_args, end = parse_call_args(tokens, j) if any(tok.name == 'NL' for tok in tokens[i:end]): return diff --git a/pyupgrade/_plugins/open_mode.py b/pyupgrade/_plugins/open_mode.py index a20b95c9..f0bfc98c 100644 --- a/pyupgrade/_plugins/open_mode.py +++ b/pyupgrade/_plugins/open_mode.py @@ -16,7 +16,7 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import delete_argument -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import parse_call_args @@ -41,7 +41,7 @@ class FunctionArg(NamedTuple): def _fix_open_mode(i: int, tokens: list[Token], *, arg_idx: int) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') func_args, end = parse_call_args(tokens, j) mode = tokens_to_src(tokens[slice(*func_args[arg_idx])]) mode_stripped = mode.split('=')[-1] diff --git a/pyupgrade/_plugins/oserror_aliases.py b/pyupgrade/_plugins/oserror_aliases.py index 9b57037a..57d272b1 100644 --- a/pyupgrade/_plugins/oserror_aliases.py +++ b/pyupgrade/_plugins/oserror_aliases.py @@ -12,7 +12,7 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import arg_str -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import parse_call_args from pyupgrade._token_helpers import replace_name @@ -30,7 +30,7 @@ def _fix_oserror_except( except_index = i while tokens[except_index].src != 'except': except_index -= 1 - start = find_open_paren(tokens, except_index) + start = find_op(tokens, except_index, '(') func_args, end = parse_call_args(tokens, start) # save the exceptions and remove the block diff --git a/pyupgrade/_plugins/set_literals.py b/pyupgrade/_plugins/set_literals.py index 25630199..d8233ab2 100644 --- a/pyupgrade/_plugins/set_literals.py +++ b/pyupgrade/_plugins/set_literals.py @@ -11,8 +11,9 @@ from pyupgrade._data import register from pyupgrade._data import State from pyupgrade._data import TokenFunc -from pyupgrade._token_helpers import BRACES from pyupgrade._token_helpers import immediately_paren +from pyupgrade._token_helpers import is_close +from pyupgrade._token_helpers import is_open from pyupgrade._token_helpers import remove_brace from pyupgrade._token_helpers import victims @@ -25,13 +26,12 @@ def _fix_set_empty_literal(i: int, tokens: list[Token]) -> None: return j = i + 2 - brace_stack = ['('] - while brace_stack: - token = tokens[j].src - if token == BRACES[brace_stack[-1]]: - brace_stack.pop() - elif token in BRACES: - brace_stack.append(token) + depth = 1 + while depth: + if is_open(tokens[j]): + depth += 1 + elif is_close(tokens[j]): + depth -= 1 j += 1 # Remove the inner tokens diff --git a/pyupgrade/_plugins/shlex_join.py b/pyupgrade/_plugins/shlex_join.py index 3b5410e5..14067560 100644 --- a/pyupgrade/_plugins/shlex_join.py +++ b/pyupgrade/_plugins/shlex_join.py @@ -12,15 +12,15 @@ from pyupgrade._data import register from pyupgrade._data import State from pyupgrade._data import TokenFunc -from pyupgrade._token_helpers import find_open_paren -from pyupgrade._token_helpers import find_token +from pyupgrade._token_helpers import find_name +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import victims def _fix_shlex_join(i: int, tokens: list[Token], *, arg: ast.expr) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') comp_victims = victims(tokens, j, arg, gen=True) - k = find_token(tokens, comp_victims.arg_index, 'in') + 1 + k = find_name(tokens, comp_victims.arg_index, 'in') + 1 while tokens[k].name in NON_CODING_TOKENS: k += 1 tokens[comp_victims.ends[0]:comp_victims.ends[-1] + 1] = [Token('OP', ')')] diff --git a/pyupgrade/_plugins/six_calls.py b/pyupgrade/_plugins/six_calls.py index 3fe12959..2434a66f 100644 --- a/pyupgrade/_plugins/six_calls.py +++ b/pyupgrade/_plugins/six_calls.py @@ -14,7 +14,7 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import find_and_replace_call -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import parse_call_args from pyupgrade._token_helpers import replace_call @@ -53,7 +53,7 @@ def _fix_six_b(i: int, tokens: list[Token]) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') if ( tokens[j + 1].name == 'STRING' and tokens[j + 1].src.isascii() and diff --git a/pyupgrade/_plugins/six_metaclasses.py b/pyupgrade/_plugins/six_metaclasses.py index c4619529..9d9e6eb9 100644 --- a/pyupgrade/_plugins/six_metaclasses.py +++ b/pyupgrade/_plugins/six_metaclasses.py @@ -15,20 +15,20 @@ from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import arg_str from pyupgrade._token_helpers import find_block_start -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import parse_call_args from pyupgrade._token_helpers import remove_decorator from pyupgrade._token_helpers import replace_call def _fix_add_metaclass(i: int, tokens: list[Token]) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') func_args, end = parse_call_args(tokens, j) metaclass = f'metaclass={arg_str(tokens, *func_args[0])}' # insert `metaclass={args[0]}` into `class:` # search forward for the `class` token j = i + 1 - while tokens[j].src != 'class': + while not tokens[j].matches(name='NAME', src='class'): j += 1 class_token = j # then search forward for a `:` token, not inside a brace @@ -55,7 +55,7 @@ def _fix_add_metaclass(i: int, tokens: list[Token]) -> None: def _fix_with_metaclass(i: int, tokens: list[Token]) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') func_args, end = parse_call_args(tokens, j) if len(func_args) == 1: tmpl = 'metaclass={args[0]}' diff --git a/pyupgrade/_plugins/subprocess_run.py b/pyupgrade/_plugins/subprocess_run.py index 4116c29a..0452e48d 100644 --- a/pyupgrade/_plugins/subprocess_run.py +++ b/pyupgrade/_plugins/subprocess_run.py @@ -13,7 +13,7 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import delete_argument -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import parse_call_args from pyupgrade._token_helpers import replace_argument @@ -25,7 +25,7 @@ def _use_capture_output( stdout_arg_idx: int, stderr_arg_idx: int, ) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') func_args, _ = parse_call_args(tokens, j) if stdout_arg_idx < stderr_arg_idx: delete_argument(stderr_arg_idx, tokens, func_args) @@ -51,7 +51,7 @@ def _replace_universal_newlines_with_text( *, arg_idx: int, ) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') func_args, _ = parse_call_args(tokens, j) for i in range(*func_args[arg_idx]): if tokens[i].src == 'universal_newlines': diff --git a/pyupgrade/_plugins/type_of_primitive.py b/pyupgrade/_plugins/type_of_primitive.py index 327a583f..3dd834b3 100644 --- a/pyupgrade/_plugins/type_of_primitive.py +++ b/pyupgrade/_plugins/type_of_primitive.py @@ -12,7 +12,7 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import find_closing_bracket -from pyupgrade._token_helpers import find_open_paren +from pyupgrade._token_helpers import find_op _TYPES = { bool: 'bool', @@ -30,7 +30,7 @@ def _rewrite_type_of_primitive( *, src: str, ) -> None: - open_paren = find_open_paren(tokens, i + 1) + open_paren = find_op(tokens, i + 1, '(') j = find_closing_bracket(tokens, open_paren) tokens[i] = tokens[i]._replace(src=src) del tokens[i + 1:j + 1] diff --git a/pyupgrade/_plugins/typing_pep604.py b/pyupgrade/_plugins/typing_pep604.py index 6e196020..c7b53bc0 100644 --- a/pyupgrade/_plugins/typing_pep604.py +++ b/pyupgrade/_plugins/typing_pep604.py @@ -14,14 +14,14 @@ from pyupgrade._data import register from pyupgrade._data import State from pyupgrade._data import TokenFunc -from pyupgrade._token_helpers import CLOSING from pyupgrade._token_helpers import find_closing_bracket -from pyupgrade._token_helpers import find_token -from pyupgrade._token_helpers import OPENING +from pyupgrade._token_helpers import find_op +from pyupgrade._token_helpers import is_close +from pyupgrade._token_helpers import is_open def _fix_optional(i: int, tokens: list[Token]) -> None: - j = find_token(tokens, i, '[') + j = find_op(tokens, i, '[') k = find_closing_bracket(tokens, j) if tokens[j].line == tokens[k].line: tokens[k] = Token('CODE', ' | None') @@ -44,7 +44,7 @@ def _fix_union( commas = [] coding_depth = None - j = find_token(tokens, i, '[') + j = find_op(tokens, i, '[') k = j + 1 while depth: # it's possible our first coding token is a close paren @@ -59,12 +59,12 @@ def _fix_union( else: coding_depth = depth - if tokens[k].src in OPENING: + if is_open(tokens[k]): if tokens[k].src == '(': open_parens.append((depth, k)) depth += 1 - elif tokens[k].src in CLOSING: + elif is_close(tokens[k]): if tokens[k].src == ')': paren_depth, open_paren = open_parens.pop() parens_done.append((paren_depth, (open_paren, k))) diff --git a/pyupgrade/_plugins/typing_pep646_unpack.py b/pyupgrade/_plugins/typing_pep646_unpack.py index 1259c60b..632814a7 100644 --- a/pyupgrade/_plugins/typing_pep646_unpack.py +++ b/pyupgrade/_plugins/typing_pep646_unpack.py @@ -12,12 +12,12 @@ from pyupgrade._data import State from pyupgrade._data import TokenFunc from pyupgrade._token_helpers import find_closing_bracket -from pyupgrade._token_helpers import find_token +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import remove_brace def _replace_unpack_with_star(i: int, tokens: list[Token]) -> None: - start = find_token(tokens, i, '[') + start = find_op(tokens, i, '[') end = find_closing_bracket(tokens, start) remove_brace(tokens, end) diff --git a/pyupgrade/_token_helpers.py b/pyupgrade/_token_helpers.py index 9dafd85c..3d28bc7e 100644 --- a/pyupgrade/_token_helpers.py +++ b/pyupgrade/_token_helpers.py @@ -10,8 +10,8 @@ from tokenize_rt import tokens_to_src from tokenize_rt import UNIMPORTANT_WS -BRACES = {'(': ')', '[': ']', '{': '}'} -OPENING, CLOSING = frozenset(BRACES), frozenset(BRACES.values()) +_OPENING = frozenset('([{') +_CLOSING = frozenset(')]}') KEYWORDS = frozenset(keyword.kwlist) @@ -26,41 +26,43 @@ class Victims(NamedTuple): arg_index: int -def _search_until(tokens: list[Token], idx: int, arg: ast.expr) -> int: - while ( - idx < len(tokens) and - not ( - tokens[idx].line == arg.lineno and - tokens[idx].utf8_byte_offset == arg.col_offset - ) - ): - idx += 1 - return idx +def is_open(token: Token) -> bool: + return token.name == 'OP' and token.src in _OPENING + + +def is_close(token: Token) -> bool: + return token.name == 'OP' and token.src in _CLOSING -def find_token(tokens: list[Token], i: int, src: str) -> int: - while tokens[i].src != src: +def _find_token(tokens: list[Token], i: int, name: str, src: str) -> int: + while not tokens[i].matches(name=name, src=src): i += 1 return i -def find_open_paren(tokens: list[Token], i: int) -> int: - return find_token(tokens, i, '(') +def find_name(tokens: list[Token], i: int, src: str) -> int: + return _find_token(tokens, i, 'NAME', src) + + +def find_op(tokens: list[Token], i: int, src: str) -> int: + return _find_token(tokens, i, 'OP', src) def find_end(tokens: list[Token], i: int) -> int: while tokens[i].name != 'NEWLINE': i += 1 - i += 1 - return i + return i + 1 def _arg_token_index(tokens: list[Token], i: int, arg: ast.expr) -> int: - idx = _search_until(tokens, i, arg) + 1 - while idx < len(tokens) and tokens[idx].name in NON_CODING_TOKENS: - idx += 1 - return idx + offset = (arg.lineno, arg.col_offset) + while tokens[i].offset != offset: + i += 1 + i += 1 + while tokens[i].name in NON_CODING_TOKENS: + i += 1 + return i def victims( @@ -75,45 +77,44 @@ def victims( first_comma_index = None arg_depth = None arg_index = _arg_token_index(tokens, start, arg) - brace_stack = [tokens[start].src] + depth = 1 i = start + 1 - while brace_stack: - token = tokens[i].src - is_start_brace = token in BRACES - is_end_brace = token == BRACES[brace_stack[-1]] + while depth: + is_start_brace = is_open(tokens[i]) + is_end_brace = is_close(tokens[i]) if i == arg_index: - arg_depth = len(brace_stack) + arg_depth = depth if is_start_brace: - brace_stack.append(token) + depth += 1 # Remove all braces before the first element of the inner # comprehension's target. if is_start_brace and arg_depth is None: - start_depths.append(len(brace_stack)) + start_depths.append(depth) starts.append(i) if ( - token == ',' and - len(brace_stack) == arg_depth and + tokens[i].matches(name='OP', src=',') and + depth == arg_depth and first_comma_index is None ): first_comma_index = i - if is_end_brace and len(brace_stack) in start_depths: + if is_end_brace and depth in start_depths: if tokens[i - 2].src == ',' and tokens[i - 1].src == ' ': ends.extend((i - 2, i - 1, i)) elif tokens[i - 1].src == ',': ends.extend((i - 1, i)) else: ends.append(i) - if len(brace_stack) > 1 and tokens[i + 1].src == ',': + if depth > 1 and tokens[i + 1].src == ',': ends.append(i + 1) if is_end_brace: - brace_stack.pop() + depth -= 1 i += 1 # May need to remove a trailing comma for a comprehension @@ -128,13 +129,13 @@ def victims( def find_closing_bracket(tokens: list[Token], i: int) -> int: - assert tokens[i].src in OPENING + assert tokens[i].src in _OPENING depth = 1 i += 1 while depth: - if tokens[i].src in OPENING: + if is_open(tokens[i]): depth += 1 - elif tokens[i].src in CLOSING: + elif is_close(tokens[i]): depth -= 1 i += 1 return i - 1 @@ -142,10 +143,10 @@ def find_closing_bracket(tokens: list[Token], i: int) -> int: def find_block_start(tokens: list[Token], i: int) -> int: depth = 0 - while depth or tokens[i].src != ':': - if tokens[i].src in OPENING: + while depth or not tokens[i].matches(name='OP', src=':'): + if is_open(tokens[i]): depth += 1 - elif tokens[i].src in CLOSING: + elif is_close(tokens[i]): depth -= 1 i += 1 return i @@ -343,22 +344,20 @@ def parse_call_args( i: int, ) -> tuple[list[tuple[int, int]], int]: args = [] - stack = [i] + depth = 1 i += 1 arg_start = i - while stack: - token = tokens[i] - - if len(stack) == 1 and token.src == ',': + while depth: + if depth == 1 and tokens[i].src == ',': args.append((arg_start, i)) arg_start = i + 1 - elif token.src in BRACES: - stack.append(i) - elif token.src == BRACES[tokens[stack[-1]].src]: - stack.pop() + elif is_open(tokens[i]): + depth += 1 + elif is_close(tokens[i]): + depth -= 1 # if we're at the end, append that argument - if not stack and tokens_to_src(tokens[arg_start:i]).strip(): + if not depth and tokens_to_src(tokens[arg_start:i]).strip(): args.append((arg_start, i)) i += 1 @@ -410,10 +409,7 @@ def replace_call( # Remove trailing comma end_rest = end - 1 - while ( - tokens[end_rest - 1].name == 'OP' and - tokens[end_rest - 1].src == ',' - ): + if tokens[end_rest - 1].matches(name='OP', src=','): end_rest -= 1 rest = tokens_to_src(tokens[start_rest:end_rest]) @@ -428,7 +424,7 @@ def find_and_replace_call( template: str, parens: tuple[int, ...] = (), ) -> None: - j = find_open_paren(tokens, i) + j = find_op(tokens, i, '(') func_args, end = parse_call_args(tokens, j) replace_call(tokens, i, end, func_args, template, parens=parens) @@ -437,7 +433,7 @@ def replace_name(i: int, tokens: list[Token], *, name: str, new: str) -> None: # preserve token offset in case we need to match it later new_token = tokens[i]._replace(name='CODE', src=new) j = i - while tokens[j].src != name: + while not tokens[j].matches(name='NAME', src=name): # timid: if we see a parenthesis here, skip it if tokens[j].src == ')': return diff --git a/setup.cfg b/setup.cfg index d06ba618..8c3e98a3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ classifiers = [options] packages = find: install_requires = - tokenize-rt>=3.2.0 + tokenize-rt>=5.1.0 python_requires = >=3.8.1 [options.packages.find] diff --git a/tests/features/capture_output_test.py b/tests/features/capture_output_test.py index 13748e3a..62ba198d 100644 --- a/tests/features/capture_output_test.py +++ b/tests/features/capture_output_test.py @@ -111,6 +111,34 @@ def test_fix_capture_output_noop(s, version): ')', id='both universal_newlines and capture_output rewrite', ), + pytest.param( + 'subprocess.run(\n' + ' f"{x}(",\n' + ' stdout=subprocess.PIPE,\n' + ' stderr=subprocess.PIPE,\n' + ')', + + 'subprocess.run(\n' + ' f"{x}(",\n' + ' capture_output=True,\n' + ')', + + id='3.12: fstring with open brace', + ), + pytest.param( + 'subprocess.run(\n' + ' f"{x})",\n' + ' stdout=subprocess.PIPE,\n' + ' stderr=subprocess.PIPE,\n' + ')', + + 'subprocess.run(\n' + ' f"{x})",\n' + ' capture_output=True,\n' + ')', + + id='3.12: fstring with close brace', + ), ), ) def test_fix_capture_output(s, expected): diff --git a/tests/features/default_encoding_test.py b/tests/features/default_encoding_test.py index d5cc1930..f039fdca 100644 --- a/tests/features/default_encoding_test.py +++ b/tests/features/default_encoding_test.py @@ -33,6 +33,11 @@ ' "y\\u2603"\n' ').encode()\n', ), + pytest.param( + 'f"{x}(".encode("utf-8")', + 'f"{x}(".encode()', + id='3.12+ handle open brace in fstring', + ), ), ) def test_fix_encode(s, expected): diff --git a/tests/features/extra_parens_test.py b/tests/features/extra_parens_test.py index 9ce0ea3c..653c8f40 100644 --- a/tests/features/extra_parens_test.py +++ b/tests/features/extra_parens_test.py @@ -58,6 +58,16 @@ def test_fix_extra_parens_noop(s): id='extra parens on coroutines are instead reduced to 2', ), + pytest.param( + 'f((f"{x})"))', + 'f(f"{x})")', + id='3.12: handle close brace in fstring body', + ), + pytest.param( + 'f((f"{x}("))', + 'f(f"{x}(")', + id='3.12: handle open brace in fstring body', + ), ), ) def test_fix_extra_parens(s, expected): diff --git a/tests/features/set_literals_test.py b/tests/features/set_literals_test.py index ea3bb5c9..c2ea301f 100644 --- a/tests/features/set_literals_test.py +++ b/tests/features/set_literals_test.py @@ -90,6 +90,16 @@ def test_fix_sets_noop(s): '}\n', ), pytest.param('set((\n))', 'set()', id='empty literal with newline'), + pytest.param( + 'set((f"{x}(",))', + '{f"{x}("}', + id='3.12 fstring containing open brace', + ), + pytest.param( + 'set((f"{x})",))', + '{f"{x})"}', + id='3.12 fstring containing close brace', + ), ), ) def test_sets(s, expected): diff --git a/tests/features/six_test.py b/tests/features/six_test.py index 746012d8..2a71dfc8 100644 --- a/tests/features/six_test.py +++ b/tests/features/six_test.py @@ -301,6 +301,16 @@ def test_fix_six_noop(s): id='add_metaclass, indented', ), + pytest.param( + '@six.add_metaclass(M)\n' + '@unrelated(f"class{x}")\n' + 'class C: pass\n', + + '@unrelated(f"class{x}")\n' + 'class C(metaclass=M): pass\n', + + id='add_metaclass, 3.12: fstring between add_metaclass and class', + ), pytest.param( 'print(six.itervalues({1:2}))\n', 'print({1:2}.values())\n', diff --git a/tests/features/typing_pep604_test.py b/tests/features/typing_pep604_test.py index 035429c7..eff5bf31 100644 --- a/tests/features/typing_pep604_test.py +++ b/tests/features/typing_pep604_test.py @@ -206,6 +206,42 @@ def f(x: int | str) -> None: ... id='nested unions', ), + pytest.param( + 'from typing import Annotated, Union\n' + 'x: Union[str, Annotated[int, f"{x})"]]\n', + + 'from typing import Annotated, Union\n' + 'x: str | Annotated[int, f"{x})"]\n', + + id='union, 3.12: ignore close brace in fstring', + ), + pytest.param( + 'from typing import Annotated, Union\n' + 'x: Union[str, Annotated[int, f"{x}("]]\n', + + 'from typing import Annotated, Union\n' + 'x: str | Annotated[int, f"{x}("]\n', + + id='union, 3.12: ignore open brace in fstring', + ), + pytest.param( + 'from typing import Annotated, Optional\n' + 'x: Optional[Annotated[int, f"{x}("]]\n', + + 'from typing import Annotated, Optional\n' + 'x: Annotated[int, f"{x}("] | None\n', + + id='optional, 3.12: ignore open brace in fstring', + ), + pytest.param( + 'from typing import Annotated, Optional\n' + 'x: Optional[Annotated[int, f"{x})"]]\n', + + 'from typing import Annotated, Optional\n' + 'x: Annotated[int, f"{x})"] | None\n', + + id='optional, 3.12: ignore close brace in fstring', + ), ), ) def test_fix_pep604_types(s, expected): diff --git a/tests/features/yield_from_test.py b/tests/features/yield_from_test.py index dbc4bd4b..2679f0f0 100644 --- a/tests/features/yield_from_test.py +++ b/tests/features/yield_from_test.py @@ -142,6 +142,36 @@ ' yield from x\n', id='leave one loop alone (referenced after assignment)', ), + pytest.param( + 'def f(y):\n' + ' for x in f"{y}:":\n' + ' yield x\n', + + 'def f(y):\n' + ' yield from f"{y}:"\n', + + id='3.12: colon in fstring', + ), + pytest.param( + 'def f(y):\n' + ' for x in f"{y}(":\n' + ' yield x\n', + + 'def f(y):\n' + ' yield from f"{y}("\n', + + id='3.12: open brace in fstring', + ), + pytest.param( + 'def f(y):\n' + ' for x in f"{y})":\n' + ' yield x\n', + + 'def f(y):\n' + ' yield from f"{y})"\n', + + id='3.13: close brace in fstring', + ), ), ) def test_fix_yield_from(s, expected): From 87426e2462a88f5a6eb960a33e5cb5f30894f295 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 30 Jul 2023 19:45:03 -0400 Subject: [PATCH 3/6] fix weird-ws empty set literals --- pyupgrade/_plugins/set_literals.py | 22 +++++----------------- tests/features/set_literals_test.py | 3 ++- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/pyupgrade/_plugins/set_literals.py b/pyupgrade/_plugins/set_literals.py index d8233ab2..631dc88a 100644 --- a/pyupgrade/_plugins/set_literals.py +++ b/pyupgrade/_plugins/set_literals.py @@ -11,9 +11,9 @@ from pyupgrade._data import register from pyupgrade._data import State from pyupgrade._data import TokenFunc +from pyupgrade._token_helpers import find_closing_bracket +from pyupgrade._token_helpers import find_op from pyupgrade._token_helpers import immediately_paren -from pyupgrade._token_helpers import is_close -from pyupgrade._token_helpers import is_open from pyupgrade._token_helpers import remove_brace from pyupgrade._token_helpers import victims @@ -21,21 +21,9 @@ def _fix_set_empty_literal(i: int, tokens: list[Token]) -> None: - # TODO: this could be implemented with a little extra logic - if not immediately_paren('set', tokens, i): - return - - j = i + 2 - depth = 1 - while depth: - if is_open(tokens[j]): - depth += 1 - elif is_close(tokens[j]): - depth -= 1 - j += 1 - - # Remove the inner tokens - del tokens[i + 2:j - 1] + i = find_op(tokens, i, '(') + j = find_closing_bracket(tokens, i) + del tokens[i + 1:j] def _fix_set_literal(i: int, tokens: list[Token], *, arg: ast.expr) -> None: diff --git a/tests/features/set_literals_test.py b/tests/features/set_literals_test.py index c2ea301f..15251812 100644 --- a/tests/features/set_literals_test.py +++ b/tests/features/set_literals_test.py @@ -13,7 +13,7 @@ 'set()', # Don't touch weird looking function calls -- use autopep8 or such # first - 'set (())', 'set ((1, 2))', + 'set ((1, 2))', ), ) def test_fix_sets_noop(s): @@ -26,6 +26,7 @@ def test_fix_sets_noop(s): # Take a set literal with an empty tuple / list and remove the arg ('set(())', 'set()'), ('set([])', 'set()'), + pytest.param('set (())', 'set ()', id='empty, weird ws'), # Remove spaces in empty set literals ('set(( ))', 'set()'), # Some "normal" test cases From a62260b4deb230178615f174e46a746c7d33126b Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 30 Jul 2023 19:47:13 -0400 Subject: [PATCH 4/6] v3.10.0 --- .pre-commit-config.yaml | 2 +- README.md | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 37b3b4a4..a7fce7e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: hooks: - id: add-trailing-comma - repo: https://github.com/asottile/pyupgrade - rev: v3.9.0 + rev: v3.10.0 hooks: - id: pyupgrade args: [--py38-plus] diff --git a/README.md b/README.md index de726270..1910c21b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Sample `.pre-commit-config.yaml`: ```yaml - repo: https://github.com/asottile/pyupgrade - rev: v3.9.0 + rev: v3.10.0 hooks: - id: pyupgrade ``` diff --git a/setup.cfg b/setup.cfg index 8c3e98a3..ecf97e47 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pyupgrade -version = 3.9.0 +version = 3.10.0 description = A tool to automatically upgrade syntax for newer versions. long_description = file: README.md long_description_content_type = text/markdown From 76d3124d229da05bfd190a4f757fdf171b62cc61 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 30 Jul 2023 19:55:16 -0400 Subject: [PATCH 5/6] correct minimum requirement --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ecf97e47..0cefb5e1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ classifiers = [options] packages = find: install_requires = - tokenize-rt>=5.1.0 + tokenize-rt>=5.2.0 python_requires = >=3.8.1 [options.packages.find] From b3e813e5d60a472ba815a45d107bbdf106405973 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 30 Jul 2023 19:58:55 -0400 Subject: [PATCH 6/6] v3.10.1 --- .pre-commit-config.yaml | 2 +- README.md | 2 +- setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7fce7e1..29b4001a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: hooks: - id: add-trailing-comma - repo: https://github.com/asottile/pyupgrade - rev: v3.10.0 + rev: v3.10.1 hooks: - id: pyupgrade args: [--py38-plus] diff --git a/README.md b/README.md index 1910c21b..77f9d31e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Sample `.pre-commit-config.yaml`: ```yaml - repo: https://github.com/asottile/pyupgrade - rev: v3.10.0 + rev: v3.10.1 hooks: - id: pyupgrade ``` diff --git a/setup.cfg b/setup.cfg index 0cefb5e1..16b50851 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = pyupgrade -version = 3.10.0 +version = 3.10.1 description = A tool to automatically upgrade syntax for newer versions. long_description = file: README.md long_description_content_type = text/markdown