diff --git a/AUTHORS.txt b/AUTHORS.txt index 451918f..60f9fba 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -54,6 +54,7 @@ Simon Ruggier (@sruggier) Tim Gates (@timgates42) Batuhan Taskaya (@isidentical) Jocelyn Boullier (@Kazy) +Sahil Rajput (@SahilRajputdotexe) Note: (@user) means a github user name. diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a4811fb..4c6d959 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,9 @@ Changelog Unreleased ++++++++++ +- Add support for the ``match`` statement (PEP 634) in the grammars for + Python 3.10 and later + 0.8.7 (2026-05-02) ++++++++++++++++++ diff --git a/conftest.py b/conftest.py index 2cba256..2b218a2 100644 --- a/conftest.py +++ b/conftest.py @@ -147,6 +147,12 @@ def works_ge_py39(each_version): return Checker(each_version, version_info >= (3, 9)) +@pytest.fixture +def works_ge_py310(each_version): + version_info = parse_version_string(each_version) + return Checker(each_version, version_info >= (3, 10)) + + @pytest.fixture def works_ge_py312(each_version): version_info = parse_version_string(each_version) diff --git a/parso/parser.py b/parso/parser.py index 3746643..995e9d6 100644 --- a/parso/parser.py +++ b/parso/parser.py @@ -23,7 +23,7 @@ complexity of the ``Parser`` (there's another parser sitting inside ``Statement``, which produces ``Array`` and ``Call``). """ -from typing import Dict, Type +from typing import Dict, FrozenSet, Type from parso import tree from parso.pgen2.generator import ReservedString @@ -88,7 +88,7 @@ def __repr__(self): def _token_to_transition(grammar, type_, value): # Map from token to label - if type_.value.contains_syntax: + if type_.value.contains_syntax and value.__class__ is not PlainName: # Check for reserved words (keywords) try: return grammar.reserved_syntax_strings[value] @@ -98,6 +98,14 @@ def _token_to_transition(grammar, type_, value): return type_ +class PlainName(str): + """ + The value of a soft keyword that is used as a name. It is never looked up + as a keyword. + """ + __slots__ = () + + class BaseParser: """Parser engine. @@ -116,6 +124,10 @@ class BaseParser: leaf_map: Dict[str, Type[tree.Leaf]] = {} default_leaf = tree.Leaf + # Keywords that are only reserved where the grammar asks for them and are + # plain names everywhere else. + soft_keywords: FrozenSet[str] = frozenset() + def __init__(self, pgen_grammar, start_nonterminal='file_input', error_recovery=False): self._pgen_grammar = pgen_grammar self._start_nonterminal = start_nonterminal @@ -180,7 +192,13 @@ def _add_token(self, token): plan = stack[-1].dfa.transitions[transition] break except KeyError: - if stack[-1].dfa.is_final: + if ( + value in self.soft_keywords + and type_ in stack[-1].dfa.transitions + ): + value = PlainName(value) + transition = type_ + elif stack[-1].dfa.is_final: self._pop() else: self.error_recovery(token) diff --git a/parso/python/diff.py b/parso/python/diff.py index 232be86..228d06f 100644 --- a/parso/python/diff.py +++ b/parso/python/diff.py @@ -186,11 +186,13 @@ def _ends_with_newline(leaf, suffix=''): def _flows_finished(pgen_grammar, stack): """ - if, while, for and try might not be finished, because another part might - still be parsed. + if, while, for, try and match might not be finished, because another part + might still be parsed. """ for stack_node in stack: - if stack_node.nonterminal in ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt'): + if stack_node.nonterminal in ( + 'if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'match_stmt' + ): return False return True @@ -223,6 +225,8 @@ def _suite_or_file_input_is_valid(pgen_grammar, stack): def _is_flow_node(node): if node.type == 'async_stmt': node = node.children[1] + if node.type == 'match_stmt': + return True try: value = node.children[0].value except AttributeError: diff --git a/parso/python/errors.py b/parso/python/errors.py index b498af3..9ff6102 100644 --- a/parso/python/errors.py +++ b/parso/python/errors.py @@ -3,13 +3,14 @@ import sys import warnings import re +from ast import literal_eval from contextlib import contextmanager from parso.normalizer import Normalizer, NormalizerConfig, Issue, Rule from parso.python.tokenize import _get_token_collection _BLOCK_STMTS = ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt') -_STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist') +_STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist', 'subject_expr') # This is the maximal block size given by python. _MAX_BLOCK_SIZE = 20 _MAX_INDENT_COUNT = 100 @@ -1012,6 +1013,224 @@ def is_issue(self, try_stmt): self.add_issue(default_except, message=self.message) +def _bound_names(pattern): + """ + Yields the names a pattern binds in source order. An or-pattern binds the + names of its first alternative, all alternatives have to bind the same. + """ + type_ = pattern.type + if type_ == 'name': + if pattern.value != '_': + yield pattern + elif type_ == 'or_pattern': + yield from _bound_names(pattern.children[0]) + elif type_ == 'as_pattern': + yield from _bound_names(pattern.children[0]) + if pattern.children[-1].value != '_': + yield pattern.children[-1] + elif type_ == 'star_pattern': + if pattern.children[1].value != '_': + yield pattern.children[1] + elif type_ in ('class_arg', 'mapping_item'): + yield from _bound_names(pattern.children[-1]) + elif type_ == 'name_pattern': + if _is_class_pattern(pattern): + for argument in _class_pattern_arguments(pattern): + yield from _bound_names(argument) + elif hasattr(pattern, 'children'): + for child in pattern.children: + yield from _bound_names(child) + + +def _is_class_pattern(name_pattern): + return name_pattern.children[-1].value == ')' + + +def _class_pattern_arguments(name_pattern): + for index, child in enumerate(name_pattern.children): + if child.type == 'operator' and child.value == '(': + return name_pattern.children[index + 1:-1] + + +def _irrefutable_name(pattern): + """ + Returns the capture name or the wildcard that makes a pattern match every + subject, or ``None`` if the pattern can fail. + """ + type_ = pattern.type + if type_ == 'name': + return pattern + if type_ == 'as_pattern': + return _irrefutable_name(pattern.children[0]) + if type_ == 'or_pattern': + for alternative in pattern.children[::2]: + name = _irrefutable_name(alternative) + if name is not None: + return name + if type_ == 'paren_pattern' and len(pattern.children) == 3: + return _irrefutable_name(pattern.children[1]) + return None + + +def _unreachable_message(name): + if name.value == '_': + return "wildcard makes remaining patterns unreachable" + return "name capture %r makes remaining patterns unreachable" % name.value + + +@ErrorFinder.register_rule(type='match_stmt') +class _MatchStmtRule(SyntaxRule): + def is_issue(self, match_stmt): + cases = [c for c in match_stmt.children if c.type == 'case_block'] + for case in cases[:-1]: + if any(c.type == 'guard' for c in case.children): + continue + pattern = case.children[1] + if pattern.type == 'or_pattern': + # The other alternatives are checked in the or-pattern. + pattern = pattern.children[-1] + name = _irrefutable_name(pattern) + if name is not None: + self.add_issue(name, message=_unreachable_message(name)) + + +@ErrorFinder.register_rule(type='case_block') +class _CaseBlockRule(SyntaxRule): + message = "multiple assignments to name %r in pattern" + message_star = "invalid syntax" + + def is_issue(self, case_block): + if case_block.children[1].type == 'star_pattern': + self.add_issue(case_block.children[1], message=self.message_star) + return + seen = set() + for name in _bound_names(case_block.children[1]): + if name.value in seen: + self.add_issue(name, message=self.message % name.value) + break + seen.add(name.value) + + +@ErrorFinder.register_rule(type='or_pattern') +class _OrPatternRule(SyntaxRule): + message = "alternative patterns bind different names" + + def is_issue(self, or_pattern): + alternatives = or_pattern.children[::2] + for alternative in alternatives[:-1]: + name = _irrefutable_name(alternative) + if name is not None: + self.add_issue(name, message=_unreachable_message(name)) + return + + first = {name.value for name in _bound_names(alternatives[0])} + for alternative in alternatives[1:]: + if {name.value for name in _bound_names(alternative)} != first: + self.add_issue(alternative, message=self.message) + return + + +@ErrorFinder.register_rule(type='as_pattern') +class _AsPatternRule(SyntaxRule): + message = "cannot use '_' as a target" + + def is_issue(self, as_pattern): + return as_pattern.children[-1].value == '_' + + def get_node(self, as_pattern): + return as_pattern.children[-1] + + +@ErrorFinder.register_rule(types=('patterns', 'list_pattern', 'paren_pattern')) +class _StarPatternRule(SyntaxRule): + message = "multiple starred names in sequence pattern" + message_group = "invalid syntax" + + def is_issue(self, sequence): + stars = [c for c in sequence.children if c.type == 'star_pattern'] + if len(stars) > 1: + self.add_issue(stars[1], message=self.message) + elif sequence.type == 'paren_pattern' and stars and len(sequence.children) == 3: + self.add_issue(stars[0], message=self.message_group) + + +@ErrorFinder.register_rule(type='literal_pattern') +class _ComplexLiteralRule(SyntaxRule): + def is_issue(self, literal): + numbers = [c for c in literal.children if c.type == 'number'] + if len(numbers) == 2: + real, imaginary = numbers + if real.value[-1] in 'jJ': + self.add_issue(real, message="real number required in complex literal") + elif imaginary.value[-1] not in 'jJ': + self.add_issue( + imaginary, message="imaginary number required in complex literal" + ) + + +@ErrorFinder.register_rule(type='mapping_pattern') +class _MappingPatternRule(SyntaxRule): + message = "invalid syntax" + message_duplicate = "mapping pattern checks duplicate key (%r)" + + @staticmethod + def _is_valid_key(key): + if key.type in ('number', 'string', 'literal_pattern'): + return True + if key.type == 'keyword': + return key.value in ('None', 'True', 'False') + return key.type == 'name_pattern' and not _is_class_pattern(key) + + def is_issue(self, mapping_pattern): + keys = set() + for child in mapping_pattern.children: + if child.type == 'name' and child.value == '_': + self.add_issue(child, message=self.message) + if child.type != 'mapping_item': + continue + + key = child.children[0] + if not self._is_valid_key(key): + self.add_issue(key, message=self.message) + continue + if key.type == 'name_pattern': + continue + try: + value = literal_eval(key.get_code(include_prefix=False)) + except (ValueError, SyntaxError): + continue + if value in keys: + self.add_issue(key, message=self.message_duplicate % (value,)) + keys.add(value) + + +@ErrorFinder.register_rule(type='name_pattern') +class _ClassPatternRule(SyntaxRule): + message = "invalid syntax" + message_positional = "positional patterns follow keyword patterns" + message_repeated = "attribute name repeated in class pattern: %s" + + def is_issue(self, name_pattern): + if not _is_class_pattern(name_pattern): + return + + keywords = set() + for argument in _class_pattern_arguments(name_pattern): + if argument.type == 'operator': + continue + if argument.type != 'class_arg': + if keywords: + self.add_issue(argument, message=self.message_positional) + continue + + keyword = argument.children[0] + if keyword.type != 'name': + self.add_issue(keyword, message=self.message) + elif keyword.value in keywords: + self.add_issue(keyword, message=self.message_repeated % keyword.value) + keywords.add(keyword.value) + + @ErrorFinder.register_rule(type='fstring') class _FStringRule(SyntaxRule): _fstring_grammar = None diff --git a/parso/python/grammar310.txt b/parso/python/grammar310.txt index f092050..01bb0f8 100644 --- a/parso/python/grammar310.txt +++ b/parso/python/grammar310.txt @@ -79,7 +79,7 @@ global_stmt: 'global' NAME (',' NAME)* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] @@ -95,6 +95,25 @@ with_item: test ['as' expr] except_clause: 'except' [test ['as' NAME]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +match_stmt: 'match' subject_expr ':' NEWLINE INDENT case_block+ DEDENT +subject_expr: (namedexpr_test | star_expr) (',' (namedexpr_test | star_expr))* [','] +case_block: 'case' patterns [guard] ':' suite +guard: 'if' namedexpr_test +patterns: pattern_item (',' pattern_item)* [','] +pattern_item: star_pattern | as_pattern +star_pattern: '*' NAME +as_pattern: or_pattern ['as' NAME] +or_pattern: closed_pattern ('|' closed_pattern)* +closed_pattern: (literal_pattern | 'None' | 'True' | 'False' | name_pattern + | paren_pattern | list_pattern | mapping_pattern) +literal_pattern: STRING+ | ['-'] NUMBER [('+' | '-') NUMBER] +name_pattern: NAME ('.' NAME)* ['(' [class_arg (',' class_arg)* [',']] ')'] +class_arg: as_pattern ['=' as_pattern] +paren_pattern: '(' [pattern_item (',' pattern_item)* [',']] ')' +list_pattern: '[' [pattern_item (',' pattern_item)* [',']] ']' +mapping_pattern: '{' [(mapping_item ',')* ('**' NAME | mapping_item) [',']] '}' +mapping_item: closed_pattern ':' as_pattern + namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef lambdef: 'lambda' [varargslist] ':' test diff --git a/parso/python/grammar311.txt b/parso/python/grammar311.txt index f092050..01bb0f8 100644 --- a/parso/python/grammar311.txt +++ b/parso/python/grammar311.txt @@ -79,7 +79,7 @@ global_stmt: 'global' NAME (',' NAME)* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] @@ -95,6 +95,25 @@ with_item: test ['as' expr] except_clause: 'except' [test ['as' NAME]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +match_stmt: 'match' subject_expr ':' NEWLINE INDENT case_block+ DEDENT +subject_expr: (namedexpr_test | star_expr) (',' (namedexpr_test | star_expr))* [','] +case_block: 'case' patterns [guard] ':' suite +guard: 'if' namedexpr_test +patterns: pattern_item (',' pattern_item)* [','] +pattern_item: star_pattern | as_pattern +star_pattern: '*' NAME +as_pattern: or_pattern ['as' NAME] +or_pattern: closed_pattern ('|' closed_pattern)* +closed_pattern: (literal_pattern | 'None' | 'True' | 'False' | name_pattern + | paren_pattern | list_pattern | mapping_pattern) +literal_pattern: STRING+ | ['-'] NUMBER [('+' | '-') NUMBER] +name_pattern: NAME ('.' NAME)* ['(' [class_arg (',' class_arg)* [',']] ')'] +class_arg: as_pattern ['=' as_pattern] +paren_pattern: '(' [pattern_item (',' pattern_item)* [',']] ')' +list_pattern: '[' [pattern_item (',' pattern_item)* [',']] ']' +mapping_pattern: '{' [(mapping_item ',')* ('**' NAME | mapping_item) [',']] '}' +mapping_item: closed_pattern ':' as_pattern + namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef lambdef: 'lambda' [varargslist] ':' test diff --git a/parso/python/grammar312.txt b/parso/python/grammar312.txt index e81a792..9a24556 100644 --- a/parso/python/grammar312.txt +++ b/parso/python/grammar312.txt @@ -83,7 +83,7 @@ global_stmt: 'global' NAME (',' NAME)* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] @@ -99,6 +99,25 @@ with_item: test ['as' expr] except_clause: 'except' [test ['as' NAME]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +match_stmt: 'match' subject_expr ':' NEWLINE INDENT case_block+ DEDENT +subject_expr: (namedexpr_test | star_expr) (',' (namedexpr_test | star_expr))* [','] +case_block: 'case' patterns [guard] ':' suite +guard: 'if' namedexpr_test +patterns: pattern_item (',' pattern_item)* [','] +pattern_item: star_pattern | as_pattern +star_pattern: '*' NAME +as_pattern: or_pattern ['as' NAME] +or_pattern: closed_pattern ('|' closed_pattern)* +closed_pattern: (literal_pattern | 'None' | 'True' | 'False' | name_pattern + | paren_pattern | list_pattern | mapping_pattern) +literal_pattern: STRING+ | ['-'] NUMBER [('+' | '-') NUMBER] +name_pattern: NAME ('.' NAME)* ['(' [class_arg (',' class_arg)* [',']] ')'] +class_arg: as_pattern ['=' as_pattern] +paren_pattern: '(' [pattern_item (',' pattern_item)* [',']] ')' +list_pattern: '[' [pattern_item (',' pattern_item)* [',']] ']' +mapping_pattern: '{' [(mapping_item ',')* ('**' NAME | mapping_item) [',']] '}' +mapping_item: closed_pattern ':' as_pattern + namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef lambdef: 'lambda' [varargslist] ':' test diff --git a/parso/python/grammar313.txt b/parso/python/grammar313.txt index 04cd2d5..0e48fba 100644 --- a/parso/python/grammar313.txt +++ b/parso/python/grammar313.txt @@ -84,7 +84,7 @@ global_stmt: 'global' NAME (',' NAME)* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] @@ -100,6 +100,25 @@ with_item: test ['as' expr] except_clause: 'except' [test ['as' NAME]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +match_stmt: 'match' subject_expr ':' NEWLINE INDENT case_block+ DEDENT +subject_expr: (namedexpr_test | star_expr) (',' (namedexpr_test | star_expr))* [','] +case_block: 'case' patterns [guard] ':' suite +guard: 'if' namedexpr_test +patterns: pattern_item (',' pattern_item)* [','] +pattern_item: star_pattern | as_pattern +star_pattern: '*' NAME +as_pattern: or_pattern ['as' NAME] +or_pattern: closed_pattern ('|' closed_pattern)* +closed_pattern: (literal_pattern | 'None' | 'True' | 'False' | name_pattern + | paren_pattern | list_pattern | mapping_pattern) +literal_pattern: STRING+ | ['-'] NUMBER [('+' | '-') NUMBER] +name_pattern: NAME ('.' NAME)* ['(' [class_arg (',' class_arg)* [',']] ')'] +class_arg: as_pattern ['=' as_pattern] +paren_pattern: '(' [pattern_item (',' pattern_item)* [',']] ')' +list_pattern: '[' [pattern_item (',' pattern_item)* [',']] ']' +mapping_pattern: '{' [(mapping_item ',')* ('**' NAME | mapping_item) [',']] '}' +mapping_item: closed_pattern ':' as_pattern + namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef lambdef: 'lambda' [varargslist] ':' test diff --git a/parso/python/grammar314.txt b/parso/python/grammar314.txt index 04cd2d5..0e48fba 100644 --- a/parso/python/grammar314.txt +++ b/parso/python/grammar314.txt @@ -84,7 +84,7 @@ global_stmt: 'global' NAME (',' NAME)* nonlocal_stmt: 'nonlocal' NAME (',' NAME)* assert_stmt: 'assert' test [',' test] -compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt | match_stmt async_stmt: 'async' (funcdef | with_stmt | for_stmt) if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] @@ -100,6 +100,25 @@ with_item: test ['as' expr] except_clause: 'except' [test ['as' NAME]] suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT +match_stmt: 'match' subject_expr ':' NEWLINE INDENT case_block+ DEDENT +subject_expr: (namedexpr_test | star_expr) (',' (namedexpr_test | star_expr))* [','] +case_block: 'case' patterns [guard] ':' suite +guard: 'if' namedexpr_test +patterns: pattern_item (',' pattern_item)* [','] +pattern_item: star_pattern | as_pattern +star_pattern: '*' NAME +as_pattern: or_pattern ['as' NAME] +or_pattern: closed_pattern ('|' closed_pattern)* +closed_pattern: (literal_pattern | 'None' | 'True' | 'False' | name_pattern + | paren_pattern | list_pattern | mapping_pattern) +literal_pattern: STRING+ | ['-'] NUMBER [('+' | '-') NUMBER] +name_pattern: NAME ('.' NAME)* ['(' [class_arg (',' class_arg)* [',']] ')'] +class_arg: as_pattern ['=' as_pattern] +paren_pattern: '(' [pattern_item (',' pattern_item)* [',']] ')' +list_pattern: '[' [pattern_item (',' pattern_item)* [',']] ']' +mapping_pattern: '{' [(mapping_item ',')* ('**' NAME | mapping_item) [',']] '}' +mapping_item: closed_pattern ':' as_pattern + namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef lambdef: 'lambda' [varargslist] ':' test diff --git a/parso/python/parser.py b/parso/python/parser.py index fa45e8b..022aaf2 100644 --- a/parso/python/parser.py +++ b/parso/python/parser.py @@ -1,11 +1,14 @@ from parso.python import tree from parso.python.token import PythonTokenTypes -from parso.parser import BaseParser +from parso.parser import BaseParser, PlainName NAME = PythonTokenTypes.NAME INDENT = PythonTokenTypes.INDENT DEDENT = PythonTokenTypes.DEDENT +NEWLINE = PythonTokenTypes.NEWLINE +ERROR_DEDENT = PythonTokenTypes.ERROR_DEDENT +ENDMARKER = PythonTokenTypes.ENDMARKER class Parser(BaseParser): @@ -61,6 +64,8 @@ class structure of different scopes. PythonTokenTypes.FSTRING_END: tree.FStringEnd, } + soft_keywords = frozenset(['match', 'case']) + def __init__(self, pgen_grammar, error_recovery=True, start_nonterminal='file_input'): super().__init__(pgen_grammar, start_nonterminal, error_recovery=error_recovery) @@ -70,6 +75,9 @@ def __init__(self, pgen_grammar, error_recovery=True, start_nonterminal='file_in self._indent_counter = 0 def parse(self, tokens): + if 'match' in self._pgen_grammar.reserved_syntax_strings: + tokens = self._match_statement_tokenize(tokens) + if self._error_recovery: if self._start_nonterminal != 'file_input': raise NotImplementedError @@ -78,6 +86,35 @@ def parse(self, tokens): return super().parse(tokens) + def _match_statement_tokenize(self, tokens): + """ + ``match`` is only a keyword at the start of a match statement, which + is a line that ends with a colon. At the start of a statement the + grammar accepts both the keyword and a name, so the logical line is + looked at before deciding. Everywhere else the parser falls back to a + name by itself. + """ + previous_type = NEWLINE + tokens = iter(tokens) + for token in tokens: + if ( + token.string == 'match' + and token.type == NAME + and previous_type in (NEWLINE, INDENT, DEDENT, ERROR_DEDENT) + ): + line = [token] + for token in tokens: + line.append(token) + if token.type in (NEWLINE, ENDMARKER): + break + if len(line) < 3 or line[-2].string != ':': + line[0] = line[0]._replace(string=PlainName('match')) + yield from line + previous_type = line[-1].type + else: + yield token + previous_type = token.type + def convert_node(self, nonterminal, children): """ Convert raw node information to a PythonBaseNode instance. @@ -95,12 +132,16 @@ def convert_node(self, nonterminal, children): # ones and therefore have pseudo start/end positions and no # prefixes. Just ignore them. children = [children[0]] + children[2:-1] + elif nonterminal == 'match_stmt': + children = children[:4] + children[5:-1] node = self.default_node(nonterminal, children) return node def convert_leaf(self, type, value, prefix, start_pos): # print('leaf', repr(value), token.tok_name[type]) if type == NAME: + if value.__class__ is PlainName: + return tree.Name(str(value), start_pos, prefix) if value in self._pgen_grammar.reserved_syntax_strings: return tree.Keyword(value, start_pos, prefix) else: @@ -179,7 +220,13 @@ def current_suite(stack): pass def _stack_removal(self, start_index): - all_nodes = [node for stack_node in self.stack[start_index:] for node in stack_node.nodes] + all_nodes = [ + node + for stack_node in self.stack[start_index:] + for node in stack_node.nodes + # The INDENT of a match statement is only a virtual leaf. + if stack_node.nonterminal != 'match_stmt' or node.type != 'operator' or node.value + ] if all_nodes: node = tree.PythonErrorNode(all_nodes) diff --git a/parso/python/pep8.py b/parso/python/pep8.py index 47db445..e18d821 100644 --- a/parso/python/pep8.py +++ b/parso/python/pep8.py @@ -9,8 +9,10 @@ _IMPORT_TYPES = ('import_name', 'import_from') _SUITE_INTRODUCERS = ('classdef', 'funcdef', 'if_stmt', 'while_stmt', - 'for_stmt', 'try_stmt', 'with_stmt') + 'for_stmt', 'try_stmt', 'with_stmt', 'match_stmt', + 'case_block') _NON_STAR_TYPES = ('term', 'import_from', 'power') +_KEYWORD_ARGUMENT_TYPES = ('argument', 'param', 'class_arg') _OPENING_BRACKETS = '(', '[', '{' _CLOSING_BRACKETS = ')', ']', '}' _FACTOR = '+', '-', '~' @@ -254,7 +256,7 @@ def _visit_node(self, node): in_introducer = typ in _SUITE_INTRODUCERS if in_introducer: self._in_suite_introducer = True - elif typ == 'suite': + if typ == 'suite' or typ == 'case_block': if self._indentation_tos.type == IndentationTypes.BACKSLASH: self._indentation_tos = self._indentation_tos.parent @@ -266,7 +268,9 @@ def _visit_node(self, node): elif implicit_indentation_possible: self._implicit_indentation_possible = True yield - if typ == 'suite': + if typ == 'case_block': + self._indentation_tos = self._indentation_tos.parent + elif typ == 'suite': assert self._indentation_tos.type == IndentationTypes.SUITE self._indentation_tos = self._indentation_tos.parent # If we dedent, no lines are needed anymore. @@ -623,8 +627,8 @@ def add_not_spaces(*args): elif prev == '@' and prev.parent.type == 'decorator': pass # TODO should probably raise an error if there's a space here elif part in _NEEDS_SPACE or prev in _NEEDS_SPACE: - if part == '=' and part.parent.type in ('argument', 'param') \ - or prev == '=' and prev.parent.type in ('argument', 'param'): + if part == '=' and part.parent.type in _KEYWORD_ARGUMENT_TYPES \ + or prev == '=' and prev.parent.type in _KEYWORD_ARGUMENT_TYPES: if part == '=': param = part.parent else: diff --git a/parso/python/tree.py b/parso/python/tree.py index 60a982d..159837c 100644 --- a/parso/python/tree.py +++ b/parso/python/tree.py @@ -51,7 +51,8 @@ from parso.utils import split_lines _FLOW_CONTAINERS = set(['if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', - 'with_stmt', 'async_stmt', 'suite']) + 'with_stmt', 'async_stmt', 'suite', 'match_stmt', + 'case_block']) _RETURN_STMT_CONTAINERS = set(['suite', 'simple_stmt']) | _FLOW_CONTAINERS _FUNC_CONTAINERS = set( diff --git a/test/failing_examples.py b/test/failing_examples.py index af0c711..3ce2160 100644 --- a/test/failing_examples.py +++ b/test/failing_examples.py @@ -408,6 +408,34 @@ def y(): "f'{1=!b}'", ] +if sys.version_info[:2] >= (3, 10): + FAILING_EXAMPLES += [ + 'match x:\n case {**a, **b}: pass', + 'match x:\n case {**a, 1: b}: pass', + 'match x:\n case {**_}: pass', + 'match x:\n case {a: 1}: pass', + 'match x:\n case {1: a, 1: b}: pass', + 'match x:\n case {1: a, 1.0: b}: pass', + "match x:\n case {'a': 1, 'a': 2}: pass", + 'match x:\n case [a, a]: pass', + 'match x:\n case ({1: a}, a): pass', + 'match x:\n case [a, *a]: pass', + 'match x:\n case [*a, *b]: pass', + 'match x:\n case *a: pass', + 'match x:\n case (*a): pass', + 'match x:\n case _: pass\n case 1: pass', + 'match x:\n case y: pass\n case 1: pass', + 'match x:\n case 1 | y: pass\n case 2: pass', + 'match x:\n case _ | 1: pass', + 'match x:\n case [y | 1]: pass', + 'match x:\n case [a] | [b]: pass', + 'match x:\n case 1 as _: pass', + 'match x:\n case 1+2: pass', + 'match x:\n case 1j+2j: pass', + 'match x:\n case P(x=1, y): pass', + 'match x:\n case P(x=1, x=2): pass', + ] + if sys.version_info[:2] < (3, 12): FAILING_EXAMPLES += [ # f-string expression part cannot include a backslash before 3.12 diff --git a/test/test_diff_parser.py b/test/test_diff_parser.py index 222236e..c546ca5 100644 --- a/test/test_diff_parser.py +++ b/test/test_diff_parser.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from textwrap import dedent import logging +import sys import pytest @@ -1744,3 +1745,41 @@ def encoding(self): True - differ.initialize(code1) differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True) + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 10), reason="match needs Python 3.10") +def test_match_statement(differ): + code1 = dedent('''\ + import os + match x: + case 1: + a = 1 + case [b, *c]: + d = 2 + e = 3 + ''') + differ.initialize(code1) + + code2 = code1.replace('a = 1', 'a = 2') + differ.parse(code2, parsers=ANY, copies=ANY) + + code3 = code1.replace('case 1:', 'case 1 | 2:\n pass\n case 3:') + differ.parse(code3, parsers=ANY, copies=ANY) + + code4 = code1.replace('case [b, *c]:', 'case [b, *c]\n') + differ.parse(code4, parsers=ANY, copies=ANY, expect_error_leaves=True) + + differ.parse(code1, parsers=ANY, copies=ANY) + differ.parse(code1 + 'match = 1\n', parsers=1, copies=1) + + +@pytest.mark.skipif(sys.version_info[:2] < (3, 10), reason="match needs Python 3.10") +def test_match_statement_is_not_continued(differ): + code1 = dedent('''\ + match x: + case 1: + pass + ''') + differ.initialize(code1 + 'y = 1\n') + differ.parse(code1 + ' case 2:\n pass\ny = 1\n', parsers=1, copies=ANY) + differ.parse(code1 + 'case 2:\n pass\n', parsers=ANY, copies=ANY, expect_error_leaves=True) diff --git a/test/test_parser.py b/test/test_parser.py index d8a0fad..d510d6b 100644 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -3,7 +3,7 @@ import pytest -from parso import parse +from parso import parse, ParserSyntaxError from parso.python import tree from parso.utils import split_lines @@ -267,3 +267,150 @@ def test_pep695_class_no_bases(works_ge_py312): ) def test_pep696_type_param_defaults(works_ge_py313, code): works_ge_py313.parse(code) + + +@pytest.mark.parametrize( + 'code', [ + 'match x:\n case 1: pass\n', + 'match x:\n case 1:\n pass\n case _:\n pass\n', + 'match x,:\n case (1,): pass\n', + 'match x, *y:\n case a, *b: pass\n', + 'match (x := f()):\n case 1: pass\n', + 'match [x]:\n case [1]: pass\n', + 'match -x:\n case -1: pass\n', + 'match x:\n case None | True | False: pass\n', + 'match x:\n case "a" "b" | b"c": pass\n', + 'match x:\n case -1 | 1 | -1j | 1+2j | -1-2j: pass\n', + 'match x:\n case a: pass\n', + 'match x:\n case Color.RED | a.b.c: pass\n', + 'match x:\n case [] | () | {}: pass\n', + 'match x:\n case [a, b, *rest]: pass\n', + 'match x:\n case (a, *_, b): pass\n', + 'match x:\n case a, b: pass\n', + 'match x:\n case (a): pass\n', + 'match x:\n case {"k": v, 1: w, K.v: u, **rest}: pass\n', + 'match x:\n case {**rest,}: pass\n', + 'match x:\n case P(): pass\n', + 'match x:\n case P(1, a, x=2, y=b,): pass\n', + 'match x:\n case a.P(x=[1, {2: c}]): pass\n', + 'match x:\n case [a] | (a,) as b: pass\n', + 'match x:\n case (1 | 2) as a: pass\n', + 'match x:\n case a if a > 1: pass\n', + 'match x:\n case _ if x: pass\n case 1: pass\n', + 'match x:\n case y if y:\n pass\n case _:\n pass\n', + 'match x:\n case 1:\n match y:\n case 2: pass\n', + 'def f(x):\n match x:\n case 1: return 1\n', + 'class C:\n match = 1\n match x:\n case 1: pass\n', + 'if x:\n match x:\n case 1: pass\nelse:\n pass\n', + 'for x in y:\n match x:\n case 1: break\n case 2: continue\n', + 'match x:\n case 1: pass\nmatch = 1\n', + 'match = 1\nmatch match:\n case 1: pass\n', + 'match x:\n case match: pass\n', + 'match x:\n case case: pass\n', + 'match x:\n case match.case: pass\n', + 'match x:\n case [match, case]: pass\n', + 'match x:\n case P(match=case): pass\n', + 'match x:\n case 1:\n case = 1\n match = 2\n match(case)\n', + 'match x:\n # comment\n\n case 1: pass # comment\n\n # other\n case 2: pass\n', + 'match (x,\n y):\n case (1,\n 2): pass\n', + 'match x: # comment\n case 1: pass', + ] +) +def test_match_statement(works_ge_py310, code): + works_ge_py310.parse(code) + works_ge_py310.assert_no_error_in_passing(code) + + +@pytest.mark.parametrize( + 'code', [ + 'match = 1\n', + 'case = 1\n', + 'match, case = 1, 2\n', + 'match: int = 1\n', + 'match.group(1)\n', + 'match[0] = 1\n', + 'match(x)\n', + 'match (x)\n', + 'match -x\n', + 'match = lambda case: case\n', + 'x = match\n', + 'x = {match: case}\n', + 're.match(x, case)\n', + 'print(match, case)\n', + 'def match(case): return case\n', + 'class match: case = 1\n', + 'lambda match, case: match\n', + 'import match\nfrom case import match as case\n', + 'case(x)\n', + 'case: int\n', + 'x = 1; match = 2; case = 3\n', + 'if x: match = 1\n', + ] +) +def test_match_and_case_as_names(works_in_py, code): + works_in_py.parse(code) + works_in_py.assert_no_error_in_passing(code) + + +@pytest.mark.parametrize('version', ['3.8', '3.9']) +def test_match_statement_before_py310(version): + code = 'match x:\n case 1: pass\n' + with pytest.raises(ParserSyntaxError): + parse(code, version=version, error_recovery=False) + + +def test_match_statement_tree(): + code = dedent('''\ + match x, y: + case [1, *rest] if rest: + pass + case {"k": v} | P(a=v): + pass + + match = 1 + ''') + module = parse(code, version='3.10') + assert module.get_code() == code + match_stmt, assignment, _ = module.children + assert match_stmt.type == 'match_stmt' + keyword, subject, colon, newline, *cases = match_stmt.children + assert (keyword.type, keyword.value) == ('keyword', 'match') + assert subject.type == 'subject_expr' + assert [case.type for case in cases] == ['case_block', 'case_block'] + assert [case.children[0].type for case in cases] == ['keyword', 'keyword'] + assert cases[0].children[2].type == 'guard' + assert cases[1].children[1].type == 'or_pattern' + assert all(child.type != 'operator' or child.value for child in match_stmt.children) + assert assignment.children[0].children[0].type == 'name' + + +def test_match_statement_scopes(): + module = parse(dedent('''\ + match x: + case 1: + def f(): return 1 + case 2: + class C: pass + import os + '''), version='3.10') + assert [f.name.value for f in module.iter_funcdefs()] == ['f'] + assert [c.name.value for c in module.iter_classdefs()] == ['C'] + assert [i.get_code().strip() for i in module.iter_imports()] == ['import os'] + function, = module.iter_funcdefs() + assert len(list(function.iter_return_stmts())) == 1 + + +def test_match_statement_error_recovery(): + code = dedent('''\ + match x: + case 1: + pass + foo bar + case 2: + pass + match y + match z: + a = 1 + ''') + module = parse(code, version='3.10') + assert module.get_code() == code diff --git a/test/test_pep8.py b/test/test_pep8.py index 06cffb4..1a8d2ba 100644 --- a/test/test_pep8.py +++ b/test/test_pep8.py @@ -40,3 +40,23 @@ def test_shebang(): assert not issues('#!\n') assert not issues('#!/foo\n') assert not issues('#! python\n') + + +def test_match_statement(): + grammar = parso.load_grammar(version='3.10') + + def match_issues(code): + return [i.code for i in grammar._get_normalizer_issues(grammar.parse(code))] + + code = ( + 'match command:\n' + ' case [action, obj] if obj:\n' + ' match action:\n' + ' case Point(x=0, y=0):\n' + ' pass\n' + ' case _:\n' + ' pass\n' + ) + assert match_issues(code) == [] + assert match_issues(code.replace('y=0', 'y = 0')) == [251, 251] + assert match_issues(code.replace('_:\n pass', '_:\n pass')) == [111]