Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 24 additions & 5 deletions parso/python/tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,15 @@ def _get_token_collection(version_info):
fstring_string_multi_line = _compile(
r'(?:\{\{|\}\}|\\N\{' + unicode_character_name + r'\}|\\[^N]|[^{}\\])+'
)
# Raw f-strings don't process backslash escapes at all, so a backslash is
# just ordinary literal text and must not swallow a following '{'/'}' (the
# `\N{...}` unicode-name escape doesn't exist either). Only the
# backslash-newline line continuation still needs special handling, because
# an unescaped newline would otherwise end a single-line string.
fstring_raw_string_single_line = _compile(
r'(?:\{\{|\}\}|\\(?:\r\n?|\n)|[^{}\r\n])+'
)
fstring_raw_string_multi_line = _compile(r'(?:\{\{|\}\}|[^{}])+')
fstring_format_spec_single_line = _compile(r'(?:\\(?:\r\n?|\n)|[^{}\r\n])+')
fstring_format_spec_multi_line = _compile(r'[^{}]+')

Expand Down Expand Up @@ -253,8 +262,9 @@ def __repr__(self):


class FStringNode:
def __init__(self, quote):
def __init__(self, quote, raw):
self.quote = quote
self.raw = raw
self.parentheses_count = 0
self.previous_lines = ''
self.last_string_start_pos: Any = None
Expand Down Expand Up @@ -308,10 +318,16 @@ def _find_fstring_string(endpats, fstring_stack, line, lnum, pos):
else:
regex = fstring_format_spec_single_line
else:
if allow_multiline:
regex = fstring_string_multi_line
if tos.raw:
if allow_multiline:
regex = fstring_raw_string_multi_line
else:
regex = fstring_raw_string_single_line
else:
regex = fstring_string_single_line
if allow_multiline:
regex = fstring_string_multi_line
else:
regex = fstring_string_single_line

match = regex.match(line, pos)
if match is None:
Expand Down Expand Up @@ -593,7 +609,10 @@ def dedent_if_necessary(start):
else: # ordinary string
yield PythonToken(STRING, token, spos, prefix)
elif token in fstring_pattern_map: # The start of an fstring.
fstring_stack.append(FStringNode(fstring_pattern_map[token]))
fstring_stack.append(FStringNode(
fstring_pattern_map[token],
raw='r' in token.rstrip('"\'').lower(),
))
yield PythonToken(FSTRING_START, token, spos, prefix)
elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n', '\\\r'): # continued stmt
additional_prefix += prefix + line[start:]
Expand Down
14 changes: 14 additions & 0 deletions test/test_fstring.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ def grammar():
'f"\\N{SOYOMBO LETTER -A}"',
'f"\\N{DOMINO TILE HORIZONTAL-00-00}"',
'f"""\\N{NO ENTRY}"""',

# raw f-strings: backslashes are literal and don't escape braces
'rf"script\\{c}.png"',
'rf"script\\\\{c}.png"',
'fr"script\\{c}.png"',
'Rf"script\\{c}.png"',
'fR"script\\{c}.png"',
'rf"{{}}\\{c}"',
'rf"""script\\{c}.png"""',
'rf"""abc\\\ndef{1}"""',
]
)
def test_valid(code, grammar):
Expand Down Expand Up @@ -120,6 +130,10 @@ def test_valid(code, grammar):
'f"\\N{ BULLET }"',
'f"\\N{NO ENTRY}"',
'f"""\\N{NO\nENTRY}"""',

# in a raw f-string a backslash doesn't escape braces, so a lone
# backslash followed by an unmatched '}' is still an unmatched brace
'rf"\\}"',
]
)
def test_invalid(code, grammar):
Expand Down
Loading