LinuxLists.cc - [PATCH 0/8] Fix Python string escapes

2023-08-14 07:20:38

Subject: [PATCH 0/8] Fix Python string escapes

Python 3.6 introduced a DeprecationWarning for invalid escape sequences.
This is upgraded to a SyntaxWarning in Python 3.12 (3.12.0rc1), and is
intended to eventually be a syntax error.

This series aims to fix these now to get ahead of it before it's an error.

Most of the patches are generated using the below Python script. It
uses the builtin ast module to parse a Python file, locate all strings,
find the corresponding source code, and check for bad escape sequences.

If it finds things to fix then it applies the fixes and reparses the
file into a fixed AST. It finally compares the original and fixed ASTs
to ensure no semantic difference has been introduced (dumping is done to
remove node location information, which is expected to be different).

There are some limitations of the ast module, in particular it throws
away a lot of information about the string source. f-strings especially
interact poorly here (the slices between formats are presented as
separate strings, but the node range of each is the entire f-string),
so are skipped. f-strings are handled manually in the final patch.

A lot of the fixes are for regex patterns, which could be changed to use
r-strings instead. But that is less easy to automate, so I avoided doing
so in this series. AST verification should still be possible though,
because being a plain or r-string is stripped away in the AST.

---
#!/usr/bin/env python3

"""
Fix all bad escape characters in strings
"""

import ast
from pathlib import Path

def get_offset(source: str, row: int, col: int) -> int:
"""
Turn a row + column pair into a byte offset
"""
offset = 0

cur_row = 1 # 1-indexed rows
cur_col = 0 # 0-indexed columns

for c in source:
if cur_row == row and cur_col == col:
return offset

offset += 1

if c == "\n":
cur_row += 1
cur_col = 0
else:
cur_col += 1

raise Exception("Failed to get offset")

parse_failures: list[Path] = []
fix_failures = 0
bad_escapes = 0
fstrings: set[Path] = set()

for pyfile in Path(".").glob("**/*.py"):
content = pyfile.read_text("utf-8")

try:
syntax = ast.parse(content, filename=pyfile)
except:
print(f"{pyfile}: ERROR Failed to parse, is it Python3?")
parse_failures.append(pyfile)
continue

fixes: list[tuple[int, int, str]] = []

for node in ast.walk(syntax):
if not isinstance(node, ast.Constant):
continue

if not isinstance(node.value, str):
continue

if node.value.count("\\") == 0:
continue

assert(isinstance(node.end_lineno, int))
assert(isinstance(node.end_col_offset, int))

start = get_offset(content, node.lineno, node.col_offset)
end = get_offset(content, node.end_lineno, node.end_col_offset)
raw = content[start:end]

# backslashes in r-strings are already literal
if raw.startswith("r"):
continue

# f-strings are difficult to handle correctly
if raw.startswith("f"):
fstrings.add(pyfile)
continue

fixed = "" # The fixed representation of the string
escaped = False # If the current character is escaped by a previous backslash
allowed = '\n\\\'"abfnrtv01234567xNuU' # characters allowed after a backslash

for c in raw:
if escaped:
if c not in allowed:
fixed += '\\'

fixed += c
escaped = False
continue

fixed += c

if c == '\\':
escaped = True

if fixed != raw:
print(f"{pyfile}:{node.lineno}:{node.col_offset}: FOUND {raw}")
fixes.append((start, end, fixed))

if len(fixes) == 0:
continue

bad_escapes += len(fixes)

# Apply fixes in reverse order to keep offsets valid
for start, end, fix in reversed(sorted(fixes, key=lambda k: k[0])):
print(f"{pyfile}:[{start}-{end}]: APPLY {fix}")
content = content[:start] + fix + content[end:]

fixed_syntax = ast.parse(content, filename=f"{pyfile}+fixed")

if ast.dump(syntax) != ast.dump(fixed_syntax):
print(f"{pyfile}: ERROR Fixed syntax tree yields different AST")
fix_failures += 1
continue

pyfile.write_text(content)

print(f"---------------------------------")
print(f"Parse failures: {len(parse_failures)}")
for f in sorted(parse_failures):
print(f" - {f}")

print(f"Bad escapes fixed: {bad_escapes}")
print(f"Fixes that broke the AST: {fix_failures}")
print(f"Files with skipped f-strings: {len(fstrings)}")
for f in sorted(fstrings):
print(f" - {f}")

---

Benjamin Gray (8):
ia64: fix Python string escapes
Documentation/sphinx: fix Python string escapes
drivers/comedi: fix Python string escapes
scripts: fix Python string escapes
tools/perf: fix Python string escapes
tools/power: fix Python string escapes
selftests/bpf: fix Python string escapes
selftests/bpf: fix Python string escapes in f-strings

Documentation/sphinx/cdomain.py | 2 +-
Documentation/sphinx/kernel_abi.py | 2 +-
Documentation/sphinx/kernel_feat.py | 2 +-
Documentation/sphinx/kerneldoc.py | 2 +-
Documentation/sphinx/maintainers_include.py | 8 +--
arch/ia64/scripts/unwcheck.py | 2 +-
.../ni_routing/tools/convert_csv_to_c.py | 2 +-
scripts/bpf_doc.py | 56 +++++++++----------
scripts/clang-tools/gen_compile_commands.py | 2 +-
scripts/gdb/linux/symbols.py | 2 +-
tools/perf/pmu-events/jevents.py | 2 +-
.../scripts/python/arm-cs-trace-disasm.py | 4 +-
tools/perf/scripts/python/compaction-times.py | 2 +-
.../scripts/python/exported-sql-viewer.py | 4 +-
tools/power/pm-graph/bootgraph.py | 12 ++--
.../selftests/bpf/test_bpftool_synctypes.py | 26 ++++-----
tools/testing/selftests/bpf/test_offload.py | 2 +-
17 files changed, 66 insertions(+), 66 deletions(-)

--
2.41.0

2023-08-14 07:26:55

by Benjamin Gray

[permalink] [raw]

Subject: [PATCH 3/8] drivers/comedi: fix Python string escapes

2023-08-14 07:27:44

by Benjamin Gray

[permalink] [raw]

Subject: [PATCH 7/8] selftests/bpf: fix Python string escapes

2023-08-14 07:29:38

by Benjamin Gray

[permalink] [raw]

Subject: [PATCH 6/8] tools/power: fix Python string escapes

2023-08-14 08:03:33

by Benjamin Gray

[permalink] [raw]

Subject: [PATCH 5/8] tools/perf: fix Python string escapes

2023-08-14 08:37:24

by Benjamin Gray

[permalink] [raw]

Subject: [PATCH 4/8] scripts: fix Python string escapes

Python 3.6 introduced a DeprecationWarning for invalid escape sequences.
This is upgraded to a SyntaxWarning in Python 3.12, and will eventually
be a syntax error.

Fix these now to get ahead of it before it's an error.

Signed-off-by: Benjamin Gray <[email protected]>
---
scripts/bpf_doc.py | 56 ++++++++++-----------
scripts/clang-tools/gen_compile_commands.py | 2 +-
scripts/gdb/linux/symbols.py | 2 +-
3 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py
index eaae2ce78381..40a2643f94d2 100755
--- a/scripts/bpf_doc.py
+++ b/scripts/bpf_doc.py
@@ -59,9 +59,9 @@ class Helper(APIElement):
Break down helper function protocol into smaller chunks: return type,
name, distincts arguments.
"""
- arg_re = re.compile('((\w+ )*?(\w+|...))( (\**)(\w+))?$')
+ arg_re = re.compile('((\\w+ )*?(\\w+|...))( (\\**)(\\w+))?$')
res = {}
- proto_re = re.compile('(.+) (\**)(\w+)$((([^,]+)(, )?){1,5})$$')
+ proto_re = re.compile('(.+) (\\**)(\\w+)\$((([^,]+)(, )?){1,5})\$$')

capture = proto_re.match(self.proto)
res['ret_type'] = capture.group(1)
@@ -114,11 +114,11 @@ class HeaderParser(object):
return Helper(proto=proto, desc=desc, ret=ret)

def parse_symbol(self):
- p = re.compile(' \* ?(BPF\w+)$')
+ p = re.compile(' \\* ?(BPF\\w+)$')
capture = p.match(self.line)
if not capture:
raise NoSyscallCommandFound
- end_re = re.compile(' \* ?NOTES$')
+ end_re = re.compile(' \\* ?NOTES$')
end = end_re.match(self.line)
if end:
raise NoSyscallCommandFound
@@ -133,7 +133,7 @@ class HeaderParser(object):
# - Same as above, with "const" and/or "struct" in front of type
# - "..." (undefined number of arguments, for bpf_trace_printk())
# There is at least one term ("void"), and at most five arguments.
- p = re.compile(' \* ?((.+) \**\w+$(((const )?(struct )?(\w+|\.\.\.)( \**\w+)?)(, )?){1,5}$)$')
+ p = re.compile(' \\* ?((.+) \\**\\w+\$(((const )?(struct )?(\\w+|\\.\\.\\.)( \\**\\w+)?)(, )?){1,5}\$)$')
capture = p.match(self.line)
if not capture:
raise NoHelperFound
@@ -141,7 +141,7 @@ class HeaderParser(object):
return capture.group(1)

def parse_desc(self, proto):
- p = re.compile(' \* ?(?:\t| {5,8})Description$')
+ p = re.compile(' \\* ?(?:\t| {5,8})Description$')
capture = p.match(self.line)
if not capture:
raise Exception("No description section found for " + proto)
@@ -154,7 +154,7 @@ class HeaderParser(object):
if self.line == ' *\n':
desc += '\n'
else:
- p = re.compile(' \* ?(?:\t| {5,8})(?:\t| {8})(.*)')
+ p = re.compile(' \\* ?(?:\t| {5,8})(?:\t| {8})(.*)')
capture = p.match(self.line)
if capture:
desc_present = True
@@ -167,7 +167,7 @@ class HeaderParser(object):
return desc

def parse_ret(self, proto):
- p = re.compile(' \* ?(?:\t| {5,8})Return$')
+ p = re.compile(' \\* ?(?:\t| {5,8})Return$')
capture = p.match(self.line)
if not capture:
raise Exception("No return section found for " + proto)
@@ -180,7 +180,7 @@ class HeaderParser(object):
if self.line == ' *\n':
ret += '\n'
else:
- p = re.compile(' \* ?(?:\t| {5,8})(?:\t| {8})(.*)')
+ p = re.compile(' \\* ?(?:\t| {5,8})(?:\t| {8})(.*)')
capture = p.match(self.line)
if capture:
ret_present = True
@@ -219,12 +219,12 @@ class HeaderParser(object):
self.seek_to('enum bpf_cmd {',
'Could not find start of bpf_cmd enum', 0)
# Searches for either one or more BPF\w+ enums
- bpf_p = re.compile('\s*(BPF\w+)+')
+ bpf_p = re.compile('\\s*(BPF\\w+)+')
# Searches for an enum entry assigned to another entry,
# for e.g. BPF_PROG_RUN = BPF_PROG_TEST_RUN, which is
# not documented hence should be skipped in check to
# determine if the right number of syscalls are documented
- assign_p = re.compile('\s*(BPF\w+)\s*=\s*(BPF\w+)')
+ assign_p = re.compile('\\s*(BPF\\w+)\\s*=\\s*(BPF\\w+)')
bpf_cmd_str = ''
while True:
capture = assign_p.match(self.line)
@@ -239,7 +239,7 @@ class HeaderParser(object):
break
self.line = self.reader.readline()
# Find the number of occurences of BPF\w+
- self.enum_syscalls = re.findall('(BPF\w+)+', bpf_cmd_str)
+ self.enum_syscalls = re.findall('(BPF\\w+)+', bpf_cmd_str)

def parse_desc_helpers(self):
self.seek_to(helpersDocStart,
@@ -263,7 +263,7 @@ class HeaderParser(object):
self.seek_to('#define ___BPF_FUNC_MAPPER(FN, ctx...)',
'Could not find start of eBPF helper definition list')
# Searches for one FN(\w+) define or a backslash for newline
- p = re.compile('\s*FN$(\w+), (\d+), ##ctx$|\\\\')
+ p = re.compile('\\s*FN\$(\\w+), (\\d+), ##ctx\$|\\\\')
fn_defines_str = ''
i = 0
while True:
@@ -278,7 +278,7 @@ class HeaderParser(object):
break
self.line = self.reader.readline()
# Find the number of occurences of FN(\w+)
- self.define_unique_helpers = re.findall('FN$\w+, \d+, ##ctx$', fn_defines_str)
+ self.define_unique_helpers = re.findall('FN\$\\w+, \\d+, ##ctx\$', fn_defines_str)

def validate_helpers(self):
last_helper = ''
@@ -425,7 +425,7 @@ class PrinterRST(Printer):
try:
cmd = ['git', 'log', '-1', '--pretty=format:%cs', '--no-patch',
'-L',
- '/{}/,/\*\//:include/uapi/linux/bpf.h'.format(delimiter)]
+ '/{}/,/\\*\\//:include/uapi/linux/bpf.h'.format(delimiter)]
date = subprocess.run(cmd, cwd=linuxRoot,
capture_output=True, check=True)
return date.stdout.decode().rstrip()
@@ -516,7 +516,7 @@ as "Dual BSD/GPL", may be used). Some helper functions are only accessible to
programs that are compatible with the GNU Privacy License (GPL).

In order to use such helpers, the eBPF program must be loaded with the correct
-license string passed (via **attr**) to the **bpf**\ () system call, and this
+license string passed (via **attr**) to the **bpf**\\ () system call, and this
generally translates into the C source code of the program containing a line
similar to the following:

@@ -550,7 +550,7 @@ may be interested in:
* The bpftool utility can be used to probe the availability of helper functions
on the system (as well as supported program and map types, and a number of
other parameters). To do so, run **bpftool feature probe** (see
- **bpftool-feature**\ (8) for details). Add the **unprivileged** keyword to
+ **bpftool-feature**\\ (8) for details). Add the **unprivileged** keyword to
list features available to unprivileged users.

Compatibility between helper functions and program types can generally be found
@@ -562,23 +562,23 @@ other functions, themselves allowing access to additional helpers. The
requirement for GPL license is also in those **struct bpf_func_proto**.

Compatibility between helper functions and map types can be found in the
-**check_map_func_compatibility**\ () function in file *kernel/bpf/verifier.c*.
+**check_map_func_compatibility**\\ () function in file *kernel/bpf/verifier.c*.

Helper functions that invalidate the checks on **data** and **data_end**
pointers for network processing are listed in function
-**bpf_helper_changes_pkt_data**\ () in file *net/core/filter.c*.
+**bpf_helper_changes_pkt_data**\\ () in file *net/core/filter.c*.

SEE ALSO
========

-**bpf**\ (2),
-**bpftool**\ (8),
-**cgroups**\ (7),
-**ip**\ (8),
-**perf_event_open**\ (2),
-**sendmsg**\ (2),
-**socket**\ (7),
-**tc-bpf**\ (8)'''
+**bpf**\\ (2),
+**bpftool**\\ (8),
+**cgroups**\\ (7),
+**ip**\\ (8),
+**perf_event_open**\\ (2),
+**sendmsg**\\ (2),
+**socket**\\ (7),
+**tc-bpf**\\ (8)'''
print(footer)

def print_proto(self, helper):
@@ -598,7 +598,7 @@ SEE ALSO
one_arg = '{}{}'.format(comma, a['type'])
if a['name']:
if a['star']:
- one_arg += ' {}**\ '.format(a['star'].replace('*', '\\*'))
+ one_arg += ' {}**\\ '.format(a['star'].replace('*', '\\*'))
else:
one_arg += '** '
one_arg += '*{}*\\ **'.format(a['name'])
diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py
index a84cc5737c2c..649a80005c83 100755
--- a/scripts/clang-tools/gen_compile_commands.py
+++ b/scripts/clang-tools/gen_compile_commands.py
@@ -170,7 +170,7 @@ def process_line(root_directory, command_prefix, file_path):
# escape the pound sign '#', either as '\#' or '$(pound)' (depending on the
# kernel version). The compile_commands.json file is not interepreted
# by Make, so this code replaces the escaped version with '#'.
- prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#')
+ prefix = command_prefix.replace('\\#', '#').replace('$(pound)', '#')

# Use os.path.abspath() to normalize the path resolving '.' and '..' .
abs_path = os.path.abspath(os.path.join(root_directory, file_path))
diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py
index fdad3f32c747..bae7cbff187e 100644
--- a/scripts/gdb/linux/symbols.py
+++ b/scripts/gdb/linux/symbols.py
@@ -82,7 +82,7 @@ lx-symbols command."""
self.module_files_updated = True

def _get_module_file(self, module_name):
- module_pattern = ".*/{0}\.ko(?:.debug)?$".format(
+ module_pattern = ".*/{0}\\.ko(?:.debug)?$".format(
module_name.replace("_", r"[_\-]"))
for name in self.module_files:
if re.match(module_pattern, name) and os.path.exists(name):
--
2.41.0