Hi Greg,
That's the second version of the logic added at get_abi.pl meant to
validate ABI definitions.
While it would be doable to validate the ABI by searching __ATTR and
similar macros around the driver, this would probably be very complex
and would take a while to parse.
Instead, let's add a new feature at scripts/get_abi.pl which does
check the real ABI found at the sysfs contents of a running system
with Documentation/ABI.
patch 1 changes get_abi.pl logic to handle multiple What: lines, in
order to make the script more robust;
patch 2 adds the basic logic. It runs really quicky (up to 2
seconds), but it doesn't use sysfs softlinks.
Patch 3 adds support for parsing softlinks. It makes the script a
lot slower, making it take a couple of minutes to process the entire
sysfs files. It could be optimized in the future by using a graph,
but, for now, let's keep it simple.
Patch 4 adds an optional parameter to allow filtering the results
using a regex given by the user. When this parameter is used
(which should be the normal usecase), it will only try to find softlinks
if the sysfs node matches a regex.
Patch 5 improves the report by avoiding it to ignore What: that
ends with a wildcard.
While this series is independent, if you wanna test, I recommend to
apply first this series:
https://lore.kernel.org/lkml/[email protected]/T/#t
As it fix some bad What: descriptions, and some wildcard ambiquities.
-
v2:
- multiple What: for the same description are now properly handled;
- some special cases are now better handled;
- some bugs got fixed.
The full series, with the ABI changes and some ABI improvements can be found
at:
https://git.kernel.org/pub/scm/linux/kernel/git/mchehab/devel.git/commit/?h=get_undefined&id=1838d8fb149170f6c19feda0645d6c3157f46f4f
Mauro Carvalho Chehab (5):
scripts: get_abi.pl: Better handle multiple What parameters
scripts: get_abi.pl: Check for missing symbols at the ABI specs
scripts: get_abi.pl: detect softlinks
scripts: get_abi.pl: add an option to filter undefined results
scripts: get_abi.pl: don't skip what that ends with wildcards
scripts/get_abi.pl | 258 +++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 252 insertions(+), 6 deletions(-)
--
2.31.1
The way sysfs works is that the same leave may be present under
/sys/devices, /sys/bus and /sys/class, etc, linked via soft
symlinks.
To make it harder to parse, the ABI definition usually refers
only to one of those locations.
So, improve the logic in order to retrieve the symlinks.
Signed-off-by: Mauro Carvalho Chehab <[email protected]>
---
scripts/get_abi.pl | 207 ++++++++++++++++++++++++++++++++++++---------
1 file changed, 165 insertions(+), 42 deletions(-)
diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl
index ded6dd8b9f71..8bde9944ee12 100755
--- a/scripts/get_abi.pl
+++ b/scripts/get_abi.pl
@@ -8,8 +8,10 @@ use Pod::Usage;
use Getopt::Long;
use File::Find;
use Fcntl ':mode';
+use Cwd 'abs_path';
my $help = 0;
+my $hint = 0;
my $man = 0;
my $debug = 0;
my $enable_lineno = 0;
@@ -28,6 +30,7 @@ GetOptions(
"rst-source!" => \$description_is_rst,
"dir=s" => \$prefix,
'help|?' => \$help,
+ "show-hints" => \$hint,
man => \$man
) or pod2usage(2);
@@ -526,7 +529,7 @@ sub search_symbols {
}
# Exclude /sys/kernel/debug and /sys/kernel/tracing from the search path
-sub skip_debugfs {
+sub dont_parse_special_attributes {
if (($File::Find::dir =~ m,^/sys/kernel,)) {
return grep {!/(debug|tracing)/ } @_;
}
@@ -539,64 +542,178 @@ sub skip_debugfs {
}
my %leaf;
+my %aliases;
+my @files;
-my $escape_symbols = qr { ([\x01-\x08\x0e-\x1f\x21-\x29\x2b-\x2d\x3a-\x40\x7b-\xff]) }x;
+my $escape_symbols = qr { ([\x01-\x08\x0e-\x1f\x21-\x29\x2b-\x2d\x3a-\x40\x7b-\xfe]) }x;
sub parse_existing_sysfs {
my $file = $File::Find::name;
+ my $mode = (lstat($file))[2];
+ my $abs_file = abs_path($file);
- my $mode = (stat($file))[2];
- return if ($mode & S_IFDIR);
-
- my $leave = $file;
- $leave =~ s,.*/,,;
-
- if (defined($leaf{$leave})) {
- # FIXME: need to check if the path makes sense
- my $what = $leaf{$leave};
-
- $what =~ s/,/ /g;
-
- $what =~ s/\<[^\>]+\>/.*/g;
- $what =~ s/\{[^\}]+\}/.*/g;
- $what =~ s/\[[^\]]+\]/.*/g;
- $what =~ s,/\.\.\./,/.*/,g;
- $what =~ s,/\*/,/.*/,g;
-
- $what =~ s/\s+/ /g;
-
- # Escape all other symbols
- $what =~ s/$escape_symbols/\\$1/g;
-
- foreach my $i (split / /,$what) {
- if ($file =~ m#^$i$#) {
-# print "$file: $i: OK!\n";
- return;
- }
- }
-
- print "$file: $leave is defined at $what\n";
-
+ if (S_ISLNK($mode)) {
+ $aliases{$file} = $abs_file;
return;
}
- print "$file not found.\n";
+ return if (S_ISDIR($mode));
+
+ # Trivial: file is defined exactly the same way at ABI What:
+ return if (defined($data{$file}));
+ return if (defined($data{$abs_file}));
+
+ push @files, $abs_file;
+}
+
+sub check_undefined_symbols {
+ foreach my $file (sort @files) {
+
+ # sysfs-module is special, as its definitions are inside
+ # a text. For now, just ignore them.
+ next if ($file =~ m#^/sys/module/#);
+
+ # Ignore cgroup and firmware
+ next if ($file =~ m#^/sys/(fs/cgroup|firmware)/#);
+
+ my $defined = 0;
+ my $exact = 0;
+ my $whats = "";
+
+ my $leave = $file;
+ $leave =~ s,.*/,,;
+
+ my $path = $file;
+ $path =~ s,(.*/).*,$1,;
+
+ if (defined($leaf{$leave})) {
+ my $what = $leaf{$leave};
+ $whats .= " $what" if (!($whats =~ m/$what/));
+
+ foreach my $w (split / /, $what) {
+ if ($file =~ m#^$w$#) {
+ $exact = 1;
+ last;
+ }
+ }
+ # Check for aliases
+ #
+ # TODO: this algorithm is O(w * n²). It can be
+ # improved in the future in order to handle it
+ # faster, by changing parse_existing_sysfs to
+ # store the sysfs inside a tree, at the expense
+ # on making the code less readable and/or using some
+ # additional perl library.
+ foreach my $a (keys %aliases) {
+ my $new = $aliases{$a};
+ my $len = length($new);
+
+ if (substr($file, 0, $len) eq $new) {
+ my $newf = $a . substr($file, $len);
+
+ foreach my $w (split / /, $what) {
+ if ($newf =~ m#^$w$#) {
+ $exact = 1;
+ last;
+ }
+ }
+ }
+ }
+
+ $defined++;
+ }
+ next if ($exact);
+
+ # Ignore some sysfs nodes
+ next if ($file =~ m#/(sections|notes)/#);
+
+ # Would need to check at
+ # Documentation/admin-guide/kernel-parameters.txt, but this
+ # is not easily parseable.
+ next if ($file =~ m#/parameters/#);
+
+ if ($hint && $defined) {
+ print "$leave at $path might be one of:$whats\n";
+ next;
+ }
+ print "$file not found.\n";
+ }
}
sub undefined_symbols {
+ find({
+ wanted =>\&parse_existing_sysfs,
+ preprocess =>\&dont_parse_special_attributes,
+ no_chdir => 1
+ }, $sysfs_prefix);
+
foreach my $w (sort keys %data) {
foreach my $what (split /\xac /,$w) {
+ next if (!($what =~ m/^$sysfs_prefix/));
+
+ # Convert what into regular expressions
+
+ $what =~ s,/\.\.\./,/*/,g;
+ $what =~ s,\*,.*,g;
+
+ # Temporarily change [0-9]+ type of patterns
+ $what =~ s/\[0\-9\]\+/\xff/g;
+
+ # Temporarily change [\d+-\d+] type of patterns
+ $what =~ s/\[0\-\d+\]/\xff/g;
+ $what =~ s/\[(\d+)\]/\xf4$1\xf5/g;
+
+ # Temporarily change [0-9] type of patterns
+ $what =~ s/\[(\d)\-(\d)\]/\xf4$1-$2\xf5/g;
+
+ # Handle multiple option patterns
+ $what =~ s/[\{\<\[]([\w_]+)(?:[,|]+([\w_]+)){1,}[\}\>\]]/($1|$2)/g;
+
+ # Handle wildcards
+ $what =~ s/\<[^\>]+\>/.*/g;
+ $what =~ s/\{[^\}]+\}/.*/g;
+ $what =~ s/\[[^\]]+\]/.*/g;
+
+ $what =~ s/[XYZ]/.*/g;
+
+ # Recover [0-9] type of patterns
+ $what =~ s/\xf4/[/g;
+ $what =~ s/\xf5/]/g;
+
+ # Remove duplicated spaces
+ $what =~ s/\s+/ /g;
+
+ # Special case: this ABI has a parenthesis on it
+ $what =~ s/sqrt\(x^2\+y^2\+z^2\)/sqrt\(x^2\+y^2\+z^2\)/;
+
+ # Special case: drop comparition as in:
+ # What: foo = <something>
+ # (this happens on a few IIO definitions)
+ $what =~ s,\s*\=.*$,,;
+
my $leave = $what;
$leave =~ s,.*/,,;
- if (defined($leaf{$leave})) {
- $leaf{$leave} .= " " . $what;
- } else {
- $leaf{$leave} = $what;
+ next if ($leave =~ m/^\.\*/ || $leave eq "");
+
+ # Escape all other symbols
+ $what =~ s/$escape_symbols/\\$1/g;
+ $what =~ s/\\\\/\\/g;
+ $what =~ s/\\([\[\]\(\)\|])/$1/g;
+ $what =~ s/(\d+)\\(-\d+)/$1$2/g;
+
+ $leave =~ s/[\(\)]//g;
+
+ foreach my $l (split /\|/, $leave) {
+ if (defined($leaf{$l})) {
+ next if ($leaf{$l} =~ m/$what/);
+ $leaf{$l} .= " " . $what;
+ } else {
+ $leaf{$l} = $what;
+ }
}
}
}
-
- find({wanted =>\&parse_existing_sysfs, preprocess =>\&skip_debugfs, no_chdir => 1}, $sysfs_prefix);
+ check_undefined_symbols;
}
# Ensure that the prefix will always end with a slash
@@ -646,7 +763,8 @@ abi_book.pl - parse the Linux ABI files and produce a ReST book.
=head1 SYNOPSIS
B<abi_book.pl> [--debug] [--enable-lineno] [--man] [--help]
- [--(no-)rst-source] [--dir=<dir>] <COMAND> [<ARGUMENT>]
+ [--(no-)rst-source] [--dir=<dir>] [--show-hints]
+ <COMAND> [<ARGUMENT>]
Where <COMMAND> can be:
@@ -688,6 +806,11 @@ Enable output of #define LINENO lines.
Put the script in verbose mode, useful for debugging. Can be called multiple
times, to increase verbosity.
+=item B<--show-hints>
+
+Show hints about possible definitions for the missing ABI symbols.
+Used only when B<undefined>.
+
=item B<--help>
Prints a brief help message and exits.
--
2.31.1
Using a comma here is problematic, as some What: expressions
may already contain a comma. So, use \xac character instead.
Signed-off-by: Mauro Carvalho Chehab <[email protected]>
---
scripts/get_abi.pl | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl
index d7aa82094296..3b25d0a855ad 100755
--- a/scripts/get_abi.pl
+++ b/scripts/get_abi.pl
@@ -129,7 +129,7 @@ sub parse_abi {
push @{$symbols{$content}->{file}}, " $file:" . ($ln - 1);
if ($tag =~ m/what/) {
- $what .= ", " . $content;
+ $what .= "\xac " . $content;
} else {
if ($what) {
parse_error($file, $ln, "What '$what' doesn't have a description", "") if (!$data{$what}->{description});
@@ -239,7 +239,7 @@ sub parse_abi {
if ($what) {
parse_error($file, $ln, "What '$what' doesn't have a description", "") if (!$data{$what}->{description});
- foreach my $w(split /, /,$what) {
+ foreach my $w(split /\xac /,$what) {
$symbols{$w}->{xref} = $what;
};
}
@@ -328,7 +328,7 @@ sub output_rest {
printf ".. _%s:\n\n", $data{$what}->{label};
- my @names = split /, /,$w;
+ my @names = split /\xac /,$w;
my $len = 0;
foreach my $name (@names) {
--
2.31.1
The search algorithm used inside check_undefined_symbols
has an optimization: it seeks only whats that have the same
leave name. This helps not only to speedup the search, but
it also allows providing a hint about a partial match.
There's a drawback, however: when "what:" finishes with a
wildcard, the logic will skip the what, reporting it as
"not found".
Fix it by grouping the remaining cases altogether, and
disabing any hints for such cases.
Signed-off-by: Mauro Carvalho Chehab <[email protected]>
---
scripts/get_abi.pl | 74 +++++++++++++++++++++++++++-------------------
1 file changed, 43 insertions(+), 31 deletions(-)
diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl
index 4f31d2e34dd0..e001b6612ac7 100755
--- a/scripts/get_abi.pl
+++ b/scripts/get_abi.pl
@@ -589,44 +589,47 @@ sub check_undefined_symbols {
$found_string = 1;
}
+ if ($leave =~ /^\d+$/ || !defined($leaf{$leave})) {
+ $leave = "others";
+ }
+
print "--> $file\n" if ($found_string && $hint);
- if (defined($leaf{$leave})) {
- my $what = $leaf{$leave};
- $whats .= " $what" if (!($whats =~ m/$what/));
+ my $what = $leaf{$leave};
+ $whats .= " $what" if (!($whats =~ m/$what/));
- foreach my $w (split / /, $what) {
- if ($file =~ m#^$w$#) {
- $exact = 1;
- last;
- }
+ foreach my $w (split / /, $what) {
+ if ($file =~ m#^$w$#) {
+ $exact = 1;
+ last;
}
- # Check for aliases
- #
- # TODO: this algorithm is O(w * n²). It can be
- # improved in the future in order to handle it
- # faster, by changing parse_existing_sysfs to
- # store the sysfs inside a tree, at the expense
- # on making the code less readable and/or using some
- # additional perl library.
- foreach my $a (keys %aliases) {
- my $new = $aliases{$a};
- my $len = length($new);
+ }
+ # Check for aliases
+ #
+ # TODO: this algorithm is O(w * n²). It can be
+ # improved in the future in order to handle it
+ # faster, by changing parse_existing_sysfs to
+ # store the sysfs inside a tree, at the expense
+ # on making the code less readable and/or using some
+ # additional perl library.
+ foreach my $a (keys %aliases) {
+ my $new = $aliases{$a};
+ my $len = length($new);
- if (substr($file, 0, $len) eq $new) {
- my $newf = $a . substr($file, $len);
+ if (substr($file, 0, $len) eq $new) {
+ my $newf = $a . substr($file, $len);
- print " $newf\n" if ($found_string && $hint);
- foreach my $w (split / /, $what) {
- if ($newf =~ m#^$w$#) {
- $exact = 1;
- last;
- }
+ print " $newf\n" if ($found_string && $hint);
+ foreach my $w (split / /, $what) {
+ if ($newf =~ m#^$w$#) {
+ $exact = 1;
+ last;
}
}
}
-
- $defined++;
}
+
+ $defined++;
+
next if ($exact);
# Ignore some sysfs nodes
@@ -637,7 +640,7 @@ sub check_undefined_symbols {
# is not easily parseable.
next if ($file =~ m#/parameters/#);
- if ($hint && $defined) {
+ if ($hint && $defined && $leave ne "others") {
print "$leave at $path might be one of:$whats\n" if (!$search_string || $found_string);
next;
}
@@ -699,7 +702,16 @@ sub undefined_symbols {
my $leave = $what;
$leave =~ s,.*/,,;
- next if ($leave =~ m/^\.\*/ || $leave eq "");
+ # $leave is used to improve search performance at
+ # check_undefined_symbols, as the algorithm there can seek
+ # for a small number of "what". It also allows giving a
+ # hint about a leave with the same name somewhere else.
+ # However, there are a few occurences where the leave is
+ # either a wildcard or a number. Just group such cases
+ # altogether.
+ if ($leave =~ m/^\.\*/ || $leave eq "" || $leave =~ /^\d+$/) {
+ $leave = "others" ;
+ }
# Escape all other symbols
$what =~ s/$escape_symbols/\\$1/g;
--
2.31.1
Check for the symbols that exists under /sys but aren't
defined at Documentation/ABI.
Signed-off-by: Mauro Carvalho Chehab <[email protected]>
---
scripts/get_abi.pl | 90 ++++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 88 insertions(+), 2 deletions(-)
diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl
index 3b25d0a855ad..ded6dd8b9f71 100755
--- a/scripts/get_abi.pl
+++ b/scripts/get_abi.pl
@@ -13,7 +13,9 @@ my $help = 0;
my $man = 0;
my $debug = 0;
my $enable_lineno = 0;
+my $show_warnings = 1;
my $prefix="Documentation/ABI";
+my $sysfs_prefix="/sys";
#
# If true, assumes that the description is formatted with ReST
@@ -36,7 +38,7 @@ pod2usage(2) if (scalar @ARGV < 1 || @ARGV > 2);
my ($cmd, $arg) = @ARGV;
-pod2usage(2) if ($cmd ne "search" && $cmd ne "rest" && $cmd ne "validate");
+pod2usage(2) if ($cmd ne "search" && $cmd ne "rest" && $cmd ne "validate" && $cmd ne "undefined");
pod2usage(2) if ($cmd eq "search" && !$arg);
require Data::Dumper if ($debug);
@@ -50,6 +52,8 @@ my %symbols;
sub parse_error($$$$) {
my ($file, $ln, $msg, $data) = @_;
+ return if (!$show_warnings);
+
$data =~ s/\s+$/\n/;
print STDERR "Warning: file $file#$ln:\n\t$msg";
@@ -521,11 +525,88 @@ sub search_symbols {
}
}
+# Exclude /sys/kernel/debug and /sys/kernel/tracing from the search path
+sub skip_debugfs {
+ if (($File::Find::dir =~ m,^/sys/kernel,)) {
+ return grep {!/(debug|tracing)/ } @_;
+ }
+
+ if (($File::Find::dir =~ m,^/sys/fs,)) {
+ return grep {!/(pstore|bpf|fuse)/ } @_;
+ }
+
+ return @_
+}
+
+my %leaf;
+
+my $escape_symbols = qr { ([\x01-\x08\x0e-\x1f\x21-\x29\x2b-\x2d\x3a-\x40\x7b-\xff]) }x;
+sub parse_existing_sysfs {
+ my $file = $File::Find::name;
+
+ my $mode = (stat($file))[2];
+ return if ($mode & S_IFDIR);
+
+ my $leave = $file;
+ $leave =~ s,.*/,,;
+
+ if (defined($leaf{$leave})) {
+ # FIXME: need to check if the path makes sense
+ my $what = $leaf{$leave};
+
+ $what =~ s/,/ /g;
+
+ $what =~ s/\<[^\>]+\>/.*/g;
+ $what =~ s/\{[^\}]+\}/.*/g;
+ $what =~ s/\[[^\]]+\]/.*/g;
+ $what =~ s,/\.\.\./,/.*/,g;
+ $what =~ s,/\*/,/.*/,g;
+
+ $what =~ s/\s+/ /g;
+
+ # Escape all other symbols
+ $what =~ s/$escape_symbols/\\$1/g;
+
+ foreach my $i (split / /,$what) {
+ if ($file =~ m#^$i$#) {
+# print "$file: $i: OK!\n";
+ return;
+ }
+ }
+
+ print "$file: $leave is defined at $what\n";
+
+ return;
+ }
+
+ print "$file not found.\n";
+}
+
+sub undefined_symbols {
+ foreach my $w (sort keys %data) {
+ foreach my $what (split /\xac /,$w) {
+ my $leave = $what;
+ $leave =~ s,.*/,,;
+
+ if (defined($leaf{$leave})) {
+ $leaf{$leave} .= " " . $what;
+ } else {
+ $leaf{$leave} = $what;
+ }
+ }
+ }
+
+ find({wanted =>\&parse_existing_sysfs, preprocess =>\&skip_debugfs, no_chdir => 1}, $sysfs_prefix);
+}
+
# Ensure that the prefix will always end with a slash
# While this is not needed for find, it makes the patch nicer
# with --enable-lineno
$prefix =~ s,/?$,/,;
+if ($cmd eq "undefined" || $cmd eq "search") {
+ $show_warnings = 0;
+}
#
# Parses all ABI files located at $prefix dir
#
@@ -536,7 +617,9 @@ print STDERR Data::Dumper->Dump([\%data], [qw(*data)]) if ($debug);
#
# Handles the command
#
-if ($cmd eq "search") {
+if ($cmd eq "undefined") {
+ undefined_symbols;
+} elsif ($cmd eq "search") {
search_symbols;
} else {
if ($cmd eq "rest") {
@@ -575,6 +658,9 @@ B<rest> - output the ABI in ReST markup language
B<validate> - validate the ABI contents
+B<undefined> - existing symbols at the system that aren't
+ defined at Documentation/ABI
+
=back
=head1 OPTIONS
--
2.31.1