Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
From:   Song Liu <songliubraving@fb.com>
To:     Namhyung Kim <namhyung@kernel.org>
CC:     linux-kernel <linux-kernel@vger.kernel.org>,
        Kernel Team <Kernel-team@fb.com>,
        Arnaldo Carvalho de Melo <acme@kernel.org>,
        "Arnaldo Carvalho de Melo" <acme@redhat.com>,
        Jiri Olsa <jolsa@kernel.org>
Subject: Re: [PATCH v2 3/3] perf-test: add a test for perf-stat --bpf-counters
 option
Thread-Topic: [PATCH v2 3/3] perf-test: add a test for perf-stat
 --bpf-counters option
Thread-Index: AQHXGqn6I/OKoZ/+FEuD284VkHxFJKqJRLeAgAAZgoA=
Date:   Thu, 18 Mar 2021 07:39:08 +0000
Message-ID: <85FE8519-615F-464F-A2BE-06A16D6BAFD0@fb.com>
References: <20210316211837.910506-1-songliubraving@fb.com>
 <20210316211837.910506-4-songliubraving@fb.com>
 <CAM9d7ciqTGRx6kZ3c-+tXi3AwvOj5EG1ig4oTiUy9sDuTA=KEw@mail.gmail.com>
In-Reply-To: <CAM9d7ciqTGRx6kZ3c-+tXi3AwvOj5EG1ig4oTiUy9sDuTA=KEw@mail.gmail.com>
Accept-Language: en-US
Content-Language: en-US
Content-Type: text/plain; charset="us-ascii"
Content-ID: <36AEB2D4A7A15F45A1E02E1E17DF779C@namprd15.prod.outlook.com>
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0
X-MS-Exchange-CrossTenant-Network-Message-Id: bdcb03dd-e0de-41a6-312f-08d8e9e0ea7a
X-MS-Exchange-CrossTenant-originalarrivaltime: 18 Mar 2021 07:39:08.6539
 (UTC)
X-MS-Exchange-CrossTenant-fromentityheader: Hosted
X-MS-Exchange-CrossTenant-id: 8ae927fe-1255-47a7-a2af-5f3a069daaa2
X-MS-Exchange-CrossTenant-mailboxtype: HOSTED
X-MS-Exchange-CrossTenant-userprincipalname: NDgQeDa/boGJAu8onpPK9M0XyGwaC/6Uen0SH0t/ntR0WF/lsEVcJBjEeHWC7CkcmEf8s2LMb0AFxUeautky9w==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: BYAPR15MB2773
X-OriginatorOrg: fb.com
X-Proofpoint-Virus-Version: vendor=fsecure engine=2.50.10434:6.0.369,18.0.761
 definitions=2021-03-18_02:2021-03-17,2021-03-18 signatures=0
X-Proofpoint-Spam-Details: rule=fb_default_notspam policy=fb_default score=0 mlxlogscore=999
 clxscore=1015 phishscore=0 priorityscore=1501 malwarescore=0 mlxscore=0
 bulkscore=0 spamscore=0 impostorscore=0 lowpriorityscore=0 adultscore=0
 suspectscore=0 classifier=spam adjust=0 reason=mlx scancount=1
 engine=8.12.0-2009150000 definitions=main-2103180057
X-FB-Internal: deliver
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org


> On Mar 17, 2021, at 11:07 PM, Namhyung Kim <namhyung@kernel.org> wrote:
>=20
> On Wed, Mar 17, 2021 at 6:18 AM Song Liu <songliubraving@fb.com> wrote:
>>=20
>> Add a test to compare the output of perf-stat with and without option
>> --bpf-counters. If the difference is more than 10%, the test is consider=
ed
>> as failed.
>>=20
>> For stable results between two runs (w/ and w/o --bpf-counters), the tes=
t
>> program should: 1) be long enough for better signal-noise-ratio; 2) not
>> depend on the behavior of IO subsystem (for less noise from caching). So
>> far, the best option we found is stressapptest.
>>=20
>> Signed-off-by: Song Liu <songliubraving@fb.com>
>> ---
>> tools/perf/tests/shell/stat_bpf_counters.sh | 34 +++++++++++++++++++++
>> 1 file changed, 34 insertions(+)
>> create mode 100755 tools/perf/tests/shell/stat_bpf_counters.sh
>>=20
>> diff --git a/tools/perf/tests/shell/stat_bpf_counters.sh b/tools/perf/te=
sts/shell/stat_bpf_counters.sh
>> new file mode 100755
>> index 0000000000000..c0bcb38d6b53c
>> --- /dev/null
>> +++ b/tools/perf/tests/shell/stat_bpf_counters.sh
>> @@ -0,0 +1,34 @@
>> +#!/bin/sh
>> +# perf stat --bpf-counters test
>> +# SPDX-License-Identifier: GPL-2.0
>> +
>> +set -e
>> +
>> +# check whether $2 is within +/- 10% of $1
>> +compare_number()
>> +{
>> +       first_num=3D$1
>> +       second_num=3D$2
>> +
>> +       # upper bound is first_num * 110%
>> +       upper=3D$(( $first_num + $first_num / 10 ))
>> +       # lower bound is first_num * 90%
>> +       lower=3D$(( $first_num - $first_num / 10 ))
>> +
>> +       if [ $second_num -gt $upper ] || [ $second_num -lt $lower ]; the=
n
>> +               echo "The difference between $first_num and $second_num =
are greater than 10%."
>> +               exit 1
>> +       fi
>> +}
>> +
>> +# skip if --bpf-counters is not supported
>> +perf stat --bpf-counters true > /dev/null 2>&1 || exit 2
>> +
>> +# skip if stressapptest is not available
>> +stressapptest -s 1 -M 100 -m 1 > /dev/null 2>&1 || exit 2
>=20
> I don't know how popular it is, but we can print some info
> in case we miss it.

I just realized that perf-bench-sched-messaging is a good test to use,=20
so we don't need stressapptest. Attached the updated version below.

>=20
>> +
>> +base_cycles=3D$(perf stat --no-big-num -e cycles -- stressapptest -s 3 =
-M 100 -m 1 2>&1 | grep -e cycles | awk '{print $1}')
>> +bpf_cycles=3D$(perf stat --no-big-num --bpf-counters -e cycles -- stres=
sapptest -s 3 -M 100 -m 1 2>&1 | grep -e cycles | awk '{print $1}')
>=20
> I think just awk '/cycles/ {print $1}' should work.

Thanks! Fixed in the new version.=20

Song


cat tools/perf/tests/shell/stat_bpf_counters.sh
#!/bin/sh
# perf stat --bpf-counters test
# SPDX-License-Identifier: GPL-2.0

set -e

# check whether $2 is within +/- 10% of $1
compare_number()
{
        first_num=3D$1
        second_num=3D$2

        # upper bound is first_num * 110%
        upper=3D$(( $first_num + $first_num / 10 ))
        # lower bound is first_num * 90%
        lower=3D$(( $first_num - $first_num / 10 ))

        if [ $second_num -gt $upper ] || [ $second_num -lt $lower ]; then
                echo "The difference between $first_num and $second_num are=
 greater than 10%."
                exit 1
        fi
}

# skip if --bpf-counters is not supported
perf stat --bpf-counters true > /dev/null 2>&1 || exit 2

base_cycles=3D$(perf stat --no-big-num -e cycles -- perf bench sched messag=
ing -g 1 -l 100 -t 2>&1 | awk '/cycles/ {print $1}')
bpf_cycles=3D$(perf stat --no-big-num --bpf-counters -e cycles -- perf benc=
h sched messaging -g 1 -l 100 -t 2>&1 | awk '/cycles/ {print $1}')

compare_number $base_cycles $bpf_cycles
exit 0