Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
Received-SPF: Pass (protection.outlook.com: domain of nvidia.com designates
 216.228.112.35 as permitted sender) receiver=protection.outlook.com;
 client-ip=216.228.112.35; helo=mail.nvidia.com;
Subject: Re: [PATCH v13 5/9] PCI: Allow userspace to query and set device
 reset mechanism
To:     Amey Narkhede <ameynarkhede03@gmail.com>,
        Bjorn Helgaas <bhelgaas@google.com>
CC:     <alex.williamson@redhat.com>,
        Raphael Norwitz <raphael.norwitz@nutanix.com>,
        <linux-pci@vger.kernel.org>, <linux-kernel@vger.kernel.org>,
        <kw@linux.com>, Sinan Kaya <okaya@kernel.org>,
        Len Brown <lenb@kernel.org>,
        "Rafael J . Wysocki" <rjw@rjwysocki.net>
References: <20210801142518.1224-1-ameynarkhede03@gmail.com>
 <20210801142518.1224-6-ameynarkhede03@gmail.com>
From:   Shanker R Donthineni <sdonthineni@nvidia.com>
Message-ID: <b3a5aef9-45ba-5f61-a756-c024c3a728ad@nvidia.com>
Date:   Tue, 3 Aug 2021 08:45:44 -0500
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
 Thunderbird/68.10.0
MIME-Version: 1.0
In-Reply-To: <20210801142518.1224-6-ameynarkhede03@gmail.com>
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: 8bit
Content-Language: en-US
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 03 Aug 2021 13:45:48.5728
 (UTC)
X-MS-Exchange-CrossTenant-Network-Message-Id: 6c5e96ee-2c02-4003-4d17-08d956850064
X-MS-Exchange-CrossTenant-Id: 43083d15-7273-40c1-b7db-39efd9ccc17a
X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp: TenantId=43083d15-7273-40c1-b7db-39efd9ccc17a;Ip=[216.228.112.35];Helo=[mail.nvidia.com]
X-MS-Exchange-CrossTenant-AuthSource: CO1NAM11FT039.eop-nam11.prod.protection.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Anonymous
X-MS-Exchange-CrossTenant-FromEntityHeader: HybridOnPrem
X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM6PR12MB2779
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org


On 8/1/21 9:25 AM, Amey Narkhede wrote:
> External email: Use caution opening links or attachments
>
>
> Add reset_method sysfs attribute to enable user to query and set user
> preferred device reset methods and their ordering.
>
> Co-developed-by: Alex Williamson <alex.williamson@redhat.com>
> Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
> Signed-off-by: Amey Narkhede <ameynarkhede03@gmail.com>
> ---
>  Documentation/ABI/testing/sysfs-bus-pci |  19 +++++
>  drivers/pci/pci-sysfs.c                 |   1 +
>  drivers/pci/pci.c                       | 105 ++++++++++++++++++++++++
>  drivers/pci/pci.h                       |   2 +
>  4 files changed, 127 insertions(+)
>
> diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci
> index ef00fada2efb..ef66b62bf025 100644
> --- a/Documentation/ABI/testing/sysfs-bus-pci
> +++ b/Documentation/ABI/testing/sysfs-bus-pci
> @@ -121,6 +121,25 @@ Description:
>                 child buses, and re-discover devices removed earlier
>                 from this part of the device tree.
>
> +What:          /sys/bus/pci/devices/.../reset_method
> +Date:          March 2021
> +Contact:       Amey Narkhede <ameynarkhede03@gmail.com>
> +Description:
> +               Some devices allow an individual function to be reset
> +               without affecting other functions in the same slot.
> +
> +               For devices that have this support, a file named
> +               reset_method will be present in sysfs. Initially reading
> +               this file will give names of the device supported reset
> +               methods and their ordering. After write, this file will
> +               give names and ordering of currently enabled reset methods.
> +               Writing the name or space separated list of names of any of
> +               the device supported reset methods to this file will set
> +               the reset methods and their ordering to be used when
> +               resetting the device. Writing empty string to this file
> +               will disable ability to reset the device and writing
> +               "default" will return to the original value.
> +
>  What:          /sys/bus/pci/devices/.../reset
>  Date:          July 2009
>  Contact:       Michael S. Tsirkin <mst@redhat.com>
> diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
> index 316f70c3e3b4..54ee7193b463 100644
> --- a/drivers/pci/pci-sysfs.c
> +++ b/drivers/pci/pci-sysfs.c
> @@ -1491,6 +1491,7 @@ const struct attribute_group *pci_dev_groups[] = {
>         &pci_dev_config_attr_group,
>         &pci_dev_rom_attr_group,
>         &pci_dev_reset_attr_group,
> +       &pci_dev_reset_method_attr_group,
>         &pci_dev_vpd_attr_group,
>  #ifdef CONFIG_DMI
>         &pci_dev_smbios_attr_group,
> diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
> index 932dd21e759b..c496cd164aca 100644
> --- a/drivers/pci/pci.c
> +++ b/drivers/pci/pci.c
> @@ -5132,6 +5132,111 @@ static const struct pci_reset_fn_method pci_reset_fn_methods[] = {
>         { pci_reset_bus_function, .name = "bus" },
>  };
>
> +static ssize_t reset_method_show(struct device *dev,
> +                                struct device_attribute *attr,
> +                                char *buf)
> +{
> +       struct pci_dev *pdev = to_pci_dev(dev);
> +       ssize_t len = 0;
> +       int i, m;
> +
> +       for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
> +               m = pdev->reset_methods[i];
> +               if (!m)
> +                       break;
> +
> +               len += sysfs_emit_at(buf, len, "%s%s", len ? " " : "",
> +                                    pci_reset_fn_methods[m].name);
> +       }
> +
> +       if (len)
> +               len += sysfs_emit_at(buf, len, "\n");
> +
> +       return len;
> +}
> +
> +static ssize_t reset_method_store(struct device *dev,
> +                                 struct device_attribute *attr,
> +                                 const char *buf, size_t count)
> +{
> +       struct pci_dev *pdev = to_pci_dev(dev);
> +       int i = 0;
> +       char *name, *options = NULL;
> +
> +       if (count >= (PAGE_SIZE - 1))
> +               return -EINVAL;
> +
> +       if (sysfs_streq(buf, "")) {
> +               pdev->reset_methods[0] = 0;
> +               pci_warn(pdev, "All device reset methods disabled by user");
> +               return count;
> +       }
> +
> +       if (sysfs_streq(buf, "default")) {
> +               pci_init_reset_methods(pdev);
> +               return count;
> +       }
> +
> +       options = kstrndup(buf, count, GFP_KERNEL);
> +       if (!options)
> +               return -ENOMEM;
> +
> +       while ((name = strsep(&options, " ")) != NULL) {
> +               int m;
> +
> +               if (sysfs_streq(name, ""))
> +                       continue;
> +
> +               name = strim(name);
> +
> +               for (m = 1; m < PCI_NUM_RESET_METHODS && i < PCI_NUM_RESET_METHODS; m++) {
> +                       if (sysfs_streq(name, pci_reset_fn_methods[m].name) &&
> +                           !pci_reset_fn_methods[m].reset_fn(pdev, 1)) {
> +                               pdev->reset_methods[i++] = m;
> +                               break;
> +                       }
> +               }
> +
Checking reset method logic isn't optimized, iterating through all entries if the
device doesn't support a requested method.

Something like this:
        for (m = 1; m < PCI_NUM_RESET_METHODS && i < PCI_NUM_RESET_METHODS; m++) {
                if (!sysfs_streq(name, pci_reset_fn_methods[m].name))
                        continue;
                if(!pci_reset_fn_methods[m].reset_fn(pdev, 1))
                        pdev->reset_methods[i++] = m;
                break;
        }

I think we should avoid duplicate entries in pdev->reset_methods.
Example:
   root# cat reset_method
   acpi flr bus

   root# echo "acpi flr bus flr" > reset_method
   root# cat reset_method
   acpi flr bus flr


> +               if (m == PCI_NUM_RESET_METHODS) {
> +                       kfree(options);
> +                       return -EINVAL;
Set the last entry to zero in pdev->reset_methods otherwise the inconsistent
methods are enabled.
Example:
   root# cat reset_method
   acpi flr bus

   root# echo "flr a" > reset_method
   root# cat reset_method
   flr flr bus

> +
> +               }
> +       }
> +
> +       if (i < PCI_NUM_RESET_METHODS)
> +               pdev->reset_methods[i] = 0;
> +
Last entry can be set unconditionally after removing the duplicate entries.
Refactored code to filter duplicate entries and warn the user about the invalid
& unsupported reset methods.

static ssize_t reset_method_store(struct device *dev,
                                  struct device_attribute *attr,
                                  const char *buf, size_t count)
{
        struct pci_dev *pdev = to_pci_dev(dev);
        char *name, *options = NULL;
        int i, m, n = 0;

        if (count >= (PAGE_SIZE - 1))
                return -EINVAL;

        if (sysfs_streq(buf, ""))
                goto done;

        if (sysfs_streq(buf, "default")) {
                pci_init_reset_methods(pdev);
                return count;
        }

        options = kstrndup(buf, count, GFP_KERNEL);
        if (!options)
                return -ENOMEM;

        while ((name = strsep(&options, " ")) != NULL) {
                if (sysfs_streq(name, ""))
                        continue;
                name = strim(name);

                /* Validate reset method */
                for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
                        if (sysfs_streq(name, pci_reset_fn_methods[m].name))
                                break;
                }
                if (m == PCI_NUM_RESET_METHODS) {
                        pci_warn(pdev, "Skip invalid reset method '%s'", name);
                        continue;
                }

                /* Check if the reset method is already enabled */
                for (i = 0; i < n; i++) {
                        if (pdev->reset_methods[i] == m)
                                break;
                }
                if (i < n)
                        continue;

                /* Probe the requested reset method */
                if (pci_reset_fn_methods[m].reset_fn(pdev, 1))
                        pci_warn(pdev, "Unsupported reset method '%s'", name);

                pdev->reset_methods[n++] = m;
                BUG_ON(n == PCI_NUM_RESET_METHODS)
       }
        kfree(options);

done:
        pdev->reset_methods[n] = 0;
        if (pdev->reset_methods[0] == 0) {
                pci_warn(pdev, "All device reset methods are disabled");
        } else if ((pdev->reset_methods[0] != 1) &&
                   !pci_reset_fn_methods[1].reset_fn(pdev, 1)) {
                pci_warn(pdev, "Device specific reset disabled/de-prioritized by user");
        }
        return count;
}