From:   Shay Drory <shayd@nvidia.com>
To:     "David S . Miller" <davem@davemloft.net>,
        Jakub Kicinski <kuba@kernel.org>
Cc:     jiri@nvidia.com, saeedm@nvidia.com, netdev@vger.kernel.org,
        linux-kernel@vger.kernel.org, Shay Drory <shayd@nvidia.com>,
        Moshe Shemesh <moshe@nvidia.com>
Subject: [PATCH net-next v4 3/7] net/mlx5: Let user configure io_eq_size param
Date:   Thu,  9 Dec 2021 12:09:25 +0200
Message-Id: <20211209100929.28115-4-shayd@nvidia.com>
In-Reply-To: <20211209100929.28115-1-shayd@nvidia.com>
References: <20211209100929.28115-1-shayd@nvidia.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?gq5yVg+eMzBdbbBqVzOZjCNF9x+0D9mqPSV5pinmZWiumeCjAso5HZjWFmYc?=
 =?us-ascii?Q?8tzn2WTWt5zmiw81xlM0U8gzS1TJQb3lACNKe0YZK0yQEAIORXg1IvyPcu+t?=
 =?us-ascii?Q?i+dXQl3zIlacOY+96+oozgcGgVlXjD+uLwfqNXoScXsSwtrsdNCWta2bvN2a?=
 =?us-ascii?Q?8edko9B9TFniEQSByClZ2w2nkOt5f0muTLJpniIpYxVIjLjzKdq9H/g4yxof?=
 =?us-ascii?Q?QvJxVj6AILlmP7pP62xQb+Bfc2Y9R7eilt6sPo/CqWCTPnSMrELG0rY32YHt?=
 =?us-ascii?Q?1ZlAkorv2nLKbYM6KVx/3Dk+zIYYxO7XdpxWs20KPcbVXn49hiUS5tcWUBD/?=
 =?us-ascii?Q?5ZeVnBV1TsO3dzvbwENyevDdZdgQMyULgu4bIbA3T9VfF3T58tdhN3p1hZE2?=
 =?us-ascii?Q?mlJo3msnzYCVycg4M353rQul3pLE3jRbXj4GH5z120GBqg2Kc3LZvDW2L7GK?=
 =?us-ascii?Q?jnOVnZWtSipjbUnTKua93zvd4sH3O22U4CvCEY9jc+k0y6Vm/bIHPYyoo4hG?=
 =?us-ascii?Q?LVkF8OJUzsao38uwOln+cPVF13sUT3Mt/qF/l4hNROCWeC+9acpmNtno7p/f?=
 =?us-ascii?Q?hDRO8G/i3/T2h5JPadN8W/kNmTovVBAL2pIjU7oUxFBetOipf+1zwCZsgNm2?=
 =?us-ascii?Q?AtlkjcLPVFDgLdACOE+5aSYQzEQ/0yid+i9uJMlnJqy93GUeMkC0XV+Ithv2?=
 =?us-ascii?Q?AJxBUnxHQvKLq1wjXpJA97hB3UNtg3eaoY5hiHxLWIgtY/C5huaIJaTTYjqM?=
 =?us-ascii?Q?1bXLvBL3z2N8ebzt8f16c/zdBdojDq4Z+tSptmEE5YjokEPrGMCDPPISNr4S?=
 =?us-ascii?Q?kf8fO5X1EbkPEPXj5pNOjFt3dzvABUSBqj7t/5DbwiNyJRqMjcC8mAQhC337?=
 =?us-ascii?Q?dq2NfI4N88AWFFgdAXAnb+WXusNqQOwpKZAOr2MbxdaywuM+v7AuxYxss/2q?=
 =?us-ascii?Q?1oYx5A7TzyW9hmGrA3fwpWPxxUsjoJfKvDuZtmrM9TzICS2HqK5CLwzRbPjL?=
 =?us-ascii?Q?o3eVEK4flH/lGntI/9NAiZxEkh1F1STFt0ZON9tBCs8Xy6TJJT/+ZtP0ufBs?=
 =?us-ascii?Q?lCp6az56MCyFWtEJ9KaUKBRIV1uCRUAopGkBd/lWrwL5WcWZ5NMZFi7EJYX0?=
 =?us-ascii?Q?4zgtNNOW+eIAGu7EvI2pQYU902ROvQNM8tHrg5cGK7U5s72PreHKb8SNgI+9?=
 =?us-ascii?Q?mHV612g0MWMW274E7DXtHCM63HBBdB9FHqXKAeqW8snigJjLezSqol30+A3U?=
 =?us-ascii?Q?mjJ2fXAW4QEyHVXUWLfgpoOU2eaj5im+pznXXi1HfyyfS0/vRMs56nDP7jI/?=
 =?us-ascii?Q?itaQqMMpnoe/r+WBwx0N8lr0FD9eaU9UzW/ddTJLbwgUh53X9pn/gc1OlmUl?=
 =?us-ascii?Q?e0GqcbtsfW2S1Pe1D0bxDT0QGbVQ6ZsBcUHuI5NbSptzJMq0FLwff7LBcpH/?=
 =?us-ascii?Q?zNL7bZrmdmVSoPHtpZkgvuwS9z8iOTPG07diCI87L/zDM81t/8rDmDgRQQVW?=
 =?us-ascii?Q?Kdt2/4Sg3kWZKV99WiPf19tjUc+JfNYqOHwtXBk4gjjabXKLg5rq9f6co8F4?=
 =?us-ascii?Q?aIjzoTyRMdFPWI8avEKYCvNbnIxIqmVMgGftXd3VBZsqZK38G6c9fS6qf4wh?=
 =?us-ascii?Q?BCaJC7GhE1O/yq41FLkFdvs=3D?=
X-OriginatorOrg: Nvidia.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 15eccf68-c18f-4a24-39e9-08d9bafc0f13
X-MS-Exchange-CrossTenant-AuthSource: DM4PR12MB5373.namprd12.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 09 Dec 2021 10:09:59.8861
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 43083d15-7273-40c1-b7db-39efd9ccc17a
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: ufed+p2OORwFwUNFkcgwJ29MpwTIYMqPWCyWg4dr+UsvaLrZp1V3gwtV6bfhu9I7TfG/iUqwprxuiYiy1xsZzA==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM4PR12MB5311
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Currently, each I/O EQ is taking 128KB of memory. This size
is not needed in all use cases, and is critical with large scale.
Hence, allow user to configure the size of I/O EQs.

For example, to reduce I/O EQ size to 64, execute:
$ devlink dev param set pci/0000:00:0b.0 name io_eq_size value 64 \
              cmode driverinit
$ devlink dev reload pci/0000:00:0b.0

Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
---
 Documentation/networking/devlink/mlx5.rst      |  4 ++++
 .../net/ethernet/mellanox/mlx5/core/devlink.c  | 14 ++++++++++++++
 drivers/net/ethernet/mellanox/mlx5/core/eq.c   | 18 +++++++++++++++++-
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/Documentation/networking/devlink/mlx5.rst b/Documentation/networking/devlink/mlx5.rst
index 4e4b97f7971a..291e7f63af73 100644
--- a/Documentation/networking/devlink/mlx5.rst
+++ b/Documentation/networking/devlink/mlx5.rst
@@ -14,8 +14,12 @@ Parameters
 
    * - Name
      - Mode
+     - Validation
    * - ``enable_roce``
      - driverinit
+   * - ``io_eq_size``
+     - driverinit
+     - The range is between 64 and 4096.
 
 The ``mlx5`` driver also implements the following driver-specific
 parameters.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index 1c98652b244a..177c6e9159f8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -546,6 +546,13 @@ static int mlx5_devlink_enable_remote_dev_reset_get(struct devlink *devlink, u32
 	return 0;
 }
 
+static int mlx5_devlink_eq_depth_validate(struct devlink *devlink, u32 id,
+					  union devlink_param_value val,
+					  struct netlink_ext_ack *extack)
+{
+	return (val.vu16 >= 64 && val.vu16 <= 4096) ? 0 : -EINVAL;
+}
+
 static const struct devlink_param mlx5_devlink_params[] = {
 	DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_FLOW_STEERING_MODE,
 			     "flow_steering_mode", DEVLINK_PARAM_TYPE_STRING,
@@ -570,6 +577,8 @@ static const struct devlink_param mlx5_devlink_params[] = {
 	DEVLINK_PARAM_GENERIC(ENABLE_REMOTE_DEV_RESET, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
 			      mlx5_devlink_enable_remote_dev_reset_get,
 			      mlx5_devlink_enable_remote_dev_reset_set, NULL),
+	DEVLINK_PARAM_GENERIC(IO_EQ_SIZE, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
+			      NULL, NULL, mlx5_devlink_eq_depth_validate),
 };
 
 static void mlx5_devlink_set_params_init_values(struct devlink *devlink)
@@ -608,6 +617,11 @@ static void mlx5_devlink_set_params_init_values(struct devlink *devlink)
 						   value);
 	}
 #endif
+
+	value.vu32 = MLX5_COMP_EQ_SIZE;
+	devlink_param_driverinit_value_set(devlink,
+					   DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
+					   value);
 }
 
 static const struct devlink_param enable_eth_param =
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 792e0d6aa861..7686d7c9c824 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -19,6 +19,7 @@
 #include "lib/clock.h"
 #include "diag/fw_tracer.h"
 #include "mlx5_irq.h"
+#include "devlink.h"
 
 enum {
 	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
@@ -796,6 +797,21 @@ static void destroy_comp_eqs(struct mlx5_core_dev *dev)
 	}
 }
 
+static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
+{
+	struct devlink *devlink = priv_to_devlink(dev);
+	union devlink_param_value val;
+	int err;
+
+	err = devlink_param_driverinit_value_get(devlink,
+						 DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
+						 &val);
+	if (!err)
+		return val.vu32;
+	mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
+	return MLX5_COMP_EQ_SIZE;
+}
+
 static int create_comp_eqs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *table = dev->priv.eq_table;
@@ -807,7 +823,7 @@ static int create_comp_eqs(struct mlx5_core_dev *dev)
 
 	INIT_LIST_HEAD(&table->comp_eqs_list);
 	ncomp_eqs = table->num_comp_eqs;
-	nent = MLX5_COMP_EQ_SIZE;
+	nent = comp_eq_depth_devlink_param_get(dev);
 	for (i = 0; i < ncomp_eqs; i++) {
 		struct mlx5_eq_param param = {};
 		int vecidx = i;
-- 
2.21.3