From:   Shay Drory <shayd@nvidia.com>
To:     "David S . Miller" <davem@davemloft.net>,
        Jakub Kicinski <kuba@kernel.org>
Cc:     jiri@nvidia.com, saeedm@nvidia.com, netdev@vger.kernel.org,
        linux-kernel@vger.kernel.org, Shay Drory <shayd@nvidia.com>,
        Moshe Shemesh <moshe@nvidia.com>
Subject: [PATCH net-next v4 5/7] net/mlx5: Let user configure event_eq_size param
Date:   Thu,  9 Dec 2021 12:09:27 +0200
Message-Id: <20211209100929.28115-6-shayd@nvidia.com>
In-Reply-To: <20211209100929.28115-1-shayd@nvidia.com>
References: <20211209100929.28115-1-shayd@nvidia.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?omiN4RAUf4piPM9Cwx1hxtYJnY4iY+Fb3Bhq3iZVBWEglsoSOslHFWTGVSoI?=
 =?us-ascii?Q?Id+3NCEijoQK6RiS30WMntk3Wyd1W+CDKUEfCBemPKG6EcS4iwiDWW+xbJts?=
 =?us-ascii?Q?t6sMmD4GyGCfYym9YitpkBXTIAahHIi7NZKCuP2GTWu1WDGkv8RjTvKfBdeL?=
 =?us-ascii?Q?bMwtZFW35FLbFhZUk5Qr787HODzAIrYf2IsacLIVEcMtlGFwQSasFZ/gYa91?=
 =?us-ascii?Q?fOwDXK+jFocRV0ARn63aoYoYTeHzitGA2XD+dMnA39Q2rJQ8I/qTuJZIKbUh?=
 =?us-ascii?Q?nB80uHdUvFJwcDv4Ss/MKeT9Jti6OMOlV3+w7BVRWL4FeTgEbs8eEulA/R3h?=
 =?us-ascii?Q?3583B04qX8Au/lhw4UlQ3rjZDS5Z5s7M0FhLpVGiITCebf7Mf19O7evQDpSa?=
 =?us-ascii?Q?NxWpZTOkKHatfNiBJ8QhloeIFeExZ3z+3O+5NGbW8nWP1lci/rMQH1obyT7T?=
 =?us-ascii?Q?08TxcPjcdXZOnVM6e/8BUeIRh9Md+60TA8LuAK2BtfDKN6toFrygtUzXk2Zb?=
 =?us-ascii?Q?GuUNLI/vmzHfDpeDk/bT7cVt+v5Bjcx8+TPFvSSJYvc5ePqYjOIz0q1kDVxp?=
 =?us-ascii?Q?g22w4zUBux6NNuk4ACgy2guizq0wqF7kZdaJgVO/5f8bo4Cjda9ST8nKC0M2?=
 =?us-ascii?Q?3omK1E2lP+HAi9shBi4JQXJpWq7g9M4gckqGUjQksylR5QUp+DJ0vzw07aTR?=
 =?us-ascii?Q?5pSTXNq9NEtw7WtGGmtD/nvMdpFRieyeQq4OKs8Jan3CEprZaRJ/OFT+SZ2f?=
 =?us-ascii?Q?uFKqh6fJXHxYin5J2UP5dcAfJ+jb9fVllCT4Ipp/NJ/cpu29z0uRQ/InupUh?=
 =?us-ascii?Q?v9eXL3fqa59m9Njde8tAmfXkHJY+0SMLmQ/j7qv3JC3yxSlbNt4hpt9Tn9tU?=
 =?us-ascii?Q?81EW/6i5Tig9fF1Mpe/h00WsdzOxkcEpW4TIrpAaa/CmyYgcI9zDNlFgXc/B?=
 =?us-ascii?Q?UMJJGG2GFy/yxI8WJvmYQ06GK9NF6qCiM6Z7TkgrS3jEMUiL9WGR6qn9itVt?=
 =?us-ascii?Q?KBv2ZJVKtdiFylSb4aG5aQ0sZY5mZUd27cEn4o3cmhip4Q0u0VFssW0Pbtla?=
 =?us-ascii?Q?dKjkIXNuhfrIOeoNViU5ztze/joG5xqgTHrNp0JX3Yx0QhM9ZXMuI20xCDtY?=
 =?us-ascii?Q?ocwr+YqsuQKdimIlhRY3Ru984EDVuKWM0DoEAg/chTt5hC+S+ApD9nnd1rm0?=
 =?us-ascii?Q?0+Ey00jd/KRRNwqx78+V4tm2lqJKPogaRWZOFflTd2RdgARG2JFqdbPShWGF?=
 =?us-ascii?Q?+LGYgy8p0KL7KGLeGguzpBBEvUSNqMP3owu/9qAtU4G8r7w4up15LIrW8XX3?=
 =?us-ascii?Q?4WiY6gZv4z8A7Rbyqr9cjgULQw68nnRczmpgDeFpkYLMnI1db4gtK+QwdN6A?=
 =?us-ascii?Q?zcU3NwPo1L1u3EdwJMFTd6uxH38EjjXofuRgNkmP49J0k+243nMy4mh9O5fx?=
 =?us-ascii?Q?ZgfDLMH3D2Iy8pVzSIBL4RewlfT0IGW9/Z69ljkoFzkUb1bQMSgh3uDmKviR?=
 =?us-ascii?Q?QDaN01/aJJE6gR/axk60yq8UGhZkAS//cFYA85VmpaEBxFrNguyJ3v62+uSM?=
 =?us-ascii?Q?sYgwmQ6WcYrvacFJoN0JpnOnFBr5pkNNN4TYgJTnzXyKWVN0x1wFzPBUSxAe?=
 =?us-ascii?Q?0tOBZQemGSi5XpdBg7dSgB4=3D?=
X-OriginatorOrg: Nvidia.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 488501ea-0338-46f3-9705-08d9bafc11bc
X-MS-Exchange-CrossTenant-AuthSource: DM4PR12MB5373.namprd12.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 09 Dec 2021 10:10:04.4042
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 43083d15-7273-40c1-b7db-39efd9ccc17a
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: EUsbCcbe1/PbGO6M8hMLTMbi8pFVgodgXHTIva8l40n5GsAlFuFISjKN9O15QzPnEhuTgUv9BNWVozog/XMjSg==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM4PR12MB5311
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Event EQ is an EQ which received the notification of almost all the
events generated by the NIC.
Currently, each event EQ is taking 512KB of memory. This size is not
needed in most use cases, and is critical with large scale. Hence,
allow user to configure the size of the event EQ.

For example to reduce event EQ size to 64, execute::
$ devlink dev param set pci/0000:00:0b.0 name event_eq_size value 64 \
              cmode driverinit
$ devlink dev reload pci/0000:00:0b.0

Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
---
 Documentation/networking/devlink/mlx5.rst        |  3 +++
 .../net/ethernet/mellanox/mlx5/core/devlink.c    |  7 +++++++
 drivers/net/ethernet/mellanox/mlx5/core/eq.c     | 16 +++++++++++++++-
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/Documentation/networking/devlink/mlx5.rst b/Documentation/networking/devlink/mlx5.rst
index 291e7f63af73..38089f0aefcf 100644
--- a/Documentation/networking/devlink/mlx5.rst
+++ b/Documentation/networking/devlink/mlx5.rst
@@ -20,6 +20,9 @@ Parameters
    * - ``io_eq_size``
      - driverinit
      - The range is between 64 and 4096.
+   * - ``event_eq_size``
+     - driverinit
+     - The range is between 64 and 4096.
 
 The ``mlx5`` driver also implements the following driver-specific
 parameters.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index 177c6e9159f8..37b7600c5545 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -579,6 +579,8 @@ static const struct devlink_param mlx5_devlink_params[] = {
 			      mlx5_devlink_enable_remote_dev_reset_set, NULL),
 	DEVLINK_PARAM_GENERIC(IO_EQ_SIZE, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
 			      NULL, NULL, mlx5_devlink_eq_depth_validate),
+	DEVLINK_PARAM_GENERIC(EVENT_EQ_SIZE, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
+			      NULL, NULL, mlx5_devlink_eq_depth_validate),
 };
 
 static void mlx5_devlink_set_params_init_values(struct devlink *devlink)
@@ -622,6 +624,11 @@ static void mlx5_devlink_set_params_init_values(struct devlink *devlink)
 	devlink_param_driverinit_value_set(devlink,
 					   DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
 					   value);
+
+	value.vu32 = MLX5_NUM_ASYNC_EQE;
+	devlink_param_driverinit_value_set(devlink,
+					   DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
+					   value);
 }
 
 static const struct devlink_param enable_eth_param =
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 7686d7c9c824..b695aad71ee1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -623,6 +623,20 @@ static void cleanup_async_eq(struct mlx5_core_dev *dev,
 			      name, err);
 }
 
+static u16 async_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
+{
+	struct devlink *devlink = priv_to_devlink(dev);
+	union devlink_param_value val;
+	int err;
+
+	err = devlink_param_driverinit_value_get(devlink,
+						 DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
+						 &val);
+	if (!err)
+		return val.vu32;
+	mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
+	return MLX5_NUM_ASYNC_EQE;
+}
 static int create_async_eqs(struct mlx5_core_dev *dev)
 {
 	struct mlx5_eq_table *table = dev->priv.eq_table;
@@ -647,7 +661,7 @@ static int create_async_eqs(struct mlx5_core_dev *dev)
 
 	param = (struct mlx5_eq_param) {
 		.irq_index = MLX5_IRQ_EQ_CTRL,
-		.nent = MLX5_NUM_ASYNC_EQE,
+		.nent = async_eq_depth_devlink_param_get(dev),
 	};
 
 	gather_async_events_mask(dev, param.mask);
-- 
2.21.3