From:   Shay Drory <shayd@nvidia.com>
To:     "David S . Miller" <davem@davemloft.net>,
        Jakub Kicinski <kuba@kernel.org>
Cc:     jiri@nvidia.com, saeedm@nvidia.com, netdev@vger.kernel.org,
        linux-kernel@vger.kernel.org, Shay Drory <shayd@nvidia.com>,
        Moshe Shemesh <moshe@nvidia.com>,
        Parav Pandit <parav@nvidia.com>
Subject: [PATCH net-next v3 7/7] net/mlx5: Let user configure max_macs generic param
Date:   Wed,  8 Dec 2021 16:17:22 +0200
Message-Id: <20211208141722.13646-8-shayd@nvidia.com>
In-Reply-To: <20211208141722.13646-1-shayd@nvidia.com>
References: <20211208141722.13646-1-shayd@nvidia.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?X1R71b6gNuOFxy4SF08zMSGqfN/U0VSL4bDvYLVLSosBmYPO15odHjCdGBgY?=
 =?us-ascii?Q?jwd8CdyQv+52XmoqQanxZHHC6TAg4T4LHyn7IgaCBUfx7bkRIqykJyS18J3M?=
 =?us-ascii?Q?HjsPVO5VVp/PMywTKPzquVigGfQbbOlutqghSDlO0Ps+blldDPHIGCsU25mY?=
 =?us-ascii?Q?ONIzCQq2KgPTi/VwzNbNxVumHWz1Z/w7kRZ7G9Z1X8OvycxEV7uD3srw0mGB?=
 =?us-ascii?Q?wqPmtUoQHikgcMl1miEhT51j6IZjJSjJj9FIBU2nZxBzrd6kXtchXg6nx+dv?=
 =?us-ascii?Q?M5Mvwq765JslCtKCTpkV+b4I2pzCMo6gwW2sVS8MLdGnKsna/xqLMg7oe3Ax?=
 =?us-ascii?Q?9Mll1r3AGunK2tI34M0xiX1hh+Rh+LKVqfIuURX/f8351cmyLrPxQbClRKxB?=
 =?us-ascii?Q?xTGl1FhcYW9LkejZ/zjmOAz2/T1ktibxoPFEHBrEinSgGEsxQk9cjIZXCzCt?=
 =?us-ascii?Q?N5wykpmInfHruD3zeR7Nrx0jeTQkKLn0mggQcHuNHpbyeqNTfGBaulY892Q3?=
 =?us-ascii?Q?EWBbU0PMpInFRzWIDbTbGQIzn6Zdaa16Bfr1x2PyS/JSqC4uBDdjpN6LEbkH?=
 =?us-ascii?Q?96SsSos5Pjc/+CM6sMd6VUItza/IakR/PeAwRYPwxmUgwckRbCTtezxHUhRF?=
 =?us-ascii?Q?3EDIShhGBoV5DrfNHs2UwUXLanhQYTnLZy7xPK+D4pc44nxV2jiv92fpTKQY?=
 =?us-ascii?Q?4tK6/cTFcbNMEYFr6zLceMEf4cCDK+8Fnmdfl+N2qwzC2lZcLi5ipdAVoqUm?=
 =?us-ascii?Q?aE4ZhNJljsxTCSOL6GXM+75x3hu6VP4+S55yCbqul0LjSKD4VCqUTrWja+Bc?=
 =?us-ascii?Q?p5MnZpHOBBd/WWlbh7FW/k9kRG4IC/O++0kQq5J7khynTS8hQLd/XKQe0jMU?=
 =?us-ascii?Q?rUAA3XeMpedXGE7ukd67ck5UVVVMEJakbyNErSgsTQQuYXgnLeyY6nGe4T+G?=
 =?us-ascii?Q?f0W6XrM4wxfSySkTo8zZrpB//6BuZdCEOAKshzt7FDAWM3w1GbIM/ykXOXzX?=
 =?us-ascii?Q?Abupdz972oEVnqeYhXYvMrvDs6AQuXP3dSqgwtiiNuI9VZ2/U7X0cEr7Z6pc?=
 =?us-ascii?Q?hheqUEf/ssjxaJHHEY7HiJZmZNJp/gDA/CFyFnVu8H3DsbfwbVW+r4pkurpN?=
 =?us-ascii?Q?TLjS7Nt/DfGazsb+ZYmLM43QSyldSgDWh+7OkRdYZcGX08wRbPUqNhSmXh7m?=
 =?us-ascii?Q?HcOQvVoq6sqE8N2rUGF3bE/Z7VvSxvLMuw+bC4k3NCOY1qx2LuSUnSdurVNp?=
 =?us-ascii?Q?/YdK3R8iHgIP6vGRlXN+A9j1nJcGznLkX3SfyqQvmPRoHucjl6GMBkla+XIh?=
 =?us-ascii?Q?TJ2rA4qcXgm/RCsAwaYysuf6zRaWlDmalSPQJ3V0cw+3ASSJzfE/pCCGxwi/?=
 =?us-ascii?Q?pLrYQZUXuf0Qc7A3wUZjIZ+n6hy92DyjymaqukjR7j/ixGA7FxfBvEox7Dn7?=
 =?us-ascii?Q?bNb3T1dSRMTsn8aLcZg7eyB4SoigFGMbpo7O9DhpinYvH48nr3jWswQNC9HO?=
 =?us-ascii?Q?d/SBraVUbPS/6XNWZsf2yt7OaQckxVzu767aHYW/OXo366zJhic94y9nzsfp?=
 =?us-ascii?Q?z+CyQCDw5K1yVNe3LS4mxuJ/6uJY32ZgJn5ywf1IJmVTWO2Lk0/3JrFQgspl?=
 =?us-ascii?Q?64lufhLymunzW0ObHRJHH+A=3D?=
X-OriginatorOrg: Nvidia.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 80a01573-d14a-48ec-f0cf-08d9ba55890e
X-MS-Exchange-CrossTenant-AuthSource: DM4PR12MB5373.namprd12.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 08 Dec 2021 14:17:58.6041
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 43083d15-7273-40c1-b7db-39efd9ccc17a
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: pUnjnUg5tJixRJ7Qxh/n+sDVgvo83UDmY31jvVA+uwz8LoRwuzEZB1KSE+Z7IcBTPDvSjP9mVxf5CNNjx34Rdw==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM6PR12MB5568
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Currently, max_macs is taking 70Kbytes of memory per function. This
size is not needed in all use cases, and is critical with large scale.
Hence, allow user to configure the number of max_macs.

For example, to reduce the number of max_macs to 1, execute::
$ devlink dev param set pci/0000:00:0b.0 name max_macs value 1 \
              cmode driverinit
$ devlink dev reload pci/0000:00:0b.0

Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Moshe Shemesh <moshe@nvidia.com>
Reviewed-by: Parav Pandit <parav@nvidia.com>
---
 Documentation/networking/devlink/mlx5.rst     |  3 +
 .../net/ethernet/mellanox/mlx5/core/devlink.c | 67 +++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/main.c    | 21 ++++++
 3 files changed, 91 insertions(+)

diff --git a/Documentation/networking/devlink/mlx5.rst b/Documentation/networking/devlink/mlx5.rst
index 38089f0aefcf..38e94ed65936 100644
--- a/Documentation/networking/devlink/mlx5.rst
+++ b/Documentation/networking/devlink/mlx5.rst
@@ -23,6 +23,9 @@ Parameters
    * - ``event_eq_size``
      - driverinit
      - The range is between 64 and 4096.
+   * - ``max_macs``
+     - driverinit
+     - The range is between 1 and 2^31. Only power of 2 values are supported.
 
 The ``mlx5`` driver also implements the following driver-specific
 parameters.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index 37b7600c5545..d1093bb2d436 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -773,6 +773,66 @@ static void mlx5_devlink_auxdev_params_unregister(struct devlink *devlink)
 	mlx5_devlink_eth_param_unregister(devlink);
 }
 
+static int mlx5_devlink_max_uc_list_validate(struct devlink *devlink, u32 id,
+					     union devlink_param_value val,
+					     struct netlink_ext_ack *extack)
+{
+	struct mlx5_core_dev *dev = devlink_priv(devlink);
+
+	if (val.vu32 == 0) {
+		NL_SET_ERR_MSG_MOD(extack, "max_macs value must be greater than 0");
+		return -EINVAL;
+	}
+
+	if (!is_power_of_2(val.vu32)) {
+		NL_SET_ERR_MSG_MOD(extack, "Only power of 2 values are supported for max_macs");
+		return -EINVAL;
+	}
+
+	if (ilog2(val.vu32) >
+	    MLX5_CAP_GEN_MAX(dev, log_max_current_uc_list)) {
+		NL_SET_ERR_MSG_MOD(extack, "max_macs value is out of the supported range");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct devlink_param max_uc_list_param =
+	DEVLINK_PARAM_GENERIC(MAX_MACS, BIT(DEVLINK_PARAM_CMODE_DRIVERINIT),
+			      NULL, NULL, mlx5_devlink_max_uc_list_validate);
+
+static int mlx5_devlink_max_uc_list_param_register(struct devlink *devlink)
+{
+	struct mlx5_core_dev *dev = devlink_priv(devlink);
+	union devlink_param_value value;
+	int err;
+
+	if (!MLX5_CAP_GEN_MAX(dev, log_max_current_uc_list_wr_supported))
+		return 0;
+
+	err = devlink_param_register(devlink, &max_uc_list_param);
+	if (err)
+		return err;
+
+	value.vu32 = 1 << MLX5_CAP_GEN(dev, log_max_current_uc_list);
+	devlink_param_driverinit_value_set(devlink,
+					   DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
+					   value);
+	return 0;
+}
+
+static void
+mlx5_devlink_max_uc_list_param_unregister(struct devlink *devlink)
+{
+	struct mlx5_core_dev *dev = devlink_priv(devlink);
+
+	if (!MLX5_CAP_GEN_MAX(dev, log_max_current_uc_list_wr_supported))
+		return;
+
+	devlink_param_unregister(devlink, &max_uc_list_param);
+}
+
 #define MLX5_TRAP_DROP(_id, _group_id)					\
 	DEVLINK_TRAP_GENERIC(DROP, DROP, _id,				\
 			     DEVLINK_TRAP_GROUP_GENERIC_ID_##_group_id, \
@@ -832,6 +892,10 @@ int mlx5_devlink_register(struct devlink *devlink)
 	if (err)
 		goto auxdev_reg_err;
 
+	err = mlx5_devlink_max_uc_list_param_register(devlink);
+	if (err)
+		goto max_uc_list_err;
+
 	err = mlx5_devlink_traps_register(devlink);
 	if (err)
 		goto traps_reg_err;
@@ -842,6 +906,8 @@ int mlx5_devlink_register(struct devlink *devlink)
 	return 0;
 
 traps_reg_err:
+	mlx5_devlink_max_uc_list_param_unregister(devlink);
+max_uc_list_err:
 	mlx5_devlink_auxdev_params_unregister(devlink);
 auxdev_reg_err:
 	devlink_params_unregister(devlink, mlx5_devlink_params,
@@ -852,6 +918,7 @@ int mlx5_devlink_register(struct devlink *devlink)
 void mlx5_devlink_unregister(struct devlink *devlink)
 {
 	mlx5_devlink_traps_unregister(devlink);
+	mlx5_devlink_max_uc_list_param_unregister(devlink);
 	mlx5_devlink_auxdev_params_unregister(devlink);
 	devlink_params_unregister(devlink, mlx5_devlink_params,
 				  ARRAY_SIZE(mlx5_devlink_params));
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index d97c9e86d7b3..b1a82226623c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -484,10 +484,26 @@ static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx)
 	return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ODP);
 }
 
+static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev)
+{
+	struct devlink *devlink = priv_to_devlink(dev);
+	union devlink_param_value val;
+	int err;
+
+	err = devlink_param_driverinit_value_get(devlink,
+						 DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
+						 &val);
+	if (!err)
+		return val.vu32;
+	mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err);
+	return err;
+}
+
 static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
 {
 	struct mlx5_profile *prof = &dev->profile;
 	void *set_hca_cap;
+	int max_uc_list;
 	int err;
 
 	err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
@@ -561,6 +577,11 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
 	if (MLX5_CAP_GEN(dev, roce_rw_supported))
 		MLX5_SET(cmd_hca_cap, set_hca_cap, roce, mlx5_is_roce_init_enabled(dev));
 
+	max_uc_list = max_uc_list_get_devlink_param(dev);
+	if (max_uc_list > 0)
+		MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_current_uc_list,
+			 ilog2(max_uc_list));
+
 	return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
 }
 
-- 
2.21.3