From: Shay Drory Date: Thu, 12 Aug 2021 08:53:34 +0000 (+0300) Subject: net/mlx5: Let user configure io_eq_size param X-Git-Tag: v5.16-rc1~159^2~85^2~4 X-Git-Url: http://git.osdn.net/view?a=commitdiff_plain;h=46ae40b94d8826591472798114a723cc7feac7a7;p=tomoyo%2Ftomoyo-test1.git net/mlx5: Let user configure io_eq_size param Currently, each I/O EQ is taking 128KB of memory. This size is not needed in all use cases, and is critical with large scale. Hence, allow user to configure the size of I/O EQs. For example, to reduce I/O EQ size to 64, execute: $ devlink resource set pci/0000:00:0b.0 path /io_eq_size/ size 64 $ devlink dev reload pci/0000:00:0b.0 Signed-off-by: Shay Drory Reviewed-by: Moshe Shemesh Reviewed-by: Parav Pandit Signed-off-by: Saeed Mahameed --- diff --git a/Documentation/networking/devlink/mlx5.rst b/Documentation/networking/devlink/mlx5.rst index 4e4b97f7971a..4e6020570292 100644 --- a/Documentation/networking/devlink/mlx5.rst +++ b/Documentation/networking/devlink/mlx5.rst @@ -46,6 +46,18 @@ parameters. The ``mlx5`` driver supports reloading via ``DEVLINK_CMD_RELOAD`` +Resources +========= + +.. list-table:: Driver-specific resources implemented + :widths: 5 5 5 85 + + * - Name + - Description + * - ``comp_eq_size`` + - Control the size of I/O completion EQs. + * The default value is 1024, and the range is between 64 and 4096. + Info versions ============= diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index bdb271b604d9..79c15ee62cde 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -16,7 +16,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \ fs_counters.o fs_ft_pool.o rl.o lag/lag.o dev.o events.o wq.o lib/gid.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ - diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \ + diag/fw_tracer.o diag/crdump.o devlink.o devlink_res.o diag/rsc_dump.o \ fw_reset.o qos.o lib/tout.o # diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h index 30bf4882779b..4192f23b1446 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h @@ -6,6 +6,13 @@ #include +enum mlx5_devlink_resource_id { + MLX5_DL_RES_COMP_EQ = 1, + + __MLX5_ID_RES_MAX, + MLX5_ID_RES_MAX = __MLX5_ID_RES_MAX - 1, +}; + enum mlx5_devlink_param_id { MLX5_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, MLX5_DEVLINK_PARAM_ID_FLOW_STEERING_MODE, @@ -31,6 +38,10 @@ int mlx5_devlink_trap_get_num_active(struct mlx5_core_dev *dev); int mlx5_devlink_traps_get_action(struct mlx5_core_dev *dev, int trap_id, enum devlink_trap_action *action); +void mlx5_devlink_res_register(struct mlx5_core_dev *dev); +void mlx5_devlink_res_unregister(struct mlx5_core_dev *dev); +size_t mlx5_devlink_res_size(struct mlx5_core_dev *dev, enum mlx5_devlink_resource_id id); + struct devlink *mlx5_devlink_alloc(struct device *dev); void mlx5_devlink_free(struct devlink *devlink); int mlx5_devlink_register(struct devlink *devlink); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c new file mode 100644 index 000000000000..3beedfb8534a --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink_res.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. */ + +#include "devlink.h" +#include "mlx5_core.h" + +enum { + MLX5_EQ_MIN_SIZE = 64, + MLX5_EQ_MAX_SIZE = 4096, + MLX5_COMP_EQ_SIZE = 1024, +}; + +static int comp_eq_res_register(struct mlx5_core_dev *dev) +{ + struct devlink_resource_size_params comp_eq_size; + struct devlink *devlink = priv_to_devlink(dev); + + devlink_resource_size_params_init(&comp_eq_size, MLX5_EQ_MIN_SIZE, + MLX5_EQ_MAX_SIZE, 1, DEVLINK_RESOURCE_UNIT_ENTRY); + return devlink_resource_register(devlink, "io_eq_size", MLX5_COMP_EQ_SIZE, + MLX5_DL_RES_COMP_EQ, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &comp_eq_size); +} + +void mlx5_devlink_res_register(struct mlx5_core_dev *dev) +{ + int err; + + err = comp_eq_res_register(dev); + if (err) + mlx5_core_err(dev, "Failed to register resources, err = %d\n", err); +} + +void mlx5_devlink_res_unregister(struct mlx5_core_dev *dev) +{ + devlink_resources_unregister(priv_to_devlink(dev), NULL); +} + +static const size_t default_vals[MLX5_ID_RES_MAX + 1] = { + [MLX5_DL_RES_COMP_EQ] = MLX5_COMP_EQ_SIZE, +}; + +size_t mlx5_devlink_res_size(struct mlx5_core_dev *dev, enum mlx5_devlink_resource_id id) +{ + struct devlink *devlink = priv_to_devlink(dev); + u64 size; + int err; + + err = devlink_resource_size_get(devlink, id, &size); + if (!err) + return size; + mlx5_core_err(dev, "Failed to get param. using default. err = %d, id = %u\n", + err, id); + return default_vals[id]; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c index 792e0d6aa861..4dda6e2a4cbc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c @@ -19,6 +19,7 @@ #include "lib/clock.h" #include "diag/fw_tracer.h" #include "mlx5_irq.h" +#include "devlink.h" enum { MLX5_EQE_OWNER_INIT_VAL = 0x1, @@ -807,7 +808,7 @@ static int create_comp_eqs(struct mlx5_core_dev *dev) INIT_LIST_HEAD(&table->comp_eqs_list); ncomp_eqs = table->num_comp_eqs; - nent = MLX5_COMP_EQ_SIZE; + nent = mlx5_devlink_res_size(dev, MLX5_DL_RES_COMP_EQ); for (i = 0; i < ncomp_eqs; i++) { struct mlx5_eq_param param = {}; int vecidx = i; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index f8446395163a..96fdbc0c87bf 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -922,6 +922,8 @@ static int mlx5_init_once(struct mlx5_core_dev *dev) dev->hv_vhca = mlx5_hv_vhca_create(dev); dev->rsc_dump = mlx5_rsc_dump_create(dev); + mlx5_devlink_res_register(dev); + return 0; err_sf_table_cleanup: @@ -957,6 +959,7 @@ err_devcom: static void mlx5_cleanup_once(struct mlx5_core_dev *dev) { + mlx5_devlink_res_unregister(dev); mlx5_rsc_dump_destroy(dev); mlx5_hv_vhca_destroy(dev->hv_vhca); mlx5_fw_tracer_destroy(dev->tracer); diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index f617dfbcd9fd..47c07f95bbe1 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -798,10 +798,6 @@ struct mlx5_db { }; enum { - MLX5_COMP_EQ_SIZE = 1024, -}; - -enum { MLX5_PTYS_IB = 1 << 0, MLX5_PTYS_EN = 1 << 2, };