Merge tag 'mlx5-updates-2023-03-20' of git://git.kernel.org/pub/scm/linux/kernel...

Saeed Mahameed says:

====================
mlx5-updates-2023-03-20

mlx5 dynamic msix

This patch series adds support for dynamic msix vectors allocation in mlx5.

Eli Cohen Says:

================

The following series of patches modifies mlx5_core to work with the
dynamic MSIX API. Currently, mlx5_core allocates all the interrupt
vectors it needs and distributes them amongst the consumers. With the
introduction of dynamic MSIX support, which allows for allocation of
interrupts more than once, we now allocate vectors as we need them.
This allows other drivers running on top of mlx5_core to allocate
interrupt vectors for their own use. An example for this is mlx5_vdpa,
which uses these vectors to propagate interrupts directly from the
hardware to the vCPU [1].

As a preparation for using this series, a use after free issue is fixed
in lib/cpu_rmap.c and the allocator for rmap entries has been modified.
A complementary API for irq_cpu_rmap_add() has also been introduced.

[1] https://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux.git/patch/?id=0f2bf1fcae96a83b8c5581854713c9fc3407556e

================

* tag 'mlx5-updates-2023-03-20' of git://git.kernel.org/pub/scm/linux/kernel/git/saeed/linux:
  net/mlx5: Provide external API for allocating vectors
  net/mlx5: Use one completion vector if eth is disabled
  net/mlx5: Refactor calculation of required completion vectors
  net/mlx5: Move devlink registration before mlx5_load
  net/mlx5: Use dynamic msix vectors allocation
  net/mlx5: Refactor completion irq request/release code
  net/mlx5: Improve naming of pci function vectors
  net/mlx5: Use newer affinity descriptor
  net/mlx5: Modify struct mlx5_irq to use struct msi_map
  net/mlx5: Fix wrong comment
  net/mlx5e: Coding style fix, add empty line
  lib: cpu_rmap: Add irq_cpu_rmap_remove to complement irq_cpu_rmap_add
  lib: cpu_rmap: Use allocator for rmap entries
  lib: cpu_rmap: Avoid use after free on rmap->obj array entries
====================

Link: https://lore.kernel.org/r/20230324231341.29808-1-saeed@kernel.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --combined drivers/net/ethernet/mellanox/mlx5/core/devlink.h
index 5dcfb4d,f2601f9..defba5b
--- 1/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
--- 2/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h
@@@ -12,8 -12,6 +12,8 @@@ enum mlx5_devlink_param_id 
        MLX5_DEVLINK_PARAM_ID_ESW_LARGE_GROUP_NUM,
        MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA,
        MLX5_DEVLINK_PARAM_ID_ESW_MULTIPORT,
 +      MLX5_DEVLINK_PARAM_ID_HAIRPIN_NUM_QUEUES,
 +      MLX5_DEVLINK_PARAM_ID_HAIRPIN_QUEUE_SIZE,
  };
  
  struct mlx5_trap_ctx {
@@@ -46,4 -44,15 +46,15 @@@ void mlx5_devlink_free(struct devlink *
  int mlx5_devlink_params_register(struct devlink *devlink);
  void mlx5_devlink_params_unregister(struct devlink *devlink);
  
+ static inline bool mlx5_core_is_eth_enabled(struct mlx5_core_dev *dev)
+ {
+       union devlink_param_value val;
+       int err;
+ 
+       err = devl_param_driverinit_value_get(priv_to_devlink(dev),
+                                             DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH,
+                                             &val);
+       return err ? false : val.vbool;
+ }
+ 
  #endif /* __MLX5_DEVLINK_H__ */

diff --combined drivers/net/ethernet/mellanox/mlx5/core/main.c
index 597174c,fb9ac7d..f95df73
--- 1/drivers/net/ethernet/mellanox/mlx5/core/main.c
--- 2/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@@ -46,13 -46,9 +46,10 @@@
  #include <linux/kmod.h>
  #include <linux/mlx5/mlx5_ifc.h>
  #include <linux/mlx5/vport.h>
- #ifdef CONFIG_RFS_ACCEL
- #include <linux/cpu_rmap.h>
- #endif
  #include <linux/version.h>
  #include <net/devlink.h>
  #include "mlx5_core.h"
 +#include "thermal.h"
  #include "lib/eq.h"
  #include "fs_core.h"
  #include "lib/mpfs.h"
@@@ -192,7 -188,7 +189,7 @@@ static int wait_fw_init(struct mlx5_cor
                if (!(fw_initializing >> 31))
                        break;
                if (time_after(jiffies, end) ||
 -                  test_and_clear_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
 +                  test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
                        err = -EBUSY;
                        break;
                }
@@@ -918,6 -914,7 +915,6 @@@ static int mlx5_pci_init(struct mlx5_co
        return 0;
  
  err_clr_master:
 -      pci_clear_master(dev->pdev);
        release_bar(dev->pdev);
  err_disable:
        mlx5_pci_disable_device(dev);
@@@ -932,6 -929,7 +929,6 @@@ static void mlx5_pci_close(struct mlx5_
         */
        mlx5_drain_health_wq(dev);
        iounmap(dev->iseg);
 -      pci_clear_master(dev->pdev);
        release_bar(dev->pdev);
        mlx5_pci_disable_device(dev);
  }
@@@ -1401,16 -1399,16 +1398,16 @@@ int mlx5_init_one(struct mlx5_core_dev 
                goto function_teardown;
        }
  
+       err = mlx5_devlink_params_register(priv_to_devlink(dev));
+       if (err)
+               goto err_devlink_params_reg;
+ 
        err = mlx5_load(dev);
        if (err)
                goto err_load;
  
        set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
  
-       err = mlx5_devlink_params_register(priv_to_devlink(dev));
-       if (err)
-               goto err_devlink_params_reg;
- 
        err = mlx5_register_device(dev);
        if (err)
                goto err_register;
@@@ -1420,11 -1418,11 +1417,11 @@@
        return 0;
  
  err_register:
-       mlx5_devlink_params_unregister(priv_to_devlink(dev));
- err_devlink_params_reg:
        clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
        mlx5_unload(dev);
  err_load:
+       mlx5_devlink_params_unregister(priv_to_devlink(dev));
+ err_devlink_params_reg:
        mlx5_cleanup_once(dev);
  function_teardown:
        mlx5_function_teardown(dev, true);
@@@ -1443,7 -1441,6 +1440,6 @@@ void mlx5_uninit_one(struct mlx5_core_d
        mutex_lock(&dev->intf_state_mutex);
  
        mlx5_unregister_device(dev);
-       mlx5_devlink_params_unregister(priv_to_devlink(dev));
  
        if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
                mlx5_core_warn(dev, "%s: interface is down, NOP\n",
@@@ -1454,6 -1451,7 +1450,7 @@@
  
        clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
        mlx5_unload(dev);
+       mlx5_devlink_params_unregister(priv_to_devlink(dev));
        mlx5_cleanup_once(dev);
        mlx5_function_teardown(dev, true);
  out:
@@@ -1767,10 -1765,6 +1764,10 @@@ static int probe_one(struct pci_dev *pd
        if (err)
                dev_err(&pdev->dev, "mlx5_crdump_enable failed with error code %d\n", err);
  
 +      err = mlx5_thermal_init(dev);
 +      if (err)
 +              dev_err(&pdev->dev, "mlx5_thermal_init failed with error code %d\n", err);
 +
        pci_save_state(pdev);
        devlink_register(devlink);
        return 0;
@@@ -1799,7 -1793,6 +1796,7 @@@ static void remove_one(struct pci_dev *
        mlx5_drain_fw_reset(dev);
        devlink_unregister(devlink);
        mlx5_sriov_disable(pdev);
 +      mlx5_thermal_uninit(dev);
        mlx5_crdump_disable(dev);
        mlx5_drain_health_wq(dev);
        mlx5_uninit_one(dev);

diff --combined include/linux/mlx5/driver.h
index 7a89811,df0f821..f243bd1
--- 1/include/linux/mlx5/driver.h
--- 2/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@@ -134,7 -134,6 +134,7 @@@ enum 
        MLX5_REG_PCAM            = 0x507f,
        MLX5_REG_NODE_DESC       = 0x6001,
        MLX5_REG_HOST_ENDIANNESS = 0x7004,
 +      MLX5_REG_MTMP            = 0x900A,
        MLX5_REG_MCIA            = 0x9014,
        MLX5_REG_MFRL            = 0x9028,
        MLX5_REG_MLCR            = 0x902b,
@@@ -732,7 -731,6 +732,7 @@@ struct mlx5_fw_tracer
  struct mlx5_vxlan;
  struct mlx5_geneve;
  struct mlx5_hv_vhca;
 +struct mlx5_thermal;
  
  #define MLX5_LOG_SW_ICM_BLOCK_SIZE(dev) (MLX5_CAP_DEV_MEM(dev, log_sw_icm_alloc_granularity))
  #define MLX5_SW_ICM_BLOCK_SIZE(dev) (1 << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev))
@@@ -810,7 -808,6 +810,7 @@@ struct mlx5_core_dev 
        struct mlx5_rsc_dump    *rsc_dump;
        u32                      vsc_addr;
        struct mlx5_hv_vhca     *hv_vhca;
 +      struct mlx5_thermal     *thermal;
  };
  
  struct mlx5_db {
@@@ -1311,4 -1308,10 +1311,10 @@@ enum 
        MLX5_OCTWORD = 16,
  };
  
+ struct msi_map mlx5_msix_alloc(struct mlx5_core_dev *dev,
+                              irqreturn_t (*handler)(int, void *),
+                              const struct irq_affinity_desc *affdesc,
+                              const char *name);
+ void mlx5_msix_free(struct mlx5_core_dev *dev, struct msi_map map);
+ 
  #endif /* MLX5_DRIVER_H */