net/mlx5: Introduce inter-device communication mechanism

This introduces devcom, a generic mechanism for performing operations
on both physical functions of the same Connect-X card.

The first user of this API is merged eswitch, which will be introduced
in subsequent patches.

Signed-off-by: Aviv Heller <avivh@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index 9678051..9de9aba 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -15,7 +15,7 @@ mlx5_core-y :=        main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
                health.o mcg.o cq.o alloc.o qp.o port.o mr.o pd.o \
                mad.o transobj.o vport.o sriov.o fs_cmd.o fs_core.o \
                fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \
-               diag/fs_tracepoint.o diag/fw_tracer.o
+               lib/devcom.o diag/fs_tracepoint.o diag/fw_tracer.o
 
 #
 # Netdev basic

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
new file mode 100644 (file)
index 0000000..bced2ef
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2018 Mellanox Technologies */
+
+#include <linux/mlx5/vport.h>
+#include "lib/devcom.h"
+
+static LIST_HEAD(devcom_list);
+
+#define devcom_for_each_component(priv, comp, iter) \
+       for (iter = 0; \
+            comp = &(priv)->components[iter], iter < MLX5_DEVCOM_NUM_COMPONENTS; \
+            iter++)
+
+struct mlx5_devcom_component {
+       struct {
+               void *data;
+       } device[MLX5_MAX_PORTS];
+
+       mlx5_devcom_event_handler_t handler;
+       struct rw_semaphore sem;
+       bool paired;
+};
+
+struct mlx5_devcom_list {
+       struct list_head list;
+
+       struct mlx5_devcom_component components[MLX5_DEVCOM_NUM_COMPONENTS];
+       struct mlx5_core_dev *devs[MLX5_MAX_PORTS];
+};
+
+struct mlx5_devcom {
+       struct mlx5_devcom_list *priv;
+       int idx;
+};
+
+static struct mlx5_devcom_list *mlx5_devcom_list_alloc(void)
+{
+       struct mlx5_devcom_component *comp;
+       struct mlx5_devcom_list *priv;
+       int i;
+
+       priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+       if (!priv)
+               return NULL;
+
+       devcom_for_each_component(priv, comp, i)
+               init_rwsem(&comp->sem);
+
+       return priv;
+}
+
+static struct mlx5_devcom *mlx5_devcom_alloc(struct mlx5_devcom_list *priv,
+                                            u8 idx)
+{
+       struct mlx5_devcom *devcom;
+
+       devcom = kzalloc(sizeof(*devcom), GFP_KERNEL);
+       if (!devcom)
+               return NULL;
+
+       devcom->priv = priv;
+       devcom->idx = idx;
+       return devcom;
+}
+
+/* Must be called with intf_mutex held */
+struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev)
+{
+       struct mlx5_devcom_list *priv = NULL, *iter;
+       struct mlx5_devcom *devcom = NULL;
+       bool new_priv = false;
+       u64 sguid0, sguid1;
+       int idx, i;
+
+       if (!mlx5_core_is_pf(dev))
+               return NULL;
+
+       sguid0 = mlx5_query_nic_system_image_guid(dev);
+       list_for_each_entry(iter, &devcom_list, list) {
+               struct mlx5_core_dev *tmp_dev = NULL;
+
+               idx = -1;
+               for (i = 0; i < MLX5_MAX_PORTS; i++) {
+                       if (iter->devs[i])
+                               tmp_dev = iter->devs[i];
+                       else
+                               idx = i;
+               }
+
+               if (idx == -1)
+                       continue;
+
+               sguid1 = mlx5_query_nic_system_image_guid(tmp_dev);
+               if (sguid0 != sguid1)
+                       continue;
+
+               priv = iter;
+               break;
+       }
+
+       if (!priv) {
+               priv = mlx5_devcom_list_alloc();
+               if (!priv)
+                       return ERR_PTR(-ENOMEM);
+
+               idx = 0;
+               new_priv = true;
+       }
+
+       priv->devs[idx] = dev;
+       devcom = mlx5_devcom_alloc(priv, idx);
+       if (!devcom) {
+               kfree(priv);
+               return ERR_PTR(-ENOMEM);
+       }
+
+       if (new_priv)
+               list_add(&priv->list, &devcom_list);
+
+       return devcom;
+}
+
+/* Must be called with intf_mutex held */
+void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom)
+{
+       struct mlx5_devcom_list *priv;
+       int i;
+
+       if (IS_ERR_OR_NULL(devcom))
+               return;
+
+       priv = devcom->priv;
+       priv->devs[devcom->idx] = NULL;
+
+       kfree(devcom);
+
+       for (i = 0; i < MLX5_MAX_PORTS; i++)
+               if (priv->devs[i])
+                       break;
+
+       if (i != MLX5_MAX_PORTS)
+               return;
+
+       list_del(&priv->list);
+       kfree(priv);
+}
+
+void mlx5_devcom_register_component(struct mlx5_devcom *devcom,
+                                   enum mlx5_devcom_components id,
+                                   mlx5_devcom_event_handler_t handler,
+                                   void *data)
+{
+       struct mlx5_devcom_component *comp;
+
+       if (IS_ERR_OR_NULL(devcom))
+               return;
+
+       WARN_ON(!data);
+
+       comp = &devcom->priv->components[id];
+       down_write(&comp->sem);
+       comp->handler = handler;
+       comp->device[devcom->idx].data = data;
+       up_write(&comp->sem);
+}
+
+void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,
+                                     enum mlx5_devcom_components id)
+{
+       struct mlx5_devcom_component *comp;
+
+       if (IS_ERR_OR_NULL(devcom))
+               return;
+
+       comp = &devcom->priv->components[id];
+       down_write(&comp->sem);
+       comp->device[devcom->idx].data = NULL;
+       up_write(&comp->sem);
+}
+
+int mlx5_devcom_send_event(struct mlx5_devcom *devcom,
+                          enum mlx5_devcom_components id,
+                          int event,
+                          void *event_data)
+{
+       struct mlx5_devcom_component *comp;
+       int err = -ENODEV, i;
+
+       if (IS_ERR_OR_NULL(devcom))
+               return err;
+
+       comp = &devcom->priv->components[id];
+       down_write(&comp->sem);
+       for (i = 0; i < MLX5_MAX_PORTS; i++)
+               if (i != devcom->idx && comp->device[i].data) {
+                       err = comp->handler(event, comp->device[i].data,
+                                           event_data);
+                       break;
+               }
+
+       up_write(&comp->sem);
+       return err;
+}
+
+void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,
+                           enum mlx5_devcom_components id,
+                           bool paired)
+{
+       struct mlx5_devcom_component *comp;
+
+       comp = &devcom->priv->components[id];
+       WARN_ON(!rwsem_is_locked(&comp->sem));
+
+       comp->paired = paired;
+}
+
+bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,
+                          enum mlx5_devcom_components id)
+{
+       if (IS_ERR_OR_NULL(devcom))
+               return false;
+
+       return devcom->priv->components[id].paired;
+}
+
+void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,
+                               enum mlx5_devcom_components id)
+{
+       struct mlx5_devcom_component *comp;
+       int i;
+
+       if (IS_ERR_OR_NULL(devcom))
+               return NULL;
+
+       comp = &devcom->priv->components[id];
+       down_read(&comp->sem);
+       if (!comp->paired) {
+               up_read(&comp->sem);
+               return NULL;
+       }
+
+       for (i = 0; i < MLX5_MAX_PORTS; i++)
+               if (i != devcom->idx)
+                       break;
+
+       return comp->device[i].data;
+}
+
+void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,
+                                  enum mlx5_devcom_components id)
+{
+       struct mlx5_devcom_component *comp = &devcom->priv->components[id];
+
+       up_read(&comp->sem);
+}

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
new file mode 100644 (file)
index 0000000..f2d338b
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2018 Mellanox Technologies */
+
+#ifndef __LIB_MLX5_DEVCOM_H__
+#define __LIB_MLX5_DEVCOM_H__
+
+#include <linux/mlx5/driver.h>
+
+enum mlx5_devcom_components {
+       MLX5_DEVCOM_NUM_COMPONENTS,
+};
+
+typedef int (*mlx5_devcom_event_handler_t)(int event,
+                                          void *my_data,
+                                          void *event_data);
+
+struct mlx5_devcom *mlx5_devcom_register_device(struct mlx5_core_dev *dev);
+void mlx5_devcom_unregister_device(struct mlx5_devcom *devcom);
+
+void mlx5_devcom_register_component(struct mlx5_devcom *devcom,
+                                   enum mlx5_devcom_components id,
+                                   mlx5_devcom_event_handler_t handler,
+                                   void *data);
+void mlx5_devcom_unregister_component(struct mlx5_devcom *devcom,
+                                     enum mlx5_devcom_components id);
+
+int mlx5_devcom_send_event(struct mlx5_devcom *devcom,
+                          enum mlx5_devcom_components id,
+                          int event,
+                          void *event_data);
+
+void mlx5_devcom_set_paired(struct mlx5_devcom *devcom,
+                           enum mlx5_devcom_components id,
+                           bool paired);
+bool mlx5_devcom_is_paired(struct mlx5_devcom *devcom,
+                          enum mlx5_devcom_components id);
+
+void *mlx5_devcom_get_peer_data(struct mlx5_devcom *devcom,
+                               enum mlx5_devcom_components id);
+void mlx5_devcom_release_peer_data(struct mlx5_devcom *devcom,
+                                  enum mlx5_devcom_components id);
+
+#endif
+

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 7789955..c235531 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -63,6 +63,7 @@
 #include "accel/tls.h"
 #include "lib/clock.h"
 #include "lib/vxlan.h"
+#include "lib/devcom.h"
 #include "diag/fw_tracer.h"
 
 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
@@ -722,16 +723,21 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
        struct pci_dev *pdev = dev->pdev;
        int err;
 
+       priv->devcom = mlx5_devcom_register_device(dev);
+       if (IS_ERR(priv->devcom))
+               dev_err(&pdev->dev, "failed to register with devcom (0x%p)\n",
+                       priv->devcom);
+
        err = mlx5_query_board_id(dev);
        if (err) {
                dev_err(&pdev->dev, "query board id failed\n");
-               goto out;
+               goto err_devcom;
        }
 
        err = mlx5_eq_table_init(dev);
        if (err) {
                dev_err(&pdev->dev, "failed to initialize eq\n");
-               goto out;
+               goto err_devcom;
        }
 
        err = mlx5_events_init(dev);
@@ -807,8 +813,9 @@ err_events_cleanup:
        mlx5_events_cleanup(dev);
 err_eq_cleanup:
        mlx5_eq_table_cleanup(dev);
+err_devcom:
+       mlx5_devcom_unregister_device(dev->priv.devcom);
 
-out:
        return err;
 }
 
@@ -828,6 +835,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
        mlx5_cq_debugfs_cleanup(dev);
        mlx5_events_cleanup(dev);
        mlx5_eq_table_cleanup(dev);
+       mlx5_devcom_unregister_device(dev->priv.devcom);
 }
 
 static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index cc29e88..cd7af5d 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -486,6 +486,7 @@ struct mlx5_events;
 struct mlx5_mpfs;
 struct mlx5_eswitch;
 struct mlx5_lag;
+struct mlx5_devcom;
 struct mlx5_eq_table;
 
 struct mlx5_rate_limit {
@@ -560,6 +561,7 @@ struct mlx5_priv {
        struct mlx5_eswitch     *eswitch;
        struct mlx5_core_sriov  sriov;
        struct mlx5_lag         *lag;
+       struct mlx5_devcom      *devcom;
        unsigned long           pci_dev_data;
        struct mlx5_fc_stats            fc_stats;
        struct mlx5_rl_table            rl_table;