drivers/net/ethernet/mellanox/mlxsw/core.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/device.h>
   7 #include <linux/export.h>
   8 #include <linux/err.h>
   9 #include <linux/if_link.h>
  10 #include <linux/netdevice.h>
  11 #include <linux/completion.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/types.h>
  15 #include <linux/string.h>
  16 #include <linux/gfp.h>
  17 #include <linux/random.h>
  18 #include <linux/jiffies.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rcupdate.h>
  21 #include <linux/slab.h>
  22 #include <linux/workqueue.h>
  23 #include <asm/byteorder.h>
  24 #include <net/devlink.h>
  25 #include <trace/events/devlink.h>
  26
  27 #include "core.h"
  28 #include "item.h"
  29 #include "cmd.h"
  30 #include "port.h"
  31 #include "trap.h"
  32 #include "emad.h"
  33 #include "reg.h"
  34 #include "resources.h"
  35
  36 static LIST_HEAD(mlxsw_core_driver_list);
  37 static DEFINE_SPINLOCK(mlxsw_core_driver_list_lock);
  38
  39 static const char mlxsw_core_driver_name[] = "mlxsw_core";
  40
  41 static struct workqueue_struct *mlxsw_wq;
  42 static struct workqueue_struct *mlxsw_owq;
  43
  44 struct mlxsw_core_port {
  45         struct devlink_port devlink_port;
  46         void *port_driver_priv;
  47         u8 local_port;
  48 };
  49
  50 void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port)
  51 {
  52         return mlxsw_core_port->port_driver_priv;
  53 }
  54 EXPORT_SYMBOL(mlxsw_core_port_driver_priv);
  55
  56 static bool mlxsw_core_port_check(struct mlxsw_core_port *mlxsw_core_port)
  57 {
  58         return mlxsw_core_port->port_driver_priv != NULL;
  59 }
  60
  61 struct mlxsw_core {
  62         struct mlxsw_driver *driver;
  63         const struct mlxsw_bus *bus;
  64         void *bus_priv;
  65         const struct mlxsw_bus_info *bus_info;
  66         struct workqueue_struct *emad_wq;
  67         struct list_head rx_listener_list;
  68         struct list_head event_listener_list;
  69         struct {
  70                 atomic64_t tid;
  71                 struct list_head trans_list;
  72                 spinlock_t trans_list_lock; /* protects trans_list writes */
  73                 bool use_emad;
  74         } emad;
  75         struct {
  76                 u8 *mapping; /* lag_id+port_index to local_port mapping */
  77         } lag;
  78         struct mlxsw_res res;
  79         struct mlxsw_hwmon *hwmon;
  80         struct mlxsw_thermal *thermal;
  81         struct mlxsw_core_port *ports;
  82         unsigned int max_ports;
  83         bool fw_flash_in_progress;
  84         unsigned long driver_priv[0];
  85         /* driver_priv has to be always the last item */
  86 };
  87
  88 #define MLXSW_PORT_MAX_PORTS_DEFAULT    0x40
  89
  90 static int mlxsw_ports_init(struct mlxsw_core *mlxsw_core)
  91 {
  92         /* Switch ports are numbered from 1 to queried value */
  93         if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_SYSTEM_PORT))
  94                 mlxsw_core->max_ports = MLXSW_CORE_RES_GET(mlxsw_core,
  95                                                            MAX_SYSTEM_PORT) + 1;
  96         else
  97                 mlxsw_core->max_ports = MLXSW_PORT_MAX_PORTS_DEFAULT + 1;
  98
  99         mlxsw_core->ports = kcalloc(mlxsw_core->max_ports,
 100                                     sizeof(struct mlxsw_core_port), GFP_KERNEL);
 101         if (!mlxsw_core->ports)
 102                 return -ENOMEM;
 103
 104         return 0;
 105 }
 106
 107 static void mlxsw_ports_fini(struct mlxsw_core *mlxsw_core)
 108 {
 109         kfree(mlxsw_core->ports);
 110 }
 111
 112 unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core)
 113 {
 114         return mlxsw_core->max_ports;
 115 }
 116 EXPORT_SYMBOL(mlxsw_core_max_ports);
 117
 118 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
 119 {
 120         return mlxsw_core->driver_priv;
 121 }
 122 EXPORT_SYMBOL(mlxsw_core_driver_priv);
 123
 124 bool mlxsw_core_res_query_enabled(const struct mlxsw_core *mlxsw_core)
 125 {
 126         return mlxsw_core->driver->res_query_enabled;
 127 }
 128 EXPORT_SYMBOL(mlxsw_core_res_query_enabled);
 129
 130 bool
 131 mlxsw_core_fw_rev_minor_subminor_validate(const struct mlxsw_fw_rev *rev,
 132                                           const struct mlxsw_fw_rev *req_rev)
 133 {
 134         return rev->minor > req_rev->minor ||
 135                (rev->minor == req_rev->minor &&
 136                 rev->subminor >= req_rev->subminor);
 137 }
 138 EXPORT_SYMBOL(mlxsw_core_fw_rev_minor_subminor_validate);
 139
 140 struct mlxsw_rx_listener_item {
 141         struct list_head list;
 142         struct mlxsw_rx_listener rxl;
 143         void *priv;
 144 };
 145
 146 struct mlxsw_event_listener_item {
 147         struct list_head list;
 148         struct mlxsw_event_listener el;
 149         void *priv;
 150 };
 151
 152 /******************
 153  * EMAD processing
 154  ******************/
 155
 156 /* emad_eth_hdr_dmac
 157  * Destination MAC in EMAD's Ethernet header.
 158  * Must be set to 01:02:c9:00:00:01
 159  */
 160 MLXSW_ITEM_BUF(emad, eth_hdr, dmac, 0x00, 6);
 161
 162 /* emad_eth_hdr_smac
 163  * Source MAC in EMAD's Ethernet header.
 164  * Must be set to 00:02:c9:01:02:03
 165  */
 166 MLXSW_ITEM_BUF(emad, eth_hdr, smac, 0x06, 6);
 167
 168 /* emad_eth_hdr_ethertype
 169  * Ethertype in EMAD's Ethernet header.
 170  * Must be set to 0x8932
 171  */
 172 MLXSW_ITEM32(emad, eth_hdr, ethertype, 0x0C, 16, 16);
 173
 174 /* emad_eth_hdr_mlx_proto
 175  * Mellanox protocol.
 176  * Must be set to 0x0.
 177  */
 178 MLXSW_ITEM32(emad, eth_hdr, mlx_proto, 0x0C, 8, 8);
 179
 180 /* emad_eth_hdr_ver
 181  * Mellanox protocol version.
 182  * Must be set to 0x0.
 183  */
 184 MLXSW_ITEM32(emad, eth_hdr, ver, 0x0C, 4, 4);
 185
 186 /* emad_op_tlv_type
 187  * Type of the TLV.
 188  * Must be set to 0x1 (operation TLV).
 189  */
 190 MLXSW_ITEM32(emad, op_tlv, type, 0x00, 27, 5);
 191
 192 /* emad_op_tlv_len
 193  * Length of the operation TLV in u32.
 194  * Must be set to 0x4.
 195  */
 196 MLXSW_ITEM32(emad, op_tlv, len, 0x00, 16, 11);
 197
 198 /* emad_op_tlv_dr
 199  * Direct route bit. Setting to 1 indicates the EMAD is a direct route
 200  * EMAD. DR TLV must follow.
 201  *
 202  * Note: Currently not supported and must not be set.
 203  */
 204 MLXSW_ITEM32(emad, op_tlv, dr, 0x00, 15, 1);
 205
 206 /* emad_op_tlv_status
 207  * Returned status in case of EMAD response. Must be set to 0 in case
 208  * of EMAD request.
 209  * 0x0 - success
 210  * 0x1 - device is busy. Requester should retry
 211  * 0x2 - Mellanox protocol version not supported
 212  * 0x3 - unknown TLV
 213  * 0x4 - register not supported
 214  * 0x5 - operation class not supported
 215  * 0x6 - EMAD method not supported
 216  * 0x7 - bad parameter (e.g. port out of range)
 217  * 0x8 - resource not available
 218  * 0x9 - message receipt acknowledgment. Requester should retry
 219  * 0x70 - internal error
 220  */
 221 MLXSW_ITEM32(emad, op_tlv, status, 0x00, 8, 7);
 222
 223 /* emad_op_tlv_register_id
 224  * Register ID of register within register TLV.
 225  */
 226 MLXSW_ITEM32(emad, op_tlv, register_id, 0x04, 16, 16);
 227
 228 /* emad_op_tlv_r
 229  * Response bit. Setting to 1 indicates Response, otherwise request.
 230  */
 231 MLXSW_ITEM32(emad, op_tlv, r, 0x04, 15, 1);
 232
 233 /* emad_op_tlv_method
 234  * EMAD method type.
 235  * 0x1 - query
 236  * 0x2 - write
 237  * 0x3 - send (currently not supported)
 238  * 0x4 - event
 239  */
 240 MLXSW_ITEM32(emad, op_tlv, method, 0x04, 8, 7);
 241
 242 /* emad_op_tlv_class
 243  * EMAD operation class. Must be set to 0x1 (REG_ACCESS).
 244  */
 245 MLXSW_ITEM32(emad, op_tlv, class, 0x04, 0, 8);
 246
 247 /* emad_op_tlv_tid
 248  * EMAD transaction ID. Used for pairing request and response EMADs.
 249  */
 250 MLXSW_ITEM64(emad, op_tlv, tid, 0x08, 0, 64);
 251
 252 /* emad_reg_tlv_type
 253  * Type of the TLV.
 254  * Must be set to 0x3 (register TLV).
 255  */
 256 MLXSW_ITEM32(emad, reg_tlv, type, 0x00, 27, 5);
 257
 258 /* emad_reg_tlv_len
 259  * Length of the operation TLV in u32.
 260  */
 261 MLXSW_ITEM32(emad, reg_tlv, len, 0x00, 16, 11);
 262
 263 /* emad_end_tlv_type
 264  * Type of the TLV.
 265  * Must be set to 0x0 (end TLV).
 266  */
 267 MLXSW_ITEM32(emad, end_tlv, type, 0x00, 27, 5);
 268
 269 /* emad_end_tlv_len
 270  * Length of the end TLV in u32.
 271  * Must be set to 1.
 272  */
 273 MLXSW_ITEM32(emad, end_tlv, len, 0x00, 16, 11);
 274
 275 enum mlxsw_core_reg_access_type {
 276         MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
 277         MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
 278 };
 279
 280 static inline const char *
 281 mlxsw_core_reg_access_type_str(enum mlxsw_core_reg_access_type type)
 282 {
 283         switch (type) {
 284         case MLXSW_CORE_REG_ACCESS_TYPE_QUERY:
 285                 return "query";
 286         case MLXSW_CORE_REG_ACCESS_TYPE_WRITE:
 287                 return "write";
 288         }
 289         BUG();
 290 }
 291
 292 static void mlxsw_emad_pack_end_tlv(char *end_tlv)
 293 {
 294         mlxsw_emad_end_tlv_type_set(end_tlv, MLXSW_EMAD_TLV_TYPE_END);
 295         mlxsw_emad_end_tlv_len_set(end_tlv, MLXSW_EMAD_END_TLV_LEN);
 296 }
 297
 298 static void mlxsw_emad_pack_reg_tlv(char *reg_tlv,
 299                                     const struct mlxsw_reg_info *reg,
 300                                     char *payload)
 301 {
 302         mlxsw_emad_reg_tlv_type_set(reg_tlv, MLXSW_EMAD_TLV_TYPE_REG);
 303         mlxsw_emad_reg_tlv_len_set(reg_tlv, reg->len / sizeof(u32) + 1);
 304         memcpy(reg_tlv + sizeof(u32), payload, reg->len);
 305 }
 306
 307 static void mlxsw_emad_pack_op_tlv(char *op_tlv,
 308                                    const struct mlxsw_reg_info *reg,
 309                                    enum mlxsw_core_reg_access_type type,
 310                                    u64 tid)
 311 {
 312         mlxsw_emad_op_tlv_type_set(op_tlv, MLXSW_EMAD_TLV_TYPE_OP);
 313         mlxsw_emad_op_tlv_len_set(op_tlv, MLXSW_EMAD_OP_TLV_LEN);
 314         mlxsw_emad_op_tlv_dr_set(op_tlv, 0);
 315         mlxsw_emad_op_tlv_status_set(op_tlv, 0);
 316         mlxsw_emad_op_tlv_register_id_set(op_tlv, reg->id);
 317         mlxsw_emad_op_tlv_r_set(op_tlv, MLXSW_EMAD_OP_TLV_REQUEST);
 318         if (type == MLXSW_CORE_REG_ACCESS_TYPE_QUERY)
 319                 mlxsw_emad_op_tlv_method_set(op_tlv,
 320                                              MLXSW_EMAD_OP_TLV_METHOD_QUERY);
 321         else
 322                 mlxsw_emad_op_tlv_method_set(op_tlv,
 323                                              MLXSW_EMAD_OP_TLV_METHOD_WRITE);
 324         mlxsw_emad_op_tlv_class_set(op_tlv,
 325                                     MLXSW_EMAD_OP_TLV_CLASS_REG_ACCESS);
 326         mlxsw_emad_op_tlv_tid_set(op_tlv, tid);
 327 }
 328
 329 static int mlxsw_emad_construct_eth_hdr(struct sk_buff *skb)
 330 {
 331         char *eth_hdr = skb_push(skb, MLXSW_EMAD_ETH_HDR_LEN);
 332
 333         mlxsw_emad_eth_hdr_dmac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_DMAC);
 334         mlxsw_emad_eth_hdr_smac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_SMAC);
 335         mlxsw_emad_eth_hdr_ethertype_set(eth_hdr, MLXSW_EMAD_EH_ETHERTYPE);
 336         mlxsw_emad_eth_hdr_mlx_proto_set(eth_hdr, MLXSW_EMAD_EH_MLX_PROTO);
 337         mlxsw_emad_eth_hdr_ver_set(eth_hdr, MLXSW_EMAD_EH_PROTO_VERSION);
 338
 339         skb_reset_mac_header(skb);
 340
 341         return 0;
 342 }
 343
 344 static void mlxsw_emad_construct(struct sk_buff *skb,
 345                                  const struct mlxsw_reg_info *reg,
 346                                  char *payload,
 347                                  enum mlxsw_core_reg_access_type type,
 348                                  u64 tid)
 349 {
 350         char *buf;
 351
 352         buf = skb_push(skb, MLXSW_EMAD_END_TLV_LEN * sizeof(u32));
 353         mlxsw_emad_pack_end_tlv(buf);
 354
 355         buf = skb_push(skb, reg->len + sizeof(u32));
 356         mlxsw_emad_pack_reg_tlv(buf, reg, payload);
 357
 358         buf = skb_push(skb, MLXSW_EMAD_OP_TLV_LEN * sizeof(u32));
 359         mlxsw_emad_pack_op_tlv(buf, reg, type, tid);
 360
 361         mlxsw_emad_construct_eth_hdr(skb);
 362 }
 363
 364 static char *mlxsw_emad_op_tlv(const struct sk_buff *skb)
 365 {
 366         return ((char *) (skb->data + MLXSW_EMAD_ETH_HDR_LEN));
 367 }
 368
 369 static char *mlxsw_emad_reg_tlv(const struct sk_buff *skb)
 370 {
 371         return ((char *) (skb->data + MLXSW_EMAD_ETH_HDR_LEN +
 372                                       MLXSW_EMAD_OP_TLV_LEN * sizeof(u32)));
 373 }
 374
 375 static char *mlxsw_emad_reg_payload(const char *op_tlv)
 376 {
 377         return ((char *) (op_tlv + (MLXSW_EMAD_OP_TLV_LEN + 1) * sizeof(u32)));
 378 }
 379
 380 static u64 mlxsw_emad_get_tid(const struct sk_buff *skb)
 381 {
 382         char *op_tlv;
 383
 384         op_tlv = mlxsw_emad_op_tlv(skb);
 385         return mlxsw_emad_op_tlv_tid_get(op_tlv);
 386 }
 387
 388 static bool mlxsw_emad_is_resp(const struct sk_buff *skb)
 389 {
 390         char *op_tlv;
 391
 392         op_tlv = mlxsw_emad_op_tlv(skb);
 393         return (mlxsw_emad_op_tlv_r_get(op_tlv) == MLXSW_EMAD_OP_TLV_RESPONSE);
 394 }
 395
 396 static int mlxsw_emad_process_status(char *op_tlv,
 397                                      enum mlxsw_emad_op_tlv_status *p_status)
 398 {
 399         *p_status = mlxsw_emad_op_tlv_status_get(op_tlv);
 400
 401         switch (*p_status) {
 402         case MLXSW_EMAD_OP_TLV_STATUS_SUCCESS:
 403                 return 0;
 404         case MLXSW_EMAD_OP_TLV_STATUS_BUSY:
 405         case MLXSW_EMAD_OP_TLV_STATUS_MESSAGE_RECEIPT_ACK:
 406                 return -EAGAIN;
 407         case MLXSW_EMAD_OP_TLV_STATUS_VERSION_NOT_SUPPORTED:
 408         case MLXSW_EMAD_OP_TLV_STATUS_UNKNOWN_TLV:
 409         case MLXSW_EMAD_OP_TLV_STATUS_REGISTER_NOT_SUPPORTED:
 410         case MLXSW_EMAD_OP_TLV_STATUS_CLASS_NOT_SUPPORTED:
 411         case MLXSW_EMAD_OP_TLV_STATUS_METHOD_NOT_SUPPORTED:
 412         case MLXSW_EMAD_OP_TLV_STATUS_BAD_PARAMETER:
 413         case MLXSW_EMAD_OP_TLV_STATUS_RESOURCE_NOT_AVAILABLE:
 414         case MLXSW_EMAD_OP_TLV_STATUS_INTERNAL_ERROR:
 415         default:
 416                 return -EIO;
 417         }
 418 }
 419
 420 static int
 421 mlxsw_emad_process_status_skb(struct sk_buff *skb,
 422                               enum mlxsw_emad_op_tlv_status *p_status)
 423 {
 424         return mlxsw_emad_process_status(mlxsw_emad_op_tlv(skb), p_status);
 425 }
 426
 427 struct mlxsw_reg_trans {
 428         struct list_head list;
 429         struct list_head bulk_list;
 430         struct mlxsw_core *core;
 431         struct sk_buff *tx_skb;
 432         struct mlxsw_tx_info tx_info;
 433         struct delayed_work timeout_dw;
 434         unsigned int retries;
 435         u64 tid;
 436         struct completion completion;
 437         atomic_t active;
 438         mlxsw_reg_trans_cb_t *cb;
 439         unsigned long cb_priv;
 440         const struct mlxsw_reg_info *reg;
 441         enum mlxsw_core_reg_access_type type;
 442         int err;
 443         enum mlxsw_emad_op_tlv_status emad_status;
 444         struct rcu_head rcu;
 445 };
 446
 447 #define MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS   3000
 448 #define MLXSW_EMAD_TIMEOUT_MS                   200
 449
 450 static void mlxsw_emad_trans_timeout_schedule(struct mlxsw_reg_trans *trans)
 451 {
 452         unsigned long timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_MS);
 453
 454         if (trans->core->fw_flash_in_progress)
 455                 timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS);
 456
 457         queue_delayed_work(trans->core->emad_wq, &trans->timeout_dw, timeout);
 458 }
 459
 460 static int mlxsw_emad_transmit(struct mlxsw_core *mlxsw_core,
 461                                struct mlxsw_reg_trans *trans)
 462 {
 463         struct sk_buff *skb;
 464         int err;
 465
 466         skb = skb_copy(trans->tx_skb, GFP_KERNEL);
 467         if (!skb)
 468                 return -ENOMEM;
 469
 470         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), false, 0,
 471                             skb->data + mlxsw_core->driver->txhdr_len,
 472                             skb->len - mlxsw_core->driver->txhdr_len);
 473
 474         atomic_set(&trans->active, 1);
 475         err = mlxsw_core_skb_transmit(mlxsw_core, skb, &trans->tx_info);
 476         if (err) {
 477                 dev_kfree_skb(skb);
 478                 return err;
 479         }
 480         mlxsw_emad_trans_timeout_schedule(trans);
 481         return 0;
 482 }
 483
 484 static void mlxsw_emad_trans_finish(struct mlxsw_reg_trans *trans, int err)
 485 {
 486         struct mlxsw_core *mlxsw_core = trans->core;
 487
 488         dev_kfree_skb(trans->tx_skb);
 489         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 490         list_del_rcu(&trans->list);
 491         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 492         trans->err = err;
 493         complete(&trans->completion);
 494 }
 495
 496 static void mlxsw_emad_transmit_retry(struct mlxsw_core *mlxsw_core,
 497                                       struct mlxsw_reg_trans *trans)
 498 {
 499         int err;
 500
 501         if (trans->retries < MLXSW_EMAD_MAX_RETRY) {
 502                 trans->retries++;
 503                 err = mlxsw_emad_transmit(trans->core, trans);
 504                 if (err == 0)
 505                         return;
 506         } else {
 507                 err = -EIO;
 508         }
 509         mlxsw_emad_trans_finish(trans, err);
 510 }
 511
 512 static void mlxsw_emad_trans_timeout_work(struct work_struct *work)
 513 {
 514         struct mlxsw_reg_trans *trans = container_of(work,
 515                                                      struct mlxsw_reg_trans,
 516                                                      timeout_dw.work);
 517
 518         if (!atomic_dec_and_test(&trans->active))
 519                 return;
 520
 521         mlxsw_emad_transmit_retry(trans->core, trans);
 522 }
 523
 524 static void mlxsw_emad_process_response(struct mlxsw_core *mlxsw_core,
 525                                         struct mlxsw_reg_trans *trans,
 526                                         struct sk_buff *skb)
 527 {
 528         int err;
 529
 530         if (!atomic_dec_and_test(&trans->active))
 531                 return;
 532
 533         err = mlxsw_emad_process_status_skb(skb, &trans->emad_status);
 534         if (err == -EAGAIN) {
 535                 mlxsw_emad_transmit_retry(mlxsw_core, trans);
 536         } else {
 537                 if (err == 0) {
 538                         char *op_tlv = mlxsw_emad_op_tlv(skb);
 539
 540                         if (trans->cb)
 541                                 trans->cb(mlxsw_core,
 542                                           mlxsw_emad_reg_payload(op_tlv),
 543                                           trans->reg->len, trans->cb_priv);
 544                 }
 545                 mlxsw_emad_trans_finish(trans, err);
 546         }
 547 }
 548
 549 /* called with rcu read lock held */
 550 static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u8 local_port,
 551                                         void *priv)
 552 {
 553         struct mlxsw_core *mlxsw_core = priv;
 554         struct mlxsw_reg_trans *trans;
 555
 556         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
 557                             skb->data, skb->len);
 558
 559         if (!mlxsw_emad_is_resp(skb))
 560                 goto free_skb;
 561
 562         list_for_each_entry_rcu(trans, &mlxsw_core->emad.trans_list, list) {
 563                 if (mlxsw_emad_get_tid(skb) == trans->tid) {
 564                         mlxsw_emad_process_response(mlxsw_core, trans, skb);
 565                         break;
 566                 }
 567         }
 568
 569 free_skb:
 570         dev_kfree_skb(skb);
 571 }
 572
 573 static const struct mlxsw_listener mlxsw_emad_rx_listener =
 574         MLXSW_RXL(mlxsw_emad_rx_listener_func, ETHEMAD, TRAP_TO_CPU, false,
 575                   EMAD, DISCARD);
 576
 577 static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core)
 578 {
 579         struct workqueue_struct *emad_wq;
 580         u64 tid;
 581         int err;
 582
 583         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 584                 return 0;
 585
 586         emad_wq = alloc_workqueue("mlxsw_core_emad", 0, 0);
 587         if (!emad_wq)
 588                 return -ENOMEM;
 589         mlxsw_core->emad_wq = emad_wq;
 590
 591         /* Set the upper 32 bits of the transaction ID field to a random
 592          * number. This allows us to discard EMADs addressed to other
 593          * devices.
 594          */
 595         get_random_bytes(&tid, 4);
 596         tid <<= 32;
 597         atomic64_set(&mlxsw_core->emad.tid, tid);
 598
 599         INIT_LIST_HEAD(&mlxsw_core->emad.trans_list);
 600         spin_lock_init(&mlxsw_core->emad.trans_list_lock);
 601
 602         err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_emad_rx_listener,
 603                                        mlxsw_core);
 604         if (err)
 605                 return err;
 606
 607         err = mlxsw_core->driver->basic_trap_groups_set(mlxsw_core);
 608         if (err)
 609                 goto err_emad_trap_set;
 610         mlxsw_core->emad.use_emad = true;
 611
 612         return 0;
 613
 614 err_emad_trap_set:
 615         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 616                                    mlxsw_core);
 617         destroy_workqueue(mlxsw_core->emad_wq);
 618         return err;
 619 }
 620
 621 static void mlxsw_emad_fini(struct mlxsw_core *mlxsw_core)
 622 {
 623
 624         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 625                 return;
 626
 627         mlxsw_core->emad.use_emad = false;
 628         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 629                                    mlxsw_core);
 630         destroy_workqueue(mlxsw_core->emad_wq);
 631 }
 632
 633 static struct sk_buff *mlxsw_emad_alloc(const struct mlxsw_core *mlxsw_core,
 634                                         u16 reg_len)
 635 {
 636         struct sk_buff *skb;
 637         u16 emad_len;
 638
 639         emad_len = (reg_len + sizeof(u32) + MLXSW_EMAD_ETH_HDR_LEN +
 640                     (MLXSW_EMAD_OP_TLV_LEN + MLXSW_EMAD_END_TLV_LEN) *
 641                     sizeof(u32) + mlxsw_core->driver->txhdr_len);
 642         if (emad_len > MLXSW_EMAD_MAX_FRAME_LEN)
 643                 return NULL;
 644
 645         skb = netdev_alloc_skb(NULL, emad_len);
 646         if (!skb)
 647                 return NULL;
 648         memset(skb->data, 0, emad_len);
 649         skb_reserve(skb, emad_len);
 650
 651         return skb;
 652 }
 653
 654 static int mlxsw_emad_reg_access(struct mlxsw_core *mlxsw_core,
 655                                  const struct mlxsw_reg_info *reg,
 656                                  char *payload,
 657                                  enum mlxsw_core_reg_access_type type,
 658                                  struct mlxsw_reg_trans *trans,
 659                                  struct list_head *bulk_list,
 660                                  mlxsw_reg_trans_cb_t *cb,
 661                                  unsigned long cb_priv, u64 tid)
 662 {
 663         struct sk_buff *skb;
 664         int err;
 665
 666         dev_dbg(mlxsw_core->bus_info->dev, "EMAD reg access (tid=%llx,reg_id=%x(%s),type=%s)\n",
 667                 tid, reg->id, mlxsw_reg_id_str(reg->id),
 668                 mlxsw_core_reg_access_type_str(type));
 669
 670         skb = mlxsw_emad_alloc(mlxsw_core, reg->len);
 671         if (!skb)
 672                 return -ENOMEM;
 673
 674         list_add_tail(&trans->bulk_list, bulk_list);
 675         trans->core = mlxsw_core;
 676         trans->tx_skb = skb;
 677         trans->tx_info.local_port = MLXSW_PORT_CPU_PORT;
 678         trans->tx_info.is_emad = true;
 679         INIT_DELAYED_WORK(&trans->timeout_dw, mlxsw_emad_trans_timeout_work);
 680         trans->tid = tid;
 681         init_completion(&trans->completion);
 682         trans->cb = cb;
 683         trans->cb_priv = cb_priv;
 684         trans->reg = reg;
 685         trans->type = type;
 686
 687         mlxsw_emad_construct(skb, reg, payload, type, trans->tid);
 688         mlxsw_core->driver->txhdr_construct(skb, &trans->tx_info);
 689
 690         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 691         list_add_tail_rcu(&trans->list, &mlxsw_core->emad.trans_list);
 692         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 693         err = mlxsw_emad_transmit(mlxsw_core, trans);
 694         if (err)
 695                 goto err_out;
 696         return 0;
 697
 698 err_out:
 699         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 700         list_del_rcu(&trans->list);
 701         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 702         list_del(&trans->bulk_list);
 703         dev_kfree_skb(trans->tx_skb);
 704         return err;
 705 }
 706
 707 /*****************
 708  * Core functions
 709  *****************/
 710
 711 int mlxsw_core_driver_register(struct mlxsw_driver *mlxsw_driver)
 712 {
 713         spin_lock(&mlxsw_core_driver_list_lock);
 714         list_add_tail(&mlxsw_driver->list, &mlxsw_core_driver_list);
 715         spin_unlock(&mlxsw_core_driver_list_lock);
 716         return 0;
 717 }
 718 EXPORT_SYMBOL(mlxsw_core_driver_register);
 719
 720 void mlxsw_core_driver_unregister(struct mlxsw_driver *mlxsw_driver)
 721 {
 722         spin_lock(&mlxsw_core_driver_list_lock);
 723         list_del(&mlxsw_driver->list);
 724         spin_unlock(&mlxsw_core_driver_list_lock);
 725 }
 726 EXPORT_SYMBOL(mlxsw_core_driver_unregister);
 727
 728 static struct mlxsw_driver *__driver_find(const char *kind)
 729 {
 730         struct mlxsw_driver *mlxsw_driver;
 731
 732         list_for_each_entry(mlxsw_driver, &mlxsw_core_driver_list, list) {
 733                 if (strcmp(mlxsw_driver->kind, kind) == 0)
 734                         return mlxsw_driver;
 735         }
 736         return NULL;
 737 }
 738
 739 static struct mlxsw_driver *mlxsw_core_driver_get(const char *kind)
 740 {
 741         struct mlxsw_driver *mlxsw_driver;
 742
 743         spin_lock(&mlxsw_core_driver_list_lock);
 744         mlxsw_driver = __driver_find(kind);
 745         spin_unlock(&mlxsw_core_driver_list_lock);
 746         return mlxsw_driver;
 747 }
 748
 749 static int mlxsw_devlink_port_split(struct devlink *devlink,
 750                                     unsigned int port_index,
 751                                     unsigned int count,
 752                                     struct netlink_ext_ack *extack)
 753 {
 754         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 755
 756         if (port_index >= mlxsw_core->max_ports) {
 757                 NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
 758                 return -EINVAL;
 759         }
 760         if (!mlxsw_core->driver->port_split)
 761                 return -EOPNOTSUPP;
 762         return mlxsw_core->driver->port_split(mlxsw_core, port_index, count,
 763                                               extack);
 764 }
 765
 766 static int mlxsw_devlink_port_unsplit(struct devlink *devlink,
 767                                       unsigned int port_index,
 768                                       struct netlink_ext_ack *extack)
 769 {
 770         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 771
 772         if (port_index >= mlxsw_core->max_ports) {
 773                 NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
 774                 return -EINVAL;
 775         }
 776         if (!mlxsw_core->driver->port_unsplit)
 777                 return -EOPNOTSUPP;
 778         return mlxsw_core->driver->port_unsplit(mlxsw_core, port_index,
 779                                                 extack);
 780 }
 781
 782 static int
 783 mlxsw_devlink_sb_pool_get(struct devlink *devlink,
 784                           unsigned int sb_index, u16 pool_index,
 785                           struct devlink_sb_pool_info *pool_info)
 786 {
 787         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 788         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 789
 790         if (!mlxsw_driver->sb_pool_get)
 791                 return -EOPNOTSUPP;
 792         return mlxsw_driver->sb_pool_get(mlxsw_core, sb_index,
 793                                          pool_index, pool_info);
 794 }
 795
 796 static int
 797 mlxsw_devlink_sb_pool_set(struct devlink *devlink,
 798                           unsigned int sb_index, u16 pool_index, u32 size,
 799                           enum devlink_sb_threshold_type threshold_type,
 800                           struct netlink_ext_ack *extack)
 801 {
 802         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 803         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 804
 805         if (!mlxsw_driver->sb_pool_set)
 806                 return -EOPNOTSUPP;
 807         return mlxsw_driver->sb_pool_set(mlxsw_core, sb_index,
 808                                          pool_index, size, threshold_type,
 809                                          extack);
 810 }
 811
 812 static void *__dl_port(struct devlink_port *devlink_port)
 813 {
 814         return container_of(devlink_port, struct mlxsw_core_port, devlink_port);
 815 }
 816
 817 static int mlxsw_devlink_port_type_set(struct devlink_port *devlink_port,
 818                                        enum devlink_port_type port_type)
 819 {
 820         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 821         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 822         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 823
 824         if (!mlxsw_driver->port_type_set)
 825                 return -EOPNOTSUPP;
 826
 827         return mlxsw_driver->port_type_set(mlxsw_core,
 828                                            mlxsw_core_port->local_port,
 829                                            port_type);
 830 }
 831
 832 static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port,
 833                                           unsigned int sb_index, u16 pool_index,
 834                                           u32 *p_threshold)
 835 {
 836         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 837         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 838         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 839
 840         if (!mlxsw_driver->sb_port_pool_get ||
 841             !mlxsw_core_port_check(mlxsw_core_port))
 842                 return -EOPNOTSUPP;
 843         return mlxsw_driver->sb_port_pool_get(mlxsw_core_port, sb_index,
 844                                               pool_index, p_threshold);
 845 }
 846
 847 static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port,
 848                                           unsigned int sb_index, u16 pool_index,
 849                                           u32 threshold,
 850                                           struct netlink_ext_ack *extack)
 851 {
 852         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 853         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 854         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 855
 856         if (!mlxsw_driver->sb_port_pool_set ||
 857             !mlxsw_core_port_check(mlxsw_core_port))
 858                 return -EOPNOTSUPP;
 859         return mlxsw_driver->sb_port_pool_set(mlxsw_core_port, sb_index,
 860                                               pool_index, threshold, extack);
 861 }
 862
 863 static int
 864 mlxsw_devlink_sb_tc_pool_bind_get(struct devlink_port *devlink_port,
 865                                   unsigned int sb_index, u16 tc_index,
 866                                   enum devlink_sb_pool_type pool_type,
 867                                   u16 *p_pool_index, u32 *p_threshold)
 868 {
 869         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 870         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 871         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 872
 873         if (!mlxsw_driver->sb_tc_pool_bind_get ||
 874             !mlxsw_core_port_check(mlxsw_core_port))
 875                 return -EOPNOTSUPP;
 876         return mlxsw_driver->sb_tc_pool_bind_get(mlxsw_core_port, sb_index,
 877                                                  tc_index, pool_type,
 878                                                  p_pool_index, p_threshold);
 879 }
 880
 881 static int
 882 mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
 883                                   unsigned int sb_index, u16 tc_index,
 884                                   enum devlink_sb_pool_type pool_type,
 885                                   u16 pool_index, u32 threshold,
 886                                   struct netlink_ext_ack *extack)
 887 {
 888         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 889         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 890         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 891
 892         if (!mlxsw_driver->sb_tc_pool_bind_set ||
 893             !mlxsw_core_port_check(mlxsw_core_port))
 894                 return -EOPNOTSUPP;
 895         return mlxsw_driver->sb_tc_pool_bind_set(mlxsw_core_port, sb_index,
 896                                                  tc_index, pool_type,
 897                                                  pool_index, threshold, extack);
 898 }
 899
 900 static int mlxsw_devlink_sb_occ_snapshot(struct devlink *devlink,
 901                                          unsigned int sb_index)
 902 {
 903         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 904         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 905
 906         if (!mlxsw_driver->sb_occ_snapshot)
 907                 return -EOPNOTSUPP;
 908         return mlxsw_driver->sb_occ_snapshot(mlxsw_core, sb_index);
 909 }
 910
 911 static int mlxsw_devlink_sb_occ_max_clear(struct devlink *devlink,
 912                                           unsigned int sb_index)
 913 {
 914         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 915         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 916
 917         if (!mlxsw_driver->sb_occ_max_clear)
 918                 return -EOPNOTSUPP;
 919         return mlxsw_driver->sb_occ_max_clear(mlxsw_core, sb_index);
 920 }
 921
 922 static int
 923 mlxsw_devlink_sb_occ_port_pool_get(struct devlink_port *devlink_port,
 924                                    unsigned int sb_index, u16 pool_index,
 925                                    u32 *p_cur, u32 *p_max)
 926 {
 927         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 928         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 929         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 930
 931         if (!mlxsw_driver->sb_occ_port_pool_get ||
 932             !mlxsw_core_port_check(mlxsw_core_port))
 933                 return -EOPNOTSUPP;
 934         return mlxsw_driver->sb_occ_port_pool_get(mlxsw_core_port, sb_index,
 935                                                   pool_index, p_cur, p_max);
 936 }
 937
 938 static int
 939 mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
 940                                       unsigned int sb_index, u16 tc_index,
 941                                       enum devlink_sb_pool_type pool_type,
 942                                       u32 *p_cur, u32 *p_max)
 943 {
 944         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 945         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 946         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 947
 948         if (!mlxsw_driver->sb_occ_tc_port_bind_get ||
 949             !mlxsw_core_port_check(mlxsw_core_port))
 950                 return -EOPNOTSUPP;
 951         return mlxsw_driver->sb_occ_tc_port_bind_get(mlxsw_core_port,
 952                                                      sb_index, tc_index,
 953                                                      pool_type, p_cur, p_max);
 954 }
 955
 956 static int
 957 mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req,
 958                        struct netlink_ext_ack *extack)
 959 {
 960         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 961         char fw_info_psid[MLXSW_REG_MGIR_FW_INFO_PSID_SIZE];
 962         u32 hw_rev, fw_major, fw_minor, fw_sub_minor;
 963         char mgir_pl[MLXSW_REG_MGIR_LEN];
 964         char buf[32];
 965         int err;
 966
 967         err = devlink_info_driver_name_put(req,
 968                                            mlxsw_core->bus_info->device_kind);
 969         if (err)
 970                 return err;
 971
 972         mlxsw_reg_mgir_pack(mgir_pl);
 973         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl);
 974         if (err)
 975                 return err;
 976         mlxsw_reg_mgir_unpack(mgir_pl, &hw_rev, fw_info_psid, &fw_major,
 977                               &fw_minor, &fw_sub_minor);
 978
 979         sprintf(buf, "%X", hw_rev);
 980         err = devlink_info_version_fixed_put(req, "hw.revision", buf);
 981         if (err)
 982                 return err;
 983
 984         err = devlink_info_version_fixed_put(req, "fw.psid", fw_info_psid);
 985         if (err)
 986                 return err;
 987
 988         sprintf(buf, "%d.%d.%d", fw_major, fw_minor, fw_sub_minor);
 989         err = devlink_info_version_running_put(req, "fw.version", buf);
 990         if (err)
 991                 return err;
 992
 993         return 0;
 994 }
 995
 996 static int
 997 mlxsw_devlink_core_bus_device_reload_down(struct devlink *devlink,
 998                                           bool netns_change,
 999                                           struct netlink_ext_ack *extack)
1000 {
1001         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1002
1003         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET))
1004                 return -EOPNOTSUPP;
1005
1006         mlxsw_core_bus_device_unregister(mlxsw_core, true);
1007         return 0;
1008 }
1009
1010 static int
1011 mlxsw_devlink_core_bus_device_reload_up(struct devlink *devlink,
1012                                         struct netlink_ext_ack *extack)
1013 {
1014         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1015
1016         return mlxsw_core_bus_device_register(mlxsw_core->bus_info,
1017                                               mlxsw_core->bus,
1018                                               mlxsw_core->bus_priv, true,
1019                                               devlink, extack);
1020 }
1021
1022 static int mlxsw_devlink_flash_update(struct devlink *devlink,
1023                                       const char *file_name,
1024                                       const char *component,
1025                                       struct netlink_ext_ack *extack)
1026 {
1027         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1028         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1029
1030         if (!mlxsw_driver->flash_update)
1031                 return -EOPNOTSUPP;
1032         return mlxsw_driver->flash_update(mlxsw_core, file_name,
1033                                           component, extack);
1034 }
1035
1036 static int mlxsw_devlink_trap_init(struct devlink *devlink,
1037                                    const struct devlink_trap *trap,
1038                                    void *trap_ctx)
1039 {
1040         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1041         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1042
1043         if (!mlxsw_driver->trap_init)
1044                 return -EOPNOTSUPP;
1045         return mlxsw_driver->trap_init(mlxsw_core, trap, trap_ctx);
1046 }
1047
1048 static void mlxsw_devlink_trap_fini(struct devlink *devlink,
1049                                     const struct devlink_trap *trap,
1050                                     void *trap_ctx)
1051 {
1052         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1053         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1054
1055         if (!mlxsw_driver->trap_fini)
1056                 return;
1057         mlxsw_driver->trap_fini(mlxsw_core, trap, trap_ctx);
1058 }
1059
1060 static int mlxsw_devlink_trap_action_set(struct devlink *devlink,
1061                                          const struct devlink_trap *trap,
1062                                          enum devlink_trap_action action)
1063 {
1064         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1065         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1066
1067         if (!mlxsw_driver->trap_action_set)
1068                 return -EOPNOTSUPP;
1069         return mlxsw_driver->trap_action_set(mlxsw_core, trap, action);
1070 }
1071
1072 static int
1073 mlxsw_devlink_trap_group_init(struct devlink *devlink,
1074                               const struct devlink_trap_group *group)
1075 {
1076         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1077         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1078
1079         if (!mlxsw_driver->trap_group_init)
1080                 return -EOPNOTSUPP;
1081         return mlxsw_driver->trap_group_init(mlxsw_core, group);
1082 }
1083
1084 static const struct devlink_ops mlxsw_devlink_ops = {
1085         .reload_down            = mlxsw_devlink_core_bus_device_reload_down,
1086         .reload_up              = mlxsw_devlink_core_bus_device_reload_up,
1087         .port_type_set                  = mlxsw_devlink_port_type_set,
1088         .port_split                     = mlxsw_devlink_port_split,
1089         .port_unsplit                   = mlxsw_devlink_port_unsplit,
1090         .sb_pool_get                    = mlxsw_devlink_sb_pool_get,
1091         .sb_pool_set                    = mlxsw_devlink_sb_pool_set,
1092         .sb_port_pool_get               = mlxsw_devlink_sb_port_pool_get,
1093         .sb_port_pool_set               = mlxsw_devlink_sb_port_pool_set,
1094         .sb_tc_pool_bind_get            = mlxsw_devlink_sb_tc_pool_bind_get,
1095         .sb_tc_pool_bind_set            = mlxsw_devlink_sb_tc_pool_bind_set,
1096         .sb_occ_snapshot                = mlxsw_devlink_sb_occ_snapshot,
1097         .sb_occ_max_clear               = mlxsw_devlink_sb_occ_max_clear,
1098         .sb_occ_port_pool_get           = mlxsw_devlink_sb_occ_port_pool_get,
1099         .sb_occ_tc_port_bind_get        = mlxsw_devlink_sb_occ_tc_port_bind_get,
1100         .info_get                       = mlxsw_devlink_info_get,
1101         .flash_update                   = mlxsw_devlink_flash_update,
1102         .trap_init                      = mlxsw_devlink_trap_init,
1103         .trap_fini                      = mlxsw_devlink_trap_fini,
1104         .trap_action_set                = mlxsw_devlink_trap_action_set,
1105         .trap_group_init                = mlxsw_devlink_trap_group_init,
1106 };
1107
1108 static int
1109 __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1110                                  const struct mlxsw_bus *mlxsw_bus,
1111                                  void *bus_priv, bool reload,
1112                                  struct devlink *devlink,
1113                                  struct netlink_ext_ack *extack)
1114 {
1115         const char *device_kind = mlxsw_bus_info->device_kind;
1116         struct mlxsw_core *mlxsw_core;
1117         struct mlxsw_driver *mlxsw_driver;
1118         struct mlxsw_res *res;
1119         size_t alloc_size;
1120         int err;
1121
1122         mlxsw_driver = mlxsw_core_driver_get(device_kind);
1123         if (!mlxsw_driver)
1124                 return -EINVAL;
1125
1126         if (!reload) {
1127                 alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
1128                 devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size);
1129                 if (!devlink) {
1130                         err = -ENOMEM;
1131                         goto err_devlink_alloc;
1132                 }
1133         }
1134
1135         mlxsw_core = devlink_priv(devlink);
1136         INIT_LIST_HEAD(&mlxsw_core->rx_listener_list);
1137         INIT_LIST_HEAD(&mlxsw_core->event_listener_list);
1138         mlxsw_core->driver = mlxsw_driver;
1139         mlxsw_core->bus = mlxsw_bus;
1140         mlxsw_core->bus_priv = bus_priv;
1141         mlxsw_core->bus_info = mlxsw_bus_info;
1142
1143         res = mlxsw_driver->res_query_enabled ? &mlxsw_core->res : NULL;
1144         err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile, res);
1145         if (err)
1146                 goto err_bus_init;
1147
1148         if (mlxsw_driver->resources_register && !reload) {
1149                 err = mlxsw_driver->resources_register(mlxsw_core);
1150                 if (err)
1151                         goto err_register_resources;
1152         }
1153
1154         err = mlxsw_ports_init(mlxsw_core);
1155         if (err)
1156                 goto err_ports_init;
1157
1158         if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) &&
1159             MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
1160                 alloc_size = sizeof(u8) *
1161                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) *
1162                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS);
1163                 mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
1164                 if (!mlxsw_core->lag.mapping) {
1165                         err = -ENOMEM;
1166                         goto err_alloc_lag_mapping;
1167                 }
1168         }
1169
1170         err = mlxsw_emad_init(mlxsw_core);
1171         if (err)
1172                 goto err_emad_init;
1173
1174         if (!reload) {
1175                 err = devlink_register(devlink, mlxsw_bus_info->dev);
1176                 if (err)
1177                         goto err_devlink_register;
1178         }
1179
1180         if (mlxsw_driver->params_register && !reload) {
1181                 err = mlxsw_driver->params_register(mlxsw_core);
1182                 if (err)
1183                         goto err_register_params;
1184         }
1185
1186         if (mlxsw_driver->init) {
1187                 err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info, extack);
1188                 if (err)
1189                         goto err_driver_init;
1190         }
1191
1192         err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon);
1193         if (err)
1194                 goto err_hwmon_init;
1195
1196         err = mlxsw_thermal_init(mlxsw_core, mlxsw_bus_info,
1197                                  &mlxsw_core->thermal);
1198         if (err)
1199                 goto err_thermal_init;
1200
1201         if (mlxsw_driver->params_register && !reload)
1202                 devlink_params_publish(devlink);
1203
1204         return 0;
1205
1206 err_thermal_init:
1207         mlxsw_hwmon_fini(mlxsw_core->hwmon);
1208 err_hwmon_init:
1209         if (mlxsw_core->driver->fini)
1210                 mlxsw_core->driver->fini(mlxsw_core);
1211 err_driver_init:
1212         if (mlxsw_driver->params_unregister && !reload)
1213                 mlxsw_driver->params_unregister(mlxsw_core);
1214 err_register_params:
1215         if (!reload)
1216                 devlink_unregister(devlink);
1217 err_devlink_register:
1218         mlxsw_emad_fini(mlxsw_core);
1219 err_emad_init:
1220         kfree(mlxsw_core->lag.mapping);
1221 err_alloc_lag_mapping:
1222         mlxsw_ports_fini(mlxsw_core);
1223 err_ports_init:
1224         if (!reload)
1225                 devlink_resources_unregister(devlink, NULL);
1226 err_register_resources:
1227         mlxsw_bus->fini(bus_priv);
1228 err_bus_init:
1229         if (!reload)
1230                 devlink_free(devlink);
1231 err_devlink_alloc:
1232         return err;
1233 }
1234
1235 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1236                                    const struct mlxsw_bus *mlxsw_bus,
1237                                    void *bus_priv, bool reload,
1238                                    struct devlink *devlink,
1239                                    struct netlink_ext_ack *extack)
1240 {
1241         bool called_again = false;
1242         int err;
1243
1244 again:
1245         err = __mlxsw_core_bus_device_register(mlxsw_bus_info, mlxsw_bus,
1246                                                bus_priv, reload,
1247                                                devlink, extack);
1248         /* -EAGAIN is returned in case the FW was updated. FW needs
1249          * a reset, so lets try to call __mlxsw_core_bus_device_register()
1250          * again.
1251          */
1252         if (err == -EAGAIN && !called_again) {
1253                 called_again = true;
1254                 goto again;
1255         }
1256
1257         return err;
1258 }
1259 EXPORT_SYMBOL(mlxsw_core_bus_device_register);
1260
1261 void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
1262                                       bool reload)
1263 {
1264         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1265
1266         if (devlink_is_reload_failed(devlink)) {
1267                 if (!reload)
1268                         /* Only the parts that were not de-initialized in the
1269                          * failed reload attempt need to be de-initialized.
1270                          */
1271                         goto reload_fail_deinit;
1272                 else
1273                         return;
1274         }
1275
1276         if (mlxsw_core->driver->params_unregister && !reload)
1277                 devlink_params_unpublish(devlink);
1278         mlxsw_thermal_fini(mlxsw_core->thermal);
1279         mlxsw_hwmon_fini(mlxsw_core->hwmon);
1280         if (mlxsw_core->driver->fini)
1281                 mlxsw_core->driver->fini(mlxsw_core);
1282         if (mlxsw_core->driver->params_unregister && !reload)
1283                 mlxsw_core->driver->params_unregister(mlxsw_core);
1284         if (!reload)
1285                 devlink_unregister(devlink);
1286         mlxsw_emad_fini(mlxsw_core);
1287         kfree(mlxsw_core->lag.mapping);
1288         mlxsw_ports_fini(mlxsw_core);
1289         if (!reload)
1290                 devlink_resources_unregister(devlink, NULL);
1291         mlxsw_core->bus->fini(mlxsw_core->bus_priv);
1292
1293         return;
1294
1295 reload_fail_deinit:
1296         if (mlxsw_core->driver->params_unregister)
1297                 mlxsw_core->driver->params_unregister(mlxsw_core);
1298         devlink_unregister(devlink);
1299         devlink_resources_unregister(devlink, NULL);
1300         devlink_free(devlink);
1301 }
1302 EXPORT_SYMBOL(mlxsw_core_bus_device_unregister);
1303
1304 bool mlxsw_core_skb_transmit_busy(struct mlxsw_core *mlxsw_core,
1305                                   const struct mlxsw_tx_info *tx_info)
1306 {
1307         return mlxsw_core->bus->skb_transmit_busy(mlxsw_core->bus_priv,
1308                                                   tx_info);
1309 }
1310 EXPORT_SYMBOL(mlxsw_core_skb_transmit_busy);
1311
1312 int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
1313                             const struct mlxsw_tx_info *tx_info)
1314 {
1315         return mlxsw_core->bus->skb_transmit(mlxsw_core->bus_priv, skb,
1316                                              tx_info);
1317 }
1318 EXPORT_SYMBOL(mlxsw_core_skb_transmit);
1319
1320 void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core,
1321                                 struct sk_buff *skb, u8 local_port)
1322 {
1323         if (mlxsw_core->driver->ptp_transmitted)
1324                 mlxsw_core->driver->ptp_transmitted(mlxsw_core, skb,
1325                                                     local_port);
1326 }
1327 EXPORT_SYMBOL(mlxsw_core_ptp_transmitted);
1328
1329 static bool __is_rx_listener_equal(const struct mlxsw_rx_listener *rxl_a,
1330                                    const struct mlxsw_rx_listener *rxl_b)
1331 {
1332         return (rxl_a->func == rxl_b->func &&
1333                 rxl_a->local_port == rxl_b->local_port &&
1334                 rxl_a->trap_id == rxl_b->trap_id);
1335 }
1336
1337 static struct mlxsw_rx_listener_item *
1338 __find_rx_listener_item(struct mlxsw_core *mlxsw_core,
1339                         const struct mlxsw_rx_listener *rxl,
1340                         void *priv)
1341 {
1342         struct mlxsw_rx_listener_item *rxl_item;
1343
1344         list_for_each_entry(rxl_item, &mlxsw_core->rx_listener_list, list) {
1345                 if (__is_rx_listener_equal(&rxl_item->rxl, rxl) &&
1346                     rxl_item->priv == priv)
1347                         return rxl_item;
1348         }
1349         return NULL;
1350 }
1351
1352 int mlxsw_core_rx_listener_register(struct mlxsw_core *mlxsw_core,
1353                                     const struct mlxsw_rx_listener *rxl,
1354                                     void *priv)
1355 {
1356         struct mlxsw_rx_listener_item *rxl_item;
1357
1358         rxl_item = __find_rx_listener_item(mlxsw_core, rxl, priv);
1359         if (rxl_item)
1360                 return -EEXIST;
1361         rxl_item = kmalloc(sizeof(*rxl_item), GFP_KERNEL);
1362         if (!rxl_item)
1363                 return -ENOMEM;
1364         rxl_item->rxl = *rxl;
1365         rxl_item->priv = priv;
1366
1367         list_add_rcu(&rxl_item->list, &mlxsw_core->rx_listener_list);
1368         return 0;
1369 }
1370 EXPORT_SYMBOL(mlxsw_core_rx_listener_register);
1371
1372 void mlxsw_core_rx_listener_unregister(struct mlxsw_core *mlxsw_core,
1373                                        const struct mlxsw_rx_listener *rxl,
1374                                        void *priv)
1375 {
1376         struct mlxsw_rx_listener_item *rxl_item;
1377
1378         rxl_item = __find_rx_listener_item(mlxsw_core, rxl, priv);
1379         if (!rxl_item)
1380                 return;
1381         list_del_rcu(&rxl_item->list);
1382         synchronize_rcu();
1383         kfree(rxl_item);
1384 }
1385 EXPORT_SYMBOL(mlxsw_core_rx_listener_unregister);
1386
1387 static void mlxsw_core_event_listener_func(struct sk_buff *skb, u8 local_port,
1388                                            void *priv)
1389 {
1390         struct mlxsw_event_listener_item *event_listener_item = priv;
1391         struct mlxsw_reg_info reg;
1392         char *payload;
1393         char *op_tlv = mlxsw_emad_op_tlv(skb);
1394         char *reg_tlv = mlxsw_emad_reg_tlv(skb);
1395
1396         reg.id = mlxsw_emad_op_tlv_register_id_get(op_tlv);
1397         reg.len = (mlxsw_emad_reg_tlv_len_get(reg_tlv) - 1) * sizeof(u32);
1398         payload = mlxsw_emad_reg_payload(op_tlv);
1399         event_listener_item->el.func(&reg, payload, event_listener_item->priv);
1400         dev_kfree_skb(skb);
1401 }
1402
1403 static bool __is_event_listener_equal(const struct mlxsw_event_listener *el_a,
1404                                       const struct mlxsw_event_listener *el_b)
1405 {
1406         return (el_a->func == el_b->func &&
1407                 el_a->trap_id == el_b->trap_id);
1408 }
1409
1410 static struct mlxsw_event_listener_item *
1411 __find_event_listener_item(struct mlxsw_core *mlxsw_core,
1412                            const struct mlxsw_event_listener *el,
1413                            void *priv)
1414 {
1415         struct mlxsw_event_listener_item *el_item;
1416
1417         list_for_each_entry(el_item, &mlxsw_core->event_listener_list, list) {
1418                 if (__is_event_listener_equal(&el_item->el, el) &&
1419                     el_item->priv == priv)
1420                         return el_item;
1421         }
1422         return NULL;
1423 }
1424
1425 int mlxsw_core_event_listener_register(struct mlxsw_core *mlxsw_core,
1426                                        const struct mlxsw_event_listener *el,
1427                                        void *priv)
1428 {
1429         int err;
1430         struct mlxsw_event_listener_item *el_item;
1431         const struct mlxsw_rx_listener rxl = {
1432                 .func = mlxsw_core_event_listener_func,
1433                 .local_port = MLXSW_PORT_DONT_CARE,
1434                 .trap_id = el->trap_id,
1435         };
1436
1437         el_item = __find_event_listener_item(mlxsw_core, el, priv);
1438         if (el_item)
1439                 return -EEXIST;
1440         el_item = kmalloc(sizeof(*el_item), GFP_KERNEL);
1441         if (!el_item)
1442                 return -ENOMEM;
1443         el_item->el = *el;
1444         el_item->priv = priv;
1445
1446         err = mlxsw_core_rx_listener_register(mlxsw_core, &rxl, el_item);
1447         if (err)
1448                 goto err_rx_listener_register;
1449
1450         /* No reason to save item if we did not manage to register an RX
1451          * listener for it.
1452          */
1453         list_add_rcu(&el_item->list, &mlxsw_core->event_listener_list);
1454
1455         return 0;
1456
1457 err_rx_listener_register:
1458         kfree(el_item);
1459         return err;
1460 }
1461 EXPORT_SYMBOL(mlxsw_core_event_listener_register);
1462
1463 void mlxsw_core_event_listener_unregister(struct mlxsw_core *mlxsw_core,
1464                                           const struct mlxsw_event_listener *el,
1465                                           void *priv)
1466 {
1467         struct mlxsw_event_listener_item *el_item;
1468         const struct mlxsw_rx_listener rxl = {
1469                 .func = mlxsw_core_event_listener_func,
1470                 .local_port = MLXSW_PORT_DONT_CARE,
1471                 .trap_id = el->trap_id,
1472         };
1473
1474         el_item = __find_event_listener_item(mlxsw_core, el, priv);
1475         if (!el_item)
1476                 return;
1477         mlxsw_core_rx_listener_unregister(mlxsw_core, &rxl, el_item);
1478         list_del(&el_item->list);
1479         kfree(el_item);
1480 }
1481 EXPORT_SYMBOL(mlxsw_core_event_listener_unregister);
1482
1483 static int mlxsw_core_listener_register(struct mlxsw_core *mlxsw_core,
1484                                         const struct mlxsw_listener *listener,
1485                                         void *priv)
1486 {
1487         if (listener->is_event)
1488                 return mlxsw_core_event_listener_register(mlxsw_core,
1489                                                 &listener->u.event_listener,
1490                                                 priv);
1491         else
1492                 return mlxsw_core_rx_listener_register(mlxsw_core,
1493                                                 &listener->u.rx_listener,
1494                                                 priv);
1495 }
1496
1497 static void mlxsw_core_listener_unregister(struct mlxsw_core *mlxsw_core,
1498                                       const struct mlxsw_listener *listener,
1499                                       void *priv)
1500 {
1501         if (listener->is_event)
1502                 mlxsw_core_event_listener_unregister(mlxsw_core,
1503                                                      &listener->u.event_listener,
1504                                                      priv);
1505         else
1506                 mlxsw_core_rx_listener_unregister(mlxsw_core,
1507                                                   &listener->u.rx_listener,
1508                                                   priv);
1509 }
1510
1511 int mlxsw_core_trap_register(struct mlxsw_core *mlxsw_core,
1512                              const struct mlxsw_listener *listener, void *priv)
1513 {
1514         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1515         int err;
1516
1517         err = mlxsw_core_listener_register(mlxsw_core, listener, priv);
1518         if (err)
1519                 return err;
1520
1521         mlxsw_reg_hpkt_pack(hpkt_pl, listener->action, listener->trap_id,
1522                             listener->trap_group, listener->is_ctrl);
1523         err = mlxsw_reg_write(mlxsw_core,  MLXSW_REG(hpkt), hpkt_pl);
1524         if (err)
1525                 goto err_trap_set;
1526
1527         return 0;
1528
1529 err_trap_set:
1530         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1531         return err;
1532 }
1533 EXPORT_SYMBOL(mlxsw_core_trap_register);
1534
1535 void mlxsw_core_trap_unregister(struct mlxsw_core *mlxsw_core,
1536                                 const struct mlxsw_listener *listener,
1537                                 void *priv)
1538 {
1539         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1540
1541         if (!listener->is_event) {
1542                 mlxsw_reg_hpkt_pack(hpkt_pl, listener->unreg_action,
1543                                     listener->trap_id, listener->trap_group,
1544                                     listener->is_ctrl);
1545                 mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
1546         }
1547
1548         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1549 }
1550 EXPORT_SYMBOL(mlxsw_core_trap_unregister);
1551
1552 int mlxsw_core_trap_action_set(struct mlxsw_core *mlxsw_core,
1553                                const struct mlxsw_listener *listener,
1554                                enum mlxsw_reg_hpkt_action action)
1555 {
1556         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1557
1558         mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
1559                             listener->trap_group, listener->is_ctrl);
1560         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
1561 }
1562 EXPORT_SYMBOL(mlxsw_core_trap_action_set);
1563
1564 static u64 mlxsw_core_tid_get(struct mlxsw_core *mlxsw_core)
1565 {
1566         return atomic64_inc_return(&mlxsw_core->emad.tid);
1567 }
1568
1569 static int mlxsw_core_reg_access_emad(struct mlxsw_core *mlxsw_core,
1570                                       const struct mlxsw_reg_info *reg,
1571                                       char *payload,
1572                                       enum mlxsw_core_reg_access_type type,
1573                                       struct list_head *bulk_list,
1574                                       mlxsw_reg_trans_cb_t *cb,
1575                                       unsigned long cb_priv)
1576 {
1577         u64 tid = mlxsw_core_tid_get(mlxsw_core);
1578         struct mlxsw_reg_trans *trans;
1579         int err;
1580
1581         trans = kzalloc(sizeof(*trans), GFP_KERNEL);
1582         if (!trans)
1583                 return -ENOMEM;
1584
1585         err = mlxsw_emad_reg_access(mlxsw_core, reg, payload, type, trans,
1586                                     bulk_list, cb, cb_priv, tid);
1587         if (err) {
1588                 kfree(trans);
1589                 return err;
1590         }
1591         return 0;
1592 }
1593
1594 int mlxsw_reg_trans_query(struct mlxsw_core *mlxsw_core,
1595                           const struct mlxsw_reg_info *reg, char *payload,
1596                           struct list_head *bulk_list,
1597                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1598 {
1599         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1600                                           MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
1601                                           bulk_list, cb, cb_priv);
1602 }
1603 EXPORT_SYMBOL(mlxsw_reg_trans_query);
1604
1605 int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core,
1606                           const struct mlxsw_reg_info *reg, char *payload,
1607                           struct list_head *bulk_list,
1608                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1609 {
1610         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1611                                           MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
1612                                           bulk_list, cb, cb_priv);
1613 }
1614 EXPORT_SYMBOL(mlxsw_reg_trans_write);
1615
1616 static int mlxsw_reg_trans_wait(struct mlxsw_reg_trans *trans)
1617 {
1618         struct mlxsw_core *mlxsw_core = trans->core;
1619         int err;
1620
1621         wait_for_completion(&trans->completion);
1622         cancel_delayed_work_sync(&trans->timeout_dw);
1623         err = trans->err;
1624
1625         if (trans->retries)
1626                 dev_warn(mlxsw_core->bus_info->dev, "EMAD retries (%d/%d) (tid=%llx)\n",
1627                          trans->retries, MLXSW_EMAD_MAX_RETRY, trans->tid);
1628         if (err) {
1629                 dev_err(mlxsw_core->bus_info->dev, "EMAD reg access failed (tid=%llx,reg_id=%x(%s),type=%s,status=%x(%s))\n",
1630                         trans->tid, trans->reg->id,
1631                         mlxsw_reg_id_str(trans->reg->id),
1632                         mlxsw_core_reg_access_type_str(trans->type),
1633                         trans->emad_status,
1634                         mlxsw_emad_op_tlv_status_str(trans->emad_status));
1635                 trace_devlink_hwerr(priv_to_devlink(mlxsw_core),
1636                                     trans->emad_status,
1637                                     mlxsw_emad_op_tlv_status_str(trans->emad_status));
1638         }
1639
1640         list_del(&trans->bulk_list);
1641         kfree_rcu(trans, rcu);
1642         return err;
1643 }
1644
1645 int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list)
1646 {
1647         struct mlxsw_reg_trans *trans;
1648         struct mlxsw_reg_trans *tmp;
1649         int sum_err = 0;
1650         int err;
1651
1652         list_for_each_entry_safe(trans, tmp, bulk_list, bulk_list) {
1653                 err = mlxsw_reg_trans_wait(trans);
1654                 if (err && sum_err == 0)
1655                         sum_err = err; /* first error to be returned */
1656         }
1657         return sum_err;
1658 }
1659 EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait);
1660
1661 static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
1662                                      const struct mlxsw_reg_info *reg,
1663                                      char *payload,
1664                                      enum mlxsw_core_reg_access_type type)
1665 {
1666         enum mlxsw_emad_op_tlv_status status;
1667         int err, n_retry;
1668         bool reset_ok;
1669         char *in_mbox, *out_mbox, *tmp;
1670
1671         dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n",
1672                 reg->id, mlxsw_reg_id_str(reg->id),
1673                 mlxsw_core_reg_access_type_str(type));
1674
1675         in_mbox = mlxsw_cmd_mbox_alloc();
1676         if (!in_mbox)
1677                 return -ENOMEM;
1678
1679         out_mbox = mlxsw_cmd_mbox_alloc();
1680         if (!out_mbox) {
1681                 err = -ENOMEM;
1682                 goto free_in_mbox;
1683         }
1684
1685         mlxsw_emad_pack_op_tlv(in_mbox, reg, type,
1686                                mlxsw_core_tid_get(mlxsw_core));
1687         tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
1688         mlxsw_emad_pack_reg_tlv(tmp, reg, payload);
1689
1690         /* There is a special treatment needed for MRSR (reset) register.
1691          * The command interface will return error after the command
1692          * is executed, so tell the lower layer to expect it
1693          * and cope accordingly.
1694          */
1695         reset_ok = reg->id == MLXSW_REG_MRSR_ID;
1696
1697         n_retry = 0;
1698 retry:
1699         err = mlxsw_cmd_access_reg(mlxsw_core, reset_ok, in_mbox, out_mbox);
1700         if (!err) {
1701                 err = mlxsw_emad_process_status(out_mbox, &status);
1702                 if (err) {
1703                         if (err == -EAGAIN && n_retry++ < MLXSW_EMAD_MAX_RETRY)
1704                                 goto retry;
1705                         dev_err(mlxsw_core->bus_info->dev, "Reg cmd access status failed (status=%x(%s))\n",
1706                                 status, mlxsw_emad_op_tlv_status_str(status));
1707                 }
1708         }
1709
1710         if (!err)
1711                 memcpy(payload, mlxsw_emad_reg_payload(out_mbox),
1712                        reg->len);
1713
1714         mlxsw_cmd_mbox_free(out_mbox);
1715 free_in_mbox:
1716         mlxsw_cmd_mbox_free(in_mbox);
1717         if (err)
1718                 dev_err(mlxsw_core->bus_info->dev, "Reg cmd access failed (reg_id=%x(%s),type=%s)\n",
1719                         reg->id, mlxsw_reg_id_str(reg->id),
1720                         mlxsw_core_reg_access_type_str(type));
1721         return err;
1722 }
1723
1724 static void mlxsw_core_reg_access_cb(struct mlxsw_core *mlxsw_core,
1725                                      char *payload, size_t payload_len,
1726                                      unsigned long cb_priv)
1727 {
1728         char *orig_payload = (char *) cb_priv;
1729
1730         memcpy(orig_payload, payload, payload_len);
1731 }
1732
1733 static int mlxsw_core_reg_access(struct mlxsw_core *mlxsw_core,
1734                                  const struct mlxsw_reg_info *reg,
1735                                  char *payload,
1736                                  enum mlxsw_core_reg_access_type type)
1737 {
1738         LIST_HEAD(bulk_list);
1739         int err;
1740
1741         /* During initialization EMAD interface is not available to us,
1742          * so we default to command interface. We switch to EMAD interface
1743          * after setting the appropriate traps.
1744          */
1745         if (!mlxsw_core->emad.use_emad)
1746                 return mlxsw_core_reg_access_cmd(mlxsw_core, reg,
1747                                                  payload, type);
1748
1749         err = mlxsw_core_reg_access_emad(mlxsw_core, reg,
1750                                          payload, type, &bulk_list,
1751                                          mlxsw_core_reg_access_cb,
1752                                          (unsigned long) payload);
1753         if (err)
1754                 return err;
1755         return mlxsw_reg_trans_bulk_wait(&bulk_list);
1756 }
1757
1758 int mlxsw_reg_query(struct mlxsw_core *mlxsw_core,
1759                     const struct mlxsw_reg_info *reg, char *payload)
1760 {
1761         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
1762                                      MLXSW_CORE_REG_ACCESS_TYPE_QUERY);
1763 }
1764 EXPORT_SYMBOL(mlxsw_reg_query);
1765
1766 int mlxsw_reg_write(struct mlxsw_core *mlxsw_core,
1767                     const struct mlxsw_reg_info *reg, char *payload)
1768 {
1769         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
1770                                      MLXSW_CORE_REG_ACCESS_TYPE_WRITE);
1771 }
1772 EXPORT_SYMBOL(mlxsw_reg_write);
1773
1774 void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
1775                             struct mlxsw_rx_info *rx_info)
1776 {
1777         struct mlxsw_rx_listener_item *rxl_item;
1778         const struct mlxsw_rx_listener *rxl;
1779         u8 local_port;
1780         bool found = false;
1781
1782         if (rx_info->is_lag) {
1783                 dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: lag_id = %d, lag_port_index = 0x%x\n",
1784                                     __func__, rx_info->u.lag_id,
1785                                     rx_info->trap_id);
1786                 /* Upper layer does not care if the skb came from LAG or not,
1787                  * so just get the local_port for the lag port and push it up.
1788                  */
1789                 local_port = mlxsw_core_lag_mapping_get(mlxsw_core,
1790                                                         rx_info->u.lag_id,
1791                                                         rx_info->lag_port_index);
1792         } else {
1793                 local_port = rx_info->u.sys_port;
1794         }
1795
1796         dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: local_port = %d, trap_id = 0x%x\n",
1797                             __func__, local_port, rx_info->trap_id);
1798
1799         if ((rx_info->trap_id >= MLXSW_TRAP_ID_MAX) ||
1800             (local_port >= mlxsw_core->max_ports))
1801                 goto drop;
1802
1803         rcu_read_lock();
1804         list_for_each_entry_rcu(rxl_item, &mlxsw_core->rx_listener_list, list) {
1805                 rxl = &rxl_item->rxl;
1806                 if ((rxl->local_port == MLXSW_PORT_DONT_CARE ||
1807                      rxl->local_port == local_port) &&
1808                     rxl->trap_id == rx_info->trap_id) {
1809                         found = true;
1810                         break;
1811                 }
1812         }
1813         rcu_read_unlock();
1814         if (!found)
1815                 goto drop;
1816
1817         rxl->func(skb, local_port, rxl_item->priv);
1818         return;
1819
1820 drop:
1821         dev_kfree_skb(skb);
1822 }
1823 EXPORT_SYMBOL(mlxsw_core_skb_receive);
1824
1825 static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core,
1826                                         u16 lag_id, u8 port_index)
1827 {
1828         return MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS) * lag_id +
1829                port_index;
1830 }
1831
1832 void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core,
1833                                 u16 lag_id, u8 port_index, u8 local_port)
1834 {
1835         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1836                                                  lag_id, port_index);
1837
1838         mlxsw_core->lag.mapping[index] = local_port;
1839 }
1840 EXPORT_SYMBOL(mlxsw_core_lag_mapping_set);
1841
1842 u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core,
1843                               u16 lag_id, u8 port_index)
1844 {
1845         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1846                                                  lag_id, port_index);
1847
1848         return mlxsw_core->lag.mapping[index];
1849 }
1850 EXPORT_SYMBOL(mlxsw_core_lag_mapping_get);
1851
1852 void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core,
1853                                   u16 lag_id, u8 local_port)
1854 {
1855         int i;
1856
1857         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); i++) {
1858                 int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1859                                                          lag_id, i);
1860
1861                 if (mlxsw_core->lag.mapping[index] == local_port)
1862                         mlxsw_core->lag.mapping[index] = 0;
1863         }
1864 }
1865 EXPORT_SYMBOL(mlxsw_core_lag_mapping_clear);
1866
1867 bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core,
1868                           enum mlxsw_res_id res_id)
1869 {
1870         return mlxsw_res_valid(&mlxsw_core->res, res_id);
1871 }
1872 EXPORT_SYMBOL(mlxsw_core_res_valid);
1873
1874 u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core,
1875                        enum mlxsw_res_id res_id)
1876 {
1877         return mlxsw_res_get(&mlxsw_core->res, res_id);
1878 }
1879 EXPORT_SYMBOL(mlxsw_core_res_get);
1880
1881 static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
1882                                   enum devlink_port_flavour flavour,
1883                                   u32 port_number, bool split,
1884                                   u32 split_port_subnumber,
1885                                   const unsigned char *switch_id,
1886                                   unsigned char switch_id_len)
1887 {
1888         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1889         struct mlxsw_core_port *mlxsw_core_port =
1890                                         &mlxsw_core->ports[local_port];
1891         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1892         int err;
1893
1894         mlxsw_core_port->local_port = local_port;
1895         devlink_port_attrs_set(devlink_port, flavour, port_number,
1896                                split, split_port_subnumber,
1897                                switch_id, switch_id_len);
1898         err = devlink_port_register(devlink, devlink_port, local_port);
1899         if (err)
1900                 memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
1901         return err;
1902 }
1903
1904 static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
1905 {
1906         struct mlxsw_core_port *mlxsw_core_port =
1907                                         &mlxsw_core->ports[local_port];
1908         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1909
1910         devlink_port_unregister(devlink_port);
1911         memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
1912 }
1913
1914 int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
1915                          u32 port_number, bool split,
1916                          u32 split_port_subnumber,
1917                          const unsigned char *switch_id,
1918                          unsigned char switch_id_len)
1919 {
1920         return __mlxsw_core_port_init(mlxsw_core, local_port,
1921                                       DEVLINK_PORT_FLAVOUR_PHYSICAL,
1922                                       port_number, split, split_port_subnumber,
1923                                       switch_id, switch_id_len);
1924 }
1925 EXPORT_SYMBOL(mlxsw_core_port_init);
1926
1927 void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
1928 {
1929         __mlxsw_core_port_fini(mlxsw_core, local_port);
1930 }
1931 EXPORT_SYMBOL(mlxsw_core_port_fini);
1932
1933 int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core,
1934                              void *port_driver_priv,
1935                              const unsigned char *switch_id,
1936                              unsigned char switch_id_len)
1937 {
1938         struct mlxsw_core_port *mlxsw_core_port =
1939                                 &mlxsw_core->ports[MLXSW_PORT_CPU_PORT];
1940         int err;
1941
1942         err = __mlxsw_core_port_init(mlxsw_core, MLXSW_PORT_CPU_PORT,
1943                                      DEVLINK_PORT_FLAVOUR_CPU,
1944                                      0, false, 0,
1945                                      switch_id, switch_id_len);
1946         if (err)
1947                 return err;
1948
1949         mlxsw_core_port->port_driver_priv = port_driver_priv;
1950         return 0;
1951 }
1952 EXPORT_SYMBOL(mlxsw_core_cpu_port_init);
1953
1954 void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core)
1955 {
1956         __mlxsw_core_port_fini(mlxsw_core, MLXSW_PORT_CPU_PORT);
1957 }
1958 EXPORT_SYMBOL(mlxsw_core_cpu_port_fini);
1959
1960 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port,
1961                              void *port_driver_priv, struct net_device *dev)
1962 {
1963         struct mlxsw_core_port *mlxsw_core_port =
1964                                         &mlxsw_core->ports[local_port];
1965         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1966
1967         mlxsw_core_port->port_driver_priv = port_driver_priv;
1968         devlink_port_type_eth_set(devlink_port, dev);
1969 }
1970 EXPORT_SYMBOL(mlxsw_core_port_eth_set);
1971
1972 void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port,
1973                             void *port_driver_priv)
1974 {
1975         struct mlxsw_core_port *mlxsw_core_port =
1976                                         &mlxsw_core->ports[local_port];
1977         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1978
1979         mlxsw_core_port->port_driver_priv = port_driver_priv;
1980         devlink_port_type_ib_set(devlink_port, NULL);
1981 }
1982 EXPORT_SYMBOL(mlxsw_core_port_ib_set);
1983
1984 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port,
1985                            void *port_driver_priv)
1986 {
1987         struct mlxsw_core_port *mlxsw_core_port =
1988                                         &mlxsw_core->ports[local_port];
1989         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1990
1991         mlxsw_core_port->port_driver_priv = port_driver_priv;
1992         devlink_port_type_clear(devlink_port);
1993 }
1994 EXPORT_SYMBOL(mlxsw_core_port_clear);
1995
1996 enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
1997                                                 u8 local_port)
1998 {
1999         struct mlxsw_core_port *mlxsw_core_port =
2000                                         &mlxsw_core->ports[local_port];
2001         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2002
2003         return devlink_port->type;
2004 }
2005 EXPORT_SYMBOL(mlxsw_core_port_type_get);
2006
2007
2008 struct devlink_port *
2009 mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core,
2010                                  u8 local_port)
2011 {
2012         struct mlxsw_core_port *mlxsw_core_port =
2013                                         &mlxsw_core->ports[local_port];
2014         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2015
2016         return devlink_port;
2017 }
2018 EXPORT_SYMBOL(mlxsw_core_port_devlink_port_get);
2019
2020 static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
2021                                     const char *buf, size_t size)
2022 {
2023         __be32 *m = (__be32 *) buf;
2024         int i;
2025         int count = size / sizeof(__be32);
2026
2027         for (i = count - 1; i >= 0; i--)
2028                 if (m[i])
2029                         break;
2030         i++;
2031         count = i ? i : 1;
2032         for (i = 0; i < count; i += 4)
2033                 dev_dbg(mlxsw_core->bus_info->dev, "%04x - %08x %08x %08x %08x\n",
2034                         i * 4, be32_to_cpu(m[i]), be32_to_cpu(m[i + 1]),
2035                         be32_to_cpu(m[i + 2]), be32_to_cpu(m[i + 3]));
2036 }
2037
2038 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
2039                    u32 in_mod, bool out_mbox_direct, bool reset_ok,
2040                    char *in_mbox, size_t in_mbox_size,
2041                    char *out_mbox, size_t out_mbox_size)
2042 {
2043         u8 status;
2044         int err;
2045
2046         BUG_ON(in_mbox_size % sizeof(u32) || out_mbox_size % sizeof(u32));
2047         if (!mlxsw_core->bus->cmd_exec)
2048                 return -EOPNOTSUPP;
2049
2050         dev_dbg(mlxsw_core->bus_info->dev, "Cmd exec (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
2051                 opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod, in_mod);
2052         if (in_mbox) {
2053                 dev_dbg(mlxsw_core->bus_info->dev, "Input mailbox:\n");
2054                 mlxsw_core_buf_dump_dbg(mlxsw_core, in_mbox, in_mbox_size);
2055         }
2056
2057         err = mlxsw_core->bus->cmd_exec(mlxsw_core->bus_priv, opcode,
2058                                         opcode_mod, in_mod, out_mbox_direct,
2059                                         in_mbox, in_mbox_size,
2060                                         out_mbox, out_mbox_size, &status);
2061
2062         if (!err && out_mbox) {
2063                 dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
2064                 mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
2065         }
2066
2067         if (reset_ok && err == -EIO &&
2068             status == MLXSW_CMD_STATUS_RUNNING_RESET) {
2069                 err = 0;
2070         } else if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
2071                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n",
2072                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
2073                         in_mod, status, mlxsw_cmd_status_str(status));
2074         } else if (err == -ETIMEDOUT) {
2075                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec timed-out (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
2076                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
2077                         in_mod);
2078         }
2079
2080         return err;
2081 }
2082 EXPORT_SYMBOL(mlxsw_cmd_exec);
2083
2084 int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay)
2085 {
2086         return queue_delayed_work(mlxsw_wq, dwork, delay);
2087 }
2088 EXPORT_SYMBOL(mlxsw_core_schedule_dw);
2089
2090 bool mlxsw_core_schedule_work(struct work_struct *work)
2091 {
2092         return queue_work(mlxsw_owq, work);
2093 }
2094 EXPORT_SYMBOL(mlxsw_core_schedule_work);
2095
2096 void mlxsw_core_flush_owq(void)
2097 {
2098         flush_workqueue(mlxsw_owq);
2099 }
2100 EXPORT_SYMBOL(mlxsw_core_flush_owq);
2101
2102 int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
2103                              const struct mlxsw_config_profile *profile,
2104                              u64 *p_single_size, u64 *p_double_size,
2105                              u64 *p_linear_size)
2106 {
2107         struct mlxsw_driver *driver = mlxsw_core->driver;
2108
2109         if (!driver->kvd_sizes_get)
2110                 return -EINVAL;
2111
2112         return driver->kvd_sizes_get(mlxsw_core, profile,
2113                                      p_single_size, p_double_size,
2114                                      p_linear_size);
2115 }
2116 EXPORT_SYMBOL(mlxsw_core_kvd_sizes_get);
2117
2118 void mlxsw_core_fw_flash_start(struct mlxsw_core *mlxsw_core)
2119 {
2120         mlxsw_core->fw_flash_in_progress = true;
2121 }
2122 EXPORT_SYMBOL(mlxsw_core_fw_flash_start);
2123
2124 void mlxsw_core_fw_flash_end(struct mlxsw_core *mlxsw_core)
2125 {
2126         mlxsw_core->fw_flash_in_progress = false;
2127 }
2128 EXPORT_SYMBOL(mlxsw_core_fw_flash_end);
2129
2130 int mlxsw_core_resources_query(struct mlxsw_core *mlxsw_core, char *mbox,
2131                                struct mlxsw_res *res)
2132 {
2133         int index, i;
2134         u64 data;
2135         u16 id;
2136         int err;
2137
2138         if (!res)
2139                 return 0;
2140
2141         mlxsw_cmd_mbox_zero(mbox);
2142
2143         for (index = 0; index < MLXSW_CMD_QUERY_RESOURCES_MAX_QUERIES;
2144              index++) {
2145                 err = mlxsw_cmd_query_resources(mlxsw_core, mbox, index);
2146                 if (err)
2147                         return err;
2148
2149                 for (i = 0; i < MLXSW_CMD_QUERY_RESOURCES_PER_QUERY; i++) {
2150                         id = mlxsw_cmd_mbox_query_resource_id_get(mbox, i);
2151                         data = mlxsw_cmd_mbox_query_resource_data_get(mbox, i);
2152
2153                         if (id == MLXSW_CMD_QUERY_RESOURCES_TABLE_END_ID)
2154                                 return 0;
2155
2156                         mlxsw_res_parse(res, id, data);
2157                 }
2158         }
2159
2160         /* If after MLXSW_RESOURCES_QUERY_MAX_QUERIES we still didn't get
2161          * MLXSW_RESOURCES_TABLE_END_ID, something went bad in the FW.
2162          */
2163         return -EIO;
2164 }
2165 EXPORT_SYMBOL(mlxsw_core_resources_query);
2166
2167 u32 mlxsw_core_read_frc_h(struct mlxsw_core *mlxsw_core)
2168 {
2169         return mlxsw_core->bus->read_frc_h(mlxsw_core->bus_priv);
2170 }
2171 EXPORT_SYMBOL(mlxsw_core_read_frc_h);
2172
2173 u32 mlxsw_core_read_frc_l(struct mlxsw_core *mlxsw_core)
2174 {
2175         return mlxsw_core->bus->read_frc_l(mlxsw_core->bus_priv);
2176 }
2177 EXPORT_SYMBOL(mlxsw_core_read_frc_l);
2178
2179 static int __init mlxsw_core_module_init(void)
2180 {
2181         int err;
2182
2183         mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, 0, 0);
2184         if (!mlxsw_wq)
2185                 return -ENOMEM;
2186         mlxsw_owq = alloc_ordered_workqueue("%s_ordered", 0,
2187                                             mlxsw_core_driver_name);
2188         if (!mlxsw_owq) {
2189                 err = -ENOMEM;
2190                 goto err_alloc_ordered_workqueue;
2191         }
2192         return 0;
2193
2194 err_alloc_ordered_workqueue:
2195         destroy_workqueue(mlxsw_wq);
2196         return err;
2197 }
2198
2199 static void __exit mlxsw_core_module_exit(void)
2200 {
2201         destroy_workqueue(mlxsw_owq);
2202         destroy_workqueue(mlxsw_wq);
2203 }
2204
2205 module_init(mlxsw_core_module_init);
2206 module_exit(mlxsw_core_module_exit);
2207
2208 MODULE_LICENSE("Dual BSD/GPL");
2209 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
2210 MODULE_DESCRIPTION("Mellanox switch device core driver");