drivers/infiniband/core/cm.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   5  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
   6  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   7  * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.
   8  */
   9
  10 #include <linux/completion.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/device.h>
  13 #include <linux/module.h>
  14 #include <linux/err.h>
  15 #include <linux/idr.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/random.h>
  18 #include <linux/rbtree.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/slab.h>
  21 #include <linux/sysfs.h>
  22 #include <linux/workqueue.h>
  23 #include <linux/kdev_t.h>
  24 #include <linux/etherdevice.h>
  25
  26 #include <rdma/ib_cache.h>
  27 #include <rdma/ib_cm.h>
  28 #include "cm_msgs.h"
  29 #include "core_priv.h"
  30
  31 MODULE_AUTHOR("Sean Hefty");
  32 MODULE_DESCRIPTION("InfiniBand CM");
  33 MODULE_LICENSE("Dual BSD/GPL");
  34
  35 static const char * const ibcm_rej_reason_strs[] = {
  36         [IB_CM_REJ_NO_QP]                       = "no QP",
  37         [IB_CM_REJ_NO_EEC]                      = "no EEC",
  38         [IB_CM_REJ_NO_RESOURCES]                = "no resources",
  39         [IB_CM_REJ_TIMEOUT]                     = "timeout",
  40         [IB_CM_REJ_UNSUPPORTED]                 = "unsupported",
  41         [IB_CM_REJ_INVALID_COMM_ID]             = "invalid comm ID",
  42         [IB_CM_REJ_INVALID_COMM_INSTANCE]       = "invalid comm instance",
  43         [IB_CM_REJ_INVALID_SERVICE_ID]          = "invalid service ID",
  44         [IB_CM_REJ_INVALID_TRANSPORT_TYPE]      = "invalid transport type",
  45         [IB_CM_REJ_STALE_CONN]                  = "stale conn",
  46         [IB_CM_REJ_RDC_NOT_EXIST]               = "RDC not exist",
  47         [IB_CM_REJ_INVALID_GID]                 = "invalid GID",
  48         [IB_CM_REJ_INVALID_LID]                 = "invalid LID",
  49         [IB_CM_REJ_INVALID_SL]                  = "invalid SL",
  50         [IB_CM_REJ_INVALID_TRAFFIC_CLASS]       = "invalid traffic class",
  51         [IB_CM_REJ_INVALID_HOP_LIMIT]           = "invalid hop limit",
  52         [IB_CM_REJ_INVALID_PACKET_RATE]         = "invalid packet rate",
  53         [IB_CM_REJ_INVALID_ALT_GID]             = "invalid alt GID",
  54         [IB_CM_REJ_INVALID_ALT_LID]             = "invalid alt LID",
  55         [IB_CM_REJ_INVALID_ALT_SL]              = "invalid alt SL",
  56         [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]   = "invalid alt traffic class",
  57         [IB_CM_REJ_INVALID_ALT_HOP_LIMIT]       = "invalid alt hop limit",
  58         [IB_CM_REJ_INVALID_ALT_PACKET_RATE]     = "invalid alt packet rate",
  59         [IB_CM_REJ_PORT_CM_REDIRECT]            = "port CM redirect",
  60         [IB_CM_REJ_PORT_REDIRECT]               = "port redirect",
  61         [IB_CM_REJ_INVALID_MTU]                 = "invalid MTU",
  62         [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
  63         [IB_CM_REJ_CONSUMER_DEFINED]            = "consumer defined",
  64         [IB_CM_REJ_INVALID_RNR_RETRY]           = "invalid RNR retry",
  65         [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]     = "duplicate local comm ID",
  66         [IB_CM_REJ_INVALID_CLASS_VERSION]       = "invalid class version",
  67         [IB_CM_REJ_INVALID_FLOW_LABEL]          = "invalid flow label",
  68         [IB_CM_REJ_INVALID_ALT_FLOW_LABEL]      = "invalid alt flow label",
  69 };
  70
  71 const char *__attribute_const__ ibcm_reject_msg(int reason)
  72 {
  73         size_t index = reason;
  74
  75         if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
  76             ibcm_rej_reason_strs[index])
  77                 return ibcm_rej_reason_strs[index];
  78         else
  79                 return "unrecognized reason";
  80 }
  81 EXPORT_SYMBOL(ibcm_reject_msg);
  82
  83 static void cm_add_one(struct ib_device *device);
  84 static void cm_remove_one(struct ib_device *device, void *client_data);
  85
  86 static struct ib_client cm_client = {
  87         .name   = "cm",
  88         .add    = cm_add_one,
  89         .remove = cm_remove_one
  90 };
  91
  92 static struct ib_cm {
  93         spinlock_t lock;
  94         struct list_head device_list;
  95         rwlock_t device_lock;
  96         struct rb_root listen_service_table;
  97         u64 listen_service_id;
  98         /* struct rb_root peer_service_table; todo: fix peer to peer */
  99         struct rb_root remote_qp_table;
 100         struct rb_root remote_id_table;
 101         struct rb_root remote_sidr_table;
 102         struct xarray local_id_table;
 103         u32 local_id_next;
 104         __be32 random_id_operand;
 105         struct list_head timewait_list;
 106         struct workqueue_struct *wq;
 107         /* Sync on cm change port state */
 108         spinlock_t state_lock;
 109 } cm;
 110
 111 /* Counter indexes ordered by attribute ID */
 112 enum {
 113         CM_REQ_COUNTER,
 114         CM_MRA_COUNTER,
 115         CM_REJ_COUNTER,
 116         CM_REP_COUNTER,
 117         CM_RTU_COUNTER,
 118         CM_DREQ_COUNTER,
 119         CM_DREP_COUNTER,
 120         CM_SIDR_REQ_COUNTER,
 121         CM_SIDR_REP_COUNTER,
 122         CM_LAP_COUNTER,
 123         CM_APR_COUNTER,
 124         CM_ATTR_COUNT,
 125         CM_ATTR_ID_OFFSET = 0x0010,
 126 };
 127
 128 enum {
 129         CM_XMIT,
 130         CM_XMIT_RETRIES,
 131         CM_RECV,
 132         CM_RECV_DUPLICATES,
 133         CM_COUNTER_GROUPS
 134 };
 135
 136 static char const counter_group_names[CM_COUNTER_GROUPS]
 137                                      [sizeof("cm_rx_duplicates")] = {
 138         "cm_tx_msgs", "cm_tx_retries",
 139         "cm_rx_msgs", "cm_rx_duplicates"
 140 };
 141
 142 struct cm_counter_group {
 143         struct kobject obj;
 144         atomic_long_t counter[CM_ATTR_COUNT];
 145 };
 146
 147 struct cm_counter_attribute {
 148         struct attribute attr;
 149         int index;
 150 };
 151
 152 #define CM_COUNTER_ATTR(_name, _index) \
 153 struct cm_counter_attribute cm_##_name##_counter_attr = { \
 154         .attr = { .name = __stringify(_name), .mode = 0444 }, \
 155         .index = _index \
 156 }
 157
 158 static CM_COUNTER_ATTR(req, CM_REQ_COUNTER);
 159 static CM_COUNTER_ATTR(mra, CM_MRA_COUNTER);
 160 static CM_COUNTER_ATTR(rej, CM_REJ_COUNTER);
 161 static CM_COUNTER_ATTR(rep, CM_REP_COUNTER);
 162 static CM_COUNTER_ATTR(rtu, CM_RTU_COUNTER);
 163 static CM_COUNTER_ATTR(dreq, CM_DREQ_COUNTER);
 164 static CM_COUNTER_ATTR(drep, CM_DREP_COUNTER);
 165 static CM_COUNTER_ATTR(sidr_req, CM_SIDR_REQ_COUNTER);
 166 static CM_COUNTER_ATTR(sidr_rep, CM_SIDR_REP_COUNTER);
 167 static CM_COUNTER_ATTR(lap, CM_LAP_COUNTER);
 168 static CM_COUNTER_ATTR(apr, CM_APR_COUNTER);
 169
 170 static struct attribute *cm_counter_default_attrs[] = {
 171         &cm_req_counter_attr.attr,
 172         &cm_mra_counter_attr.attr,
 173         &cm_rej_counter_attr.attr,
 174         &cm_rep_counter_attr.attr,
 175         &cm_rtu_counter_attr.attr,
 176         &cm_dreq_counter_attr.attr,
 177         &cm_drep_counter_attr.attr,
 178         &cm_sidr_req_counter_attr.attr,
 179         &cm_sidr_rep_counter_attr.attr,
 180         &cm_lap_counter_attr.attr,
 181         &cm_apr_counter_attr.attr,
 182         NULL
 183 };
 184
 185 struct cm_port {
 186         struct cm_device *cm_dev;
 187         struct ib_mad_agent *mad_agent;
 188         struct kobject port_obj;
 189         u8 port_num;
 190         struct list_head cm_priv_prim_list;
 191         struct list_head cm_priv_altr_list;
 192         struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
 193 };
 194
 195 struct cm_device {
 196         struct list_head list;
 197         struct ib_device *ib_device;
 198         u8 ack_delay;
 199         int going_down;
 200         struct cm_port *port[0];
 201 };
 202
 203 struct cm_av {
 204         struct cm_port *port;
 205         union ib_gid dgid;
 206         struct rdma_ah_attr ah_attr;
 207         u16 pkey_index;
 208         u8 timeout;
 209 };
 210
 211 struct cm_work {
 212         struct delayed_work work;
 213         struct list_head list;
 214         struct cm_port *port;
 215         struct ib_mad_recv_wc *mad_recv_wc;     /* Received MADs */
 216         __be32 local_id;                        /* Established / timewait */
 217         __be32 remote_id;
 218         struct ib_cm_event cm_event;
 219         struct sa_path_rec path[0];
 220 };
 221
 222 struct cm_timewait_info {
 223         struct cm_work work;
 224         struct list_head list;
 225         struct rb_node remote_qp_node;
 226         struct rb_node remote_id_node;
 227         __be64 remote_ca_guid;
 228         __be32 remote_qpn;
 229         u8 inserted_remote_qp;
 230         u8 inserted_remote_id;
 231 };
 232
 233 struct cm_id_private {
 234         struct ib_cm_id id;
 235
 236         struct rb_node service_node;
 237         struct rb_node sidr_id_node;
 238         spinlock_t lock;        /* Do not acquire inside cm.lock */
 239         struct completion comp;
 240         refcount_t refcount;
 241         /* Number of clients sharing this ib_cm_id. Only valid for listeners.
 242          * Protected by the cm.lock spinlock. */
 243         int listen_sharecount;
 244         struct rcu_head rcu;
 245
 246         struct ib_mad_send_buf *msg;
 247         struct cm_timewait_info *timewait_info;
 248         /* todo: use alternate port on send failure */
 249         struct cm_av av;
 250         struct cm_av alt_av;
 251
 252         void *private_data;
 253         __be64 tid;
 254         __be32 local_qpn;
 255         __be32 remote_qpn;
 256         enum ib_qp_type qp_type;
 257         __be32 sq_psn;
 258         __be32 rq_psn;
 259         int timeout_ms;
 260         enum ib_mtu path_mtu;
 261         __be16 pkey;
 262         u8 private_data_len;
 263         u8 max_cm_retries;
 264         u8 peer_to_peer;
 265         u8 responder_resources;
 266         u8 initiator_depth;
 267         u8 retry_count;
 268         u8 rnr_retry_count;
 269         u8 service_timeout;
 270         u8 target_ack_delay;
 271
 272         struct list_head prim_list;
 273         struct list_head altr_list;
 274         /* Indicates that the send port mad is registered and av is set */
 275         int prim_send_port_not_ready;
 276         int altr_send_port_not_ready;
 277
 278         struct list_head work_list;
 279         atomic_t work_count;
 280 };
 281
 282 static void cm_work_handler(struct work_struct *work);
 283
 284 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
 285 {
 286         if (refcount_dec_and_test(&cm_id_priv->refcount))
 287                 complete(&cm_id_priv->comp);
 288 }
 289
 290 static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
 291                         struct ib_mad_send_buf **msg)
 292 {
 293         struct ib_mad_agent *mad_agent;
 294         struct ib_mad_send_buf *m;
 295         struct ib_ah *ah;
 296         struct cm_av *av;
 297         unsigned long flags, flags2;
 298         int ret = 0;
 299
 300         /* don't let the port to be released till the agent is down */
 301         spin_lock_irqsave(&cm.state_lock, flags2);
 302         spin_lock_irqsave(&cm.lock, flags);
 303         if (!cm_id_priv->prim_send_port_not_ready)
 304                 av = &cm_id_priv->av;
 305         else if (!cm_id_priv->altr_send_port_not_ready &&
 306                  (cm_id_priv->alt_av.port))
 307                 av = &cm_id_priv->alt_av;
 308         else {
 309                 pr_info("%s: not valid CM id\n", __func__);
 310                 ret = -ENODEV;
 311                 spin_unlock_irqrestore(&cm.lock, flags);
 312                 goto out;
 313         }
 314         spin_unlock_irqrestore(&cm.lock, flags);
 315         /* Make sure the port haven't released the mad yet */
 316         mad_agent = cm_id_priv->av.port->mad_agent;
 317         if (!mad_agent) {
 318                 pr_info("%s: not a valid MAD agent\n", __func__);
 319                 ret = -ENODEV;
 320                 goto out;
 321         }
 322         ah = rdma_create_ah(mad_agent->qp->pd, &av->ah_attr, 0);
 323         if (IS_ERR(ah)) {
 324                 ret = PTR_ERR(ah);
 325                 goto out;
 326         }
 327
 328         m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
 329                                av->pkey_index,
 330                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 331                                GFP_ATOMIC,
 332                                IB_MGMT_BASE_VERSION);
 333         if (IS_ERR(m)) {
 334                 rdma_destroy_ah(ah, 0);
 335                 ret = PTR_ERR(m);
 336                 goto out;
 337         }
 338
 339         /* Timeout set by caller if response is expected. */
 340         m->ah = ah;
 341         m->retries = cm_id_priv->max_cm_retries;
 342
 343         refcount_inc(&cm_id_priv->refcount);
 344         m->context[0] = cm_id_priv;
 345         *msg = m;
 346
 347 out:
 348         spin_unlock_irqrestore(&cm.state_lock, flags2);
 349         return ret;
 350 }
 351
 352 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
 353                                                            struct ib_mad_recv_wc *mad_recv_wc)
 354 {
 355         return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
 356                                   0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 357                                   GFP_ATOMIC,
 358                                   IB_MGMT_BASE_VERSION);
 359 }
 360
 361 static int cm_create_response_msg_ah(struct cm_port *port,
 362                                      struct ib_mad_recv_wc *mad_recv_wc,
 363                                      struct ib_mad_send_buf *msg)
 364 {
 365         struct ib_ah *ah;
 366
 367         ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
 368                                   mad_recv_wc->recv_buf.grh, port->port_num);
 369         if (IS_ERR(ah))
 370                 return PTR_ERR(ah);
 371
 372         msg->ah = ah;
 373         return 0;
 374 }
 375
 376 static void cm_free_msg(struct ib_mad_send_buf *msg)
 377 {
 378         if (msg->ah)
 379                 rdma_destroy_ah(msg->ah, 0);
 380         if (msg->context[0])
 381                 cm_deref_id(msg->context[0]);
 382         ib_free_send_mad(msg);
 383 }
 384
 385 static int cm_alloc_response_msg(struct cm_port *port,
 386                                  struct ib_mad_recv_wc *mad_recv_wc,
 387                                  struct ib_mad_send_buf **msg)
 388 {
 389         struct ib_mad_send_buf *m;
 390         int ret;
 391
 392         m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
 393         if (IS_ERR(m))
 394                 return PTR_ERR(m);
 395
 396         ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
 397         if (ret) {
 398                 cm_free_msg(m);
 399                 return ret;
 400         }
 401
 402         *msg = m;
 403         return 0;
 404 }
 405
 406 static void * cm_copy_private_data(const void *private_data,
 407                                    u8 private_data_len)
 408 {
 409         void *data;
 410
 411         if (!private_data || !private_data_len)
 412                 return NULL;
 413
 414         data = kmemdup(private_data, private_data_len, GFP_KERNEL);
 415         if (!data)
 416                 return ERR_PTR(-ENOMEM);
 417
 418         return data;
 419 }
 420
 421 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 422                                  void *private_data, u8 private_data_len)
 423 {
 424         if (cm_id_priv->private_data && cm_id_priv->private_data_len)
 425                 kfree(cm_id_priv->private_data);
 426
 427         cm_id_priv->private_data = private_data;
 428         cm_id_priv->private_data_len = private_data_len;
 429 }
 430
 431 static int cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
 432                               struct ib_grh *grh, struct cm_av *av)
 433 {
 434         struct rdma_ah_attr new_ah_attr;
 435         int ret;
 436
 437         av->port = port;
 438         av->pkey_index = wc->pkey_index;
 439
 440         /*
 441          * av->ah_attr might be initialized based on past wc during incoming
 442          * connect request or while sending out connect request. So initialize
 443          * a new ah_attr on stack. If initialization fails, old ah_attr is
 444          * used for sending any responses. If initialization is successful,
 445          * than new ah_attr is used by overwriting old one.
 446          */
 447         ret = ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
 448                                       port->port_num, wc,
 449                                       grh, &new_ah_attr);
 450         if (ret)
 451                 return ret;
 452
 453         rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
 454         return 0;
 455 }
 456
 457 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 458                                    struct ib_grh *grh, struct cm_av *av)
 459 {
 460         av->port = port;
 461         av->pkey_index = wc->pkey_index;
 462         return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
 463                                        port->port_num, wc,
 464                                        grh, &av->ah_attr);
 465 }
 466
 467 static int add_cm_id_to_port_list(struct cm_id_private *cm_id_priv,
 468                                   struct cm_av *av,
 469                                   struct cm_port *port)
 470 {
 471         unsigned long flags;
 472         int ret = 0;
 473
 474         spin_lock_irqsave(&cm.lock, flags);
 475
 476         if (&cm_id_priv->av == av)
 477                 list_add_tail(&cm_id_priv->prim_list, &port->cm_priv_prim_list);
 478         else if (&cm_id_priv->alt_av == av)
 479                 list_add_tail(&cm_id_priv->altr_list, &port->cm_priv_altr_list);
 480         else
 481                 ret = -EINVAL;
 482
 483         spin_unlock_irqrestore(&cm.lock, flags);
 484         return ret;
 485 }
 486
 487 static struct cm_port *
 488 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
 489 {
 490         struct cm_device *cm_dev;
 491         struct cm_port *port = NULL;
 492         unsigned long flags;
 493
 494         if (attr) {
 495                 read_lock_irqsave(&cm.device_lock, flags);
 496                 list_for_each_entry(cm_dev, &cm.device_list, list) {
 497                         if (cm_dev->ib_device == attr->device) {
 498                                 port = cm_dev->port[attr->port_num - 1];
 499                                 break;
 500                         }
 501                 }
 502                 read_unlock_irqrestore(&cm.device_lock, flags);
 503         } else {
 504                 /* SGID attribute can be NULL in following
 505                  * conditions.
 506                  * (a) Alternative path
 507                  * (b) IB link layer without GRH
 508                  * (c) LAP send messages
 509                  */
 510                 read_lock_irqsave(&cm.device_lock, flags);
 511                 list_for_each_entry(cm_dev, &cm.device_list, list) {
 512                         attr = rdma_find_gid(cm_dev->ib_device,
 513                                              &path->sgid,
 514                                              sa_conv_pathrec_to_gid_type(path),
 515                                              NULL);
 516                         if (!IS_ERR(attr)) {
 517                                 port = cm_dev->port[attr->port_num - 1];
 518                                 break;
 519                         }
 520                 }
 521                 read_unlock_irqrestore(&cm.device_lock, flags);
 522                 if (port)
 523                         rdma_put_gid_attr(attr);
 524         }
 525         return port;
 526 }
 527
 528 static int cm_init_av_by_path(struct sa_path_rec *path,
 529                               const struct ib_gid_attr *sgid_attr,
 530                               struct cm_av *av,
 531                               struct cm_id_private *cm_id_priv)
 532 {
 533         struct rdma_ah_attr new_ah_attr;
 534         struct cm_device *cm_dev;
 535         struct cm_port *port;
 536         int ret;
 537
 538         port = get_cm_port_from_path(path, sgid_attr);
 539         if (!port)
 540                 return -EINVAL;
 541         cm_dev = port->cm_dev;
 542
 543         ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
 544                                   be16_to_cpu(path->pkey), &av->pkey_index);
 545         if (ret)
 546                 return ret;
 547
 548         av->port = port;
 549
 550         /*
 551          * av->ah_attr might be initialized based on wc or during
 552          * request processing time which might have reference to sgid_attr.
 553          * So initialize a new ah_attr on stack.
 554          * If initialization fails, old ah_attr is used for sending any
 555          * responses. If initialization is successful, than new ah_attr
 556          * is used by overwriting the old one. So that right ah_attr
 557          * can be used to return an error response.
 558          */
 559         ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
 560                                         &new_ah_attr, sgid_attr);
 561         if (ret)
 562                 return ret;
 563
 564         av->timeout = path->packet_life_time + 1;
 565
 566         ret = add_cm_id_to_port_list(cm_id_priv, av, port);
 567         if (ret) {
 568                 rdma_destroy_ah_attr(&new_ah_attr);
 569                 return ret;
 570         }
 571         rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
 572         return 0;
 573 }
 574
 575 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
 576 {
 577         int err;
 578         u32 id;
 579
 580         err = xa_alloc_cyclic_irq(&cm.local_id_table, &id, cm_id_priv,
 581                         xa_limit_32b, &cm.local_id_next, GFP_KERNEL);
 582
 583         cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
 584         return err;
 585 }
 586
 587 static u32 cm_local_id(__be32 local_id)
 588 {
 589         return (__force u32) (local_id ^ cm.random_id_operand);
 590 }
 591
 592 static void cm_free_id(__be32 local_id)
 593 {
 594         xa_erase_irq(&cm.local_id_table, cm_local_id(local_id));
 595 }
 596
 597 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
 598 {
 599         struct cm_id_private *cm_id_priv;
 600
 601         rcu_read_lock();
 602         cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
 603         if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
 604             !refcount_inc_not_zero(&cm_id_priv->refcount))
 605                 cm_id_priv = NULL;
 606         rcu_read_unlock();
 607
 608         return cm_id_priv;
 609 }
 610
 611 /*
 612  * Trivial helpers to strip endian annotation and compare; the
 613  * endianness doesn't actually matter since we just need a stable
 614  * order for the RB tree.
 615  */
 616 static int be32_lt(__be32 a, __be32 b)
 617 {
 618         return (__force u32) a < (__force u32) b;
 619 }
 620
 621 static int be32_gt(__be32 a, __be32 b)
 622 {
 623         return (__force u32) a > (__force u32) b;
 624 }
 625
 626 static int be64_lt(__be64 a, __be64 b)
 627 {
 628         return (__force u64) a < (__force u64) b;
 629 }
 630
 631 static int be64_gt(__be64 a, __be64 b)
 632 {
 633         return (__force u64) a > (__force u64) b;
 634 }
 635
 636 static struct cm_id_private * cm_insert_listen(struct cm_id_private *cm_id_priv)
 637 {
 638         struct rb_node **link = &cm.listen_service_table.rb_node;
 639         struct rb_node *parent = NULL;
 640         struct cm_id_private *cur_cm_id_priv;
 641         __be64 service_id = cm_id_priv->id.service_id;
 642         __be64 service_mask = cm_id_priv->id.service_mask;
 643
 644         while (*link) {
 645                 parent = *link;
 646                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 647                                           service_node);
 648                 if ((cur_cm_id_priv->id.service_mask & service_id) ==
 649                     (service_mask & cur_cm_id_priv->id.service_id) &&
 650                     (cm_id_priv->id.device == cur_cm_id_priv->id.device))
 651                         return cur_cm_id_priv;
 652
 653                 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
 654                         link = &(*link)->rb_left;
 655                 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
 656                         link = &(*link)->rb_right;
 657                 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
 658                         link = &(*link)->rb_left;
 659                 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
 660                         link = &(*link)->rb_right;
 661                 else
 662                         link = &(*link)->rb_right;
 663         }
 664         rb_link_node(&cm_id_priv->service_node, parent, link);
 665         rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
 666         return NULL;
 667 }
 668
 669 static struct cm_id_private * cm_find_listen(struct ib_device *device,
 670                                              __be64 service_id)
 671 {
 672         struct rb_node *node = cm.listen_service_table.rb_node;
 673         struct cm_id_private *cm_id_priv;
 674
 675         while (node) {
 676                 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
 677                 if ((cm_id_priv->id.service_mask & service_id) ==
 678                      cm_id_priv->id.service_id &&
 679                     (cm_id_priv->id.device == device))
 680                         return cm_id_priv;
 681
 682                 if (device < cm_id_priv->id.device)
 683                         node = node->rb_left;
 684                 else if (device > cm_id_priv->id.device)
 685                         node = node->rb_right;
 686                 else if (be64_lt(service_id, cm_id_priv->id.service_id))
 687                         node = node->rb_left;
 688                 else if (be64_gt(service_id, cm_id_priv->id.service_id))
 689                         node = node->rb_right;
 690                 else
 691                         node = node->rb_right;
 692         }
 693         return NULL;
 694 }
 695
 696 static struct cm_timewait_info * cm_insert_remote_id(struct cm_timewait_info
 697                                                      *timewait_info)
 698 {
 699         struct rb_node **link = &cm.remote_id_table.rb_node;
 700         struct rb_node *parent = NULL;
 701         struct cm_timewait_info *cur_timewait_info;
 702         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 703         __be32 remote_id = timewait_info->work.remote_id;
 704
 705         while (*link) {
 706                 parent = *link;
 707                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 708                                              remote_id_node);
 709                 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
 710                         link = &(*link)->rb_left;
 711                 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
 712                         link = &(*link)->rb_right;
 713                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 714                         link = &(*link)->rb_left;
 715                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 716                         link = &(*link)->rb_right;
 717                 else
 718                         return cur_timewait_info;
 719         }
 720         timewait_info->inserted_remote_id = 1;
 721         rb_link_node(&timewait_info->remote_id_node, parent, link);
 722         rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
 723         return NULL;
 724 }
 725
 726 static struct cm_timewait_info * cm_find_remote_id(__be64 remote_ca_guid,
 727                                                    __be32 remote_id)
 728 {
 729         struct rb_node *node = cm.remote_id_table.rb_node;
 730         struct cm_timewait_info *timewait_info;
 731
 732         while (node) {
 733                 timewait_info = rb_entry(node, struct cm_timewait_info,
 734                                          remote_id_node);
 735                 if (be32_lt(remote_id, timewait_info->work.remote_id))
 736                         node = node->rb_left;
 737                 else if (be32_gt(remote_id, timewait_info->work.remote_id))
 738                         node = node->rb_right;
 739                 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
 740                         node = node->rb_left;
 741                 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
 742                         node = node->rb_right;
 743                 else
 744                         return timewait_info;
 745         }
 746         return NULL;
 747 }
 748
 749 static struct cm_timewait_info * cm_insert_remote_qpn(struct cm_timewait_info
 750                                                       *timewait_info)
 751 {
 752         struct rb_node **link = &cm.remote_qp_table.rb_node;
 753         struct rb_node *parent = NULL;
 754         struct cm_timewait_info *cur_timewait_info;
 755         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 756         __be32 remote_qpn = timewait_info->remote_qpn;
 757
 758         while (*link) {
 759                 parent = *link;
 760                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 761                                              remote_qp_node);
 762                 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
 763                         link = &(*link)->rb_left;
 764                 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
 765                         link = &(*link)->rb_right;
 766                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 767                         link = &(*link)->rb_left;
 768                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 769                         link = &(*link)->rb_right;
 770                 else
 771                         return cur_timewait_info;
 772         }
 773         timewait_info->inserted_remote_qp = 1;
 774         rb_link_node(&timewait_info->remote_qp_node, parent, link);
 775         rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 776         return NULL;
 777 }
 778
 779 static struct cm_id_private * cm_insert_remote_sidr(struct cm_id_private
 780                                                     *cm_id_priv)
 781 {
 782         struct rb_node **link = &cm.remote_sidr_table.rb_node;
 783         struct rb_node *parent = NULL;
 784         struct cm_id_private *cur_cm_id_priv;
 785         union ib_gid *port_gid = &cm_id_priv->av.dgid;
 786         __be32 remote_id = cm_id_priv->id.remote_id;
 787
 788         while (*link) {
 789                 parent = *link;
 790                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 791                                           sidr_id_node);
 792                 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
 793                         link = &(*link)->rb_left;
 794                 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
 795                         link = &(*link)->rb_right;
 796                 else {
 797                         int cmp;
 798                         cmp = memcmp(port_gid, &cur_cm_id_priv->av.dgid,
 799                                      sizeof *port_gid);
 800                         if (cmp < 0)
 801                                 link = &(*link)->rb_left;
 802                         else if (cmp > 0)
 803                                 link = &(*link)->rb_right;
 804                         else
 805                                 return cur_cm_id_priv;
 806                 }
 807         }
 808         rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
 809         rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
 810         return NULL;
 811 }
 812
 813 static void cm_reject_sidr_req(struct cm_id_private *cm_id_priv,
 814                                enum ib_cm_sidr_status status)
 815 {
 816         struct ib_cm_sidr_rep_param param;
 817
 818         memset(&param, 0, sizeof param);
 819         param.status = status;
 820         ib_send_cm_sidr_rep(&cm_id_priv->id, &param);
 821 }
 822
 823 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
 824                                  ib_cm_handler cm_handler,
 825                                  void *context)
 826 {
 827         struct cm_id_private *cm_id_priv;
 828         int ret;
 829
 830         cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
 831         if (!cm_id_priv)
 832                 return ERR_PTR(-ENOMEM);
 833
 834         cm_id_priv->id.state = IB_CM_IDLE;
 835         cm_id_priv->id.device = device;
 836         cm_id_priv->id.cm_handler = cm_handler;
 837         cm_id_priv->id.context = context;
 838         cm_id_priv->id.remote_cm_qpn = 1;
 839         ret = cm_alloc_id(cm_id_priv);
 840         if (ret)
 841                 goto error;
 842
 843         spin_lock_init(&cm_id_priv->lock);
 844         init_completion(&cm_id_priv->comp);
 845         INIT_LIST_HEAD(&cm_id_priv->work_list);
 846         INIT_LIST_HEAD(&cm_id_priv->prim_list);
 847         INIT_LIST_HEAD(&cm_id_priv->altr_list);
 848         atomic_set(&cm_id_priv->work_count, -1);
 849         refcount_set(&cm_id_priv->refcount, 1);
 850         return &cm_id_priv->id;
 851
 852 error:
 853         kfree(cm_id_priv);
 854         return ERR_PTR(-ENOMEM);
 855 }
 856 EXPORT_SYMBOL(ib_create_cm_id);
 857
 858 static struct cm_work * cm_dequeue_work(struct cm_id_private *cm_id_priv)
 859 {
 860         struct cm_work *work;
 861
 862         if (list_empty(&cm_id_priv->work_list))
 863                 return NULL;
 864
 865         work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
 866         list_del(&work->list);
 867         return work;
 868 }
 869
 870 static void cm_free_work(struct cm_work *work)
 871 {
 872         if (work->mad_recv_wc)
 873                 ib_free_recv_mad(work->mad_recv_wc);
 874         kfree(work);
 875 }
 876
 877 static inline int cm_convert_to_ms(int iba_time)
 878 {
 879         /* approximate conversion to ms from 4.096us x 2^iba_time */
 880         return 1 << max(iba_time - 8, 0);
 881 }
 882
 883 /*
 884  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
 885  * Because of how ack_timeout is stored, adding one doubles the timeout.
 886  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
 887  * increment it (round up) only if the other is within 50%.
 888  */
 889 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
 890 {
 891         int ack_timeout = packet_life_time + 1;
 892
 893         if (ack_timeout >= ca_ack_delay)
 894                 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
 895         else
 896                 ack_timeout = ca_ack_delay +
 897                               (ack_timeout >= (ca_ack_delay - 1));
 898
 899         return min(31, ack_timeout);
 900 }
 901
 902 static void cm_cleanup_timewait(struct cm_timewait_info *timewait_info)
 903 {
 904         if (timewait_info->inserted_remote_id) {
 905                 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
 906                 timewait_info->inserted_remote_id = 0;
 907         }
 908
 909         if (timewait_info->inserted_remote_qp) {
 910                 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 911                 timewait_info->inserted_remote_qp = 0;
 912         }
 913 }
 914
 915 static struct cm_timewait_info * cm_create_timewait_info(__be32 local_id)
 916 {
 917         struct cm_timewait_info *timewait_info;
 918
 919         timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
 920         if (!timewait_info)
 921                 return ERR_PTR(-ENOMEM);
 922
 923         timewait_info->work.local_id = local_id;
 924         INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
 925         timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
 926         return timewait_info;
 927 }
 928
 929 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 930 {
 931         int wait_time;
 932         unsigned long flags;
 933         struct cm_device *cm_dev;
 934
 935         cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
 936         if (!cm_dev)
 937                 return;
 938
 939         spin_lock_irqsave(&cm.lock, flags);
 940         cm_cleanup_timewait(cm_id_priv->timewait_info);
 941         list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
 942         spin_unlock_irqrestore(&cm.lock, flags);
 943
 944         /*
 945          * The cm_id could be destroyed by the user before we exit timewait.
 946          * To protect against this, we search for the cm_id after exiting
 947          * timewait before notifying the user that we've exited timewait.
 948          */
 949         cm_id_priv->id.state = IB_CM_TIMEWAIT;
 950         wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
 951
 952         /* Check if the device started its remove_one */
 953         spin_lock_irqsave(&cm.lock, flags);
 954         if (!cm_dev->going_down)
 955                 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
 956                                    msecs_to_jiffies(wait_time));
 957         spin_unlock_irqrestore(&cm.lock, flags);
 958
 959         cm_id_priv->timewait_info = NULL;
 960 }
 961
 962 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
 963 {
 964         unsigned long flags;
 965
 966         cm_id_priv->id.state = IB_CM_IDLE;
 967         if (cm_id_priv->timewait_info) {
 968                 spin_lock_irqsave(&cm.lock, flags);
 969                 cm_cleanup_timewait(cm_id_priv->timewait_info);
 970                 spin_unlock_irqrestore(&cm.lock, flags);
 971                 kfree(cm_id_priv->timewait_info);
 972                 cm_id_priv->timewait_info = NULL;
 973         }
 974 }
 975
 976 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
 977 {
 978         struct cm_id_private *cm_id_priv;
 979         struct cm_work *work;
 980
 981         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
 982 retest:
 983         spin_lock_irq(&cm_id_priv->lock);
 984         switch (cm_id->state) {
 985         case IB_CM_LISTEN:
 986                 spin_unlock_irq(&cm_id_priv->lock);
 987
 988                 spin_lock_irq(&cm.lock);
 989                 if (--cm_id_priv->listen_sharecount > 0) {
 990                         /* The id is still shared. */
 991                         cm_deref_id(cm_id_priv);
 992                         spin_unlock_irq(&cm.lock);
 993                         return;
 994                 }
 995                 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
 996                 spin_unlock_irq(&cm.lock);
 997                 break;
 998         case IB_CM_SIDR_REQ_SENT:
 999                 cm_id->state = IB_CM_IDLE;
1000                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1001                 spin_unlock_irq(&cm_id_priv->lock);
1002                 break;
1003         case IB_CM_SIDR_REQ_RCVD:
1004                 spin_unlock_irq(&cm_id_priv->lock);
1005                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_REJECT);
1006                 spin_lock_irq(&cm.lock);
1007                 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1008                         rb_erase(&cm_id_priv->sidr_id_node,
1009                                  &cm.remote_sidr_table);
1010                 spin_unlock_irq(&cm.lock);
1011                 break;
1012         case IB_CM_REQ_SENT:
1013         case IB_CM_MRA_REQ_RCVD:
1014                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1015                 spin_unlock_irq(&cm_id_priv->lock);
1016                 ib_send_cm_rej(cm_id, IB_CM_REJ_TIMEOUT,
1017                                &cm_id_priv->id.device->node_guid,
1018                                sizeof cm_id_priv->id.device->node_guid,
1019                                NULL, 0);
1020                 break;
1021         case IB_CM_REQ_RCVD:
1022                 if (err == -ENOMEM) {
1023                         /* Do not reject to allow future retries. */
1024                         cm_reset_to_idle(cm_id_priv);
1025                         spin_unlock_irq(&cm_id_priv->lock);
1026                 } else {
1027                         spin_unlock_irq(&cm_id_priv->lock);
1028                         ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
1029                                        NULL, 0, NULL, 0);
1030                 }
1031                 break;
1032         case IB_CM_REP_SENT:
1033         case IB_CM_MRA_REP_RCVD:
1034                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1035                 /* Fall through */
1036         case IB_CM_MRA_REQ_SENT:
1037         case IB_CM_REP_RCVD:
1038         case IB_CM_MRA_REP_SENT:
1039                 spin_unlock_irq(&cm_id_priv->lock);
1040                 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED,
1041                                NULL, 0, NULL, 0);
1042                 break;
1043         case IB_CM_ESTABLISHED:
1044                 spin_unlock_irq(&cm_id_priv->lock);
1045                 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT)
1046                         break;
1047                 ib_send_cm_dreq(cm_id, NULL, 0);
1048                 goto retest;
1049         case IB_CM_DREQ_SENT:
1050                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
1051                 cm_enter_timewait(cm_id_priv);
1052                 spin_unlock_irq(&cm_id_priv->lock);
1053                 break;
1054         case IB_CM_DREQ_RCVD:
1055                 spin_unlock_irq(&cm_id_priv->lock);
1056                 ib_send_cm_drep(cm_id, NULL, 0);
1057                 break;
1058         default:
1059                 spin_unlock_irq(&cm_id_priv->lock);
1060                 break;
1061         }
1062
1063         spin_lock_irq(&cm.lock);
1064         if (!list_empty(&cm_id_priv->altr_list) &&
1065             (!cm_id_priv->altr_send_port_not_ready))
1066                 list_del(&cm_id_priv->altr_list);
1067         if (!list_empty(&cm_id_priv->prim_list) &&
1068             (!cm_id_priv->prim_send_port_not_ready))
1069                 list_del(&cm_id_priv->prim_list);
1070         spin_unlock_irq(&cm.lock);
1071
1072         cm_free_id(cm_id->local_id);
1073         cm_deref_id(cm_id_priv);
1074         wait_for_completion(&cm_id_priv->comp);
1075         while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1076                 cm_free_work(work);
1077
1078         rdma_destroy_ah_attr(&cm_id_priv->av.ah_attr);
1079         rdma_destroy_ah_attr(&cm_id_priv->alt_av.ah_attr);
1080         kfree(cm_id_priv->private_data);
1081         kfree_rcu(cm_id_priv, rcu);
1082 }
1083
1084 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1085 {
1086         cm_destroy_id(cm_id, 0);
1087 }
1088 EXPORT_SYMBOL(ib_destroy_cm_id);
1089
1090 /**
1091  * __ib_cm_listen - Initiates listening on the specified service ID for
1092  *   connection and service ID resolution requests.
1093  * @cm_id: Connection identifier associated with the listen request.
1094  * @service_id: Service identifier matched against incoming connection
1095  *   and service ID resolution requests.  The service ID should be specified
1096  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1097  *   assign a service ID to the caller.
1098  * @service_mask: Mask applied to service ID used to listen across a
1099  *   range of service IDs.  If set to 0, the service ID is matched
1100  *   exactly.  This parameter is ignored if %service_id is set to
1101  *   IB_CM_ASSIGN_SERVICE_ID.
1102  */
1103 static int __ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id,
1104                           __be64 service_mask)
1105 {
1106         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
1107         int ret = 0;
1108
1109         service_mask = service_mask ? service_mask : ~cpu_to_be64(0);
1110         service_id &= service_mask;
1111         if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1112             (service_id != IB_CM_ASSIGN_SERVICE_ID))
1113                 return -EINVAL;
1114
1115         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1116         if (cm_id->state != IB_CM_IDLE)
1117                 return -EINVAL;
1118
1119         cm_id->state = IB_CM_LISTEN;
1120         ++cm_id_priv->listen_sharecount;
1121
1122         if (service_id == IB_CM_ASSIGN_SERVICE_ID) {
1123                 cm_id->service_id = cpu_to_be64(cm.listen_service_id++);
1124                 cm_id->service_mask = ~cpu_to_be64(0);
1125         } else {
1126                 cm_id->service_id = service_id;
1127                 cm_id->service_mask = service_mask;
1128         }
1129         cur_cm_id_priv = cm_insert_listen(cm_id_priv);
1130
1131         if (cur_cm_id_priv) {
1132                 cm_id->state = IB_CM_IDLE;
1133                 --cm_id_priv->listen_sharecount;
1134                 ret = -EBUSY;
1135         }
1136         return ret;
1137 }
1138
1139 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id, __be64 service_mask)
1140 {
1141         unsigned long flags;
1142         int ret;
1143
1144         spin_lock_irqsave(&cm.lock, flags);
1145         ret = __ib_cm_listen(cm_id, service_id, service_mask);
1146         spin_unlock_irqrestore(&cm.lock, flags);
1147
1148         return ret;
1149 }
1150 EXPORT_SYMBOL(ib_cm_listen);
1151
1152 /**
1153  * Create a new listening ib_cm_id and listen on the given service ID.
1154  *
1155  * If there's an existing ID listening on that same device and service ID,
1156  * return it.
1157  *
1158  * @device: Device associated with the cm_id.  All related communication will
1159  * be associated with the specified device.
1160  * @cm_handler: Callback invoked to notify the user of CM events.
1161  * @service_id: Service identifier matched against incoming connection
1162  *   and service ID resolution requests.  The service ID should be specified
1163  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1164  *   assign a service ID to the caller.
1165  *
1166  * Callers should call ib_destroy_cm_id when done with the listener ID.
1167  */
1168 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1169                                      ib_cm_handler cm_handler,
1170                                      __be64 service_id)
1171 {
1172         struct cm_id_private *cm_id_priv;
1173         struct ib_cm_id *cm_id;
1174         unsigned long flags;
1175         int err = 0;
1176
1177         /* Create an ID in advance, since the creation may sleep */
1178         cm_id = ib_create_cm_id(device, cm_handler, NULL);
1179         if (IS_ERR(cm_id))
1180                 return cm_id;
1181
1182         spin_lock_irqsave(&cm.lock, flags);
1183
1184         if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1185                 goto new_id;
1186
1187         /* Find an existing ID */
1188         cm_id_priv = cm_find_listen(device, service_id);
1189         if (cm_id_priv) {
1190                 if (cm_id->cm_handler != cm_handler || cm_id->context) {
1191                         /* Sharing an ib_cm_id with different handlers is not
1192                          * supported */
1193                         spin_unlock_irqrestore(&cm.lock, flags);
1194                         ib_destroy_cm_id(cm_id);
1195                         return ERR_PTR(-EINVAL);
1196                 }
1197                 refcount_inc(&cm_id_priv->refcount);
1198                 ++cm_id_priv->listen_sharecount;
1199                 spin_unlock_irqrestore(&cm.lock, flags);
1200
1201                 ib_destroy_cm_id(cm_id);
1202                 cm_id = &cm_id_priv->id;
1203                 return cm_id;
1204         }
1205
1206 new_id:
1207         /* Use newly created ID */
1208         err = __ib_cm_listen(cm_id, service_id, 0);
1209
1210         spin_unlock_irqrestore(&cm.lock, flags);
1211
1212         if (err) {
1213                 ib_destroy_cm_id(cm_id);
1214                 return ERR_PTR(err);
1215         }
1216         return cm_id;
1217 }
1218 EXPORT_SYMBOL(ib_cm_insert_listen);
1219
1220 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1221 {
1222         u64 hi_tid, low_tid;
1223
1224         hi_tid   = ((u64) cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1225         low_tid  = (u64)cm_id_priv->id.local_id;
1226         return cpu_to_be64(hi_tid | low_tid);
1227 }
1228
1229 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1230                               __be16 attr_id, __be64 tid)
1231 {
1232         hdr->base_version  = IB_MGMT_BASE_VERSION;
1233         hdr->mgmt_class    = IB_MGMT_CLASS_CM;
1234         hdr->class_version = IB_CM_CLASS_VERSION;
1235         hdr->method        = IB_MGMT_METHOD_SEND;
1236         hdr->attr_id       = attr_id;
1237         hdr->tid           = tid;
1238 }
1239
1240 static void cm_format_req(struct cm_req_msg *req_msg,
1241                           struct cm_id_private *cm_id_priv,
1242                           struct ib_cm_req_param *param)
1243 {
1244         struct sa_path_rec *pri_path = param->primary_path;
1245         struct sa_path_rec *alt_path = param->alternate_path;
1246         bool pri_ext = false;
1247
1248         if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1249                 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1250                                               pri_path->opa.slid);
1251
1252         cm_format_mad_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1253                           cm_form_tid(cm_id_priv));
1254
1255         IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1256                 be32_to_cpu(cm_id_priv->id.local_id));
1257         IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1258         IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1259                 be64_to_cpu(cm_id_priv->id.device->node_guid));
1260         IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1261         IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1262         IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1263                 param->remote_cm_response_timeout);
1264         cm_req_set_qp_type(req_msg, param->qp_type);
1265         IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1266         IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1267         IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1268                 param->local_cm_response_timeout);
1269         IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1270                 be16_to_cpu(param->primary_path->pkey));
1271         IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1272                 param->primary_path->mtu);
1273         IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1274
1275         if (param->qp_type != IB_QPT_XRC_INI) {
1276                 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1277                         param->responder_resources);
1278                 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1279                 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1280                         param->rnr_retry_count);
1281                 IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1282         }
1283
1284         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1285                 pri_path->sgid;
1286         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1287                 pri_path->dgid;
1288         if (pri_ext) {
1289                 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1290                         ->global.interface_id =
1291                         OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1292                 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1293                         ->global.interface_id =
1294                         OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1295         }
1296         if (pri_path->hop_limit <= 1) {
1297                 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1298                         be16_to_cpu(pri_ext ? 0 :
1299                                               htons(ntohl(sa_path_get_slid(
1300                                                       pri_path)))));
1301                 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1302                         be16_to_cpu(pri_ext ? 0 :
1303                                               htons(ntohl(sa_path_get_dlid(
1304                                                       pri_path)))));
1305         } else {
1306                 /* Work-around until there's a way to obtain remote LID info */
1307                 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1308                         be16_to_cpu(IB_LID_PERMISSIVE));
1309                 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1310                         be16_to_cpu(IB_LID_PERMISSIVE));
1311         }
1312         IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1313                 be32_to_cpu(pri_path->flow_label));
1314         IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1315         IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1316         IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1317         IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1318         IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1319                 (pri_path->hop_limit <= 1));
1320         IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1321                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1322                                pri_path->packet_life_time));
1323
1324         if (alt_path) {
1325                 bool alt_ext = false;
1326
1327                 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1328                         alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1329                                                       alt_path->opa.slid);
1330
1331                 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1332                         alt_path->sgid;
1333                 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1334                         alt_path->dgid;
1335                 if (alt_ext) {
1336                         IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1337                                         req_msg)
1338                                 ->global.interface_id =
1339                                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1340                         IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1341                                         req_msg)
1342                                 ->global.interface_id =
1343                                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1344                 }
1345                 if (alt_path->hop_limit <= 1) {
1346                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1347                                 be16_to_cpu(
1348                                         alt_ext ? 0 :
1349                                                   htons(ntohl(sa_path_get_slid(
1350                                                           alt_path)))));
1351                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1352                                 be16_to_cpu(
1353                                         alt_ext ? 0 :
1354                                                   htons(ntohl(sa_path_get_dlid(
1355                                                           alt_path)))));
1356                 } else {
1357                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1358                                 be16_to_cpu(IB_LID_PERMISSIVE));
1359                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1360                                 be16_to_cpu(IB_LID_PERMISSIVE));
1361                 }
1362                 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1363                         be32_to_cpu(alt_path->flow_label));
1364                 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1365                 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1366                         alt_path->traffic_class);
1367                 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1368                         alt_path->hop_limit);
1369                 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1370                 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1371                         (alt_path->hop_limit <= 1));
1372                 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1373                         cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1374                                        alt_path->packet_life_time));
1375         }
1376
1377         if (param->private_data && param->private_data_len)
1378                 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1379                             param->private_data_len);
1380 }
1381
1382 static int cm_validate_req_param(struct ib_cm_req_param *param)
1383 {
1384         /* peer-to-peer not supported */
1385         if (param->peer_to_peer)
1386                 return -EINVAL;
1387
1388         if (!param->primary_path)
1389                 return -EINVAL;
1390
1391         if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1392             param->qp_type != IB_QPT_XRC_INI)
1393                 return -EINVAL;
1394
1395         if (param->private_data &&
1396             param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1397                 return -EINVAL;
1398
1399         if (param->alternate_path &&
1400             (param->alternate_path->pkey != param->primary_path->pkey ||
1401              param->alternate_path->mtu != param->primary_path->mtu))
1402                 return -EINVAL;
1403
1404         return 0;
1405 }
1406
1407 int ib_send_cm_req(struct ib_cm_id *cm_id,
1408                    struct ib_cm_req_param *param)
1409 {
1410         struct cm_id_private *cm_id_priv;
1411         struct cm_req_msg *req_msg;
1412         unsigned long flags;
1413         int ret;
1414
1415         ret = cm_validate_req_param(param);
1416         if (ret)
1417                 return ret;
1418
1419         /* Verify that we're not in timewait. */
1420         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1421         spin_lock_irqsave(&cm_id_priv->lock, flags);
1422         if (cm_id->state != IB_CM_IDLE) {
1423                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1424                 ret = -EINVAL;
1425                 goto out;
1426         }
1427         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1428
1429         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1430                                                             id.local_id);
1431         if (IS_ERR(cm_id_priv->timewait_info)) {
1432                 ret = PTR_ERR(cm_id_priv->timewait_info);
1433                 goto out;
1434         }
1435
1436         ret = cm_init_av_by_path(param->primary_path,
1437                                  param->ppath_sgid_attr, &cm_id_priv->av,
1438                                  cm_id_priv);
1439         if (ret)
1440                 goto error1;
1441         if (param->alternate_path) {
1442                 ret = cm_init_av_by_path(param->alternate_path, NULL,
1443                                          &cm_id_priv->alt_av, cm_id_priv);
1444                 if (ret)
1445                         goto error1;
1446         }
1447         cm_id->service_id = param->service_id;
1448         cm_id->service_mask = ~cpu_to_be64(0);
1449         cm_id_priv->timeout_ms = cm_convert_to_ms(
1450                                     param->primary_path->packet_life_time) * 2 +
1451                                  cm_convert_to_ms(
1452                                     param->remote_cm_response_timeout);
1453         cm_id_priv->max_cm_retries = param->max_cm_retries;
1454         cm_id_priv->initiator_depth = param->initiator_depth;
1455         cm_id_priv->responder_resources = param->responder_resources;
1456         cm_id_priv->retry_count = param->retry_count;
1457         cm_id_priv->path_mtu = param->primary_path->mtu;
1458         cm_id_priv->pkey = param->primary_path->pkey;
1459         cm_id_priv->qp_type = param->qp_type;
1460
1461         ret = cm_alloc_msg(cm_id_priv, &cm_id_priv->msg);
1462         if (ret)
1463                 goto error1;
1464
1465         req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
1466         cm_format_req(req_msg, cm_id_priv, param);
1467         cm_id_priv->tid = req_msg->hdr.tid;
1468         cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
1469         cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
1470
1471         cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1472         cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1473
1474         spin_lock_irqsave(&cm_id_priv->lock, flags);
1475         ret = ib_post_send_mad(cm_id_priv->msg, NULL);
1476         if (ret) {
1477                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1478                 goto error2;
1479         }
1480         BUG_ON(cm_id->state != IB_CM_IDLE);
1481         cm_id->state = IB_CM_REQ_SENT;
1482         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1483         return 0;
1484
1485 error2: cm_free_msg(cm_id_priv->msg);
1486 error1: kfree(cm_id_priv->timewait_info);
1487 out:    return ret;
1488 }
1489 EXPORT_SYMBOL(ib_send_cm_req);
1490
1491 static int cm_issue_rej(struct cm_port *port,
1492                         struct ib_mad_recv_wc *mad_recv_wc,
1493                         enum ib_cm_rej_reason reason,
1494                         enum cm_msg_response msg_rejected,
1495                         void *ari, u8 ari_length)
1496 {
1497         struct ib_mad_send_buf *msg = NULL;
1498         struct cm_rej_msg *rej_msg, *rcv_msg;
1499         int ret;
1500
1501         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1502         if (ret)
1503                 return ret;
1504
1505         /* We just need common CM header information.  Cast to any message. */
1506         rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1507         rej_msg = (struct cm_rej_msg *) msg->mad;
1508
1509         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1510         IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1511                 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1512         IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1513                 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1514         IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1515         IBA_SET(CM_REJ_REASON, rej_msg, reason);
1516
1517         if (ari && ari_length) {
1518                 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1519                 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1520         }
1521
1522         ret = ib_post_send_mad(msg, NULL);
1523         if (ret)
1524                 cm_free_msg(msg);
1525
1526         return ret;
1527 }
1528
1529 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1530 {
1531         return ((cpu_to_be16(
1532                         IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1533                 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1534                                                req_msg))));
1535 }
1536
1537 static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,
1538                                  struct sa_path_rec *path, union ib_gid *gid)
1539 {
1540         if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1541                 path->rec_type = SA_PATH_REC_TYPE_OPA;
1542         else
1543                 path->rec_type = SA_PATH_REC_TYPE_IB;
1544 }
1545
1546 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1547                                         struct sa_path_rec *primary_path,
1548                                         struct sa_path_rec *alt_path)
1549 {
1550         u32 lid;
1551
1552         if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1553                 sa_path_set_dlid(primary_path,
1554                                  IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
1555                                          req_msg));
1556                 sa_path_set_slid(primary_path,
1557                                  IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1558                                          req_msg));
1559         } else {
1560                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1561                         CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1562                 sa_path_set_dlid(primary_path, lid);
1563
1564                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1565                         CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1566                 sa_path_set_slid(primary_path, lid);
1567         }
1568
1569         if (!cm_req_has_alt_path(req_msg))
1570                 return;
1571
1572         if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1573                 sa_path_set_dlid(alt_path,
1574                                  IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1575                                          req_msg));
1576                 sa_path_set_slid(alt_path,
1577                                  IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1578                                          req_msg));
1579         } else {
1580                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1581                         CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1582                 sa_path_set_dlid(alt_path, lid);
1583
1584                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1585                         CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1586                 sa_path_set_slid(alt_path, lid);
1587         }
1588 }
1589
1590 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1591                                      struct sa_path_rec *primary_path,
1592                                      struct sa_path_rec *alt_path)
1593 {
1594         primary_path->dgid =
1595                 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1596         primary_path->sgid =
1597                 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1598         primary_path->flow_label =
1599                 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1600         primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1601         primary_path->traffic_class =
1602                 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1603         primary_path->reversible = 1;
1604         primary_path->pkey =
1605                 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1606         primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1607         primary_path->mtu_selector = IB_SA_EQ;
1608         primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1609         primary_path->rate_selector = IB_SA_EQ;
1610         primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1611         primary_path->packet_life_time_selector = IB_SA_EQ;
1612         primary_path->packet_life_time =
1613                 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1614         primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1615         primary_path->service_id =
1616                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1617         if (sa_path_is_roce(primary_path))
1618                 primary_path->roce.route_resolved = false;
1619
1620         if (cm_req_has_alt_path(req_msg)) {
1621                 alt_path->dgid = *IBA_GET_MEM_PTR(
1622                         CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1623                 alt_path->sgid = *IBA_GET_MEM_PTR(
1624                         CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1625                 alt_path->flow_label = cpu_to_be32(
1626                         IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1627                 alt_path->hop_limit =
1628                         IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1629                 alt_path->traffic_class =
1630                         IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1631                 alt_path->reversible = 1;
1632                 alt_path->pkey =
1633                         cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1634                 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1635                 alt_path->mtu_selector = IB_SA_EQ;
1636                 alt_path->mtu =
1637                         IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1638                 alt_path->rate_selector = IB_SA_EQ;
1639                 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1640                 alt_path->packet_life_time_selector = IB_SA_EQ;
1641                 alt_path->packet_life_time =
1642                         IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1643                 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1644                 alt_path->service_id =
1645                         cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1646
1647                 if (sa_path_is_roce(alt_path))
1648                         alt_path->roce.route_resolved = false;
1649         }
1650         cm_format_path_lid_from_req(req_msg, primary_path, alt_path);
1651 }
1652
1653 static u16 cm_get_bth_pkey(struct cm_work *work)
1654 {
1655         struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1656         u8 port_num = work->port->port_num;
1657         u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1658         u16 pkey;
1659         int ret;
1660
1661         ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1662         if (ret) {
1663                 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %d, pkey index %d). %d\n",
1664                                      port_num, pkey_index, ret);
1665                 return 0;
1666         }
1667
1668         return pkey;
1669 }
1670
1671 /**
1672  * Convert OPA SGID to IB SGID
1673  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1674  * reject them as the local_gid will not match the sgid. Therefore,
1675  * change the pathrec's SGID to an IB SGID.
1676  *
1677  * @work: Work completion
1678  * @path: Path record
1679  */
1680 static void cm_opa_to_ib_sgid(struct cm_work *work,
1681                               struct sa_path_rec *path)
1682 {
1683         struct ib_device *dev = work->port->cm_dev->ib_device;
1684         u8 port_num = work->port->port_num;
1685
1686         if (rdma_cap_opa_ah(dev, port_num) &&
1687             (ib_is_opa_gid(&path->sgid))) {
1688                 union ib_gid sgid;
1689
1690                 if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1691                         dev_warn(&dev->dev,
1692                                  "Error updating sgid in CM request\n");
1693                         return;
1694                 }
1695
1696                 path->sgid = sgid;
1697         }
1698 }
1699
1700 static void cm_format_req_event(struct cm_work *work,
1701                                 struct cm_id_private *cm_id_priv,
1702                                 struct ib_cm_id *listen_id)
1703 {
1704         struct cm_req_msg *req_msg;
1705         struct ib_cm_req_event_param *param;
1706
1707         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1708         param = &work->cm_event.param.req_rcvd;
1709         param->listen_id = listen_id;
1710         param->bth_pkey = cm_get_bth_pkey(work);
1711         param->port = cm_id_priv->av.port->port_num;
1712         param->primary_path = &work->path[0];
1713         cm_opa_to_ib_sgid(work, param->primary_path);
1714         if (cm_req_has_alt_path(req_msg)) {
1715                 param->alternate_path = &work->path[1];
1716                 cm_opa_to_ib_sgid(work, param->alternate_path);
1717         } else {
1718                 param->alternate_path = NULL;
1719         }
1720         param->remote_ca_guid =
1721                 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1722         param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1723         param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1724         param->qp_type = cm_req_get_qp_type(req_msg);
1725         param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1726         param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1727         param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1728         param->local_cm_response_timeout =
1729                 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1730         param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1731         param->remote_cm_response_timeout =
1732                 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1733         param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1734         param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1735         param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1736         param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1737         work->cm_event.private_data =
1738                 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1739 }
1740
1741 static void cm_process_work(struct cm_id_private *cm_id_priv,
1742                             struct cm_work *work)
1743 {
1744         int ret;
1745
1746         /* We will typically only have the current event to report. */
1747         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1748         cm_free_work(work);
1749
1750         while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1751                 spin_lock_irq(&cm_id_priv->lock);
1752                 work = cm_dequeue_work(cm_id_priv);
1753                 spin_unlock_irq(&cm_id_priv->lock);
1754                 if (!work)
1755                         return;
1756
1757                 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1758                                                 &work->cm_event);
1759                 cm_free_work(work);
1760         }
1761         cm_deref_id(cm_id_priv);
1762         if (ret)
1763                 cm_destroy_id(&cm_id_priv->id, ret);
1764 }
1765
1766 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1767                           struct cm_id_private *cm_id_priv,
1768                           enum cm_msg_response msg_mraed, u8 service_timeout,
1769                           const void *private_data, u8 private_data_len)
1770 {
1771         cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1772         IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1773         IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1774                 be32_to_cpu(cm_id_priv->id.local_id));
1775         IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1776                 be32_to_cpu(cm_id_priv->id.remote_id));
1777         IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1778
1779         if (private_data && private_data_len)
1780                 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1781                             private_data_len);
1782 }
1783
1784 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1785                           struct cm_id_private *cm_id_priv,
1786                           enum ib_cm_rej_reason reason,
1787                           void *ari,
1788                           u8 ari_length,
1789                           const void *private_data,
1790                           u8 private_data_len)
1791 {
1792         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1793         IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1794                 be32_to_cpu(cm_id_priv->id.remote_id));
1795
1796         switch(cm_id_priv->id.state) {
1797         case IB_CM_REQ_RCVD:
1798                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1799                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1800                 break;
1801         case IB_CM_MRA_REQ_SENT:
1802                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1803                         be32_to_cpu(cm_id_priv->id.local_id));
1804                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1805                 break;
1806         case IB_CM_REP_RCVD:
1807         case IB_CM_MRA_REP_SENT:
1808                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1809                         be32_to_cpu(cm_id_priv->id.local_id));
1810                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1811                 break;
1812         default:
1813                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1814                         be32_to_cpu(cm_id_priv->id.local_id));
1815                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1816                         CM_MSG_RESPONSE_OTHER);
1817                 break;
1818         }
1819
1820         IBA_SET(CM_REJ_REASON, rej_msg, reason);
1821         if (ari && ari_length) {
1822                 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1823                 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1824         }
1825
1826         if (private_data && private_data_len)
1827                 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1828                             private_data_len);
1829 }
1830
1831 static void cm_dup_req_handler(struct cm_work *work,
1832                                struct cm_id_private *cm_id_priv)
1833 {
1834         struct ib_mad_send_buf *msg = NULL;
1835         int ret;
1836
1837         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
1838                         counter[CM_REQ_COUNTER]);
1839
1840         /* Quick state check to discard duplicate REQs. */
1841         if (cm_id_priv->id.state == IB_CM_REQ_RCVD)
1842                 return;
1843
1844         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1845         if (ret)
1846                 return;
1847
1848         spin_lock_irq(&cm_id_priv->lock);
1849         switch (cm_id_priv->id.state) {
1850         case IB_CM_MRA_REQ_SENT:
1851                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1852                               CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1853                               cm_id_priv->private_data,
1854                               cm_id_priv->private_data_len);
1855                 break;
1856         case IB_CM_TIMEWAIT:
1857                 cm_format_rej((struct cm_rej_msg *) msg->mad, cm_id_priv,
1858                               IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0);
1859                 break;
1860         default:
1861                 goto unlock;
1862         }
1863         spin_unlock_irq(&cm_id_priv->lock);
1864
1865         ret = ib_post_send_mad(msg, NULL);
1866         if (ret)
1867                 goto free;
1868         return;
1869
1870 unlock: spin_unlock_irq(&cm_id_priv->lock);
1871 free:   cm_free_msg(msg);
1872 }
1873
1874 static struct cm_id_private * cm_match_req(struct cm_work *work,
1875                                            struct cm_id_private *cm_id_priv)
1876 {
1877         struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1878         struct cm_timewait_info *timewait_info;
1879         struct cm_req_msg *req_msg;
1880         struct ib_cm_id *cm_id;
1881
1882         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1883
1884         /* Check for possible duplicate REQ. */
1885         spin_lock_irq(&cm.lock);
1886         timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1887         if (timewait_info) {
1888                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1889                                            timewait_info->work.remote_id);
1890                 spin_unlock_irq(&cm.lock);
1891                 if (cur_cm_id_priv) {
1892                         cm_dup_req_handler(work, cur_cm_id_priv);
1893                         cm_deref_id(cur_cm_id_priv);
1894                 }
1895                 return NULL;
1896         }
1897
1898         /* Check for stale connections. */
1899         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
1900         if (timewait_info) {
1901                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1902                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
1903                                            timewait_info->work.remote_id);
1904
1905                 spin_unlock_irq(&cm.lock);
1906                 cm_issue_rej(work->port, work->mad_recv_wc,
1907                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
1908                              NULL, 0);
1909                 if (cur_cm_id_priv) {
1910                         cm_id = &cur_cm_id_priv->id;
1911                         ib_send_cm_dreq(cm_id, NULL, 0);
1912                         cm_deref_id(cur_cm_id_priv);
1913                 }
1914                 return NULL;
1915         }
1916
1917         /* Find matching listen request. */
1918         listen_cm_id_priv = cm_find_listen(
1919                 cm_id_priv->id.device,
1920                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
1921         if (!listen_cm_id_priv) {
1922                 cm_cleanup_timewait(cm_id_priv->timewait_info);
1923                 spin_unlock_irq(&cm.lock);
1924                 cm_issue_rej(work->port, work->mad_recv_wc,
1925                              IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
1926                              NULL, 0);
1927                 goto out;
1928         }
1929         refcount_inc(&listen_cm_id_priv->refcount);
1930         refcount_inc(&cm_id_priv->refcount);
1931         cm_id_priv->id.state = IB_CM_REQ_RCVD;
1932         atomic_inc(&cm_id_priv->work_count);
1933         spin_unlock_irq(&cm.lock);
1934 out:
1935         return listen_cm_id_priv;
1936 }
1937
1938 /*
1939  * Work-around for inter-subnet connections.  If the LIDs are permissive,
1940  * we need to override the LID/SL data in the REQ with the LID information
1941  * in the work completion.
1942  */
1943 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
1944 {
1945         if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
1946                 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
1947                                         req_msg)) == IB_LID_PERMISSIVE) {
1948                         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1949                                 be16_to_cpu(ib_lid_be16(wc->slid)));
1950                         IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
1951                 }
1952
1953                 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1954                                         req_msg)) == IB_LID_PERMISSIVE)
1955                         IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1956                                 wc->dlid_path_bits);
1957         }
1958
1959         if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
1960                 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1961                                         req_msg)) == IB_LID_PERMISSIVE) {
1962                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1963                                 be16_to_cpu(ib_lid_be16(wc->slid)));
1964                         IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
1965                 }
1966
1967                 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1968                                         req_msg)) == IB_LID_PERMISSIVE)
1969                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1970                                 wc->dlid_path_bits);
1971         }
1972 }
1973
1974 static int cm_req_handler(struct cm_work *work)
1975 {
1976         struct ib_cm_id *cm_id;
1977         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
1978         struct cm_req_msg *req_msg;
1979         const struct ib_global_route *grh;
1980         const struct ib_gid_attr *gid_attr;
1981         int ret;
1982
1983         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1984
1985         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
1986         if (IS_ERR(cm_id))
1987                 return PTR_ERR(cm_id);
1988
1989         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1990         cm_id_priv->id.remote_id =
1991                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
1992         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
1993                                       work->mad_recv_wc->recv_buf.grh,
1994                                       &cm_id_priv->av);
1995         if (ret)
1996                 goto destroy;
1997         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1998                                                             id.local_id);
1999         if (IS_ERR(cm_id_priv->timewait_info)) {
2000                 ret = PTR_ERR(cm_id_priv->timewait_info);
2001                 goto destroy;
2002         }
2003         cm_id_priv->timewait_info->work.remote_id =
2004                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2005         cm_id_priv->timewait_info->remote_ca_guid =
2006                 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2007         cm_id_priv->timewait_info->remote_qpn =
2008                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2009
2010         listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2011         if (!listen_cm_id_priv) {
2012                 pr_debug("%s: local_id %d, no listen_cm_id_priv\n", __func__,
2013                          be32_to_cpu(cm_id->local_id));
2014                 ret = -EINVAL;
2015                 goto free_timeinfo;
2016         }
2017
2018         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2019         cm_id_priv->id.context = listen_cm_id_priv->id.context;
2020         cm_id_priv->id.service_id =
2021                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2022         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
2023
2024         cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2025
2026         memset(&work->path[0], 0, sizeof(work->path[0]));
2027         if (cm_req_has_alt_path(req_msg))
2028                 memset(&work->path[1], 0, sizeof(work->path[1]));
2029         grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2030         gid_attr = grh->sgid_attr;
2031
2032         if (gid_attr &&
2033             rdma_protocol_roce(work->port->cm_dev->ib_device,
2034                                work->port->port_num)) {
2035                 work->path[0].rec_type =
2036                         sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2037         } else {
2038                 cm_path_set_rec_type(
2039                         work->port->cm_dev->ib_device, work->port->port_num,
2040                         &work->path[0],
2041                         IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2042                                         req_msg));
2043         }
2044         if (cm_req_has_alt_path(req_msg))
2045                 work->path[1].rec_type = work->path[0].rec_type;
2046         cm_format_paths_from_req(req_msg, &work->path[0],
2047                                  &work->path[1]);
2048         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2049                 sa_path_set_dmac(&work->path[0],
2050                                  cm_id_priv->av.ah_attr.roce.dmac);
2051         work->path[0].hop_limit = grh->hop_limit;
2052         ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av,
2053                                  cm_id_priv);
2054         if (ret) {
2055                 int err;
2056
2057                 err = rdma_query_gid(work->port->cm_dev->ib_device,
2058                                      work->port->port_num, 0,
2059                                      &work->path[0].sgid);
2060                 if (err)
2061                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
2062                                        NULL, 0, NULL, 0);
2063                 else
2064                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
2065                                        &work->path[0].sgid,
2066                                        sizeof(work->path[0].sgid),
2067                                        NULL, 0);
2068                 goto rejected;
2069         }
2070         if (cm_req_has_alt_path(req_msg)) {
2071                 ret = cm_init_av_by_path(&work->path[1], NULL,
2072                                          &cm_id_priv->alt_av, cm_id_priv);
2073                 if (ret) {
2074                         ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
2075                                        &work->path[0].sgid,
2076                                        sizeof(work->path[0].sgid), NULL, 0);
2077                         goto rejected;
2078                 }
2079         }
2080         cm_id_priv->tid = req_msg->hdr.tid;
2081         cm_id_priv->timeout_ms = cm_convert_to_ms(
2082                 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2083         cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2084         cm_id_priv->remote_qpn =
2085                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2086         cm_id_priv->initiator_depth =
2087                 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2088         cm_id_priv->responder_resources =
2089                 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2090         cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2091         cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2092         cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2093         cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2094         cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2095         cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2096
2097         cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2098         cm_process_work(cm_id_priv, work);
2099         cm_deref_id(listen_cm_id_priv);
2100         return 0;
2101
2102 rejected:
2103         refcount_dec(&cm_id_priv->refcount);
2104         cm_deref_id(listen_cm_id_priv);
2105 free_timeinfo:
2106         kfree(cm_id_priv->timewait_info);
2107 destroy:
2108         ib_destroy_cm_id(cm_id);
2109         return ret;
2110 }
2111
2112 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2113                           struct cm_id_private *cm_id_priv,
2114                           struct ib_cm_rep_param *param)
2115 {
2116         cm_format_mad_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid);
2117         IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2118                 be32_to_cpu(cm_id_priv->id.local_id));
2119         IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2120                 be32_to_cpu(cm_id_priv->id.remote_id));
2121         IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2122         IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2123                 param->responder_resources);
2124         IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2125                 cm_id_priv->av.port->cm_dev->ack_delay);
2126         IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2127         IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2128         IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2129                 be64_to_cpu(cm_id_priv->id.device->node_guid));
2130
2131         if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2132                 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2133                         param->initiator_depth);
2134                 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2135                         param->flow_control);
2136                 IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2137                 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2138         } else {
2139                 IBA_SET(CM_REP_SRQ, rep_msg, 1);
2140                 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2141         }
2142
2143         if (param->private_data && param->private_data_len)
2144                 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2145                             param->private_data_len);
2146 }
2147
2148 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2149                    struct ib_cm_rep_param *param)
2150 {
2151         struct cm_id_private *cm_id_priv;
2152         struct ib_mad_send_buf *msg;
2153         struct cm_rep_msg *rep_msg;
2154         unsigned long flags;
2155         int ret;
2156
2157         if (param->private_data &&
2158             param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2159                 return -EINVAL;
2160
2161         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2162         spin_lock_irqsave(&cm_id_priv->lock, flags);
2163         if (cm_id->state != IB_CM_REQ_RCVD &&
2164             cm_id->state != IB_CM_MRA_REQ_SENT) {
2165                 pr_debug("%s: local_comm_id %d, cm_id->state: %d\n", __func__,
2166                          be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
2167                 ret = -EINVAL;
2168                 goto out;
2169         }
2170
2171         ret = cm_alloc_msg(cm_id_priv, &msg);
2172         if (ret)
2173                 goto out;
2174
2175         rep_msg = (struct cm_rep_msg *) msg->mad;
2176         cm_format_rep(rep_msg, cm_id_priv, param);
2177         msg->timeout_ms = cm_id_priv->timeout_ms;
2178         msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2179
2180         ret = ib_post_send_mad(msg, NULL);
2181         if (ret) {
2182                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2183                 cm_free_msg(msg);
2184                 return ret;
2185         }
2186
2187         cm_id->state = IB_CM_REP_SENT;
2188         cm_id_priv->msg = msg;
2189         cm_id_priv->initiator_depth = param->initiator_depth;
2190         cm_id_priv->responder_resources = param->responder_resources;
2191         cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2192         cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2193
2194 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2195         return ret;
2196 }
2197 EXPORT_SYMBOL(ib_send_cm_rep);
2198
2199 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2200                           struct cm_id_private *cm_id_priv,
2201                           const void *private_data,
2202                           u8 private_data_len)
2203 {
2204         cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2205         IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2206                 be32_to_cpu(cm_id_priv->id.local_id));
2207         IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2208                 be32_to_cpu(cm_id_priv->id.remote_id));
2209
2210         if (private_data && private_data_len)
2211                 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2212                             private_data_len);
2213 }
2214
2215 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2216                    const void *private_data,
2217                    u8 private_data_len)
2218 {
2219         struct cm_id_private *cm_id_priv;
2220         struct ib_mad_send_buf *msg;
2221         unsigned long flags;
2222         void *data;
2223         int ret;
2224
2225         if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2226                 return -EINVAL;
2227
2228         data = cm_copy_private_data(private_data, private_data_len);
2229         if (IS_ERR(data))
2230                 return PTR_ERR(data);
2231
2232         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2233         spin_lock_irqsave(&cm_id_priv->lock, flags);
2234         if (cm_id->state != IB_CM_REP_RCVD &&
2235             cm_id->state != IB_CM_MRA_REP_SENT) {
2236                 pr_debug("%s: local_id %d, cm_id->state %d\n", __func__,
2237                          be32_to_cpu(cm_id->local_id), cm_id->state);
2238                 ret = -EINVAL;
2239                 goto error;
2240         }
2241
2242         ret = cm_alloc_msg(cm_id_priv, &msg);
2243         if (ret)
2244                 goto error;
2245
2246         cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2247                       private_data, private_data_len);
2248
2249         ret = ib_post_send_mad(msg, NULL);
2250         if (ret) {
2251                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2252                 cm_free_msg(msg);
2253                 kfree(data);
2254                 return ret;
2255         }
2256
2257         cm_id->state = IB_CM_ESTABLISHED;
2258         cm_set_private_data(cm_id_priv, data, private_data_len);
2259         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2260         return 0;
2261
2262 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2263         kfree(data);
2264         return ret;
2265 }
2266 EXPORT_SYMBOL(ib_send_cm_rtu);
2267
2268 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2269 {
2270         struct cm_rep_msg *rep_msg;
2271         struct ib_cm_rep_event_param *param;
2272
2273         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2274         param = &work->cm_event.param.rep_rcvd;
2275         param->remote_ca_guid =
2276                 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2277         param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2278         param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2279         param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2280         param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2281         param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2282         param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2283         param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2284         param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2285         param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2286         param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2287         work->cm_event.private_data =
2288                 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2289 }
2290
2291 static void cm_dup_rep_handler(struct cm_work *work)
2292 {
2293         struct cm_id_private *cm_id_priv;
2294         struct cm_rep_msg *rep_msg;
2295         struct ib_mad_send_buf *msg = NULL;
2296         int ret;
2297
2298         rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2299         cm_id_priv = cm_acquire_id(
2300                 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2301                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2302         if (!cm_id_priv)
2303                 return;
2304
2305         atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2306                         counter[CM_REP_COUNTER]);
2307         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2308         if (ret)
2309                 goto deref;
2310
2311         spin_lock_irq(&cm_id_priv->lock);
2312         if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2313                 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2314                               cm_id_priv->private_data,
2315                               cm_id_priv->private_data_len);
2316         else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2317                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2318                               CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2319                               cm_id_priv->private_data,
2320                               cm_id_priv->private_data_len);
2321         else
2322                 goto unlock;
2323         spin_unlock_irq(&cm_id_priv->lock);
2324
2325         ret = ib_post_send_mad(msg, NULL);
2326         if (ret)
2327                 goto free;
2328         goto deref;
2329
2330 unlock: spin_unlock_irq(&cm_id_priv->lock);
2331 free:   cm_free_msg(msg);
2332 deref:  cm_deref_id(cm_id_priv);
2333 }
2334
2335 static int cm_rep_handler(struct cm_work *work)
2336 {
2337         struct cm_id_private *cm_id_priv;
2338         struct cm_rep_msg *rep_msg;
2339         int ret;
2340         struct cm_id_private *cur_cm_id_priv;
2341         struct ib_cm_id *cm_id;
2342         struct cm_timewait_info *timewait_info;
2343
2344         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2345         cm_id_priv = cm_acquire_id(
2346                 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2347         if (!cm_id_priv) {
2348                 cm_dup_rep_handler(work);
2349                 pr_debug("%s: remote_comm_id %d, no cm_id_priv\n", __func__,
2350                          IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2351                 return -EINVAL;
2352         }
2353
2354         cm_format_rep_event(work, cm_id_priv->qp_type);
2355
2356         spin_lock_irq(&cm_id_priv->lock);
2357         switch (cm_id_priv->id.state) {
2358         case IB_CM_REQ_SENT:
2359         case IB_CM_MRA_REQ_RCVD:
2360                 break;
2361         default:
2362                 spin_unlock_irq(&cm_id_priv->lock);
2363                 ret = -EINVAL;
2364                 pr_debug(
2365                         "%s: cm_id_priv->id.state: %d, local_comm_id %d, remote_comm_id %d\n",
2366                         __func__, cm_id_priv->id.state,
2367                         IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2368                         IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2369                 goto error;
2370         }
2371
2372         cm_id_priv->timewait_info->work.remote_id =
2373                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2374         cm_id_priv->timewait_info->remote_ca_guid =
2375                 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2376         cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2377
2378         spin_lock(&cm.lock);
2379         /* Check for duplicate REP. */
2380         if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2381                 spin_unlock(&cm.lock);
2382                 spin_unlock_irq(&cm_id_priv->lock);
2383                 ret = -EINVAL;
2384                 pr_debug("%s: Failed to insert remote id %d\n", __func__,
2385                          IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2386                 goto error;
2387         }
2388         /* Check for a stale connection. */
2389         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2390         if (timewait_info) {
2391                 rb_erase(&cm_id_priv->timewait_info->remote_id_node,
2392                          &cm.remote_id_table);
2393                 cm_id_priv->timewait_info->inserted_remote_id = 0;
2394                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2395                                            timewait_info->work.remote_id);
2396
2397                 spin_unlock(&cm.lock);
2398                 spin_unlock_irq(&cm_id_priv->lock);
2399                 cm_issue_rej(work->port, work->mad_recv_wc,
2400                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2401                              NULL, 0);
2402                 ret = -EINVAL;
2403                 pr_debug(
2404                         "%s: Stale connection. local_comm_id %d, remote_comm_id %d\n",
2405                         __func__, IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2406                         IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2407
2408                 if (cur_cm_id_priv) {
2409                         cm_id = &cur_cm_id_priv->id;
2410                         ib_send_cm_dreq(cm_id, NULL, 0);
2411                         cm_deref_id(cur_cm_id_priv);
2412                 }
2413
2414                 goto error;
2415         }
2416         spin_unlock(&cm.lock);
2417
2418         cm_id_priv->id.state = IB_CM_REP_RCVD;
2419         cm_id_priv->id.remote_id =
2420                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2421         cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2422         cm_id_priv->initiator_depth =
2423                 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2424         cm_id_priv->responder_resources =
2425                 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2426         cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2427         cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2428         cm_id_priv->target_ack_delay =
2429                 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2430         cm_id_priv->av.timeout =
2431                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2432                                        cm_id_priv->av.timeout - 1);
2433         cm_id_priv->alt_av.timeout =
2434                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2435                                        cm_id_priv->alt_av.timeout - 1);
2436
2437         /* todo: handle peer_to_peer */
2438
2439         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2440         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2441         if (!ret)
2442                 list_add_tail(&work->list, &cm_id_priv->work_list);
2443         spin_unlock_irq(&cm_id_priv->lock);
2444
2445         if (ret)
2446                 cm_process_work(cm_id_priv, work);
2447         else
2448                 cm_deref_id(cm_id_priv);
2449         return 0;
2450
2451 error:
2452         cm_deref_id(cm_id_priv);
2453         return ret;
2454 }
2455
2456 static int cm_establish_handler(struct cm_work *work)
2457 {
2458         struct cm_id_private *cm_id_priv;
2459         int ret;
2460
2461         /* See comment in cm_establish about lookup. */
2462         cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2463         if (!cm_id_priv)
2464                 return -EINVAL;
2465
2466         spin_lock_irq(&cm_id_priv->lock);
2467         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2468                 spin_unlock_irq(&cm_id_priv->lock);
2469                 goto out;
2470         }
2471
2472         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2473         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2474         if (!ret)
2475                 list_add_tail(&work->list, &cm_id_priv->work_list);
2476         spin_unlock_irq(&cm_id_priv->lock);
2477
2478         if (ret)
2479                 cm_process_work(cm_id_priv, work);
2480         else
2481                 cm_deref_id(cm_id_priv);
2482         return 0;
2483 out:
2484         cm_deref_id(cm_id_priv);
2485         return -EINVAL;
2486 }
2487
2488 static int cm_rtu_handler(struct cm_work *work)
2489 {
2490         struct cm_id_private *cm_id_priv;
2491         struct cm_rtu_msg *rtu_msg;
2492         int ret;
2493
2494         rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2495         cm_id_priv = cm_acquire_id(
2496                 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2497                 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2498         if (!cm_id_priv)
2499                 return -EINVAL;
2500
2501         work->cm_event.private_data =
2502                 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2503
2504         spin_lock_irq(&cm_id_priv->lock);
2505         if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2506             cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2507                 spin_unlock_irq(&cm_id_priv->lock);
2508                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2509                                 counter[CM_RTU_COUNTER]);
2510                 goto out;
2511         }
2512         cm_id_priv->id.state = IB_CM_ESTABLISHED;
2513
2514         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2515         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2516         if (!ret)
2517                 list_add_tail(&work->list, &cm_id_priv->work_list);
2518         spin_unlock_irq(&cm_id_priv->lock);
2519
2520         if (ret)
2521                 cm_process_work(cm_id_priv, work);
2522         else
2523                 cm_deref_id(cm_id_priv);
2524         return 0;
2525 out:
2526         cm_deref_id(cm_id_priv);
2527         return -EINVAL;
2528 }
2529
2530 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2531                           struct cm_id_private *cm_id_priv,
2532                           const void *private_data,
2533                           u8 private_data_len)
2534 {
2535         cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2536                           cm_form_tid(cm_id_priv));
2537         IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2538                 be32_to_cpu(cm_id_priv->id.local_id));
2539         IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2540                 be32_to_cpu(cm_id_priv->id.remote_id));
2541         IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2542                 be32_to_cpu(cm_id_priv->remote_qpn));
2543
2544         if (private_data && private_data_len)
2545                 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2546                             private_data_len);
2547 }
2548
2549 int ib_send_cm_dreq(struct ib_cm_id *cm_id,
2550                     const void *private_data,
2551                     u8 private_data_len)
2552 {
2553         struct cm_id_private *cm_id_priv;
2554         struct ib_mad_send_buf *msg;
2555         unsigned long flags;
2556         int ret;
2557
2558         if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2559                 return -EINVAL;
2560
2561         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2562         spin_lock_irqsave(&cm_id_priv->lock, flags);
2563         if (cm_id->state != IB_CM_ESTABLISHED) {
2564                 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2565                          be32_to_cpu(cm_id->local_id), cm_id->state);
2566                 ret = -EINVAL;
2567                 goto out;
2568         }
2569
2570         if (cm_id->lap_state == IB_CM_LAP_SENT ||
2571             cm_id->lap_state == IB_CM_MRA_LAP_RCVD)
2572                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2573
2574         ret = cm_alloc_msg(cm_id_priv, &msg);
2575         if (ret) {
2576                 cm_enter_timewait(cm_id_priv);
2577                 goto out;
2578         }
2579
2580         cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2581                        private_data, private_data_len);
2582         msg->timeout_ms = cm_id_priv->timeout_ms;
2583         msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2584
2585         ret = ib_post_send_mad(msg, NULL);
2586         if (ret) {
2587                 cm_enter_timewait(cm_id_priv);
2588                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2589                 cm_free_msg(msg);
2590                 return ret;
2591         }
2592
2593         cm_id->state = IB_CM_DREQ_SENT;
2594         cm_id_priv->msg = msg;
2595 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2596         return ret;
2597 }
2598 EXPORT_SYMBOL(ib_send_cm_dreq);
2599
2600 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2601                           struct cm_id_private *cm_id_priv,
2602                           const void *private_data,
2603                           u8 private_data_len)
2604 {
2605         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2606         IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2607                 be32_to_cpu(cm_id_priv->id.local_id));
2608         IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2609                 be32_to_cpu(cm_id_priv->id.remote_id));
2610
2611         if (private_data && private_data_len)
2612                 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2613                             private_data_len);
2614 }
2615
2616 int ib_send_cm_drep(struct ib_cm_id *cm_id,
2617                     const void *private_data,
2618                     u8 private_data_len)
2619 {
2620         struct cm_id_private *cm_id_priv;
2621         struct ib_mad_send_buf *msg;
2622         unsigned long flags;
2623         void *data;
2624         int ret;
2625
2626         if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2627                 return -EINVAL;
2628
2629         data = cm_copy_private_data(private_data, private_data_len);
2630         if (IS_ERR(data))
2631                 return PTR_ERR(data);
2632
2633         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2634         spin_lock_irqsave(&cm_id_priv->lock, flags);
2635         if (cm_id->state != IB_CM_DREQ_RCVD) {
2636                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2637                 kfree(data);
2638                 pr_debug("%s: local_id %d, cm_idcm_id->state(%d) != IB_CM_DREQ_RCVD\n",
2639                          __func__, be32_to_cpu(cm_id->local_id), cm_id->state);
2640                 return -EINVAL;
2641         }
2642
2643         cm_set_private_data(cm_id_priv, data, private_data_len);
2644         cm_enter_timewait(cm_id_priv);
2645
2646         ret = cm_alloc_msg(cm_id_priv, &msg);
2647         if (ret)
2648                 goto out;
2649
2650         cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2651                        private_data, private_data_len);
2652
2653         ret = ib_post_send_mad(msg, NULL);
2654         if (ret) {
2655                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2656                 cm_free_msg(msg);
2657                 return ret;
2658         }
2659
2660 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2661         return ret;
2662 }
2663 EXPORT_SYMBOL(ib_send_cm_drep);
2664
2665 static int cm_issue_drep(struct cm_port *port,
2666                          struct ib_mad_recv_wc *mad_recv_wc)
2667 {
2668         struct ib_mad_send_buf *msg = NULL;
2669         struct cm_dreq_msg *dreq_msg;
2670         struct cm_drep_msg *drep_msg;
2671         int ret;
2672
2673         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2674         if (ret)
2675                 return ret;
2676
2677         dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2678         drep_msg = (struct cm_drep_msg *) msg->mad;
2679
2680         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2681         IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2682                 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2683         IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2684                 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2685
2686         ret = ib_post_send_mad(msg, NULL);
2687         if (ret)
2688                 cm_free_msg(msg);
2689
2690         return ret;
2691 }
2692
2693 static int cm_dreq_handler(struct cm_work *work)
2694 {
2695         struct cm_id_private *cm_id_priv;
2696         struct cm_dreq_msg *dreq_msg;
2697         struct ib_mad_send_buf *msg = NULL;
2698         int ret;
2699
2700         dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2701         cm_id_priv = cm_acquire_id(
2702                 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2703                 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2704         if (!cm_id_priv) {
2705                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2706                                 counter[CM_DREQ_COUNTER]);
2707                 cm_issue_drep(work->port, work->mad_recv_wc);
2708                 pr_debug(
2709                         "%s: no cm_id_priv, local_comm_id %d, remote_comm_id %d\n",
2710                         __func__, IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2711                         IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2712                 return -EINVAL;
2713         }
2714
2715         work->cm_event.private_data =
2716                 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2717
2718         spin_lock_irq(&cm_id_priv->lock);
2719         if (cm_id_priv->local_qpn !=
2720             cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2721                 goto unlock;
2722
2723         switch (cm_id_priv->id.state) {
2724         case IB_CM_REP_SENT:
2725         case IB_CM_DREQ_SENT:
2726                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2727                 break;
2728         case IB_CM_ESTABLISHED:
2729                 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2730                     cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2731                         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2732                 break;
2733         case IB_CM_MRA_REP_RCVD:
2734                 break;
2735         case IB_CM_TIMEWAIT:
2736                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2737                                 counter[CM_DREQ_COUNTER]);
2738                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2739                 if (IS_ERR(msg))
2740                         goto unlock;
2741
2742                 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2743                                cm_id_priv->private_data,
2744                                cm_id_priv->private_data_len);
2745                 spin_unlock_irq(&cm_id_priv->lock);
2746
2747                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2748                     ib_post_send_mad(msg, NULL))
2749                         cm_free_msg(msg);
2750                 goto deref;
2751         case IB_CM_DREQ_RCVD:
2752                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
2753                                 counter[CM_DREQ_COUNTER]);
2754                 goto unlock;
2755         default:
2756                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
2757                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
2758                          cm_id_priv->id.state);
2759                 goto unlock;
2760         }
2761         cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2762         cm_id_priv->tid = dreq_msg->hdr.tid;
2763         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2764         if (!ret)
2765                 list_add_tail(&work->list, &cm_id_priv->work_list);
2766         spin_unlock_irq(&cm_id_priv->lock);
2767
2768         if (ret)
2769                 cm_process_work(cm_id_priv, work);
2770         else
2771                 cm_deref_id(cm_id_priv);
2772         return 0;
2773
2774 unlock: spin_unlock_irq(&cm_id_priv->lock);
2775 deref:  cm_deref_id(cm_id_priv);
2776         return -EINVAL;
2777 }
2778
2779 static int cm_drep_handler(struct cm_work *work)
2780 {
2781         struct cm_id_private *cm_id_priv;
2782         struct cm_drep_msg *drep_msg;
2783         int ret;
2784
2785         drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2786         cm_id_priv = cm_acquire_id(
2787                 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2788                 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2789         if (!cm_id_priv)
2790                 return -EINVAL;
2791
2792         work->cm_event.private_data =
2793                 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2794
2795         spin_lock_irq(&cm_id_priv->lock);
2796         if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2797             cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2798                 spin_unlock_irq(&cm_id_priv->lock);
2799                 goto out;
2800         }
2801         cm_enter_timewait(cm_id_priv);
2802
2803         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2804         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2805         if (!ret)
2806                 list_add_tail(&work->list, &cm_id_priv->work_list);
2807         spin_unlock_irq(&cm_id_priv->lock);
2808
2809         if (ret)
2810                 cm_process_work(cm_id_priv, work);
2811         else
2812                 cm_deref_id(cm_id_priv);
2813         return 0;
2814 out:
2815         cm_deref_id(cm_id_priv);
2816         return -EINVAL;
2817 }
2818
2819 int ib_send_cm_rej(struct ib_cm_id *cm_id,
2820                    enum ib_cm_rej_reason reason,
2821                    void *ari,
2822                    u8 ari_length,
2823                    const void *private_data,
2824                    u8 private_data_len)
2825 {
2826         struct cm_id_private *cm_id_priv;
2827         struct ib_mad_send_buf *msg;
2828         unsigned long flags;
2829         int ret;
2830
2831         if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2832             (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2833                 return -EINVAL;
2834
2835         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2836
2837         spin_lock_irqsave(&cm_id_priv->lock, flags);
2838         switch (cm_id->state) {
2839         case IB_CM_REQ_SENT:
2840         case IB_CM_MRA_REQ_RCVD:
2841         case IB_CM_REQ_RCVD:
2842         case IB_CM_MRA_REQ_SENT:
2843         case IB_CM_REP_RCVD:
2844         case IB_CM_MRA_REP_SENT:
2845                 ret = cm_alloc_msg(cm_id_priv, &msg);
2846                 if (!ret)
2847                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2848                                       cm_id_priv, reason, ari, ari_length,
2849                                       private_data, private_data_len);
2850
2851                 cm_reset_to_idle(cm_id_priv);
2852                 break;
2853         case IB_CM_REP_SENT:
2854         case IB_CM_MRA_REP_RCVD:
2855                 ret = cm_alloc_msg(cm_id_priv, &msg);
2856                 if (!ret)
2857                         cm_format_rej((struct cm_rej_msg *) msg->mad,
2858                                       cm_id_priv, reason, ari, ari_length,
2859                                       private_data, private_data_len);
2860
2861                 cm_enter_timewait(cm_id_priv);
2862                 break;
2863         default:
2864                 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
2865                          be32_to_cpu(cm_id_priv->id.local_id), cm_id->state);
2866                 ret = -EINVAL;
2867                 goto out;
2868         }
2869
2870         if (ret)
2871                 goto out;
2872
2873         ret = ib_post_send_mad(msg, NULL);
2874         if (ret)
2875                 cm_free_msg(msg);
2876
2877 out:    spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2878         return ret;
2879 }
2880 EXPORT_SYMBOL(ib_send_cm_rej);
2881
2882 static void cm_format_rej_event(struct cm_work *work)
2883 {
2884         struct cm_rej_msg *rej_msg;
2885         struct ib_cm_rej_event_param *param;
2886
2887         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2888         param = &work->cm_event.param.rej_rcvd;
2889         param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
2890         param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
2891         param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
2892         work->cm_event.private_data =
2893                 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
2894 }
2895
2896 static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
2897 {
2898         struct cm_timewait_info *timewait_info;
2899         struct cm_id_private *cm_id_priv;
2900         __be32 remote_id;
2901
2902         remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
2903
2904         if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
2905                 spin_lock_irq(&cm.lock);
2906                 timewait_info = cm_find_remote_id(
2907                         *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
2908                         remote_id);
2909                 if (!timewait_info) {
2910                         spin_unlock_irq(&cm.lock);
2911                         return NULL;
2912                 }
2913                 cm_id_priv =
2914                         cm_acquire_id(timewait_info->work.local_id, remote_id);
2915                 spin_unlock_irq(&cm.lock);
2916         } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
2917                    CM_MSG_RESPONSE_REQ)
2918                 cm_id_priv = cm_acquire_id(
2919                         cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
2920                         0);
2921         else
2922                 cm_id_priv = cm_acquire_id(
2923                         cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
2924                         remote_id);
2925
2926         return cm_id_priv;
2927 }
2928
2929 static int cm_rej_handler(struct cm_work *work)
2930 {
2931         struct cm_id_private *cm_id_priv;
2932         struct cm_rej_msg *rej_msg;
2933         int ret;
2934
2935         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
2936         cm_id_priv = cm_acquire_rejected_id(rej_msg);
2937         if (!cm_id_priv)
2938                 return -EINVAL;
2939
2940         cm_format_rej_event(work);
2941
2942         spin_lock_irq(&cm_id_priv->lock);
2943         switch (cm_id_priv->id.state) {
2944         case IB_CM_REQ_SENT:
2945         case IB_CM_MRA_REQ_RCVD:
2946         case IB_CM_REP_SENT:
2947         case IB_CM_MRA_REP_RCVD:
2948                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2949                 /* fall through */
2950         case IB_CM_REQ_RCVD:
2951         case IB_CM_MRA_REQ_SENT:
2952                 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
2953                         cm_enter_timewait(cm_id_priv);
2954                 else
2955                         cm_reset_to_idle(cm_id_priv);
2956                 break;
2957         case IB_CM_DREQ_SENT:
2958                 ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
2959                 /* fall through */
2960         case IB_CM_REP_RCVD:
2961         case IB_CM_MRA_REP_SENT:
2962                 cm_enter_timewait(cm_id_priv);
2963                 break;
2964         case IB_CM_ESTABLISHED:
2965                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
2966                     cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
2967                         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
2968                                 ib_cancel_mad(cm_id_priv->av.port->mad_agent,
2969                                               cm_id_priv->msg);
2970                         cm_enter_timewait(cm_id_priv);
2971                         break;
2972                 }
2973                 /* fall through */
2974         default:
2975                 spin_unlock_irq(&cm_id_priv->lock);
2976                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
2977                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
2978                          cm_id_priv->id.state);
2979                 ret = -EINVAL;
2980                 goto out;
2981         }
2982
2983         ret = atomic_inc_and_test(&cm_id_priv->work_count);
2984         if (!ret)
2985                 list_add_tail(&work->list, &cm_id_priv->work_list);
2986         spin_unlock_irq(&cm_id_priv->lock);
2987
2988         if (ret)
2989                 cm_process_work(cm_id_priv, work);
2990         else
2991                 cm_deref_id(cm_id_priv);
2992         return 0;
2993 out:
2994         cm_deref_id(cm_id_priv);
2995         return -EINVAL;
2996 }
2997
2998 int ib_send_cm_mra(struct ib_cm_id *cm_id,
2999                    u8 service_timeout,
3000                    const void *private_data,
3001                    u8 private_data_len)
3002 {
3003         struct cm_id_private *cm_id_priv;
3004         struct ib_mad_send_buf *msg;
3005         enum ib_cm_state cm_state;
3006         enum ib_cm_lap_state lap_state;
3007         enum cm_msg_response msg_response;
3008         void *data;
3009         unsigned long flags;
3010         int ret;
3011
3012         if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3013                 return -EINVAL;
3014
3015         data = cm_copy_private_data(private_data, private_data_len);
3016         if (IS_ERR(data))
3017                 return PTR_ERR(data);
3018
3019         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3020
3021         spin_lock_irqsave(&cm_id_priv->lock, flags);
3022         switch(cm_id_priv->id.state) {
3023         case IB_CM_REQ_RCVD:
3024                 cm_state = IB_CM_MRA_REQ_SENT;
3025                 lap_state = cm_id->lap_state;
3026                 msg_response = CM_MSG_RESPONSE_REQ;
3027                 break;
3028         case IB_CM_REP_RCVD:
3029                 cm_state = IB_CM_MRA_REP_SENT;
3030                 lap_state = cm_id->lap_state;
3031                 msg_response = CM_MSG_RESPONSE_REP;
3032                 break;
3033         case IB_CM_ESTABLISHED:
3034                 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3035                         cm_state = cm_id->state;
3036                         lap_state = IB_CM_MRA_LAP_SENT;
3037                         msg_response = CM_MSG_RESPONSE_OTHER;
3038                         break;
3039                 }
3040                 /* fall through */
3041         default:
3042                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
3043                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
3044                          cm_id_priv->id.state);
3045                 ret = -EINVAL;
3046                 goto error1;
3047         }
3048
3049         if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3050                 ret = cm_alloc_msg(cm_id_priv, &msg);
3051                 if (ret)
3052                         goto error1;
3053
3054                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3055                               msg_response, service_timeout,
3056                               private_data, private_data_len);
3057                 ret = ib_post_send_mad(msg, NULL);
3058                 if (ret)
3059                         goto error2;
3060         }
3061
3062         cm_id->state = cm_state;
3063         cm_id->lap_state = lap_state;
3064         cm_id_priv->service_timeout = service_timeout;
3065         cm_set_private_data(cm_id_priv, data, private_data_len);
3066         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3067         return 0;
3068
3069 error1: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3070         kfree(data);
3071         return ret;
3072
3073 error2: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3074         kfree(data);
3075         cm_free_msg(msg);
3076         return ret;
3077 }
3078 EXPORT_SYMBOL(ib_send_cm_mra);
3079
3080 static struct cm_id_private * cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3081 {
3082         switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3083         case CM_MSG_RESPONSE_REQ:
3084                 return cm_acquire_id(
3085                         cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3086                         0);
3087         case CM_MSG_RESPONSE_REP:
3088         case CM_MSG_RESPONSE_OTHER:
3089                 return cm_acquire_id(
3090                         cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3091                         cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3092         default:
3093                 return NULL;
3094         }
3095 }
3096
3097 static int cm_mra_handler(struct cm_work *work)
3098 {
3099         struct cm_id_private *cm_id_priv;
3100         struct cm_mra_msg *mra_msg;
3101         int timeout, ret;
3102
3103         mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3104         cm_id_priv = cm_acquire_mraed_id(mra_msg);
3105         if (!cm_id_priv)
3106                 return -EINVAL;
3107
3108         work->cm_event.private_data =
3109                 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3110         work->cm_event.param.mra_rcvd.service_timeout =
3111                 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3112         timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3113                   cm_convert_to_ms(cm_id_priv->av.timeout);
3114
3115         spin_lock_irq(&cm_id_priv->lock);
3116         switch (cm_id_priv->id.state) {
3117         case IB_CM_REQ_SENT:
3118                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3119                             CM_MSG_RESPONSE_REQ ||
3120                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
3121                                   cm_id_priv->msg, timeout))
3122                         goto out;
3123                 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3124                 break;
3125         case IB_CM_REP_SENT:
3126                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3127                             CM_MSG_RESPONSE_REP ||
3128                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
3129                                   cm_id_priv->msg, timeout))
3130                         goto out;
3131                 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3132                 break;
3133         case IB_CM_ESTABLISHED:
3134                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3135                             CM_MSG_RESPONSE_OTHER ||
3136                     cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3137                     ib_modify_mad(cm_id_priv->av.port->mad_agent,
3138                                   cm_id_priv->msg, timeout)) {
3139                         if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3140                                 atomic_long_inc(&work->port->
3141                                                 counter_group[CM_RECV_DUPLICATES].
3142                                                 counter[CM_MRA_COUNTER]);
3143                         goto out;
3144                 }
3145                 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3146                 break;
3147         case IB_CM_MRA_REQ_RCVD:
3148         case IB_CM_MRA_REP_RCVD:
3149                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3150                                 counter[CM_MRA_COUNTER]);
3151                 /* fall through */
3152         default:
3153                 pr_debug("%s local_id %d, cm_id_priv->id.state: %d\n",
3154                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
3155                          cm_id_priv->id.state);
3156                 goto out;
3157         }
3158
3159         cm_id_priv->msg->context[1] = (void *) (unsigned long)
3160                                       cm_id_priv->id.state;
3161         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3162         if (!ret)
3163                 list_add_tail(&work->list, &cm_id_priv->work_list);
3164         spin_unlock_irq(&cm_id_priv->lock);
3165
3166         if (ret)
3167                 cm_process_work(cm_id_priv, work);
3168         else
3169                 cm_deref_id(cm_id_priv);
3170         return 0;
3171 out:
3172         spin_unlock_irq(&cm_id_priv->lock);
3173         cm_deref_id(cm_id_priv);
3174         return -EINVAL;
3175 }
3176
3177 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3178                                         struct sa_path_rec *path)
3179 {
3180         u32 lid;
3181
3182         if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3183                 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3184                                                lap_msg));
3185                 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3186                                                lap_msg));
3187         } else {
3188                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3189                         CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3190                 sa_path_set_dlid(path, lid);
3191
3192                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3193                         CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3194                 sa_path_set_slid(path, lid);
3195         }
3196 }
3197
3198 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3199                                     struct sa_path_rec *path,
3200                                     struct cm_lap_msg *lap_msg)
3201 {
3202         path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3203         path->sgid =
3204                 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3205         path->flow_label =
3206                 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3207         path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3208         path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3209         path->reversible = 1;
3210         path->pkey = cm_id_priv->pkey;
3211         path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3212         path->mtu_selector = IB_SA_EQ;
3213         path->mtu = cm_id_priv->path_mtu;
3214         path->rate_selector = IB_SA_EQ;
3215         path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3216         path->packet_life_time_selector = IB_SA_EQ;
3217         path->packet_life_time =
3218                 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3219         path->packet_life_time -= (path->packet_life_time > 0);
3220         cm_format_path_lid_from_lap(lap_msg, path);
3221 }
3222
3223 static int cm_lap_handler(struct cm_work *work)
3224 {
3225         struct cm_id_private *cm_id_priv;
3226         struct cm_lap_msg *lap_msg;
3227         struct ib_cm_lap_event_param *param;
3228         struct ib_mad_send_buf *msg = NULL;
3229         int ret;
3230
3231         /* Currently Alternate path messages are not supported for
3232          * RoCE link layer.
3233          */
3234         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3235                                work->port->port_num))
3236                 return -EINVAL;
3237
3238         /* todo: verify LAP request and send reject APR if invalid. */
3239         lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3240         cm_id_priv = cm_acquire_id(
3241                 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3242                 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3243         if (!cm_id_priv)
3244                 return -EINVAL;
3245
3246         param = &work->cm_event.param.lap_rcvd;
3247         memset(&work->path[0], 0, sizeof(work->path[1]));
3248         cm_path_set_rec_type(work->port->cm_dev->ib_device,
3249                              work->port->port_num, &work->path[0],
3250                              IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3251                                              lap_msg));
3252         param->alternate_path = &work->path[0];
3253         cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3254         work->cm_event.private_data =
3255                 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3256
3257         spin_lock_irq(&cm_id_priv->lock);
3258         if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3259                 goto unlock;
3260
3261         switch (cm_id_priv->id.lap_state) {
3262         case IB_CM_LAP_UNINIT:
3263         case IB_CM_LAP_IDLE:
3264                 break;
3265         case IB_CM_MRA_LAP_SENT:
3266                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3267                                 counter[CM_LAP_COUNTER]);
3268                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3269                 if (IS_ERR(msg))
3270                         goto unlock;
3271
3272                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3273                               CM_MSG_RESPONSE_OTHER,
3274                               cm_id_priv->service_timeout,
3275                               cm_id_priv->private_data,
3276                               cm_id_priv->private_data_len);
3277                 spin_unlock_irq(&cm_id_priv->lock);
3278
3279                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3280                     ib_post_send_mad(msg, NULL))
3281                         cm_free_msg(msg);
3282                 goto deref;
3283         case IB_CM_LAP_RCVD:
3284                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3285                                 counter[CM_LAP_COUNTER]);
3286                 goto unlock;
3287         default:
3288                 goto unlock;
3289         }
3290
3291         ret = cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3292                                  work->mad_recv_wc->recv_buf.grh,
3293                                  &cm_id_priv->av);
3294         if (ret)
3295                 goto unlock;
3296
3297         ret = cm_init_av_by_path(param->alternate_path, NULL,
3298                                  &cm_id_priv->alt_av, cm_id_priv);
3299         if (ret)
3300                 goto unlock;
3301
3302         cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3303         cm_id_priv->tid = lap_msg->hdr.tid;
3304         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3305         if (!ret)
3306                 list_add_tail(&work->list, &cm_id_priv->work_list);
3307         spin_unlock_irq(&cm_id_priv->lock);
3308
3309         if (ret)
3310                 cm_process_work(cm_id_priv, work);
3311         else
3312                 cm_deref_id(cm_id_priv);
3313         return 0;
3314
3315 unlock: spin_unlock_irq(&cm_id_priv->lock);
3316 deref:  cm_deref_id(cm_id_priv);
3317         return -EINVAL;
3318 }
3319
3320 static int cm_apr_handler(struct cm_work *work)
3321 {
3322         struct cm_id_private *cm_id_priv;
3323         struct cm_apr_msg *apr_msg;
3324         int ret;
3325
3326         /* Currently Alternate path messages are not supported for
3327          * RoCE link layer.
3328          */
3329         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3330                                work->port->port_num))
3331                 return -EINVAL;
3332
3333         apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3334         cm_id_priv = cm_acquire_id(
3335                 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3336                 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3337         if (!cm_id_priv)
3338                 return -EINVAL; /* Unmatched reply. */
3339
3340         work->cm_event.param.apr_rcvd.ap_status =
3341                 IBA_GET(CM_APR_AR_STATUS, apr_msg);
3342         work->cm_event.param.apr_rcvd.apr_info =
3343                 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3344         work->cm_event.param.apr_rcvd.info_len =
3345                 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3346         work->cm_event.private_data =
3347                 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3348
3349         spin_lock_irq(&cm_id_priv->lock);
3350         if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3351             (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3352              cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3353                 spin_unlock_irq(&cm_id_priv->lock);
3354                 goto out;
3355         }
3356         cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3357         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3358         cm_id_priv->msg = NULL;
3359
3360         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3361         if (!ret)
3362                 list_add_tail(&work->list, &cm_id_priv->work_list);
3363         spin_unlock_irq(&cm_id_priv->lock);
3364
3365         if (ret)
3366                 cm_process_work(cm_id_priv, work);
3367         else
3368                 cm_deref_id(cm_id_priv);
3369         return 0;
3370 out:
3371         cm_deref_id(cm_id_priv);
3372         return -EINVAL;
3373 }
3374
3375 static int cm_timewait_handler(struct cm_work *work)
3376 {
3377         struct cm_timewait_info *timewait_info;
3378         struct cm_id_private *cm_id_priv;
3379         int ret;
3380
3381         timewait_info = container_of(work, struct cm_timewait_info, work);
3382         spin_lock_irq(&cm.lock);
3383         list_del(&timewait_info->list);
3384         spin_unlock_irq(&cm.lock);
3385
3386         cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3387                                    timewait_info->work.remote_id);
3388         if (!cm_id_priv)
3389                 return -EINVAL;
3390
3391         spin_lock_irq(&cm_id_priv->lock);
3392         if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3393             cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3394                 spin_unlock_irq(&cm_id_priv->lock);
3395                 goto out;
3396         }
3397         cm_id_priv->id.state = IB_CM_IDLE;
3398         ret = atomic_inc_and_test(&cm_id_priv->work_count);
3399         if (!ret)
3400                 list_add_tail(&work->list, &cm_id_priv->work_list);
3401         spin_unlock_irq(&cm_id_priv->lock);
3402
3403         if (ret)
3404                 cm_process_work(cm_id_priv, work);
3405         else
3406                 cm_deref_id(cm_id_priv);
3407         return 0;
3408 out:
3409         cm_deref_id(cm_id_priv);
3410         return -EINVAL;
3411 }
3412
3413 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3414                                struct cm_id_private *cm_id_priv,
3415                                struct ib_cm_sidr_req_param *param)
3416 {
3417         cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3418                           cm_form_tid(cm_id_priv));
3419         IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3420                 be32_to_cpu(cm_id_priv->id.local_id));
3421         IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3422                 be16_to_cpu(param->path->pkey));
3423         IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3424                 be64_to_cpu(param->service_id));
3425
3426         if (param->private_data && param->private_data_len)
3427                 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3428                             param->private_data, param->private_data_len);
3429 }
3430
3431 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3432                         struct ib_cm_sidr_req_param *param)
3433 {
3434         struct cm_id_private *cm_id_priv;
3435         struct ib_mad_send_buf *msg;
3436         unsigned long flags;
3437         int ret;
3438
3439         if (!param->path || (param->private_data &&
3440              param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3441                 return -EINVAL;
3442
3443         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3444         ret = cm_init_av_by_path(param->path, param->sgid_attr,
3445                                  &cm_id_priv->av,
3446                                  cm_id_priv);
3447         if (ret)
3448                 goto out;
3449
3450         cm_id->service_id = param->service_id;
3451         cm_id->service_mask = ~cpu_to_be64(0);
3452         cm_id_priv->timeout_ms = param->timeout_ms;
3453         cm_id_priv->max_cm_retries = param->max_cm_retries;
3454         ret = cm_alloc_msg(cm_id_priv, &msg);
3455         if (ret)
3456                 goto out;
3457
3458         cm_format_sidr_req((struct cm_sidr_req_msg *) msg->mad, cm_id_priv,
3459                            param);
3460         msg->timeout_ms = cm_id_priv->timeout_ms;
3461         msg->context[1] = (void *) (unsigned long) IB_CM_SIDR_REQ_SENT;
3462
3463         spin_lock_irqsave(&cm_id_priv->lock, flags);
3464         if (cm_id->state == IB_CM_IDLE)
3465                 ret = ib_post_send_mad(msg, NULL);
3466         else
3467                 ret = -EINVAL;
3468
3469         if (ret) {
3470                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3471                 cm_free_msg(msg);
3472                 goto out;
3473         }
3474         cm_id->state = IB_CM_SIDR_REQ_SENT;
3475         cm_id_priv->msg = msg;
3476         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3477 out:
3478         return ret;
3479 }
3480 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3481
3482 static void cm_format_sidr_req_event(struct cm_work *work,
3483                                      const struct cm_id_private *rx_cm_id,
3484                                      struct ib_cm_id *listen_id)
3485 {
3486         struct cm_sidr_req_msg *sidr_req_msg;
3487         struct ib_cm_sidr_req_event_param *param;
3488
3489         sidr_req_msg = (struct cm_sidr_req_msg *)
3490                                 work->mad_recv_wc->recv_buf.mad;
3491         param = &work->cm_event.param.sidr_req_rcvd;
3492         param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3493         param->listen_id = listen_id;
3494         param->service_id =
3495                 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3496         param->bth_pkey = cm_get_bth_pkey(work);
3497         param->port = work->port->port_num;
3498         param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3499         work->cm_event.private_data =
3500                 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3501 }
3502
3503 static int cm_sidr_req_handler(struct cm_work *work)
3504 {
3505         struct ib_cm_id *cm_id;
3506         struct cm_id_private *cm_id_priv, *cur_cm_id_priv;
3507         struct cm_sidr_req_msg *sidr_req_msg;
3508         struct ib_wc *wc;
3509         int ret;
3510
3511         cm_id = ib_create_cm_id(work->port->cm_dev->ib_device, NULL, NULL);
3512         if (IS_ERR(cm_id))
3513                 return PTR_ERR(cm_id);
3514         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3515
3516         /* Record SGID/SLID and request ID for lookup. */
3517         sidr_req_msg = (struct cm_sidr_req_msg *)
3518                                 work->mad_recv_wc->recv_buf.mad;
3519         wc = work->mad_recv_wc->wc;
3520         cm_id_priv->av.dgid.global.subnet_prefix = cpu_to_be64(wc->slid);
3521         cm_id_priv->av.dgid.global.interface_id = 0;
3522         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3523                                       work->mad_recv_wc->recv_buf.grh,
3524                                       &cm_id_priv->av);
3525         if (ret)
3526                 goto out;
3527
3528         cm_id_priv->id.remote_id =
3529                 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3530         cm_id_priv->tid = sidr_req_msg->hdr.tid;
3531         atomic_inc(&cm_id_priv->work_count);
3532
3533         spin_lock_irq(&cm.lock);
3534         cur_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3535         if (cur_cm_id_priv) {
3536                 spin_unlock_irq(&cm.lock);
3537                 atomic_long_inc(&work->port->counter_group[CM_RECV_DUPLICATES].
3538                                 counter[CM_SIDR_REQ_COUNTER]);
3539                 goto out; /* Duplicate message. */
3540         }
3541         cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3542         cur_cm_id_priv = cm_find_listen(
3543                 cm_id->device,
3544                 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg)));
3545         if (!cur_cm_id_priv) {
3546                 spin_unlock_irq(&cm.lock);
3547                 cm_reject_sidr_req(cm_id_priv, IB_SIDR_UNSUPPORTED);
3548                 goto out; /* No match. */
3549         }
3550         refcount_inc(&cur_cm_id_priv->refcount);
3551         refcount_inc(&cm_id_priv->refcount);
3552         spin_unlock_irq(&cm.lock);
3553
3554         cm_id_priv->id.cm_handler = cur_cm_id_priv->id.cm_handler;
3555         cm_id_priv->id.context = cur_cm_id_priv->id.context;
3556         cm_id_priv->id.service_id =
3557                 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3558         cm_id_priv->id.service_mask = ~cpu_to_be64(0);
3559
3560         cm_format_sidr_req_event(work, cm_id_priv, &cur_cm_id_priv->id);
3561         cm_process_work(cm_id_priv, work);
3562         cm_deref_id(cur_cm_id_priv);
3563         return 0;
3564 out:
3565         ib_destroy_cm_id(&cm_id_priv->id);
3566         return -EINVAL;
3567 }
3568
3569 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3570                                struct cm_id_private *cm_id_priv,
3571                                struct ib_cm_sidr_rep_param *param)
3572 {
3573         cm_format_mad_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3574                           cm_id_priv->tid);
3575         IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3576                 be32_to_cpu(cm_id_priv->id.remote_id));
3577         IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3578         IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3579         IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3580                 be64_to_cpu(cm_id_priv->id.service_id));
3581         IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3582
3583         if (param->info && param->info_length)
3584                 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3585                             param->info, param->info_length);
3586
3587         if (param->private_data && param->private_data_len)
3588                 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3589                             param->private_data, param->private_data_len);
3590 }
3591
3592 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3593                         struct ib_cm_sidr_rep_param *param)
3594 {
3595         struct cm_id_private *cm_id_priv;
3596         struct ib_mad_send_buf *msg;
3597         unsigned long flags;
3598         int ret;
3599
3600         if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3601             (param->private_data &&
3602              param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3603                 return -EINVAL;
3604
3605         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3606         spin_lock_irqsave(&cm_id_priv->lock, flags);
3607         if (cm_id->state != IB_CM_SIDR_REQ_RCVD) {
3608                 ret = -EINVAL;
3609                 goto error;
3610         }
3611
3612         ret = cm_alloc_msg(cm_id_priv, &msg);
3613         if (ret)
3614                 goto error;
3615
3616         cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3617                            param);
3618         ret = ib_post_send_mad(msg, NULL);
3619         if (ret) {
3620                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3621                 cm_free_msg(msg);
3622                 return ret;
3623         }
3624         cm_id->state = IB_CM_IDLE;
3625         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3626
3627         spin_lock_irqsave(&cm.lock, flags);
3628         if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3629                 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3630                 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3631         }
3632         spin_unlock_irqrestore(&cm.lock, flags);
3633         return 0;
3634
3635 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3636         return ret;
3637 }
3638 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3639
3640 static void cm_format_sidr_rep_event(struct cm_work *work,
3641                                      const struct cm_id_private *cm_id_priv)
3642 {
3643         struct cm_sidr_rep_msg *sidr_rep_msg;
3644         struct ib_cm_sidr_rep_event_param *param;
3645
3646         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3647                                 work->mad_recv_wc->recv_buf.mad;
3648         param = &work->cm_event.param.sidr_rep_rcvd;
3649         param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3650         param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3651         param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3652         param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3653                                       sidr_rep_msg);
3654         param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3655                                   sidr_rep_msg);
3656         param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3657         work->cm_event.private_data =
3658                 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3659 }
3660
3661 static int cm_sidr_rep_handler(struct cm_work *work)
3662 {
3663         struct cm_sidr_rep_msg *sidr_rep_msg;
3664         struct cm_id_private *cm_id_priv;
3665
3666         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3667                                 work->mad_recv_wc->recv_buf.mad;
3668         cm_id_priv = cm_acquire_id(
3669                 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3670         if (!cm_id_priv)
3671                 return -EINVAL; /* Unmatched reply. */
3672
3673         spin_lock_irq(&cm_id_priv->lock);
3674         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3675                 spin_unlock_irq(&cm_id_priv->lock);
3676                 goto out;
3677         }
3678         cm_id_priv->id.state = IB_CM_IDLE;
3679         ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
3680         spin_unlock_irq(&cm_id_priv->lock);
3681
3682         cm_format_sidr_rep_event(work, cm_id_priv);
3683         cm_process_work(cm_id_priv, work);
3684         return 0;
3685 out:
3686         cm_deref_id(cm_id_priv);
3687         return -EINVAL;
3688 }
3689
3690 static void cm_process_send_error(struct ib_mad_send_buf *msg,
3691                                   enum ib_wc_status wc_status)
3692 {
3693         struct cm_id_private *cm_id_priv;
3694         struct ib_cm_event cm_event;
3695         enum ib_cm_state state;
3696         int ret;
3697
3698         memset(&cm_event, 0, sizeof cm_event);
3699         cm_id_priv = msg->context[0];
3700
3701         /* Discard old sends or ones without a response. */
3702         spin_lock_irq(&cm_id_priv->lock);
3703         state = (enum ib_cm_state) (unsigned long) msg->context[1];
3704         if (msg != cm_id_priv->msg || state != cm_id_priv->id.state)
3705                 goto discard;
3706
3707         pr_debug_ratelimited("CM: failed sending MAD in state %d. (%s)\n",
3708                              state, ib_wc_status_msg(wc_status));
3709         switch (state) {
3710         case IB_CM_REQ_SENT:
3711         case IB_CM_MRA_REQ_RCVD:
3712                 cm_reset_to_idle(cm_id_priv);
3713                 cm_event.event = IB_CM_REQ_ERROR;
3714                 break;
3715         case IB_CM_REP_SENT:
3716         case IB_CM_MRA_REP_RCVD:
3717                 cm_reset_to_idle(cm_id_priv);
3718                 cm_event.event = IB_CM_REP_ERROR;
3719                 break;
3720         case IB_CM_DREQ_SENT:
3721                 cm_enter_timewait(cm_id_priv);
3722                 cm_event.event = IB_CM_DREQ_ERROR;
3723                 break;
3724         case IB_CM_SIDR_REQ_SENT:
3725                 cm_id_priv->id.state = IB_CM_IDLE;
3726                 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3727                 break;
3728         default:
3729                 goto discard;
3730         }
3731         spin_unlock_irq(&cm_id_priv->lock);
3732         cm_event.param.send_status = wc_status;
3733
3734         /* No other events can occur on the cm_id at this point. */
3735         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3736         cm_free_msg(msg);
3737         if (ret)
3738                 ib_destroy_cm_id(&cm_id_priv->id);
3739         return;
3740 discard:
3741         spin_unlock_irq(&cm_id_priv->lock);
3742         cm_free_msg(msg);
3743 }
3744
3745 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3746                             struct ib_mad_send_wc *mad_send_wc)
3747 {
3748         struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3749         struct cm_port *port;
3750         u16 attr_index;
3751
3752         port = mad_agent->context;
3753         attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3754                                   msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3755
3756         /*
3757          * If the send was in response to a received message (context[0] is not
3758          * set to a cm_id), and is not a REJ, then it is a send that was
3759          * manually retried.
3760          */
3761         if (!msg->context[0] && (attr_index != CM_REJ_COUNTER))
3762                 msg->retries = 1;
3763
3764         atomic_long_add(1 + msg->retries,
3765                         &port->counter_group[CM_XMIT].counter[attr_index]);
3766         if (msg->retries)
3767                 atomic_long_add(msg->retries,
3768                                 &port->counter_group[CM_XMIT_RETRIES].
3769                                 counter[attr_index]);
3770
3771         switch (mad_send_wc->status) {
3772         case IB_WC_SUCCESS:
3773         case IB_WC_WR_FLUSH_ERR:
3774                 cm_free_msg(msg);
3775                 break;
3776         default:
3777                 if (msg->context[0] && msg->context[1])
3778                         cm_process_send_error(msg, mad_send_wc->status);
3779                 else
3780                         cm_free_msg(msg);
3781                 break;
3782         }
3783 }
3784
3785 static void cm_work_handler(struct work_struct *_work)
3786 {
3787         struct cm_work *work = container_of(_work, struct cm_work, work.work);
3788         int ret;
3789
3790         switch (work->cm_event.event) {
3791         case IB_CM_REQ_RECEIVED:
3792                 ret = cm_req_handler(work);
3793                 break;
3794         case IB_CM_MRA_RECEIVED:
3795                 ret = cm_mra_handler(work);
3796                 break;
3797         case IB_CM_REJ_RECEIVED:
3798                 ret = cm_rej_handler(work);
3799                 break;
3800         case IB_CM_REP_RECEIVED:
3801                 ret = cm_rep_handler(work);
3802                 break;
3803         case IB_CM_RTU_RECEIVED:
3804                 ret = cm_rtu_handler(work);
3805                 break;
3806         case IB_CM_USER_ESTABLISHED:
3807                 ret = cm_establish_handler(work);
3808                 break;
3809         case IB_CM_DREQ_RECEIVED:
3810                 ret = cm_dreq_handler(work);
3811                 break;
3812         case IB_CM_DREP_RECEIVED:
3813                 ret = cm_drep_handler(work);
3814                 break;
3815         case IB_CM_SIDR_REQ_RECEIVED:
3816                 ret = cm_sidr_req_handler(work);
3817                 break;
3818         case IB_CM_SIDR_REP_RECEIVED:
3819                 ret = cm_sidr_rep_handler(work);
3820                 break;
3821         case IB_CM_LAP_RECEIVED:
3822                 ret = cm_lap_handler(work);
3823                 break;
3824         case IB_CM_APR_RECEIVED:
3825                 ret = cm_apr_handler(work);
3826                 break;
3827         case IB_CM_TIMEWAIT_EXIT:
3828                 ret = cm_timewait_handler(work);
3829                 break;
3830         default:
3831                 pr_debug("cm_event.event: 0x%x\n", work->cm_event.event);
3832                 ret = -EINVAL;
3833                 break;
3834         }
3835         if (ret)
3836                 cm_free_work(work);
3837 }
3838
3839 static int cm_establish(struct ib_cm_id *cm_id)
3840 {
3841         struct cm_id_private *cm_id_priv;
3842         struct cm_work *work;
3843         unsigned long flags;
3844         int ret = 0;
3845         struct cm_device *cm_dev;
3846
3847         cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3848         if (!cm_dev)
3849                 return -ENODEV;
3850
3851         work = kmalloc(sizeof *work, GFP_ATOMIC);
3852         if (!work)
3853                 return -ENOMEM;
3854
3855         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3856         spin_lock_irqsave(&cm_id_priv->lock, flags);
3857         switch (cm_id->state)
3858         {
3859         case IB_CM_REP_SENT:
3860         case IB_CM_MRA_REP_RCVD:
3861                 cm_id->state = IB_CM_ESTABLISHED;
3862                 break;
3863         case IB_CM_ESTABLISHED:
3864                 ret = -EISCONN;
3865                 break;
3866         default:
3867                 pr_debug("%s: local_id %d, cm_id->state: %d\n", __func__,
3868                          be32_to_cpu(cm_id->local_id), cm_id->state);
3869                 ret = -EINVAL;
3870                 break;
3871         }
3872         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3873
3874         if (ret) {
3875                 kfree(work);
3876                 goto out;
3877         }
3878
3879         /*
3880          * The CM worker thread may try to destroy the cm_id before it
3881          * can execute this work item.  To prevent potential deadlock,
3882          * we need to find the cm_id once we're in the context of the
3883          * worker thread, rather than holding a reference on it.
3884          */
3885         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3886         work->local_id = cm_id->local_id;
3887         work->remote_id = cm_id->remote_id;
3888         work->mad_recv_wc = NULL;
3889         work->cm_event.event = IB_CM_USER_ESTABLISHED;
3890
3891         /* Check if the device started its remove_one */
3892         spin_lock_irqsave(&cm.lock, flags);
3893         if (!cm_dev->going_down) {
3894                 queue_delayed_work(cm.wq, &work->work, 0);
3895         } else {
3896                 kfree(work);
3897                 ret = -ENODEV;
3898         }
3899         spin_unlock_irqrestore(&cm.lock, flags);
3900
3901 out:
3902         return ret;
3903 }
3904
3905 static int cm_migrate(struct ib_cm_id *cm_id)
3906 {
3907         struct cm_id_private *cm_id_priv;
3908         struct cm_av tmp_av;
3909         unsigned long flags;
3910         int tmp_send_port_not_ready;
3911         int ret = 0;
3912
3913         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3914         spin_lock_irqsave(&cm_id_priv->lock, flags);
3915         if (cm_id->state == IB_CM_ESTABLISHED &&
3916             (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3917              cm_id->lap_state == IB_CM_LAP_IDLE)) {
3918                 cm_id->lap_state = IB_CM_LAP_IDLE;
3919                 /* Swap address vector */
3920                 tmp_av = cm_id_priv->av;
3921                 cm_id_priv->av = cm_id_priv->alt_av;
3922                 cm_id_priv->alt_av = tmp_av;
3923                 /* Swap port send ready state */
3924                 tmp_send_port_not_ready = cm_id_priv->prim_send_port_not_ready;
3925                 cm_id_priv->prim_send_port_not_ready = cm_id_priv->altr_send_port_not_ready;
3926                 cm_id_priv->altr_send_port_not_ready = tmp_send_port_not_ready;
3927         } else
3928                 ret = -EINVAL;
3929         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3930
3931         return ret;
3932 }
3933
3934 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3935 {
3936         int ret;
3937
3938         switch (event) {
3939         case IB_EVENT_COMM_EST:
3940                 ret = cm_establish(cm_id);
3941                 break;
3942         case IB_EVENT_PATH_MIG:
3943                 ret = cm_migrate(cm_id);
3944                 break;
3945         default:
3946                 ret = -EINVAL;
3947         }
3948         return ret;
3949 }
3950 EXPORT_SYMBOL(ib_cm_notify);
3951
3952 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3953                             struct ib_mad_send_buf *send_buf,
3954                             struct ib_mad_recv_wc *mad_recv_wc)
3955 {
3956         struct cm_port *port = mad_agent->context;
3957         struct cm_work *work;
3958         enum ib_cm_event_type event;
3959         bool alt_path = false;
3960         u16 attr_id;
3961         int paths = 0;
3962         int going_down = 0;
3963
3964         switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
3965         case CM_REQ_ATTR_ID:
3966                 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
3967                                                 mad_recv_wc->recv_buf.mad);
3968                 paths = 1 + (alt_path != 0);
3969                 event = IB_CM_REQ_RECEIVED;
3970                 break;
3971         case CM_MRA_ATTR_ID:
3972                 event = IB_CM_MRA_RECEIVED;
3973                 break;
3974         case CM_REJ_ATTR_ID:
3975                 event = IB_CM_REJ_RECEIVED;
3976                 break;
3977         case CM_REP_ATTR_ID:
3978                 event = IB_CM_REP_RECEIVED;
3979                 break;
3980         case CM_RTU_ATTR_ID:
3981                 event = IB_CM_RTU_RECEIVED;
3982                 break;
3983         case CM_DREQ_ATTR_ID:
3984                 event = IB_CM_DREQ_RECEIVED;
3985                 break;
3986         case CM_DREP_ATTR_ID:
3987                 event = IB_CM_DREP_RECEIVED;
3988                 break;
3989         case CM_SIDR_REQ_ATTR_ID:
3990                 event = IB_CM_SIDR_REQ_RECEIVED;
3991                 break;
3992         case CM_SIDR_REP_ATTR_ID:
3993                 event = IB_CM_SIDR_REP_RECEIVED;
3994                 break;
3995         case CM_LAP_ATTR_ID:
3996                 paths = 1;
3997                 event = IB_CM_LAP_RECEIVED;
3998                 break;
3999         case CM_APR_ATTR_ID:
4000                 event = IB_CM_APR_RECEIVED;
4001                 break;
4002         default:
4003                 ib_free_recv_mad(mad_recv_wc);
4004                 return;
4005         }
4006
4007         attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4008         atomic_long_inc(&port->counter_group[CM_RECV].
4009                         counter[attr_id - CM_ATTR_ID_OFFSET]);
4010
4011         work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4012         if (!work) {
4013                 ib_free_recv_mad(mad_recv_wc);
4014                 return;
4015         }
4016
4017         INIT_DELAYED_WORK(&work->work, cm_work_handler);
4018         work->cm_event.event = event;
4019         work->mad_recv_wc = mad_recv_wc;
4020         work->port = port;
4021
4022         /* Check if the device started its remove_one */
4023         spin_lock_irq(&cm.lock);
4024         if (!port->cm_dev->going_down)
4025                 queue_delayed_work(cm.wq, &work->work, 0);
4026         else
4027                 going_down = 1;
4028         spin_unlock_irq(&cm.lock);
4029
4030         if (going_down) {
4031                 kfree(work);
4032                 ib_free_recv_mad(mad_recv_wc);
4033         }
4034 }
4035
4036 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4037                                 struct ib_qp_attr *qp_attr,
4038                                 int *qp_attr_mask)
4039 {
4040         unsigned long flags;
4041         int ret;
4042
4043         spin_lock_irqsave(&cm_id_priv->lock, flags);
4044         switch (cm_id_priv->id.state) {
4045         case IB_CM_REQ_SENT:
4046         case IB_CM_MRA_REQ_RCVD:
4047         case IB_CM_REQ_RCVD:
4048         case IB_CM_MRA_REQ_SENT:
4049         case IB_CM_REP_RCVD:
4050         case IB_CM_MRA_REP_SENT:
4051         case IB_CM_REP_SENT:
4052         case IB_CM_MRA_REP_RCVD:
4053         case IB_CM_ESTABLISHED:
4054                 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4055                                 IB_QP_PKEY_INDEX | IB_QP_PORT;
4056                 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4057                 if (cm_id_priv->responder_resources)
4058                         qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4059                                                     IB_ACCESS_REMOTE_ATOMIC;
4060                 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4061                 qp_attr->port_num = cm_id_priv->av.port->port_num;
4062                 ret = 0;
4063                 break;
4064         default:
4065                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4066                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
4067                          cm_id_priv->id.state);
4068                 ret = -EINVAL;
4069                 break;
4070         }
4071         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4072         return ret;
4073 }
4074
4075 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4076                                struct ib_qp_attr *qp_attr,
4077                                int *qp_attr_mask)
4078 {
4079         unsigned long flags;
4080         int ret;
4081
4082         spin_lock_irqsave(&cm_id_priv->lock, flags);
4083         switch (cm_id_priv->id.state) {
4084         case IB_CM_REQ_RCVD:
4085         case IB_CM_MRA_REQ_SENT:
4086         case IB_CM_REP_RCVD:
4087         case IB_CM_MRA_REP_SENT:
4088         case IB_CM_REP_SENT:
4089         case IB_CM_MRA_REP_RCVD:
4090         case IB_CM_ESTABLISHED:
4091                 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4092                                 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4093                 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4094                 qp_attr->path_mtu = cm_id_priv->path_mtu;
4095                 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4096                 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4097                 if (cm_id_priv->qp_type == IB_QPT_RC ||
4098                     cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4099                         *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4100                                          IB_QP_MIN_RNR_TIMER;
4101                         qp_attr->max_dest_rd_atomic =
4102                                         cm_id_priv->responder_resources;
4103                         qp_attr->min_rnr_timer = 0;
4104                 }
4105                 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4106                         *qp_attr_mask |= IB_QP_ALT_PATH;
4107                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4108                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4109                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4110                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4111                 }
4112                 ret = 0;
4113                 break;
4114         default:
4115                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4116                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
4117                          cm_id_priv->id.state);
4118                 ret = -EINVAL;
4119                 break;
4120         }
4121         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4122         return ret;
4123 }
4124
4125 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4126                                struct ib_qp_attr *qp_attr,
4127                                int *qp_attr_mask)
4128 {
4129         unsigned long flags;
4130         int ret;
4131
4132         spin_lock_irqsave(&cm_id_priv->lock, flags);
4133         switch (cm_id_priv->id.state) {
4134         /* Allow transition to RTS before sending REP */
4135         case IB_CM_REQ_RCVD:
4136         case IB_CM_MRA_REQ_SENT:
4137
4138         case IB_CM_REP_RCVD:
4139         case IB_CM_MRA_REP_SENT:
4140         case IB_CM_REP_SENT:
4141         case IB_CM_MRA_REP_RCVD:
4142         case IB_CM_ESTABLISHED:
4143                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4144                         *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4145                         qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4146                         switch (cm_id_priv->qp_type) {
4147                         case IB_QPT_RC:
4148                         case IB_QPT_XRC_INI:
4149                                 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4150                                                  IB_QP_MAX_QP_RD_ATOMIC;
4151                                 qp_attr->retry_cnt = cm_id_priv->retry_count;
4152                                 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4153                                 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4154                                 /* fall through */
4155                         case IB_QPT_XRC_TGT:
4156                                 *qp_attr_mask |= IB_QP_TIMEOUT;
4157                                 qp_attr->timeout = cm_id_priv->av.timeout;
4158                                 break;
4159                         default:
4160                                 break;
4161                         }
4162                         if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4163                                 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4164                                 qp_attr->path_mig_state = IB_MIG_REARM;
4165                         }
4166                 } else {
4167                         *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4168                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4169                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4170                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4171                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4172                         qp_attr->path_mig_state = IB_MIG_REARM;
4173                 }
4174                 ret = 0;
4175                 break;
4176         default:
4177                 pr_debug("%s: local_id %d, cm_id_priv->id.state: %d\n",
4178                          __func__, be32_to_cpu(cm_id_priv->id.local_id),
4179                          cm_id_priv->id.state);
4180                 ret = -EINVAL;
4181                 break;
4182         }
4183         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4184         return ret;
4185 }
4186
4187 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4188                        struct ib_qp_attr *qp_attr,
4189                        int *qp_attr_mask)
4190 {
4191         struct cm_id_private *cm_id_priv;
4192         int ret;
4193
4194         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4195         switch (qp_attr->qp_state) {
4196         case IB_QPS_INIT:
4197                 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4198                 break;
4199         case IB_QPS_RTR:
4200                 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4201                 break;
4202         case IB_QPS_RTS:
4203                 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4204                 break;
4205         default:
4206                 ret = -EINVAL;
4207                 break;
4208         }
4209         return ret;
4210 }
4211 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4212
4213 static ssize_t cm_show_counter(struct kobject *obj, struct attribute *attr,
4214                                char *buf)
4215 {
4216         struct cm_counter_group *group;
4217         struct cm_counter_attribute *cm_attr;
4218
4219         group = container_of(obj, struct cm_counter_group, obj);
4220         cm_attr = container_of(attr, struct cm_counter_attribute, attr);
4221
4222         return sprintf(buf, "%ld\n",
4223                        atomic_long_read(&group->counter[cm_attr->index]));
4224 }
4225
4226 static const struct sysfs_ops cm_counter_ops = {
4227         .show = cm_show_counter
4228 };
4229
4230 static struct kobj_type cm_counter_obj_type = {
4231         .sysfs_ops = &cm_counter_ops,
4232         .default_attrs = cm_counter_default_attrs
4233 };
4234
4235 static char *cm_devnode(struct device *dev, umode_t *mode)
4236 {
4237         if (mode)
4238                 *mode = 0666;
4239         return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
4240 }
4241
4242 struct class cm_class = {
4243         .owner   = THIS_MODULE,
4244         .name    = "infiniband_cm",
4245         .devnode = cm_devnode,
4246 };
4247 EXPORT_SYMBOL(cm_class);
4248
4249 static int cm_create_port_fs(struct cm_port *port)
4250 {
4251         int i, ret;
4252
4253         for (i = 0; i < CM_COUNTER_GROUPS; i++) {
4254                 ret = ib_port_register_module_stat(port->cm_dev->ib_device,
4255                                                    port->port_num,
4256                                                    &port->counter_group[i].obj,
4257                                                    &cm_counter_obj_type,
4258                                                    counter_group_names[i]);
4259                 if (ret)
4260                         goto error;
4261         }
4262
4263         return 0;
4264
4265 error:
4266         while (i--)
4267                 ib_port_unregister_module_stat(&port->counter_group[i].obj);
4268         return ret;
4269
4270 }
4271
4272 static void cm_remove_port_fs(struct cm_port *port)
4273 {
4274         int i;
4275
4276         for (i = 0; i < CM_COUNTER_GROUPS; i++)
4277                 ib_port_unregister_module_stat(&port->counter_group[i].obj);
4278
4279 }
4280
4281 static void cm_add_one(struct ib_device *ib_device)
4282 {
4283         struct cm_device *cm_dev;
4284         struct cm_port *port;
4285         struct ib_mad_reg_req reg_req = {
4286                 .mgmt_class = IB_MGMT_CLASS_CM,
4287                 .mgmt_class_version = IB_CM_CLASS_VERSION,
4288         };
4289         struct ib_port_modify port_modify = {
4290                 .set_port_cap_mask = IB_PORT_CM_SUP
4291         };
4292         unsigned long flags;
4293         int ret;
4294         int count = 0;
4295         u8 i;
4296
4297         cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4298                          GFP_KERNEL);
4299         if (!cm_dev)
4300                 return;
4301
4302         cm_dev->ib_device = ib_device;
4303         cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4304         cm_dev->going_down = 0;
4305
4306         set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4307         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4308                 if (!rdma_cap_ib_cm(ib_device, i))
4309                         continue;
4310
4311                 port = kzalloc(sizeof *port, GFP_KERNEL);
4312                 if (!port)
4313                         goto error1;
4314
4315                 cm_dev->port[i-1] = port;
4316                 port->cm_dev = cm_dev;
4317                 port->port_num = i;
4318
4319                 INIT_LIST_HEAD(&port->cm_priv_prim_list);
4320                 INIT_LIST_HEAD(&port->cm_priv_altr_list);
4321
4322                 ret = cm_create_port_fs(port);
4323                 if (ret)
4324                         goto error1;
4325
4326                 port->mad_agent = ib_register_mad_agent(ib_device, i,
4327                                                         IB_QPT_GSI,
4328                                                         &reg_req,
4329                                                         0,
4330                                                         cm_send_handler,
4331                                                         cm_recv_handler,
4332                                                         port,
4333                                                         0);
4334                 if (IS_ERR(port->mad_agent))
4335                         goto error2;
4336
4337                 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4338                 if (ret)
4339                         goto error3;
4340
4341                 count++;
4342         }
4343
4344         if (!count)
4345                 goto free;
4346
4347         ib_set_client_data(ib_device, &cm_client, cm_dev);
4348
4349         write_lock_irqsave(&cm.device_lock, flags);
4350         list_add_tail(&cm_dev->list, &cm.device_list);
4351         write_unlock_irqrestore(&cm.device_lock, flags);
4352         return;
4353
4354 error3:
4355         ib_unregister_mad_agent(port->mad_agent);
4356 error2:
4357         cm_remove_port_fs(port);
4358 error1:
4359         port_modify.set_port_cap_mask = 0;
4360         port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4361         kfree(port);
4362         while (--i) {
4363                 if (!rdma_cap_ib_cm(ib_device, i))
4364                         continue;
4365
4366                 port = cm_dev->port[i-1];
4367                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4368                 ib_unregister_mad_agent(port->mad_agent);
4369                 cm_remove_port_fs(port);
4370                 kfree(port);
4371         }
4372 free:
4373         kfree(cm_dev);
4374 }
4375
4376 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4377 {
4378         struct cm_device *cm_dev = client_data;
4379         struct cm_port *port;
4380         struct cm_id_private *cm_id_priv;
4381         struct ib_mad_agent *cur_mad_agent;
4382         struct ib_port_modify port_modify = {
4383                 .clr_port_cap_mask = IB_PORT_CM_SUP
4384         };
4385         unsigned long flags;
4386         int i;
4387
4388         if (!cm_dev)
4389                 return;
4390
4391         write_lock_irqsave(&cm.device_lock, flags);
4392         list_del(&cm_dev->list);
4393         write_unlock_irqrestore(&cm.device_lock, flags);
4394
4395         spin_lock_irq(&cm.lock);
4396         cm_dev->going_down = 1;
4397         spin_unlock_irq(&cm.lock);
4398
4399         for (i = 1; i <= ib_device->phys_port_cnt; i++) {
4400                 if (!rdma_cap_ib_cm(ib_device, i))
4401                         continue;
4402
4403                 port = cm_dev->port[i-1];
4404                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4405                 /* Mark all the cm_id's as not valid */
4406                 spin_lock_irq(&cm.lock);
4407                 list_for_each_entry(cm_id_priv, &port->cm_priv_altr_list, altr_list)
4408                         cm_id_priv->altr_send_port_not_ready = 1;
4409                 list_for_each_entry(cm_id_priv, &port->cm_priv_prim_list, prim_list)
4410                         cm_id_priv->prim_send_port_not_ready = 1;
4411                 spin_unlock_irq(&cm.lock);
4412                 /*
4413                  * We flush the queue here after the going_down set, this
4414                  * verify that no new works will be queued in the recv handler,
4415                  * after that we can call the unregister_mad_agent
4416                  */
4417                 flush_workqueue(cm.wq);
4418                 spin_lock_irq(&cm.state_lock);
4419                 cur_mad_agent = port->mad_agent;
4420                 port->mad_agent = NULL;
4421                 spin_unlock_irq(&cm.state_lock);
4422                 ib_unregister_mad_agent(cur_mad_agent);
4423                 cm_remove_port_fs(port);
4424                 kfree(port);
4425         }
4426
4427         kfree(cm_dev);
4428 }
4429
4430 static int __init ib_cm_init(void)
4431 {
4432         int ret;
4433
4434         INIT_LIST_HEAD(&cm.device_list);
4435         rwlock_init(&cm.device_lock);
4436         spin_lock_init(&cm.lock);
4437         spin_lock_init(&cm.state_lock);
4438         cm.listen_service_table = RB_ROOT;
4439         cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4440         cm.remote_id_table = RB_ROOT;
4441         cm.remote_qp_table = RB_ROOT;
4442         cm.remote_sidr_table = RB_ROOT;
4443         xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
4444         get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4445         INIT_LIST_HEAD(&cm.timewait_list);
4446
4447         ret = class_register(&cm_class);
4448         if (ret) {
4449                 ret = -ENOMEM;
4450                 goto error1;
4451         }
4452
4453         cm.wq = alloc_workqueue("ib_cm", 0, 1);
4454         if (!cm.wq) {
4455                 ret = -ENOMEM;
4456                 goto error2;
4457         }
4458
4459         ret = ib_register_client(&cm_client);
4460         if (ret)
4461                 goto error3;
4462
4463         return 0;
4464 error3:
4465         destroy_workqueue(cm.wq);
4466 error2:
4467         class_unregister(&cm_class);
4468 error1:
4469         return ret;
4470 }
4471
4472 static void __exit ib_cm_cleanup(void)
4473 {
4474         struct cm_timewait_info *timewait_info, *tmp;
4475
4476         spin_lock_irq(&cm.lock);
4477         list_for_each_entry(timewait_info, &cm.timewait_list, list)
4478                 cancel_delayed_work(&timewait_info->work.work);
4479         spin_unlock_irq(&cm.lock);
4480
4481         ib_unregister_client(&cm_client);
4482         destroy_workqueue(cm.wq);
4483
4484         list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4485                 list_del(&timewait_info->list);
4486                 kfree(timewait_info);
4487         }
4488
4489         class_unregister(&cm_class);
4490         WARN_ON(!xa_empty(&cm.local_id_table));
4491 }
4492
4493 module_init(ib_cm_init);
4494 module_exit(ib_cm_cleanup);