drivers/infiniband/core/cma.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   4  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   5  * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved.
   6  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   7  */
   8
   9 #include <linux/completion.h>
  10 #include <linux/in.h>
  11 #include <linux/in6.h>
  12 #include <linux/mutex.h>
  13 #include <linux/random.h>
  14 #include <linux/igmp.h>
  15 #include <linux/xarray.h>
  16 #include <linux/inetdevice.h>
  17 #include <linux/slab.h>
  18 #include <linux/module.h>
  19 #include <net/route.h>
  20
  21 #include <net/net_namespace.h>
  22 #include <net/netns/generic.h>
  23 #include <net/tcp.h>
  24 #include <net/ipv6.h>
  25 #include <net/ip_fib.h>
  26 #include <net/ip6_route.h>
  27
  28 #include <rdma/rdma_cm.h>
  29 #include <rdma/rdma_cm_ib.h>
  30 #include <rdma/rdma_netlink.h>
  31 #include <rdma/ib.h>
  32 #include <rdma/ib_cache.h>
  33 #include <rdma/ib_cm.h>
  34 #include <rdma/ib_sa.h>
  35 #include <rdma/iw_cm.h>
  36
  37 #include "core_priv.h"
  38 #include "cma_priv.h"
  39 #include "cma_trace.h"
  40
  41 MODULE_AUTHOR("Sean Hefty");
  42 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  43 MODULE_LICENSE("Dual BSD/GPL");
  44
  45 #define CMA_CM_RESPONSE_TIMEOUT 20
  46 #define CMA_QUERY_CLASSPORT_INFO_TIMEOUT 3000
  47 #define CMA_MAX_CM_RETRIES 15
  48 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  49 #define CMA_IBOE_PACKET_LIFETIME 18
  50 #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
  51
  52 static const char * const cma_events[] = {
  53         [RDMA_CM_EVENT_ADDR_RESOLVED]    = "address resolved",
  54         [RDMA_CM_EVENT_ADDR_ERROR]       = "address error",
  55         [RDMA_CM_EVENT_ROUTE_RESOLVED]   = "route resolved ",
  56         [RDMA_CM_EVENT_ROUTE_ERROR]      = "route error",
  57         [RDMA_CM_EVENT_CONNECT_REQUEST]  = "connect request",
  58         [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response",
  59         [RDMA_CM_EVENT_CONNECT_ERROR]    = "connect error",
  60         [RDMA_CM_EVENT_UNREACHABLE]      = "unreachable",
  61         [RDMA_CM_EVENT_REJECTED]         = "rejected",
  62         [RDMA_CM_EVENT_ESTABLISHED]      = "established",
  63         [RDMA_CM_EVENT_DISCONNECTED]     = "disconnected",
  64         [RDMA_CM_EVENT_DEVICE_REMOVAL]   = "device removal",
  65         [RDMA_CM_EVENT_MULTICAST_JOIN]   = "multicast join",
  66         [RDMA_CM_EVENT_MULTICAST_ERROR]  = "multicast error",
  67         [RDMA_CM_EVENT_ADDR_CHANGE]      = "address change",
  68         [RDMA_CM_EVENT_TIMEWAIT_EXIT]    = "timewait exit",
  69 };
  70
  71 const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
  72 {
  73         size_t index = event;
  74
  75         return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ?
  76                         cma_events[index] : "unrecognized event";
  77 }
  78 EXPORT_SYMBOL(rdma_event_msg);
  79
  80 const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
  81                                                 int reason)
  82 {
  83         if (rdma_ib_or_roce(id->device, id->port_num))
  84                 return ibcm_reject_msg(reason);
  85
  86         if (rdma_protocol_iwarp(id->device, id->port_num))
  87                 return iwcm_reject_msg(reason);
  88
  89         WARN_ON_ONCE(1);
  90         return "unrecognized transport";
  91 }
  92 EXPORT_SYMBOL(rdma_reject_msg);
  93
  94 bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
  95 {
  96         if (rdma_ib_or_roce(id->device, id->port_num))
  97                 return reason == IB_CM_REJ_CONSUMER_DEFINED;
  98
  99         if (rdma_protocol_iwarp(id->device, id->port_num))
 100                 return reason == -ECONNREFUSED;
 101
 102         WARN_ON_ONCE(1);
 103         return false;
 104 }
 105 EXPORT_SYMBOL(rdma_is_consumer_reject);
 106
 107 const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
 108                                       struct rdma_cm_event *ev, u8 *data_len)
 109 {
 110         const void *p;
 111
 112         if (rdma_is_consumer_reject(id, ev->status)) {
 113                 *data_len = ev->param.conn.private_data_len;
 114                 p = ev->param.conn.private_data;
 115         } else {
 116                 *data_len = 0;
 117                 p = NULL;
 118         }
 119         return p;
 120 }
 121 EXPORT_SYMBOL(rdma_consumer_reject_data);
 122
 123 /**
 124  * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id.
 125  * @id: Communication Identifier
 126  */
 127 struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)
 128 {
 129         struct rdma_id_private *id_priv;
 130
 131         id_priv = container_of(id, struct rdma_id_private, id);
 132         if (id->device->node_type == RDMA_NODE_RNIC)
 133                 return id_priv->cm_id.iw;
 134         return NULL;
 135 }
 136 EXPORT_SYMBOL(rdma_iw_cm_id);
 137
 138 /**
 139  * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack.
 140  * @res: rdma resource tracking entry pointer
 141  */
 142 struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)
 143 {
 144         struct rdma_id_private *id_priv =
 145                 container_of(res, struct rdma_id_private, res);
 146
 147         return &id_priv->id;
 148 }
 149 EXPORT_SYMBOL(rdma_res_to_id);
 150
 151 static void cma_add_one(struct ib_device *device);
 152 static void cma_remove_one(struct ib_device *device, void *client_data);
 153
 154 static struct ib_client cma_client = {
 155         .name   = "cma",
 156         .add    = cma_add_one,
 157         .remove = cma_remove_one
 158 };
 159
 160 static struct ib_sa_client sa_client;
 161 static LIST_HEAD(dev_list);
 162 static LIST_HEAD(listen_any_list);
 163 static DEFINE_MUTEX(lock);
 164 static struct workqueue_struct *cma_wq;
 165 static unsigned int cma_pernet_id;
 166
 167 struct cma_pernet {
 168         struct xarray tcp_ps;
 169         struct xarray udp_ps;
 170         struct xarray ipoib_ps;
 171         struct xarray ib_ps;
 172 };
 173
 174 static struct cma_pernet *cma_pernet(struct net *net)
 175 {
 176         return net_generic(net, cma_pernet_id);
 177 }
 178
 179 static
 180 struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)
 181 {
 182         struct cma_pernet *pernet = cma_pernet(net);
 183
 184         switch (ps) {
 185         case RDMA_PS_TCP:
 186                 return &pernet->tcp_ps;
 187         case RDMA_PS_UDP:
 188                 return &pernet->udp_ps;
 189         case RDMA_PS_IPOIB:
 190                 return &pernet->ipoib_ps;
 191         case RDMA_PS_IB:
 192                 return &pernet->ib_ps;
 193         default:
 194                 return NULL;
 195         }
 196 }
 197
 198 struct cma_device {
 199         struct list_head        list;
 200         struct ib_device        *device;
 201         struct completion       comp;
 202         atomic_t                refcount;
 203         struct list_head        id_list;
 204         enum ib_gid_type        *default_gid_type;
 205         u8                      *default_roce_tos;
 206 };
 207
 208 struct rdma_bind_list {
 209         enum rdma_ucm_port_space ps;
 210         struct hlist_head       owners;
 211         unsigned short          port;
 212 };
 213
 214 struct class_port_info_context {
 215         struct ib_class_port_info       *class_port_info;
 216         struct ib_device                *device;
 217         struct completion               done;
 218         struct ib_sa_query              *sa_query;
 219         u8                              port_num;
 220 };
 221
 222 static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,
 223                         struct rdma_bind_list *bind_list, int snum)
 224 {
 225         struct xarray *xa = cma_pernet_xa(net, ps);
 226
 227         return xa_insert(xa, snum, bind_list, GFP_KERNEL);
 228 }
 229
 230 static struct rdma_bind_list *cma_ps_find(struct net *net,
 231                                           enum rdma_ucm_port_space ps, int snum)
 232 {
 233         struct xarray *xa = cma_pernet_xa(net, ps);
 234
 235         return xa_load(xa, snum);
 236 }
 237
 238 static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps,
 239                           int snum)
 240 {
 241         struct xarray *xa = cma_pernet_xa(net, ps);
 242
 243         xa_erase(xa, snum);
 244 }
 245
 246 enum {
 247         CMA_OPTION_AFONLY,
 248 };
 249
 250 void cma_ref_dev(struct cma_device *cma_dev)
 251 {
 252         atomic_inc(&cma_dev->refcount);
 253 }
 254
 255 struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter  filter,
 256                                              void               *cookie)
 257 {
 258         struct cma_device *cma_dev;
 259         struct cma_device *found_cma_dev = NULL;
 260
 261         mutex_lock(&lock);
 262
 263         list_for_each_entry(cma_dev, &dev_list, list)
 264                 if (filter(cma_dev->device, cookie)) {
 265                         found_cma_dev = cma_dev;
 266                         break;
 267                 }
 268
 269         if (found_cma_dev)
 270                 cma_ref_dev(found_cma_dev);
 271         mutex_unlock(&lock);
 272         return found_cma_dev;
 273 }
 274
 275 int cma_get_default_gid_type(struct cma_device *cma_dev,
 276                              unsigned int port)
 277 {
 278         if (!rdma_is_port_valid(cma_dev->device, port))
 279                 return -EINVAL;
 280
 281         return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
 282 }
 283
 284 int cma_set_default_gid_type(struct cma_device *cma_dev,
 285                              unsigned int port,
 286                              enum ib_gid_type default_gid_type)
 287 {
 288         unsigned long supported_gids;
 289
 290         if (!rdma_is_port_valid(cma_dev->device, port))
 291                 return -EINVAL;
 292
 293         supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
 294
 295         if (!(supported_gids & 1 << default_gid_type))
 296                 return -EINVAL;
 297
 298         cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
 299                 default_gid_type;
 300
 301         return 0;
 302 }
 303
 304 int cma_get_default_roce_tos(struct cma_device *cma_dev, unsigned int port)
 305 {
 306         if (!rdma_is_port_valid(cma_dev->device, port))
 307                 return -EINVAL;
 308
 309         return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)];
 310 }
 311
 312 int cma_set_default_roce_tos(struct cma_device *cma_dev, unsigned int port,
 313                              u8 default_roce_tos)
 314 {
 315         if (!rdma_is_port_valid(cma_dev->device, port))
 316                 return -EINVAL;
 317
 318         cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] =
 319                  default_roce_tos;
 320
 321         return 0;
 322 }
 323 struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
 324 {
 325         return cma_dev->device;
 326 }
 327
 328 /*
 329  * Device removal can occur at anytime, so we need extra handling to
 330  * serialize notifying the user of device removal with other callbacks.
 331  * We do this by disabling removal notification while a callback is in process,
 332  * and reporting it after the callback completes.
 333  */
 334
 335 struct cma_multicast {
 336         struct rdma_id_private *id_priv;
 337         union {
 338                 struct ib_sa_multicast *ib;
 339         } multicast;
 340         struct list_head        list;
 341         void                    *context;
 342         struct sockaddr_storage addr;
 343         struct kref             mcref;
 344         u8                      join_state;
 345 };
 346
 347 struct cma_work {
 348         struct work_struct      work;
 349         struct rdma_id_private  *id;
 350         enum rdma_cm_state      old_state;
 351         enum rdma_cm_state      new_state;
 352         struct rdma_cm_event    event;
 353 };
 354
 355 struct cma_ndev_work {
 356         struct work_struct      work;
 357         struct rdma_id_private  *id;
 358         struct rdma_cm_event    event;
 359 };
 360
 361 struct iboe_mcast_work {
 362         struct work_struct       work;
 363         struct rdma_id_private  *id;
 364         struct cma_multicast    *mc;
 365 };
 366
 367 union cma_ip_addr {
 368         struct in6_addr ip6;
 369         struct {
 370                 __be32 pad[3];
 371                 __be32 addr;
 372         } ip4;
 373 };
 374
 375 struct cma_hdr {
 376         u8 cma_version;
 377         u8 ip_version;  /* IP version: 7:4 */
 378         __be16 port;
 379         union cma_ip_addr src_addr;
 380         union cma_ip_addr dst_addr;
 381 };
 382
 383 #define CMA_VERSION 0x00
 384
 385 struct cma_req_info {
 386         struct sockaddr_storage listen_addr_storage;
 387         struct sockaddr_storage src_addr_storage;
 388         struct ib_device *device;
 389         union ib_gid local_gid;
 390         __be64 service_id;
 391         int port;
 392         bool has_gid;
 393         u16 pkey;
 394 };
 395
 396 static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
 397 {
 398         unsigned long flags;
 399         int ret;
 400
 401         spin_lock_irqsave(&id_priv->lock, flags);
 402         ret = (id_priv->state == comp);
 403         spin_unlock_irqrestore(&id_priv->lock, flags);
 404         return ret;
 405 }
 406
 407 static int cma_comp_exch(struct rdma_id_private *id_priv,
 408                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 409 {
 410         unsigned long flags;
 411         int ret;
 412
 413         spin_lock_irqsave(&id_priv->lock, flags);
 414         if ((ret = (id_priv->state == comp)))
 415                 id_priv->state = exch;
 416         spin_unlock_irqrestore(&id_priv->lock, flags);
 417         return ret;
 418 }
 419
 420 static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
 421                                    enum rdma_cm_state exch)
 422 {
 423         unsigned long flags;
 424         enum rdma_cm_state old;
 425
 426         spin_lock_irqsave(&id_priv->lock, flags);
 427         old = id_priv->state;
 428         id_priv->state = exch;
 429         spin_unlock_irqrestore(&id_priv->lock, flags);
 430         return old;
 431 }
 432
 433 static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
 434 {
 435         return hdr->ip_version >> 4;
 436 }
 437
 438 static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 439 {
 440         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 441 }
 442
 443 static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
 444 {
 445         struct in_device *in_dev = NULL;
 446
 447         if (ndev) {
 448                 rtnl_lock();
 449                 in_dev = __in_dev_get_rtnl(ndev);
 450                 if (in_dev) {
 451                         if (join)
 452                                 ip_mc_inc_group(in_dev,
 453                                                 *(__be32 *)(mgid->raw + 12));
 454                         else
 455                                 ip_mc_dec_group(in_dev,
 456                                                 *(__be32 *)(mgid->raw + 12));
 457                 }
 458                 rtnl_unlock();
 459         }
 460         return (in_dev) ? 0 : -ENODEV;
 461 }
 462
 463 static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
 464                                struct cma_device *cma_dev)
 465 {
 466         cma_ref_dev(cma_dev);
 467         id_priv->cma_dev = cma_dev;
 468         id_priv->id.device = cma_dev->device;
 469         id_priv->id.route.addr.dev_addr.transport =
 470                 rdma_node_get_transport(cma_dev->device->node_type);
 471         list_add_tail(&id_priv->list, &cma_dev->id_list);
 472         if (id_priv->res.kern_name)
 473                 rdma_restrack_kadd(&id_priv->res);
 474         else
 475                 rdma_restrack_uadd(&id_priv->res);
 476 }
 477
 478 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 479                               struct cma_device *cma_dev)
 480 {
 481         _cma_attach_to_dev(id_priv, cma_dev);
 482         id_priv->gid_type =
 483                 cma_dev->default_gid_type[id_priv->id.port_num -
 484                                           rdma_start_port(cma_dev->device)];
 485 }
 486
 487 void cma_deref_dev(struct cma_device *cma_dev)
 488 {
 489         if (atomic_dec_and_test(&cma_dev->refcount))
 490                 complete(&cma_dev->comp);
 491 }
 492
 493 static inline void release_mc(struct kref *kref)
 494 {
 495         struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
 496
 497         kfree(mc->multicast.ib);
 498         kfree(mc);
 499 }
 500
 501 static void cma_release_dev(struct rdma_id_private *id_priv)
 502 {
 503         mutex_lock(&lock);
 504         list_del(&id_priv->list);
 505         cma_deref_dev(id_priv->cma_dev);
 506         id_priv->cma_dev = NULL;
 507         mutex_unlock(&lock);
 508 }
 509
 510 static inline struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
 511 {
 512         return (struct sockaddr *) &id_priv->id.route.addr.src_addr;
 513 }
 514
 515 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
 516 {
 517         return (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
 518 }
 519
 520 static inline unsigned short cma_family(struct rdma_id_private *id_priv)
 521 {
 522         return id_priv->id.route.addr.src_addr.ss_family;
 523 }
 524
 525 static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
 526 {
 527         struct ib_sa_mcmember_rec rec;
 528         int ret = 0;
 529
 530         if (id_priv->qkey) {
 531                 if (qkey && id_priv->qkey != qkey)
 532                         return -EINVAL;
 533                 return 0;
 534         }
 535
 536         if (qkey) {
 537                 id_priv->qkey = qkey;
 538                 return 0;
 539         }
 540
 541         switch (id_priv->id.ps) {
 542         case RDMA_PS_UDP:
 543         case RDMA_PS_IB:
 544                 id_priv->qkey = RDMA_UDP_QKEY;
 545                 break;
 546         case RDMA_PS_IPOIB:
 547                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 548                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 549                                              id_priv->id.port_num, &rec.mgid,
 550                                              &rec);
 551                 if (!ret)
 552                         id_priv->qkey = be32_to_cpu(rec.qkey);
 553                 break;
 554         default:
 555                 break;
 556         }
 557         return ret;
 558 }
 559
 560 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
 561 {
 562         dev_addr->dev_type = ARPHRD_INFINIBAND;
 563         rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
 564         ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
 565 }
 566
 567 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 568 {
 569         int ret;
 570
 571         if (addr->sa_family != AF_IB) {
 572                 ret = rdma_translate_ip(addr, dev_addr);
 573         } else {
 574                 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
 575                 ret = 0;
 576         }
 577
 578         return ret;
 579 }
 580
 581 static const struct ib_gid_attr *
 582 cma_validate_port(struct ib_device *device, u8 port,
 583                   enum ib_gid_type gid_type,
 584                   union ib_gid *gid,
 585                   struct rdma_id_private *id_priv)
 586 {
 587         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 588         int bound_if_index = dev_addr->bound_dev_if;
 589         const struct ib_gid_attr *sgid_attr;
 590         int dev_type = dev_addr->dev_type;
 591         struct net_device *ndev = NULL;
 592
 593         if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net))
 594                 return ERR_PTR(-ENODEV);
 595
 596         if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
 597                 return ERR_PTR(-ENODEV);
 598
 599         if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
 600                 return ERR_PTR(-ENODEV);
 601
 602         if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
 603                 ndev = dev_get_by_index(dev_addr->net, bound_if_index);
 604                 if (!ndev)
 605                         return ERR_PTR(-ENODEV);
 606         } else {
 607                 gid_type = IB_GID_TYPE_IB;
 608         }
 609
 610         sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
 611         if (ndev)
 612                 dev_put(ndev);
 613         return sgid_attr;
 614 }
 615
 616 static void cma_bind_sgid_attr(struct rdma_id_private *id_priv,
 617                                const struct ib_gid_attr *sgid_attr)
 618 {
 619         WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr);
 620         id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr;
 621 }
 622
 623 /**
 624  * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute
 625  * based on source ip address.
 626  * @id_priv:    cm_id which should be bound to cma device
 627  *
 628  * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute
 629  * based on source IP address. It returns 0 on success or error code otherwise.
 630  * It is applicable to active and passive side cm_id.
 631  */
 632 static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)
 633 {
 634         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 635         const struct ib_gid_attr *sgid_attr;
 636         union ib_gid gid, iboe_gid, *gidp;
 637         struct cma_device *cma_dev;
 638         enum ib_gid_type gid_type;
 639         int ret = -ENODEV;
 640         unsigned int port;
 641
 642         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 643             id_priv->id.ps == RDMA_PS_IPOIB)
 644                 return -EINVAL;
 645
 646         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 647                     &iboe_gid);
 648
 649         memcpy(&gid, dev_addr->src_dev_addr +
 650                rdma_addr_gid_offset(dev_addr), sizeof(gid));
 651
 652         mutex_lock(&lock);
 653         list_for_each_entry(cma_dev, &dev_list, list) {
 654                 rdma_for_each_port (cma_dev->device, port) {
 655                         gidp = rdma_protocol_roce(cma_dev->device, port) ?
 656                                &iboe_gid : &gid;
 657                         gid_type = cma_dev->default_gid_type[port - 1];
 658                         sgid_attr = cma_validate_port(cma_dev->device, port,
 659                                                       gid_type, gidp, id_priv);
 660                         if (!IS_ERR(sgid_attr)) {
 661                                 id_priv->id.port_num = port;
 662                                 cma_bind_sgid_attr(id_priv, sgid_attr);
 663                                 cma_attach_to_dev(id_priv, cma_dev);
 664                                 ret = 0;
 665                                 goto out;
 666                         }
 667                 }
 668         }
 669 out:
 670         mutex_unlock(&lock);
 671         return ret;
 672 }
 673
 674 /**
 675  * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute
 676  * @id_priv:            cm id to bind to cma device
 677  * @listen_id_priv:     listener cm id to match against
 678  * @req:                Pointer to req structure containaining incoming
 679  *                      request information
 680  * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when
 681  * rdma device matches for listen_id and incoming request. It also verifies
 682  * that a GID table entry is present for the source address.
 683  * Returns 0 on success, or returns error code otherwise.
 684  */
 685 static int cma_ib_acquire_dev(struct rdma_id_private *id_priv,
 686                               const struct rdma_id_private *listen_id_priv,
 687                               struct cma_req_info *req)
 688 {
 689         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 690         const struct ib_gid_attr *sgid_attr;
 691         enum ib_gid_type gid_type;
 692         union ib_gid gid;
 693
 694         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 695             id_priv->id.ps == RDMA_PS_IPOIB)
 696                 return -EINVAL;
 697
 698         if (rdma_protocol_roce(req->device, req->port))
 699                 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 700                             &gid);
 701         else
 702                 memcpy(&gid, dev_addr->src_dev_addr +
 703                        rdma_addr_gid_offset(dev_addr), sizeof(gid));
 704
 705         gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1];
 706         sgid_attr = cma_validate_port(req->device, req->port,
 707                                       gid_type, &gid, id_priv);
 708         if (IS_ERR(sgid_attr))
 709                 return PTR_ERR(sgid_attr);
 710
 711         id_priv->id.port_num = req->port;
 712         cma_bind_sgid_attr(id_priv, sgid_attr);
 713         /* Need to acquire lock to protect against reader
 714          * of cma_dev->id_list such as cma_netdev_callback() and
 715          * cma_process_remove().
 716          */
 717         mutex_lock(&lock);
 718         cma_attach_to_dev(id_priv, listen_id_priv->cma_dev);
 719         mutex_unlock(&lock);
 720         return 0;
 721 }
 722
 723 static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
 724                               const struct rdma_id_private *listen_id_priv)
 725 {
 726         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 727         const struct ib_gid_attr *sgid_attr;
 728         struct cma_device *cma_dev;
 729         enum ib_gid_type gid_type;
 730         int ret = -ENODEV;
 731         union ib_gid gid;
 732         u8 port;
 733
 734         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 735             id_priv->id.ps == RDMA_PS_IPOIB)
 736                 return -EINVAL;
 737
 738         memcpy(&gid, dev_addr->src_dev_addr +
 739                rdma_addr_gid_offset(dev_addr), sizeof(gid));
 740
 741         mutex_lock(&lock);
 742
 743         cma_dev = listen_id_priv->cma_dev;
 744         port = listen_id_priv->id.port_num;
 745         gid_type = listen_id_priv->gid_type;
 746         sgid_attr = cma_validate_port(cma_dev->device, port,
 747                                       gid_type, &gid, id_priv);
 748         if (!IS_ERR(sgid_attr)) {
 749                 id_priv->id.port_num = port;
 750                 cma_bind_sgid_attr(id_priv, sgid_attr);
 751                 ret = 0;
 752                 goto out;
 753         }
 754
 755         list_for_each_entry(cma_dev, &dev_list, list) {
 756                 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
 757                         if (listen_id_priv->cma_dev == cma_dev &&
 758                             listen_id_priv->id.port_num == port)
 759                                 continue;
 760
 761                         gid_type = cma_dev->default_gid_type[port - 1];
 762                         sgid_attr = cma_validate_port(cma_dev->device, port,
 763                                                       gid_type, &gid, id_priv);
 764                         if (!IS_ERR(sgid_attr)) {
 765                                 id_priv->id.port_num = port;
 766                                 cma_bind_sgid_attr(id_priv, sgid_attr);
 767                                 ret = 0;
 768                                 goto out;
 769                         }
 770                 }
 771         }
 772
 773 out:
 774         if (!ret)
 775                 cma_attach_to_dev(id_priv, cma_dev);
 776
 777         mutex_unlock(&lock);
 778         return ret;
 779 }
 780
 781 /*
 782  * Select the source IB device and address to reach the destination IB address.
 783  */
 784 static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
 785 {
 786         struct cma_device *cma_dev, *cur_dev;
 787         struct sockaddr_ib *addr;
 788         union ib_gid gid, sgid, *dgid;
 789         u16 pkey, index;
 790         u8 p;
 791         enum ib_port_state port_state;
 792         int i;
 793
 794         cma_dev = NULL;
 795         addr = (struct sockaddr_ib *) cma_dst_addr(id_priv);
 796         dgid = (union ib_gid *) &addr->sib_addr;
 797         pkey = ntohs(addr->sib_pkey);
 798
 799         mutex_lock(&lock);
 800         list_for_each_entry(cur_dev, &dev_list, list) {
 801                 for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
 802                         if (!rdma_cap_af_ib(cur_dev->device, p))
 803                                 continue;
 804
 805                         if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index))
 806                                 continue;
 807
 808                         if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
 809                                 continue;
 810                         for (i = 0; !rdma_query_gid(cur_dev->device,
 811                                                     p, i, &gid);
 812                              i++) {
 813                                 if (!memcmp(&gid, dgid, sizeof(gid))) {
 814                                         cma_dev = cur_dev;
 815                                         sgid = gid;
 816                                         id_priv->id.port_num = p;
 817                                         goto found;
 818                                 }
 819
 820                                 if (!cma_dev && (gid.global.subnet_prefix ==
 821                                     dgid->global.subnet_prefix) &&
 822                                     port_state == IB_PORT_ACTIVE) {
 823                                         cma_dev = cur_dev;
 824                                         sgid = gid;
 825                                         id_priv->id.port_num = p;
 826                                         goto found;
 827                                 }
 828                         }
 829                 }
 830         }
 831         mutex_unlock(&lock);
 832         return -ENODEV;
 833
 834 found:
 835         cma_attach_to_dev(id_priv, cma_dev);
 836         mutex_unlock(&lock);
 837         addr = (struct sockaddr_ib *)cma_src_addr(id_priv);
 838         memcpy(&addr->sib_addr, &sgid, sizeof(sgid));
 839         cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
 840         return 0;
 841 }
 842
 843 static void cma_deref_id(struct rdma_id_private *id_priv)
 844 {
 845         if (atomic_dec_and_test(&id_priv->refcount))
 846                 complete(&id_priv->comp);
 847 }
 848
 849 struct rdma_cm_id *__rdma_create_id(struct net *net,
 850                                     rdma_cm_event_handler event_handler,
 851                                     void *context, enum rdma_ucm_port_space ps,
 852                                     enum ib_qp_type qp_type, const char *caller)
 853 {
 854         struct rdma_id_private *id_priv;
 855
 856         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 857         if (!id_priv)
 858                 return ERR_PTR(-ENOMEM);
 859
 860         rdma_restrack_set_task(&id_priv->res, caller);
 861         id_priv->res.type = RDMA_RESTRACK_CM_ID;
 862         id_priv->state = RDMA_CM_IDLE;
 863         id_priv->id.context = context;
 864         id_priv->id.event_handler = event_handler;
 865         id_priv->id.ps = ps;
 866         id_priv->id.qp_type = qp_type;
 867         id_priv->tos_set = false;
 868         id_priv->timeout_set = false;
 869         id_priv->gid_type = IB_GID_TYPE_IB;
 870         spin_lock_init(&id_priv->lock);
 871         mutex_init(&id_priv->qp_mutex);
 872         init_completion(&id_priv->comp);
 873         atomic_set(&id_priv->refcount, 1);
 874         mutex_init(&id_priv->handler_mutex);
 875         INIT_LIST_HEAD(&id_priv->listen_list);
 876         INIT_LIST_HEAD(&id_priv->mc_list);
 877         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 878         id_priv->id.route.addr.dev_addr.net = get_net(net);
 879         id_priv->seq_num &= 0x00ffffff;
 880
 881         trace_cm_id_create(id_priv);
 882         return &id_priv->id;
 883 }
 884 EXPORT_SYMBOL(__rdma_create_id);
 885
 886 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 887 {
 888         struct ib_qp_attr qp_attr;
 889         int qp_attr_mask, ret;
 890
 891         qp_attr.qp_state = IB_QPS_INIT;
 892         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 893         if (ret)
 894                 return ret;
 895
 896         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 897         if (ret)
 898                 return ret;
 899
 900         qp_attr.qp_state = IB_QPS_RTR;
 901         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
 902         if (ret)
 903                 return ret;
 904
 905         qp_attr.qp_state = IB_QPS_RTS;
 906         qp_attr.sq_psn = 0;
 907         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
 908
 909         return ret;
 910 }
 911
 912 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 913 {
 914         struct ib_qp_attr qp_attr;
 915         int qp_attr_mask, ret;
 916
 917         qp_attr.qp_state = IB_QPS_INIT;
 918         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 919         if (ret)
 920                 return ret;
 921
 922         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
 923 }
 924
 925 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 926                    struct ib_qp_init_attr *qp_init_attr)
 927 {
 928         struct rdma_id_private *id_priv;
 929         struct ib_qp *qp;
 930         int ret;
 931
 932         id_priv = container_of(id, struct rdma_id_private, id);
 933         if (id->device != pd->device) {
 934                 ret = -EINVAL;
 935                 goto out_err;
 936         }
 937
 938         qp_init_attr->port_num = id->port_num;
 939         qp = ib_create_qp(pd, qp_init_attr);
 940         if (IS_ERR(qp)) {
 941                 ret = PTR_ERR(qp);
 942                 goto out_err;
 943         }
 944
 945         if (id->qp_type == IB_QPT_UD)
 946                 ret = cma_init_ud_qp(id_priv, qp);
 947         else
 948                 ret = cma_init_conn_qp(id_priv, qp);
 949         if (ret)
 950                 goto out_destroy;
 951
 952         id->qp = qp;
 953         id_priv->qp_num = qp->qp_num;
 954         id_priv->srq = (qp->srq != NULL);
 955         trace_cm_qp_create(id_priv, pd, qp_init_attr, 0);
 956         return 0;
 957 out_destroy:
 958         ib_destroy_qp(qp);
 959 out_err:
 960         trace_cm_qp_create(id_priv, pd, qp_init_attr, ret);
 961         return ret;
 962 }
 963 EXPORT_SYMBOL(rdma_create_qp);
 964
 965 void rdma_destroy_qp(struct rdma_cm_id *id)
 966 {
 967         struct rdma_id_private *id_priv;
 968
 969         id_priv = container_of(id, struct rdma_id_private, id);
 970         trace_cm_qp_destroy(id_priv);
 971         mutex_lock(&id_priv->qp_mutex);
 972         ib_destroy_qp(id_priv->id.qp);
 973         id_priv->id.qp = NULL;
 974         mutex_unlock(&id_priv->qp_mutex);
 975 }
 976 EXPORT_SYMBOL(rdma_destroy_qp);
 977
 978 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
 979                              struct rdma_conn_param *conn_param)
 980 {
 981         struct ib_qp_attr qp_attr;
 982         int qp_attr_mask, ret;
 983
 984         mutex_lock(&id_priv->qp_mutex);
 985         if (!id_priv->id.qp) {
 986                 ret = 0;
 987                 goto out;
 988         }
 989
 990         /* Need to update QP attributes from default values. */
 991         qp_attr.qp_state = IB_QPS_INIT;
 992         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
 993         if (ret)
 994                 goto out;
 995
 996         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
 997         if (ret)
 998                 goto out;
 999
1000         qp_attr.qp_state = IB_QPS_RTR;
1001         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1002         if (ret)
1003                 goto out;
1004
1005         BUG_ON(id_priv->cma_dev->device != id_priv->id.device);
1006
1007         if (conn_param)
1008                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
1009         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1010 out:
1011         mutex_unlock(&id_priv->qp_mutex);
1012         return ret;
1013 }
1014
1015 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
1016                              struct rdma_conn_param *conn_param)
1017 {
1018         struct ib_qp_attr qp_attr;
1019         int qp_attr_mask, ret;
1020
1021         mutex_lock(&id_priv->qp_mutex);
1022         if (!id_priv->id.qp) {
1023                 ret = 0;
1024                 goto out;
1025         }
1026
1027         qp_attr.qp_state = IB_QPS_RTS;
1028         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1029         if (ret)
1030                 goto out;
1031
1032         if (conn_param)
1033                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
1034         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1035 out:
1036         mutex_unlock(&id_priv->qp_mutex);
1037         return ret;
1038 }
1039
1040 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
1041 {
1042         struct ib_qp_attr qp_attr;
1043         int ret;
1044
1045         mutex_lock(&id_priv->qp_mutex);
1046         if (!id_priv->id.qp) {
1047                 ret = 0;
1048                 goto out;
1049         }
1050
1051         qp_attr.qp_state = IB_QPS_ERR;
1052         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
1053 out:
1054         mutex_unlock(&id_priv->qp_mutex);
1055         return ret;
1056 }
1057
1058 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
1059                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
1060 {
1061         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1062         int ret;
1063         u16 pkey;
1064
1065         if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num))
1066                 pkey = 0xffff;
1067         else
1068                 pkey = ib_addr_get_pkey(dev_addr);
1069
1070         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
1071                                   pkey, &qp_attr->pkey_index);
1072         if (ret)
1073                 return ret;
1074
1075         qp_attr->port_num = id_priv->id.port_num;
1076         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
1077
1078         if (id_priv->id.qp_type == IB_QPT_UD) {
1079                 ret = cma_set_qkey(id_priv, 0);
1080                 if (ret)
1081                         return ret;
1082
1083                 qp_attr->qkey = id_priv->qkey;
1084                 *qp_attr_mask |= IB_QP_QKEY;
1085         } else {
1086                 qp_attr->qp_access_flags = 0;
1087                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
1088         }
1089         return 0;
1090 }
1091
1092 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
1093                        int *qp_attr_mask)
1094 {
1095         struct rdma_id_private *id_priv;
1096         int ret = 0;
1097
1098         id_priv = container_of(id, struct rdma_id_private, id);
1099         if (rdma_cap_ib_cm(id->device, id->port_num)) {
1100                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
1101                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
1102                 else
1103                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
1104                                                  qp_attr_mask);
1105
1106                 if (qp_attr->qp_state == IB_QPS_RTR)
1107                         qp_attr->rq_psn = id_priv->seq_num;
1108         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
1109                 if (!id_priv->cm_id.iw) {
1110                         qp_attr->qp_access_flags = 0;
1111                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
1112                 } else
1113                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
1114                                                  qp_attr_mask);
1115                 qp_attr->port_num = id_priv->id.port_num;
1116                 *qp_attr_mask |= IB_QP_PORT;
1117         } else
1118                 ret = -ENOSYS;
1119
1120         if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set)
1121                 qp_attr->timeout = id_priv->timeout;
1122
1123         return ret;
1124 }
1125 EXPORT_SYMBOL(rdma_init_qp_attr);
1126
1127 static inline bool cma_zero_addr(const struct sockaddr *addr)
1128 {
1129         switch (addr->sa_family) {
1130         case AF_INET:
1131                 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
1132         case AF_INET6:
1133                 return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr);
1134         case AF_IB:
1135                 return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr);
1136         default:
1137                 return false;
1138         }
1139 }
1140
1141 static inline bool cma_loopback_addr(const struct sockaddr *addr)
1142 {
1143         switch (addr->sa_family) {
1144         case AF_INET:
1145                 return ipv4_is_loopback(
1146                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
1147         case AF_INET6:
1148                 return ipv6_addr_loopback(
1149                         &((struct sockaddr_in6 *)addr)->sin6_addr);
1150         case AF_IB:
1151                 return ib_addr_loopback(
1152                         &((struct sockaddr_ib *)addr)->sib_addr);
1153         default:
1154                 return false;
1155         }
1156 }
1157
1158 static inline bool cma_any_addr(const struct sockaddr *addr)
1159 {
1160         return cma_zero_addr(addr) || cma_loopback_addr(addr);
1161 }
1162
1163 static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst)
1164 {
1165         if (src->sa_family != dst->sa_family)
1166                 return -1;
1167
1168         switch (src->sa_family) {
1169         case AF_INET:
1170                 return ((struct sockaddr_in *)src)->sin_addr.s_addr !=
1171                        ((struct sockaddr_in *)dst)->sin_addr.s_addr;
1172         case AF_INET6: {
1173                 struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src;
1174                 struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst;
1175                 bool link_local;
1176
1177                 if (ipv6_addr_cmp(&src_addr6->sin6_addr,
1178                                           &dst_addr6->sin6_addr))
1179                         return 1;
1180                 link_local = ipv6_addr_type(&dst_addr6->sin6_addr) &
1181                              IPV6_ADDR_LINKLOCAL;
1182                 /* Link local must match their scope_ids */
1183                 return link_local ? (src_addr6->sin6_scope_id !=
1184                                      dst_addr6->sin6_scope_id) :
1185                                     0;
1186         }
1187
1188         default:
1189                 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
1190                                    &((struct sockaddr_ib *) dst)->sib_addr);
1191         }
1192 }
1193
1194 static __be16 cma_port(const struct sockaddr *addr)
1195 {
1196         struct sockaddr_ib *sib;
1197
1198         switch (addr->sa_family) {
1199         case AF_INET:
1200                 return ((struct sockaddr_in *) addr)->sin_port;
1201         case AF_INET6:
1202                 return ((struct sockaddr_in6 *) addr)->sin6_port;
1203         case AF_IB:
1204                 sib = (struct sockaddr_ib *) addr;
1205                 return htons((u16) (be64_to_cpu(sib->sib_sid) &
1206                                     be64_to_cpu(sib->sib_sid_mask)));
1207         default:
1208                 return 0;
1209         }
1210 }
1211
1212 static inline int cma_any_port(const struct sockaddr *addr)
1213 {
1214         return !cma_port(addr);
1215 }
1216
1217 static void cma_save_ib_info(struct sockaddr *src_addr,
1218                              struct sockaddr *dst_addr,
1219                              const struct rdma_cm_id *listen_id,
1220                              const struct sa_path_rec *path)
1221 {
1222         struct sockaddr_ib *listen_ib, *ib;
1223
1224         listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr;
1225         if (src_addr) {
1226                 ib = (struct sockaddr_ib *)src_addr;
1227                 ib->sib_family = AF_IB;
1228                 if (path) {
1229                         ib->sib_pkey = path->pkey;
1230                         ib->sib_flowinfo = path->flow_label;
1231                         memcpy(&ib->sib_addr, &path->sgid, 16);
1232                         ib->sib_sid = path->service_id;
1233                         ib->sib_scope_id = 0;
1234                 } else {
1235                         ib->sib_pkey = listen_ib->sib_pkey;
1236                         ib->sib_flowinfo = listen_ib->sib_flowinfo;
1237                         ib->sib_addr = listen_ib->sib_addr;
1238                         ib->sib_sid = listen_ib->sib_sid;
1239                         ib->sib_scope_id = listen_ib->sib_scope_id;
1240                 }
1241                 ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
1242         }
1243         if (dst_addr) {
1244                 ib = (struct sockaddr_ib *)dst_addr;
1245                 ib->sib_family = AF_IB;
1246                 if (path) {
1247                         ib->sib_pkey = path->pkey;
1248                         ib->sib_flowinfo = path->flow_label;
1249                         memcpy(&ib->sib_addr, &path->dgid, 16);
1250                 }
1251         }
1252 }
1253
1254 static void cma_save_ip4_info(struct sockaddr_in *src_addr,
1255                               struct sockaddr_in *dst_addr,
1256                               struct cma_hdr *hdr,
1257                               __be16 local_port)
1258 {
1259         if (src_addr) {
1260                 *src_addr = (struct sockaddr_in) {
1261                         .sin_family = AF_INET,
1262                         .sin_addr.s_addr = hdr->dst_addr.ip4.addr,
1263                         .sin_port = local_port,
1264                 };
1265         }
1266
1267         if (dst_addr) {
1268                 *dst_addr = (struct sockaddr_in) {
1269                         .sin_family = AF_INET,
1270                         .sin_addr.s_addr = hdr->src_addr.ip4.addr,
1271                         .sin_port = hdr->port,
1272                 };
1273         }
1274 }
1275
1276 static void cma_save_ip6_info(struct sockaddr_in6 *src_addr,
1277                               struct sockaddr_in6 *dst_addr,
1278                               struct cma_hdr *hdr,
1279                               __be16 local_port)
1280 {
1281         if (src_addr) {
1282                 *src_addr = (struct sockaddr_in6) {
1283                         .sin6_family = AF_INET6,
1284                         .sin6_addr = hdr->dst_addr.ip6,
1285                         .sin6_port = local_port,
1286                 };
1287         }
1288
1289         if (dst_addr) {
1290                 *dst_addr = (struct sockaddr_in6) {
1291                         .sin6_family = AF_INET6,
1292                         .sin6_addr = hdr->src_addr.ip6,
1293                         .sin6_port = hdr->port,
1294                 };
1295         }
1296 }
1297
1298 static u16 cma_port_from_service_id(__be64 service_id)
1299 {
1300         return (u16)be64_to_cpu(service_id);
1301 }
1302
1303 static int cma_save_ip_info(struct sockaddr *src_addr,
1304                             struct sockaddr *dst_addr,
1305                             const struct ib_cm_event *ib_event,
1306                             __be64 service_id)
1307 {
1308         struct cma_hdr *hdr;
1309         __be16 port;
1310
1311         hdr = ib_event->private_data;
1312         if (hdr->cma_version != CMA_VERSION)
1313                 return -EINVAL;
1314
1315         port = htons(cma_port_from_service_id(service_id));
1316
1317         switch (cma_get_ip_ver(hdr)) {
1318         case 4:
1319                 cma_save_ip4_info((struct sockaddr_in *)src_addr,
1320                                   (struct sockaddr_in *)dst_addr, hdr, port);
1321                 break;
1322         case 6:
1323                 cma_save_ip6_info((struct sockaddr_in6 *)src_addr,
1324                                   (struct sockaddr_in6 *)dst_addr, hdr, port);
1325                 break;
1326         default:
1327                 return -EAFNOSUPPORT;
1328         }
1329
1330         return 0;
1331 }
1332
1333 static int cma_save_net_info(struct sockaddr *src_addr,
1334                              struct sockaddr *dst_addr,
1335                              const struct rdma_cm_id *listen_id,
1336                              const struct ib_cm_event *ib_event,
1337                              sa_family_t sa_family, __be64 service_id)
1338 {
1339         if (sa_family == AF_IB) {
1340                 if (ib_event->event == IB_CM_REQ_RECEIVED)
1341                         cma_save_ib_info(src_addr, dst_addr, listen_id,
1342                                          ib_event->param.req_rcvd.primary_path);
1343                 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1344                         cma_save_ib_info(src_addr, dst_addr, listen_id, NULL);
1345                 return 0;
1346         }
1347
1348         return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id);
1349 }
1350
1351 static int cma_save_req_info(const struct ib_cm_event *ib_event,
1352                              struct cma_req_info *req)
1353 {
1354         const struct ib_cm_req_event_param *req_param =
1355                 &ib_event->param.req_rcvd;
1356         const struct ib_cm_sidr_req_event_param *sidr_param =
1357                 &ib_event->param.sidr_req_rcvd;
1358
1359         switch (ib_event->event) {
1360         case IB_CM_REQ_RECEIVED:
1361                 req->device     = req_param->listen_id->device;
1362                 req->port       = req_param->port;
1363                 memcpy(&req->local_gid, &req_param->primary_path->sgid,
1364                        sizeof(req->local_gid));
1365                 req->has_gid    = true;
1366                 req->service_id = req_param->primary_path->service_id;
1367                 req->pkey       = be16_to_cpu(req_param->primary_path->pkey);
1368                 if (req->pkey != req_param->bth_pkey)
1369                         pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
1370                                             "RDMA CMA: in the future this may cause the request to be dropped\n",
1371                                             req_param->bth_pkey, req->pkey);
1372                 break;
1373         case IB_CM_SIDR_REQ_RECEIVED:
1374                 req->device     = sidr_param->listen_id->device;
1375                 req->port       = sidr_param->port;
1376                 req->has_gid    = false;
1377                 req->service_id = sidr_param->service_id;
1378                 req->pkey       = sidr_param->pkey;
1379                 if (req->pkey != sidr_param->bth_pkey)
1380                         pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
1381                                             "RDMA CMA: in the future this may cause the request to be dropped\n",
1382                                             sidr_param->bth_pkey, req->pkey);
1383                 break;
1384         default:
1385                 return -EINVAL;
1386         }
1387
1388         return 0;
1389 }
1390
1391 static bool validate_ipv4_net_dev(struct net_device *net_dev,
1392                                   const struct sockaddr_in *dst_addr,
1393                                   const struct sockaddr_in *src_addr)
1394 {
1395         __be32 daddr = dst_addr->sin_addr.s_addr,
1396                saddr = src_addr->sin_addr.s_addr;
1397         struct fib_result res;
1398         struct flowi4 fl4;
1399         int err;
1400         bool ret;
1401
1402         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1403             ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) ||
1404             ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) ||
1405             ipv4_is_loopback(saddr))
1406                 return false;
1407
1408         memset(&fl4, 0, sizeof(fl4));
1409         fl4.flowi4_iif = net_dev->ifindex;
1410         fl4.daddr = daddr;
1411         fl4.saddr = saddr;
1412
1413         rcu_read_lock();
1414         err = fib_lookup(dev_net(net_dev), &fl4, &res, 0);
1415         ret = err == 0 && FIB_RES_DEV(res) == net_dev;
1416         rcu_read_unlock();
1417
1418         return ret;
1419 }
1420
1421 static bool validate_ipv6_net_dev(struct net_device *net_dev,
1422                                   const struct sockaddr_in6 *dst_addr,
1423                                   const struct sockaddr_in6 *src_addr)
1424 {
1425 #if IS_ENABLED(CONFIG_IPV6)
1426         const int strict = ipv6_addr_type(&dst_addr->sin6_addr) &
1427                            IPV6_ADDR_LINKLOCAL;
1428         struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr,
1429                                          &src_addr->sin6_addr, net_dev->ifindex,
1430                                          NULL, strict);
1431         bool ret;
1432
1433         if (!rt)
1434                 return false;
1435
1436         ret = rt->rt6i_idev->dev == net_dev;
1437         ip6_rt_put(rt);
1438
1439         return ret;
1440 #else
1441         return false;
1442 #endif
1443 }
1444
1445 static bool validate_net_dev(struct net_device *net_dev,
1446                              const struct sockaddr *daddr,
1447                              const struct sockaddr *saddr)
1448 {
1449         const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr;
1450         const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr;
1451         const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1452         const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr;
1453
1454         switch (daddr->sa_family) {
1455         case AF_INET:
1456                 return saddr->sa_family == AF_INET &&
1457                        validate_ipv4_net_dev(net_dev, daddr4, saddr4);
1458
1459         case AF_INET6:
1460                 return saddr->sa_family == AF_INET6 &&
1461                        validate_ipv6_net_dev(net_dev, daddr6, saddr6);
1462
1463         default:
1464                 return false;
1465         }
1466 }
1467
1468 static struct net_device *
1469 roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
1470 {
1471         const struct ib_gid_attr *sgid_attr = NULL;
1472         struct net_device *ndev;
1473
1474         if (ib_event->event == IB_CM_REQ_RECEIVED)
1475                 sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr;
1476         else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1477                 sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr;
1478
1479         if (!sgid_attr)
1480                 return NULL;
1481
1482         rcu_read_lock();
1483         ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr);
1484         if (IS_ERR(ndev))
1485                 ndev = NULL;
1486         else
1487                 dev_hold(ndev);
1488         rcu_read_unlock();
1489         return ndev;
1490 }
1491
1492 static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event,
1493                                           struct cma_req_info *req)
1494 {
1495         struct sockaddr *listen_addr =
1496                         (struct sockaddr *)&req->listen_addr_storage;
1497         struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage;
1498         struct net_device *net_dev;
1499         const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL;
1500         int err;
1501
1502         err = cma_save_ip_info(listen_addr, src_addr, ib_event,
1503                                req->service_id);
1504         if (err)
1505                 return ERR_PTR(err);
1506
1507         if (rdma_protocol_roce(req->device, req->port))
1508                 net_dev = roce_get_net_dev_by_cm_event(ib_event);
1509         else
1510                 net_dev = ib_get_net_dev_by_params(req->device, req->port,
1511                                                    req->pkey,
1512                                                    gid, listen_addr);
1513         if (!net_dev)
1514                 return ERR_PTR(-ENODEV);
1515
1516         return net_dev;
1517 }
1518
1519 static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id)
1520 {
1521         return (be64_to_cpu(service_id) >> 16) & 0xffff;
1522 }
1523
1524 static bool cma_match_private_data(struct rdma_id_private *id_priv,
1525                                    const struct cma_hdr *hdr)
1526 {
1527         struct sockaddr *addr = cma_src_addr(id_priv);
1528         __be32 ip4_addr;
1529         struct in6_addr ip6_addr;
1530
1531         if (cma_any_addr(addr) && !id_priv->afonly)
1532                 return true;
1533
1534         switch (addr->sa_family) {
1535         case AF_INET:
1536                 ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
1537                 if (cma_get_ip_ver(hdr) != 4)
1538                         return false;
1539                 if (!cma_any_addr(addr) &&
1540                     hdr->dst_addr.ip4.addr != ip4_addr)
1541                         return false;
1542                 break;
1543         case AF_INET6:
1544                 ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
1545                 if (cma_get_ip_ver(hdr) != 6)
1546                         return false;
1547                 if (!cma_any_addr(addr) &&
1548                     memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
1549                         return false;
1550                 break;
1551         case AF_IB:
1552                 return true;
1553         default:
1554                 return false;
1555         }
1556
1557         return true;
1558 }
1559
1560 static bool cma_protocol_roce(const struct rdma_cm_id *id)
1561 {
1562         struct ib_device *device = id->device;
1563         const int port_num = id->port_num ?: rdma_start_port(device);
1564
1565         return rdma_protocol_roce(device, port_num);
1566 }
1567
1568 static bool cma_is_req_ipv6_ll(const struct cma_req_info *req)
1569 {
1570         const struct sockaddr *daddr =
1571                         (const struct sockaddr *)&req->listen_addr_storage;
1572         const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1573
1574         /* Returns true if the req is for IPv6 link local */
1575         return (daddr->sa_family == AF_INET6 &&
1576                 (ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL));
1577 }
1578
1579 static bool cma_match_net_dev(const struct rdma_cm_id *id,
1580                               const struct net_device *net_dev,
1581                               const struct cma_req_info *req)
1582 {
1583         const struct rdma_addr *addr = &id->route.addr;
1584
1585         if (!net_dev)
1586                 /* This request is an AF_IB request */
1587                 return (!id->port_num || id->port_num == req->port) &&
1588                        (addr->src_addr.ss_family == AF_IB);
1589
1590         /*
1591          * If the request is not for IPv6 link local, allow matching
1592          * request to any netdevice of the one or multiport rdma device.
1593          */
1594         if (!cma_is_req_ipv6_ll(req))
1595                 return true;
1596         /*
1597          * Net namespaces must match, and if the listner is listening
1598          * on a specific netdevice than netdevice must match as well.
1599          */
1600         if (net_eq(dev_net(net_dev), addr->dev_addr.net) &&
1601             (!!addr->dev_addr.bound_dev_if ==
1602              (addr->dev_addr.bound_dev_if == net_dev->ifindex)))
1603                 return true;
1604         else
1605                 return false;
1606 }
1607
1608 static struct rdma_id_private *cma_find_listener(
1609                 const struct rdma_bind_list *bind_list,
1610                 const struct ib_cm_id *cm_id,
1611                 const struct ib_cm_event *ib_event,
1612                 const struct cma_req_info *req,
1613                 const struct net_device *net_dev)
1614 {
1615         struct rdma_id_private *id_priv, *id_priv_dev;
1616
1617         if (!bind_list)
1618                 return ERR_PTR(-EINVAL);
1619
1620         hlist_for_each_entry(id_priv, &bind_list->owners, node) {
1621                 if (cma_match_private_data(id_priv, ib_event->private_data)) {
1622                         if (id_priv->id.device == cm_id->device &&
1623                             cma_match_net_dev(&id_priv->id, net_dev, req))
1624                                 return id_priv;
1625                         list_for_each_entry(id_priv_dev,
1626                                             &id_priv->listen_list,
1627                                             listen_list) {
1628                                 if (id_priv_dev->id.device == cm_id->device &&
1629                                     cma_match_net_dev(&id_priv_dev->id,
1630                                                       net_dev, req))
1631                                         return id_priv_dev;
1632                         }
1633                 }
1634         }
1635
1636         return ERR_PTR(-EINVAL);
1637 }
1638
1639 static struct rdma_id_private *
1640 cma_ib_id_from_event(struct ib_cm_id *cm_id,
1641                      const struct ib_cm_event *ib_event,
1642                      struct cma_req_info *req,
1643                      struct net_device **net_dev)
1644 {
1645         struct rdma_bind_list *bind_list;
1646         struct rdma_id_private *id_priv;
1647         int err;
1648
1649         err = cma_save_req_info(ib_event, req);
1650         if (err)
1651                 return ERR_PTR(err);
1652
1653         *net_dev = cma_get_net_dev(ib_event, req);
1654         if (IS_ERR(*net_dev)) {
1655                 if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) {
1656                         /* Assuming the protocol is AF_IB */
1657                         *net_dev = NULL;
1658                 } else {
1659                         return ERR_CAST(*net_dev);
1660                 }
1661         }
1662
1663         /*
1664          * Net namespace might be getting deleted while route lookup,
1665          * cm_id lookup is in progress. Therefore, perform netdevice
1666          * validation, cm_id lookup under rcu lock.
1667          * RCU lock along with netdevice state check, synchronizes with
1668          * netdevice migrating to different net namespace and also avoids
1669          * case where net namespace doesn't get deleted while lookup is in
1670          * progress.
1671          * If the device state is not IFF_UP, its properties such as ifindex
1672          * and nd_net cannot be trusted to remain valid without rcu lock.
1673          * net/core/dev.c change_net_namespace() ensures to synchronize with
1674          * ongoing operations on net device after device is closed using
1675          * synchronize_net().
1676          */
1677         rcu_read_lock();
1678         if (*net_dev) {
1679                 /*
1680                  * If netdevice is down, it is likely that it is administratively
1681                  * down or it might be migrating to different namespace.
1682                  * In that case avoid further processing, as the net namespace
1683                  * or ifindex may change.
1684                  */
1685                 if (((*net_dev)->flags & IFF_UP) == 0) {
1686                         id_priv = ERR_PTR(-EHOSTUNREACH);
1687                         goto err;
1688                 }
1689
1690                 if (!validate_net_dev(*net_dev,
1691                                  (struct sockaddr *)&req->listen_addr_storage,
1692                                  (struct sockaddr *)&req->src_addr_storage)) {
1693                         id_priv = ERR_PTR(-EHOSTUNREACH);
1694                         goto err;
1695                 }
1696         }
1697
1698         bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net,
1699                                 rdma_ps_from_service_id(req->service_id),
1700                                 cma_port_from_service_id(req->service_id));
1701         id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev);
1702 err:
1703         rcu_read_unlock();
1704         if (IS_ERR(id_priv) && *net_dev) {
1705                 dev_put(*net_dev);
1706                 *net_dev = NULL;
1707         }
1708         return id_priv;
1709 }
1710
1711 static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv)
1712 {
1713         return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr);
1714 }
1715
1716 static void cma_cancel_route(struct rdma_id_private *id_priv)
1717 {
1718         if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) {
1719                 if (id_priv->query)
1720                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
1721         }
1722 }
1723
1724 static void cma_cancel_listens(struct rdma_id_private *id_priv)
1725 {
1726         struct rdma_id_private *dev_id_priv;
1727
1728         /*
1729          * Remove from listen_any_list to prevent added devices from spawning
1730          * additional listen requests.
1731          */
1732         mutex_lock(&lock);
1733         list_del(&id_priv->list);
1734
1735         while (!list_empty(&id_priv->listen_list)) {
1736                 dev_id_priv = list_entry(id_priv->listen_list.next,
1737                                          struct rdma_id_private, listen_list);
1738                 /* sync with device removal to avoid duplicate destruction */
1739                 list_del_init(&dev_id_priv->list);
1740                 list_del(&dev_id_priv->listen_list);
1741                 mutex_unlock(&lock);
1742
1743                 rdma_destroy_id(&dev_id_priv->id);
1744                 mutex_lock(&lock);
1745         }
1746         mutex_unlock(&lock);
1747 }
1748
1749 static void cma_cancel_operation(struct rdma_id_private *id_priv,
1750                                  enum rdma_cm_state state)
1751 {
1752         switch (state) {
1753         case RDMA_CM_ADDR_QUERY:
1754                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
1755                 break;
1756         case RDMA_CM_ROUTE_QUERY:
1757                 cma_cancel_route(id_priv);
1758                 break;
1759         case RDMA_CM_LISTEN:
1760                 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
1761                         cma_cancel_listens(id_priv);
1762                 break;
1763         default:
1764                 break;
1765         }
1766 }
1767
1768 static void cma_release_port(struct rdma_id_private *id_priv)
1769 {
1770         struct rdma_bind_list *bind_list = id_priv->bind_list;
1771         struct net *net = id_priv->id.route.addr.dev_addr.net;
1772
1773         if (!bind_list)
1774                 return;
1775
1776         mutex_lock(&lock);
1777         hlist_del(&id_priv->node);
1778         if (hlist_empty(&bind_list->owners)) {
1779                 cma_ps_remove(net, bind_list->ps, bind_list->port);
1780                 kfree(bind_list);
1781         }
1782         mutex_unlock(&lock);
1783 }
1784
1785 static void cma_leave_roce_mc_group(struct rdma_id_private *id_priv,
1786                                     struct cma_multicast *mc)
1787 {
1788         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1789         struct net_device *ndev = NULL;
1790
1791         if (dev_addr->bound_dev_if)
1792                 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
1793         if (ndev) {
1794                 cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid, false);
1795                 dev_put(ndev);
1796         }
1797         kref_put(&mc->mcref, release_mc);
1798 }
1799
1800 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
1801 {
1802         struct cma_multicast *mc;
1803
1804         while (!list_empty(&id_priv->mc_list)) {
1805                 mc = container_of(id_priv->mc_list.next,
1806                                   struct cma_multicast, list);
1807                 list_del(&mc->list);
1808                 if (rdma_cap_ib_mcast(id_priv->cma_dev->device,
1809                                       id_priv->id.port_num)) {
1810                         ib_sa_free_multicast(mc->multicast.ib);
1811                         kfree(mc);
1812                 } else {
1813                         cma_leave_roce_mc_group(id_priv, mc);
1814                 }
1815         }
1816 }
1817
1818 void rdma_destroy_id(struct rdma_cm_id *id)
1819 {
1820         struct rdma_id_private *id_priv;
1821         enum rdma_cm_state state;
1822
1823         id_priv = container_of(id, struct rdma_id_private, id);
1824         trace_cm_id_destroy(id_priv);
1825         state = cma_exch(id_priv, RDMA_CM_DESTROYING);
1826         cma_cancel_operation(id_priv, state);
1827
1828         /*
1829          * Wait for any active callback to finish.  New callbacks will find
1830          * the id_priv state set to destroying and abort.
1831          */
1832         mutex_lock(&id_priv->handler_mutex);
1833         mutex_unlock(&id_priv->handler_mutex);
1834
1835         rdma_restrack_del(&id_priv->res);
1836         if (id_priv->cma_dev) {
1837                 if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
1838                         if (id_priv->cm_id.ib)
1839                                 ib_destroy_cm_id(id_priv->cm_id.ib);
1840                 } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) {
1841                         if (id_priv->cm_id.iw)
1842                                 iw_destroy_cm_id(id_priv->cm_id.iw);
1843                 }
1844                 cma_leave_mc_groups(id_priv);
1845                 cma_release_dev(id_priv);
1846         }
1847
1848         cma_release_port(id_priv);
1849         cma_deref_id(id_priv);
1850         wait_for_completion(&id_priv->comp);
1851
1852         if (id_priv->internal_id)
1853                 cma_deref_id(id_priv->id.context);
1854
1855         kfree(id_priv->id.route.path_rec);
1856
1857         if (id_priv->id.route.addr.dev_addr.sgid_attr)
1858                 rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
1859
1860         put_net(id_priv->id.route.addr.dev_addr.net);
1861         kfree(id_priv);
1862 }
1863 EXPORT_SYMBOL(rdma_destroy_id);
1864
1865 static int cma_rep_recv(struct rdma_id_private *id_priv)
1866 {
1867         int ret;
1868
1869         ret = cma_modify_qp_rtr(id_priv, NULL);
1870         if (ret)
1871                 goto reject;
1872
1873         ret = cma_modify_qp_rts(id_priv, NULL);
1874         if (ret)
1875                 goto reject;
1876
1877         trace_cm_send_rtu(id_priv);
1878         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1879         if (ret)
1880                 goto reject;
1881
1882         return 0;
1883 reject:
1884         pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret);
1885         cma_modify_qp_err(id_priv);
1886         trace_cm_send_rej(id_priv);
1887         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1888                        NULL, 0, NULL, 0);
1889         return ret;
1890 }
1891
1892 static void cma_set_rep_event_data(struct rdma_cm_event *event,
1893                                    const struct ib_cm_rep_event_param *rep_data,
1894                                    void *private_data)
1895 {
1896         event->param.conn.private_data = private_data;
1897         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1898         event->param.conn.responder_resources = rep_data->responder_resources;
1899         event->param.conn.initiator_depth = rep_data->initiator_depth;
1900         event->param.conn.flow_control = rep_data->flow_control;
1901         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1902         event->param.conn.srq = rep_data->srq;
1903         event->param.conn.qp_num = rep_data->remote_qpn;
1904 }
1905
1906 static int cma_cm_event_handler(struct rdma_id_private *id_priv,
1907                                 struct rdma_cm_event *event)
1908 {
1909         int ret;
1910
1911         trace_cm_event_handler(id_priv, event);
1912         ret = id_priv->id.event_handler(&id_priv->id, event);
1913         trace_cm_event_done(id_priv, event, ret);
1914         return ret;
1915 }
1916
1917 static int cma_ib_handler(struct ib_cm_id *cm_id,
1918                           const struct ib_cm_event *ib_event)
1919 {
1920         struct rdma_id_private *id_priv = cm_id->context;
1921         struct rdma_cm_event event = {};
1922         int ret = 0;
1923
1924         mutex_lock(&id_priv->handler_mutex);
1925         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
1926              id_priv->state != RDMA_CM_CONNECT) ||
1927             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
1928              id_priv->state != RDMA_CM_DISCONNECT))
1929                 goto out;
1930
1931         switch (ib_event->event) {
1932         case IB_CM_REQ_ERROR:
1933         case IB_CM_REP_ERROR:
1934                 event.event = RDMA_CM_EVENT_UNREACHABLE;
1935                 event.status = -ETIMEDOUT;
1936                 break;
1937         case IB_CM_REP_RECEIVED:
1938                 if (cma_comp(id_priv, RDMA_CM_CONNECT) &&
1939                     (id_priv->id.qp_type != IB_QPT_UD)) {
1940                         trace_cm_send_mra(id_priv);
1941                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1942                 }
1943                 if (id_priv->id.qp) {
1944                         event.status = cma_rep_recv(id_priv);
1945                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1946                                                      RDMA_CM_EVENT_ESTABLISHED;
1947                 } else {
1948                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1949                 }
1950                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1951                                        ib_event->private_data);
1952                 break;
1953         case IB_CM_RTU_RECEIVED:
1954         case IB_CM_USER_ESTABLISHED:
1955                 event.event = RDMA_CM_EVENT_ESTABLISHED;
1956                 break;
1957         case IB_CM_DREQ_ERROR:
1958                 event.status = -ETIMEDOUT; /* fall through */
1959         case IB_CM_DREQ_RECEIVED:
1960         case IB_CM_DREP_RECEIVED:
1961                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1962                                    RDMA_CM_DISCONNECT))
1963                         goto out;
1964                 event.event = RDMA_CM_EVENT_DISCONNECTED;
1965                 break;
1966         case IB_CM_TIMEWAIT_EXIT:
1967                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1968                 break;
1969         case IB_CM_MRA_RECEIVED:
1970                 /* ignore event */
1971                 goto out;
1972         case IB_CM_REJ_RECEIVED:
1973                 pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id,
1974                                                                                 ib_event->param.rej_rcvd.reason));
1975                 cma_modify_qp_err(id_priv);
1976                 event.status = ib_event->param.rej_rcvd.reason;
1977                 event.event = RDMA_CM_EVENT_REJECTED;
1978                 event.param.conn.private_data = ib_event->private_data;
1979                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
1980                 break;
1981         default:
1982                 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
1983                        ib_event->event);
1984                 goto out;
1985         }
1986
1987         ret = cma_cm_event_handler(id_priv, &event);
1988         if (ret) {
1989                 /* Destroy the CM ID by returning a non-zero value. */
1990                 id_priv->cm_id.ib = NULL;
1991                 cma_exch(id_priv, RDMA_CM_DESTROYING);
1992                 mutex_unlock(&id_priv->handler_mutex);
1993                 rdma_destroy_id(&id_priv->id);
1994                 return ret;
1995         }
1996 out:
1997         mutex_unlock(&id_priv->handler_mutex);
1998         return ret;
1999 }
2000
2001 static struct rdma_id_private *
2002 cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,
2003                    const struct ib_cm_event *ib_event,
2004                    struct net_device *net_dev)
2005 {
2006         struct rdma_id_private *listen_id_priv;
2007         struct rdma_id_private *id_priv;
2008         struct rdma_cm_id *id;
2009         struct rdma_route *rt;
2010         const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2011         struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path;
2012         const __be64 service_id =
2013                 ib_event->param.req_rcvd.primary_path->service_id;
2014         int ret;
2015
2016         listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2017         id = __rdma_create_id(listen_id->route.addr.dev_addr.net,
2018                             listen_id->event_handler, listen_id->context,
2019                             listen_id->ps, ib_event->param.req_rcvd.qp_type,
2020                             listen_id_priv->res.kern_name);
2021         if (IS_ERR(id))
2022                 return NULL;
2023
2024         id_priv = container_of(id, struct rdma_id_private, id);
2025         if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2026                               (struct sockaddr *)&id->route.addr.dst_addr,
2027                               listen_id, ib_event, ss_family, service_id))
2028                 goto err;
2029
2030         rt = &id->route;
2031         rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
2032         rt->path_rec = kmalloc_array(rt->num_paths, sizeof(*rt->path_rec),
2033                                      GFP_KERNEL);
2034         if (!rt->path_rec)
2035                 goto err;
2036
2037         rt->path_rec[0] = *path;
2038         if (rt->num_paths == 2)
2039                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
2040
2041         if (net_dev) {
2042                 rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev);
2043         } else {
2044                 if (!cma_protocol_roce(listen_id) &&
2045                     cma_any_addr(cma_src_addr(id_priv))) {
2046                         rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
2047                         rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
2048                         ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
2049                 } else if (!cma_any_addr(cma_src_addr(id_priv))) {
2050                         ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
2051                         if (ret)
2052                                 goto err;
2053                 }
2054         }
2055         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
2056
2057         id_priv->state = RDMA_CM_CONNECT;
2058         return id_priv;
2059
2060 err:
2061         rdma_destroy_id(id);
2062         return NULL;
2063 }
2064
2065 static struct rdma_id_private *
2066 cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,
2067                   const struct ib_cm_event *ib_event,
2068                   struct net_device *net_dev)
2069 {
2070         const struct rdma_id_private *listen_id_priv;
2071         struct rdma_id_private *id_priv;
2072         struct rdma_cm_id *id;
2073         const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2074         struct net *net = listen_id->route.addr.dev_addr.net;
2075         int ret;
2076
2077         listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2078         id = __rdma_create_id(net, listen_id->event_handler, listen_id->context,
2079                               listen_id->ps, IB_QPT_UD,
2080                               listen_id_priv->res.kern_name);
2081         if (IS_ERR(id))
2082                 return NULL;
2083
2084         id_priv = container_of(id, struct rdma_id_private, id);
2085         if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2086                               (struct sockaddr *)&id->route.addr.dst_addr,
2087                               listen_id, ib_event, ss_family,
2088                               ib_event->param.sidr_req_rcvd.service_id))
2089                 goto err;
2090
2091         if (net_dev) {
2092                 rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev);
2093         } else {
2094                 if (!cma_any_addr(cma_src_addr(id_priv))) {
2095                         ret = cma_translate_addr(cma_src_addr(id_priv),
2096                                                  &id->route.addr.dev_addr);
2097                         if (ret)
2098                                 goto err;
2099                 }
2100         }
2101
2102         id_priv->state = RDMA_CM_CONNECT;
2103         return id_priv;
2104 err:
2105         rdma_destroy_id(id);
2106         return NULL;
2107 }
2108
2109 static void cma_set_req_event_data(struct rdma_cm_event *event,
2110                                    const struct ib_cm_req_event_param *req_data,
2111                                    void *private_data, int offset)
2112 {
2113         event->param.conn.private_data = private_data + offset;
2114         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
2115         event->param.conn.responder_resources = req_data->responder_resources;
2116         event->param.conn.initiator_depth = req_data->initiator_depth;
2117         event->param.conn.flow_control = req_data->flow_control;
2118         event->param.conn.retry_count = req_data->retry_count;
2119         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
2120         event->param.conn.srq = req_data->srq;
2121         event->param.conn.qp_num = req_data->remote_qpn;
2122 }
2123
2124 static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,
2125                                     const struct ib_cm_event *ib_event)
2126 {
2127         return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
2128                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
2129                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
2130                  (id->qp_type == IB_QPT_UD)) ||
2131                 (!id->qp_type));
2132 }
2133
2134 static int cma_ib_req_handler(struct ib_cm_id *cm_id,
2135                               const struct ib_cm_event *ib_event)
2136 {
2137         struct rdma_id_private *listen_id, *conn_id = NULL;
2138         struct rdma_cm_event event = {};
2139         struct cma_req_info req = {};
2140         struct net_device *net_dev;
2141         u8 offset;
2142         int ret;
2143
2144         listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev);
2145         if (IS_ERR(listen_id))
2146                 return PTR_ERR(listen_id);
2147
2148         trace_cm_req_handler(listen_id, ib_event->event);
2149         if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) {
2150                 ret = -EINVAL;
2151                 goto net_dev_put;
2152         }
2153
2154         mutex_lock(&listen_id->handler_mutex);
2155         if (listen_id->state != RDMA_CM_LISTEN) {
2156                 ret = -ECONNABORTED;
2157                 goto err1;
2158         }
2159
2160         offset = cma_user_data_offset(listen_id);
2161         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2162         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
2163                 conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev);
2164                 event.param.ud.private_data = ib_event->private_data + offset;
2165                 event.param.ud.private_data_len =
2166                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
2167         } else {
2168                 conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev);
2169                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
2170                                        ib_event->private_data, offset);
2171         }
2172         if (!conn_id) {
2173                 ret = -ENOMEM;
2174                 goto err1;
2175         }
2176
2177         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2178         ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
2179         if (ret)
2180                 goto err2;
2181
2182         conn_id->cm_id.ib = cm_id;
2183         cm_id->context = conn_id;
2184         cm_id->cm_handler = cma_ib_handler;
2185
2186         /*
2187          * Protect against the user destroying conn_id from another thread
2188          * until we're done accessing it.
2189          */
2190         atomic_inc(&conn_id->refcount);
2191         ret = cma_cm_event_handler(conn_id, &event);
2192         if (ret)
2193                 goto err3;
2194         /*
2195          * Acquire mutex to prevent user executing rdma_destroy_id()
2196          * while we're accessing the cm_id.
2197          */
2198         mutex_lock(&lock);
2199         if (cma_comp(conn_id, RDMA_CM_CONNECT) &&
2200             (conn_id->id.qp_type != IB_QPT_UD)) {
2201                 trace_cm_send_mra(cm_id->context);
2202                 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2203         }
2204         mutex_unlock(&lock);
2205         mutex_unlock(&conn_id->handler_mutex);
2206         mutex_unlock(&listen_id->handler_mutex);
2207         cma_deref_id(conn_id);
2208         if (net_dev)
2209                 dev_put(net_dev);
2210         return 0;
2211
2212 err3:
2213         cma_deref_id(conn_id);
2214         /* Destroy the CM ID by returning a non-zero value. */
2215         conn_id->cm_id.ib = NULL;
2216 err2:
2217         cma_exch(conn_id, RDMA_CM_DESTROYING);
2218         mutex_unlock(&conn_id->handler_mutex);
2219 err1:
2220         mutex_unlock(&listen_id->handler_mutex);
2221         if (conn_id)
2222                 rdma_destroy_id(&conn_id->id);
2223
2224 net_dev_put:
2225         if (net_dev)
2226                 dev_put(net_dev);
2227
2228         return ret;
2229 }
2230
2231 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr)
2232 {
2233         if (addr->sa_family == AF_IB)
2234                 return ((struct sockaddr_ib *) addr)->sib_sid;
2235
2236         return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr)));
2237 }
2238 EXPORT_SYMBOL(rdma_get_service_id);
2239
2240 void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid,
2241                     union ib_gid *dgid)
2242 {
2243         struct rdma_addr *addr = &cm_id->route.addr;
2244
2245         if (!cm_id->device) {
2246                 if (sgid)
2247                         memset(sgid, 0, sizeof(*sgid));
2248                 if (dgid)
2249                         memset(dgid, 0, sizeof(*dgid));
2250                 return;
2251         }
2252
2253         if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) {
2254                 if (sgid)
2255                         rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid);
2256                 if (dgid)
2257                         rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid);
2258         } else {
2259                 if (sgid)
2260                         rdma_addr_get_sgid(&addr->dev_addr, sgid);
2261                 if (dgid)
2262                         rdma_addr_get_dgid(&addr->dev_addr, dgid);
2263         }
2264 }
2265 EXPORT_SYMBOL(rdma_read_gids);
2266
2267 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
2268 {
2269         struct rdma_id_private *id_priv = iw_id->context;
2270         struct rdma_cm_event event = {};
2271         int ret = 0;
2272         struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2273         struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2274
2275         mutex_lock(&id_priv->handler_mutex);
2276         if (id_priv->state != RDMA_CM_CONNECT)
2277                 goto out;
2278
2279         switch (iw_event->event) {
2280         case IW_CM_EVENT_CLOSE:
2281                 event.event = RDMA_CM_EVENT_DISCONNECTED;
2282                 break;
2283         case IW_CM_EVENT_CONNECT_REPLY:
2284                 memcpy(cma_src_addr(id_priv), laddr,
2285                        rdma_addr_size(laddr));
2286                 memcpy(cma_dst_addr(id_priv), raddr,
2287                        rdma_addr_size(raddr));
2288                 switch (iw_event->status) {
2289                 case 0:
2290                         event.event = RDMA_CM_EVENT_ESTABLISHED;
2291                         event.param.conn.initiator_depth = iw_event->ird;
2292                         event.param.conn.responder_resources = iw_event->ord;
2293                         break;
2294                 case -ECONNRESET:
2295                 case -ECONNREFUSED:
2296                         event.event = RDMA_CM_EVENT_REJECTED;
2297                         break;
2298                 case -ETIMEDOUT:
2299                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2300                         break;
2301                 default:
2302                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
2303                         break;
2304                 }
2305                 break;
2306         case IW_CM_EVENT_ESTABLISHED:
2307                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2308                 event.param.conn.initiator_depth = iw_event->ird;
2309                 event.param.conn.responder_resources = iw_event->ord;
2310                 break;
2311         default:
2312                 goto out;
2313         }
2314
2315         event.status = iw_event->status;
2316         event.param.conn.private_data = iw_event->private_data;
2317         event.param.conn.private_data_len = iw_event->private_data_len;
2318         ret = cma_cm_event_handler(id_priv, &event);
2319         if (ret) {
2320                 /* Destroy the CM ID by returning a non-zero value. */
2321                 id_priv->cm_id.iw = NULL;
2322                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2323                 mutex_unlock(&id_priv->handler_mutex);
2324                 rdma_destroy_id(&id_priv->id);
2325                 return ret;
2326         }
2327
2328 out:
2329         mutex_unlock(&id_priv->handler_mutex);
2330         return ret;
2331 }
2332
2333 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
2334                                struct iw_cm_event *iw_event)
2335 {
2336         struct rdma_cm_id *new_cm_id;
2337         struct rdma_id_private *listen_id, *conn_id;
2338         struct rdma_cm_event event = {};
2339         int ret = -ECONNABORTED;
2340         struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2341         struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2342
2343         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2344         event.param.conn.private_data = iw_event->private_data;
2345         event.param.conn.private_data_len = iw_event->private_data_len;
2346         event.param.conn.initiator_depth = iw_event->ird;
2347         event.param.conn.responder_resources = iw_event->ord;
2348
2349         listen_id = cm_id->context;
2350
2351         mutex_lock(&listen_id->handler_mutex);
2352         if (listen_id->state != RDMA_CM_LISTEN)
2353                 goto out;
2354
2355         /* Create a new RDMA id for the new IW CM ID */
2356         new_cm_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
2357                                      listen_id->id.event_handler,
2358                                      listen_id->id.context,
2359                                      RDMA_PS_TCP, IB_QPT_RC,
2360                                      listen_id->res.kern_name);
2361         if (IS_ERR(new_cm_id)) {
2362                 ret = -ENOMEM;
2363                 goto out;
2364         }
2365         conn_id = container_of(new_cm_id, struct rdma_id_private, id);
2366         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2367         conn_id->state = RDMA_CM_CONNECT;
2368
2369         ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
2370         if (ret) {
2371                 mutex_unlock(&conn_id->handler_mutex);
2372                 rdma_destroy_id(new_cm_id);
2373                 goto out;
2374         }
2375
2376         ret = cma_iw_acquire_dev(conn_id, listen_id);
2377         if (ret) {
2378                 mutex_unlock(&conn_id->handler_mutex);
2379                 rdma_destroy_id(new_cm_id);
2380                 goto out;
2381         }
2382
2383         conn_id->cm_id.iw = cm_id;
2384         cm_id->context = conn_id;
2385         cm_id->cm_handler = cma_iw_handler;
2386
2387         memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
2388         memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
2389
2390         /*
2391          * Protect against the user destroying conn_id from another thread
2392          * until we're done accessing it.
2393          */
2394         atomic_inc(&conn_id->refcount);
2395         ret = cma_cm_event_handler(conn_id, &event);
2396         if (ret) {
2397                 /* User wants to destroy the CM ID */
2398                 conn_id->cm_id.iw = NULL;
2399                 cma_exch(conn_id, RDMA_CM_DESTROYING);
2400                 mutex_unlock(&conn_id->handler_mutex);
2401                 mutex_unlock(&listen_id->handler_mutex);
2402                 cma_deref_id(conn_id);
2403                 rdma_destroy_id(&conn_id->id);
2404                 return ret;
2405         }
2406
2407         mutex_unlock(&conn_id->handler_mutex);
2408         cma_deref_id(conn_id);
2409
2410 out:
2411         mutex_unlock(&listen_id->handler_mutex);
2412         return ret;
2413 }
2414
2415 static int cma_ib_listen(struct rdma_id_private *id_priv)
2416 {
2417         struct sockaddr *addr;
2418         struct ib_cm_id *id;
2419         __be64 svc_id;
2420
2421         addr = cma_src_addr(id_priv);
2422         svc_id = rdma_get_service_id(&id_priv->id, addr);
2423         id = ib_cm_insert_listen(id_priv->id.device,
2424                                  cma_ib_req_handler, svc_id);
2425         if (IS_ERR(id))
2426                 return PTR_ERR(id);
2427         id_priv->cm_id.ib = id;
2428
2429         return 0;
2430 }
2431
2432 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
2433 {
2434         int ret;
2435         struct iw_cm_id *id;
2436
2437         id = iw_create_cm_id(id_priv->id.device,
2438                              iw_conn_req_handler,
2439                              id_priv);
2440         if (IS_ERR(id))
2441                 return PTR_ERR(id);
2442
2443         id->tos = id_priv->tos;
2444         id->tos_set = id_priv->tos_set;
2445         id_priv->cm_id.iw = id;
2446
2447         memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv),
2448                rdma_addr_size(cma_src_addr(id_priv)));
2449
2450         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
2451
2452         if (ret) {
2453                 iw_destroy_cm_id(id_priv->cm_id.iw);
2454                 id_priv->cm_id.iw = NULL;
2455         }
2456
2457         return ret;
2458 }
2459
2460 static int cma_listen_handler(struct rdma_cm_id *id,
2461                               struct rdma_cm_event *event)
2462 {
2463         struct rdma_id_private *id_priv = id->context;
2464
2465         id->context = id_priv->id.context;
2466         id->event_handler = id_priv->id.event_handler;
2467         trace_cm_event_handler(id_priv, event);
2468         return id_priv->id.event_handler(id, event);
2469 }
2470
2471 static void cma_listen_on_dev(struct rdma_id_private *id_priv,
2472                               struct cma_device *cma_dev)
2473 {
2474         struct rdma_id_private *dev_id_priv;
2475         struct rdma_cm_id *id;
2476         struct net *net = id_priv->id.route.addr.dev_addr.net;
2477         int ret;
2478
2479         if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
2480                 return;
2481
2482         id = __rdma_create_id(net, cma_listen_handler, id_priv, id_priv->id.ps,
2483                               id_priv->id.qp_type, id_priv->res.kern_name);
2484         if (IS_ERR(id))
2485                 return;
2486
2487         dev_id_priv = container_of(id, struct rdma_id_private, id);
2488
2489         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
2490         memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
2491                rdma_addr_size(cma_src_addr(id_priv)));
2492
2493         _cma_attach_to_dev(dev_id_priv, cma_dev);
2494         list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
2495         atomic_inc(&id_priv->refcount);
2496         dev_id_priv->internal_id = 1;
2497         dev_id_priv->afonly = id_priv->afonly;
2498         dev_id_priv->tos_set = id_priv->tos_set;
2499         dev_id_priv->tos = id_priv->tos;
2500
2501         ret = rdma_listen(id, id_priv->backlog);
2502         if (ret)
2503                 dev_warn(&cma_dev->device->dev,
2504                          "RDMA CMA: cma_listen_on_dev, error %d\n", ret);
2505 }
2506
2507 static void cma_listen_on_all(struct rdma_id_private *id_priv)
2508 {
2509         struct cma_device *cma_dev;
2510
2511         mutex_lock(&lock);
2512         list_add_tail(&id_priv->list, &listen_any_list);
2513         list_for_each_entry(cma_dev, &dev_list, list)
2514                 cma_listen_on_dev(id_priv, cma_dev);
2515         mutex_unlock(&lock);
2516 }
2517
2518 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
2519 {
2520         struct rdma_id_private *id_priv;
2521
2522         id_priv = container_of(id, struct rdma_id_private, id);
2523         id_priv->tos = (u8) tos;
2524         id_priv->tos_set = true;
2525 }
2526 EXPORT_SYMBOL(rdma_set_service_type);
2527
2528 /**
2529  * rdma_set_ack_timeout() - Set the ack timeout of QP associated
2530  *                          with a connection identifier.
2531  * @id: Communication identifier to associated with service type.
2532  * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec.
2533  *
2534  * This function should be called before rdma_connect() on active side,
2535  * and on passive side before rdma_accept(). It is applicable to primary
2536  * path only. The timeout will affect the local side of the QP, it is not
2537  * negotiated with remote side and zero disables the timer. In case it is
2538  * set before rdma_resolve_route, the value will also be used to determine
2539  * PacketLifeTime for RoCE.
2540  *
2541  * Return: 0 for success
2542  */
2543 int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
2544 {
2545         struct rdma_id_private *id_priv;
2546
2547         if (id->qp_type != IB_QPT_RC)
2548                 return -EINVAL;
2549
2550         id_priv = container_of(id, struct rdma_id_private, id);
2551         id_priv->timeout = timeout;
2552         id_priv->timeout_set = true;
2553
2554         return 0;
2555 }
2556 EXPORT_SYMBOL(rdma_set_ack_timeout);
2557
2558 static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2559                               void *context)
2560 {
2561         struct cma_work *work = context;
2562         struct rdma_route *route;
2563
2564         route = &work->id->id.route;
2565
2566         if (!status) {
2567                 route->num_paths = 1;
2568                 *route->path_rec = *path_rec;
2569         } else {
2570                 work->old_state = RDMA_CM_ROUTE_QUERY;
2571                 work->new_state = RDMA_CM_ADDR_RESOLVED;
2572                 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
2573                 work->event.status = status;
2574                 pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",
2575                                      status);
2576         }
2577
2578         queue_work(cma_wq, &work->work);
2579 }
2580
2581 static int cma_query_ib_route(struct rdma_id_private *id_priv,
2582                               unsigned long timeout_ms, struct cma_work *work)
2583 {
2584         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2585         struct sa_path_rec path_rec;
2586         ib_sa_comp_mask comp_mask;
2587         struct sockaddr_in6 *sin6;
2588         struct sockaddr_ib *sib;
2589
2590         memset(&path_rec, 0, sizeof path_rec);
2591
2592         if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num))
2593                 path_rec.rec_type = SA_PATH_REC_TYPE_OPA;
2594         else
2595                 path_rec.rec_type = SA_PATH_REC_TYPE_IB;
2596         rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
2597         rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
2598         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2599         path_rec.numb_path = 1;
2600         path_rec.reversible = 1;
2601         path_rec.service_id = rdma_get_service_id(&id_priv->id,
2602                                                   cma_dst_addr(id_priv));
2603
2604         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
2605                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
2606                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
2607
2608         switch (cma_family(id_priv)) {
2609         case AF_INET:
2610                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
2611                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
2612                 break;
2613         case AF_INET6:
2614                 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2615                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
2616                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2617                 break;
2618         case AF_IB:
2619                 sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2620                 path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20);
2621                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2622                 break;
2623         }
2624
2625         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
2626                                                id_priv->id.port_num, &path_rec,
2627                                                comp_mask, timeout_ms,
2628                                                GFP_KERNEL, cma_query_handler,
2629                                                work, &id_priv->query);
2630
2631         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
2632 }
2633
2634 static void cma_work_handler(struct work_struct *_work)
2635 {
2636         struct cma_work *work = container_of(_work, struct cma_work, work);
2637         struct rdma_id_private *id_priv = work->id;
2638         int destroy = 0;
2639
2640         mutex_lock(&id_priv->handler_mutex);
2641         if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
2642                 goto out;
2643
2644         if (cma_cm_event_handler(id_priv, &work->event)) {
2645                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2646                 destroy = 1;
2647         }
2648 out:
2649         mutex_unlock(&id_priv->handler_mutex);
2650         cma_deref_id(id_priv);
2651         if (destroy)
2652                 rdma_destroy_id(&id_priv->id);
2653         kfree(work);
2654 }
2655
2656 static void cma_ndev_work_handler(struct work_struct *_work)
2657 {
2658         struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
2659         struct rdma_id_private *id_priv = work->id;
2660         int destroy = 0;
2661
2662         mutex_lock(&id_priv->handler_mutex);
2663         if (id_priv->state == RDMA_CM_DESTROYING ||
2664             id_priv->state == RDMA_CM_DEVICE_REMOVAL)
2665                 goto out;
2666
2667         if (cma_cm_event_handler(id_priv, &work->event)) {
2668                 cma_exch(id_priv, RDMA_CM_DESTROYING);
2669                 destroy = 1;
2670         }
2671
2672 out:
2673         mutex_unlock(&id_priv->handler_mutex);
2674         cma_deref_id(id_priv);
2675         if (destroy)
2676                 rdma_destroy_id(&id_priv->id);
2677         kfree(work);
2678 }
2679
2680 static void cma_init_resolve_route_work(struct cma_work *work,
2681                                         struct rdma_id_private *id_priv)
2682 {
2683         work->id = id_priv;
2684         INIT_WORK(&work->work, cma_work_handler);
2685         work->old_state = RDMA_CM_ROUTE_QUERY;
2686         work->new_state = RDMA_CM_ROUTE_RESOLVED;
2687         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
2688 }
2689
2690 static void cma_init_resolve_addr_work(struct cma_work *work,
2691                                        struct rdma_id_private *id_priv)
2692 {
2693         work->id = id_priv;
2694         INIT_WORK(&work->work, cma_work_handler);
2695         work->old_state = RDMA_CM_ADDR_QUERY;
2696         work->new_state = RDMA_CM_ADDR_RESOLVED;
2697         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2698 }
2699
2700 static int cma_resolve_ib_route(struct rdma_id_private *id_priv,
2701                                 unsigned long timeout_ms)
2702 {
2703         struct rdma_route *route = &id_priv->id.route;
2704         struct cma_work *work;
2705         int ret;
2706
2707         work = kzalloc(sizeof *work, GFP_KERNEL);
2708         if (!work)
2709                 return -ENOMEM;
2710
2711         cma_init_resolve_route_work(work, id_priv);
2712
2713         route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
2714         if (!route->path_rec) {
2715                 ret = -ENOMEM;
2716                 goto err1;
2717         }
2718
2719         ret = cma_query_ib_route(id_priv, timeout_ms, work);
2720         if (ret)
2721                 goto err2;
2722
2723         return 0;
2724 err2:
2725         kfree(route->path_rec);
2726         route->path_rec = NULL;
2727 err1:
2728         kfree(work);
2729         return ret;
2730 }
2731
2732 static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
2733                                            unsigned long supported_gids,
2734                                            enum ib_gid_type default_gid)
2735 {
2736         if ((network_type == RDMA_NETWORK_IPV4 ||
2737              network_type == RDMA_NETWORK_IPV6) &&
2738             test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
2739                 return IB_GID_TYPE_ROCE_UDP_ENCAP;
2740
2741         return default_gid;
2742 }
2743
2744 /*
2745  * cma_iboe_set_path_rec_l2_fields() is helper function which sets
2746  * path record type based on GID type.
2747  * It also sets up other L2 fields which includes destination mac address
2748  * netdev ifindex, of the path record.
2749  * It returns the netdev of the bound interface for this path record entry.
2750  */
2751 static struct net_device *
2752 cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv)
2753 {
2754         struct rdma_route *route = &id_priv->id.route;
2755         enum ib_gid_type gid_type = IB_GID_TYPE_ROCE;
2756         struct rdma_addr *addr = &route->addr;
2757         unsigned long supported_gids;
2758         struct net_device *ndev;
2759
2760         if (!addr->dev_addr.bound_dev_if)
2761                 return NULL;
2762
2763         ndev = dev_get_by_index(addr->dev_addr.net,
2764                                 addr->dev_addr.bound_dev_if);
2765         if (!ndev)
2766                 return NULL;
2767
2768         supported_gids = roce_gid_type_mask_support(id_priv->id.device,
2769                                                     id_priv->id.port_num);
2770         gid_type = cma_route_gid_type(addr->dev_addr.network,
2771                                       supported_gids,
2772                                       id_priv->gid_type);
2773         /* Use the hint from IP Stack to select GID Type */
2774         if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
2775                 gid_type = ib_network_to_gid_type(addr->dev_addr.network);
2776         route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
2777
2778         route->path_rec->roce.route_resolved = true;
2779         sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
2780         return ndev;
2781 }
2782
2783 int rdma_set_ib_path(struct rdma_cm_id *id,
2784                      struct sa_path_rec *path_rec)
2785 {
2786         struct rdma_id_private *id_priv;
2787         struct net_device *ndev;
2788         int ret;
2789
2790         id_priv = container_of(id, struct rdma_id_private, id);
2791         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2792                            RDMA_CM_ROUTE_RESOLVED))
2793                 return -EINVAL;
2794
2795         id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec),
2796                                      GFP_KERNEL);
2797         if (!id->route.path_rec) {
2798                 ret = -ENOMEM;
2799                 goto err;
2800         }
2801
2802         if (rdma_protocol_roce(id->device, id->port_num)) {
2803                 ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
2804                 if (!ndev) {
2805                         ret = -ENODEV;
2806                         goto err_free;
2807                 }
2808                 dev_put(ndev);
2809         }
2810
2811         id->route.num_paths = 1;
2812         return 0;
2813
2814 err_free:
2815         kfree(id->route.path_rec);
2816         id->route.path_rec = NULL;
2817 err:
2818         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
2819         return ret;
2820 }
2821 EXPORT_SYMBOL(rdma_set_ib_path);
2822
2823 static int cma_resolve_iw_route(struct rdma_id_private *id_priv)
2824 {
2825         struct cma_work *work;
2826
2827         work = kzalloc(sizeof *work, GFP_KERNEL);
2828         if (!work)
2829                 return -ENOMEM;
2830
2831         cma_init_resolve_route_work(work, id_priv);
2832         queue_work(cma_wq, &work->work);
2833         return 0;
2834 }
2835
2836 static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio)
2837 {
2838         struct net_device *dev;
2839
2840         dev = vlan_dev_real_dev(vlan_ndev);
2841         if (dev->num_tc)
2842                 return netdev_get_prio_tc_map(dev, prio);
2843
2844         return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) &
2845                 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
2846 }
2847
2848 struct iboe_prio_tc_map {
2849         int input_prio;
2850         int output_tc;
2851         bool found;
2852 };
2853
2854 static int get_lower_vlan_dev_tc(struct net_device *dev, void *data)
2855 {
2856         struct iboe_prio_tc_map *map = data;
2857
2858         if (is_vlan_dev(dev))
2859                 map->output_tc = get_vlan_ndev_tc(dev, map->input_prio);
2860         else if (dev->num_tc)
2861                 map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio);
2862         else
2863                 map->output_tc = 0;
2864         /* We are interested only in first level VLAN device, so always
2865          * return 1 to stop iterating over next level devices.
2866          */
2867         map->found = true;
2868         return 1;
2869 }
2870
2871 static int iboe_tos_to_sl(struct net_device *ndev, int tos)
2872 {
2873         struct iboe_prio_tc_map prio_tc_map = {};
2874         int prio = rt_tos2priority(tos);
2875
2876         /* If VLAN device, get it directly from the VLAN netdev */
2877         if (is_vlan_dev(ndev))
2878                 return get_vlan_ndev_tc(ndev, prio);
2879
2880         prio_tc_map.input_prio = prio;
2881         rcu_read_lock();
2882         netdev_walk_all_lower_dev_rcu(ndev,
2883                                       get_lower_vlan_dev_tc,
2884                                       &prio_tc_map);
2885         rcu_read_unlock();
2886         /* If map is found from lower device, use it; Otherwise
2887          * continue with the current netdevice to get priority to tc map.
2888          */
2889         if (prio_tc_map.found)
2890                 return prio_tc_map.output_tc;
2891         else if (ndev->num_tc)
2892                 return netdev_get_prio_tc_map(ndev, prio);
2893         else
2894                 return 0;
2895 }
2896
2897 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
2898 {
2899         struct rdma_route *route = &id_priv->id.route;
2900         struct rdma_addr *addr = &route->addr;
2901         struct cma_work *work;
2902         int ret;
2903         struct net_device *ndev;
2904
2905         u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -
2906                                         rdma_start_port(id_priv->cma_dev->device)];
2907         u8 tos = id_priv->tos_set ? id_priv->tos : default_roce_tos;
2908
2909
2910         work = kzalloc(sizeof *work, GFP_KERNEL);
2911         if (!work)
2912                 return -ENOMEM;
2913
2914         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
2915         if (!route->path_rec) {
2916                 ret = -ENOMEM;
2917                 goto err1;
2918         }
2919
2920         route->num_paths = 1;
2921
2922         ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
2923         if (!ndev) {
2924                 ret = -ENODEV;
2925                 goto err2;
2926         }
2927
2928         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
2929                     &route->path_rec->sgid);
2930         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
2931                     &route->path_rec->dgid);
2932
2933         if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
2934                 /* TODO: get the hoplimit from the inet/inet6 device */
2935                 route->path_rec->hop_limit = addr->dev_addr.hoplimit;
2936         else
2937                 route->path_rec->hop_limit = 1;
2938         route->path_rec->reversible = 1;
2939         route->path_rec->pkey = cpu_to_be16(0xffff);
2940         route->path_rec->mtu_selector = IB_SA_EQ;
2941         route->path_rec->sl = iboe_tos_to_sl(ndev, tos);
2942         route->path_rec->traffic_class = tos;
2943         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
2944         route->path_rec->rate_selector = IB_SA_EQ;
2945         route->path_rec->rate = iboe_get_rate(ndev);
2946         dev_put(ndev);
2947         route->path_rec->packet_life_time_selector = IB_SA_EQ;
2948         /* In case ACK timeout is set, use this value to calculate
2949          * PacketLifeTime.  As per IBTA 12.7.34,
2950          * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay).
2951          * Assuming a negligible local ACK delay, we can use
2952          * PacketLifeTime = local ACK timeout/2
2953          * as a reasonable approximation for RoCE networks.
2954          */
2955         route->path_rec->packet_life_time = id_priv->timeout_set ?
2956                 id_priv->timeout - 1 : CMA_IBOE_PACKET_LIFETIME;
2957
2958         if (!route->path_rec->mtu) {
2959                 ret = -EINVAL;
2960                 goto err2;
2961         }
2962
2963         cma_init_resolve_route_work(work, id_priv);
2964         queue_work(cma_wq, &work->work);
2965
2966         return 0;
2967
2968 err2:
2969         kfree(route->path_rec);
2970         route->path_rec = NULL;
2971 err1:
2972         kfree(work);
2973         return ret;
2974 }
2975
2976 int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
2977 {
2978         struct rdma_id_private *id_priv;
2979         int ret;
2980
2981         id_priv = container_of(id, struct rdma_id_private, id);
2982         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
2983                 return -EINVAL;
2984
2985         atomic_inc(&id_priv->refcount);
2986         if (rdma_cap_ib_sa(id->device, id->port_num))
2987                 ret = cma_resolve_ib_route(id_priv, timeout_ms);
2988         else if (rdma_protocol_roce(id->device, id->port_num))
2989                 ret = cma_resolve_iboe_route(id_priv);
2990         else if (rdma_protocol_iwarp(id->device, id->port_num))
2991                 ret = cma_resolve_iw_route(id_priv);
2992         else
2993                 ret = -ENOSYS;
2994
2995         if (ret)
2996                 goto err;
2997
2998         return 0;
2999 err:
3000         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
3001         cma_deref_id(id_priv);
3002         return ret;
3003 }
3004 EXPORT_SYMBOL(rdma_resolve_route);
3005
3006 static void cma_set_loopback(struct sockaddr *addr)
3007 {
3008         switch (addr->sa_family) {
3009         case AF_INET:
3010                 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
3011                 break;
3012         case AF_INET6:
3013                 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
3014                               0, 0, 0, htonl(1));
3015                 break;
3016         default:
3017                 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
3018                             0, 0, 0, htonl(1));
3019                 break;
3020         }
3021 }
3022
3023 static int cma_bind_loopback(struct rdma_id_private *id_priv)
3024 {
3025         struct cma_device *cma_dev, *cur_dev;
3026         union ib_gid gid;
3027         enum ib_port_state port_state;
3028         u16 pkey;
3029         int ret;
3030         u8 p;
3031
3032         cma_dev = NULL;
3033         mutex_lock(&lock);
3034         list_for_each_entry(cur_dev, &dev_list, list) {
3035                 if (cma_family(id_priv) == AF_IB &&
3036                     !rdma_cap_ib_cm(cur_dev->device, 1))
3037                         continue;
3038
3039                 if (!cma_dev)
3040                         cma_dev = cur_dev;
3041
3042                 for (p = 1; p <= cur_dev->device->phys_port_cnt; ++p) {
3043                         if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) &&
3044                             port_state == IB_PORT_ACTIVE) {
3045                                 cma_dev = cur_dev;
3046                                 goto port_found;
3047                         }
3048                 }
3049         }
3050
3051         if (!cma_dev) {
3052                 ret = -ENODEV;
3053                 goto out;
3054         }
3055
3056         p = 1;
3057
3058 port_found:
3059         ret = rdma_query_gid(cma_dev->device, p, 0, &gid);
3060         if (ret)
3061                 goto out;
3062
3063         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
3064         if (ret)
3065                 goto out;
3066
3067         id_priv->id.route.addr.dev_addr.dev_type =
3068                 (rdma_protocol_ib(cma_dev->device, p)) ?
3069                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
3070
3071         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3072         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
3073         id_priv->id.port_num = p;
3074         cma_attach_to_dev(id_priv, cma_dev);
3075         cma_set_loopback(cma_src_addr(id_priv));
3076 out:
3077         mutex_unlock(&lock);
3078         return ret;
3079 }
3080
3081 static void addr_handler(int status, struct sockaddr *src_addr,
3082                          struct rdma_dev_addr *dev_addr, void *context)
3083 {
3084         struct rdma_id_private *id_priv = context;
3085         struct rdma_cm_event event = {};
3086         struct sockaddr *addr;
3087         struct sockaddr_storage old_addr;
3088
3089         mutex_lock(&id_priv->handler_mutex);
3090         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
3091                            RDMA_CM_ADDR_RESOLVED))
3092                 goto out;
3093
3094         /*
3095          * Store the previous src address, so that if we fail to acquire
3096          * matching rdma device, old address can be restored back, which helps
3097          * to cancel the cma listen operation correctly.
3098          */
3099         addr = cma_src_addr(id_priv);
3100         memcpy(&old_addr, addr, rdma_addr_size(addr));
3101         memcpy(addr, src_addr, rdma_addr_size(src_addr));
3102         if (!status && !id_priv->cma_dev) {
3103                 status = cma_acquire_dev_by_src_ip(id_priv);
3104                 if (status)
3105                         pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n",
3106                                              status);
3107         } else if (status) {
3108                 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status);
3109         }
3110
3111         if (status) {
3112                 memcpy(addr, &old_addr,
3113                        rdma_addr_size((struct sockaddr *)&old_addr));
3114                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3115                                    RDMA_CM_ADDR_BOUND))
3116                         goto out;
3117                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
3118                 event.status = status;
3119         } else
3120                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3121
3122         if (cma_cm_event_handler(id_priv, &event)) {
3123                 cma_exch(id_priv, RDMA_CM_DESTROYING);
3124                 mutex_unlock(&id_priv->handler_mutex);
3125                 rdma_destroy_id(&id_priv->id);
3126                 return;
3127         }
3128 out:
3129         mutex_unlock(&id_priv->handler_mutex);
3130 }
3131
3132 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
3133 {
3134         struct cma_work *work;
3135         union ib_gid gid;
3136         int ret;
3137
3138         work = kzalloc(sizeof *work, GFP_KERNEL);
3139         if (!work)
3140                 return -ENOMEM;
3141
3142         if (!id_priv->cma_dev) {
3143                 ret = cma_bind_loopback(id_priv);
3144                 if (ret)
3145                         goto err;
3146         }
3147
3148         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3149         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
3150
3151         atomic_inc(&id_priv->refcount);
3152         cma_init_resolve_addr_work(work, id_priv);
3153         queue_work(cma_wq, &work->work);
3154         return 0;
3155 err:
3156         kfree(work);
3157         return ret;
3158 }
3159
3160 static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
3161 {
3162         struct cma_work *work;
3163         int ret;
3164
3165         work = kzalloc(sizeof *work, GFP_KERNEL);
3166         if (!work)
3167                 return -ENOMEM;
3168
3169         if (!id_priv->cma_dev) {
3170                 ret = cma_resolve_ib_dev(id_priv);
3171                 if (ret)
3172                         goto err;
3173         }
3174
3175         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
3176                 &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
3177
3178         atomic_inc(&id_priv->refcount);
3179         cma_init_resolve_addr_work(work, id_priv);
3180         queue_work(cma_wq, &work->work);
3181         return 0;
3182 err:
3183         kfree(work);
3184         return ret;
3185 }
3186
3187 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3188                          const struct sockaddr *dst_addr)
3189 {
3190         if (!src_addr || !src_addr->sa_family) {
3191                 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
3192                 src_addr->sa_family = dst_addr->sa_family;
3193                 if (IS_ENABLED(CONFIG_IPV6) &&
3194                     dst_addr->sa_family == AF_INET6) {
3195                         struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *) src_addr;
3196                         struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *) dst_addr;
3197                         src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
3198                         if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
3199                                 id->route.addr.dev_addr.bound_dev_if = dst_addr6->sin6_scope_id;
3200                 } else if (dst_addr->sa_family == AF_IB) {
3201                         ((struct sockaddr_ib *) src_addr)->sib_pkey =
3202                                 ((struct sockaddr_ib *) dst_addr)->sib_pkey;
3203                 }
3204         }
3205         return rdma_bind_addr(id, src_addr);
3206 }
3207
3208 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
3209                       const struct sockaddr *dst_addr, unsigned long timeout_ms)
3210 {
3211         struct rdma_id_private *id_priv;
3212         int ret;
3213
3214         id_priv = container_of(id, struct rdma_id_private, id);
3215         memcpy(cma_dst_addr(id_priv), dst_addr, rdma_addr_size(dst_addr));
3216         if (id_priv->state == RDMA_CM_IDLE) {
3217                 ret = cma_bind_addr(id, src_addr, dst_addr);
3218                 if (ret) {
3219                         memset(cma_dst_addr(id_priv), 0,
3220                                rdma_addr_size(dst_addr));
3221                         return ret;
3222                 }
3223         }
3224
3225         if (cma_family(id_priv) != dst_addr->sa_family) {
3226                 memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr));
3227                 return -EINVAL;
3228         }
3229
3230         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
3231                 memset(cma_dst_addr(id_priv), 0, rdma_addr_size(dst_addr));
3232                 return -EINVAL;
3233         }
3234
3235         if (cma_any_addr(dst_addr)) {
3236                 ret = cma_resolve_loopback(id_priv);
3237         } else {
3238                 if (dst_addr->sa_family == AF_IB) {
3239                         ret = cma_resolve_ib_addr(id_priv);
3240                 } else {
3241                         ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
3242                                               &id->route.addr.dev_addr,
3243                                               timeout_ms, addr_handler,
3244                                               false, id_priv);
3245                 }
3246         }
3247         if (ret)
3248                 goto err;
3249
3250         return 0;
3251 err:
3252         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
3253         return ret;
3254 }
3255 EXPORT_SYMBOL(rdma_resolve_addr);
3256
3257 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
3258 {
3259         struct rdma_id_private *id_priv;
3260         unsigned long flags;
3261         int ret;
3262
3263         id_priv = container_of(id, struct rdma_id_private, id);
3264         spin_lock_irqsave(&id_priv->lock, flags);
3265         if (reuse || id_priv->state == RDMA_CM_IDLE) {
3266                 id_priv->reuseaddr = reuse;
3267                 ret = 0;
3268         } else {
3269                 ret = -EINVAL;
3270         }
3271         spin_unlock_irqrestore(&id_priv->lock, flags);
3272         return ret;
3273 }
3274 EXPORT_SYMBOL(rdma_set_reuseaddr);
3275
3276 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
3277 {
3278         struct rdma_id_private *id_priv;
3279         unsigned long flags;
3280         int ret;
3281
3282         id_priv = container_of(id, struct rdma_id_private, id);
3283         spin_lock_irqsave(&id_priv->lock, flags);
3284         if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
3285                 id_priv->options |= (1 << CMA_OPTION_AFONLY);
3286                 id_priv->afonly = afonly;
3287                 ret = 0;
3288         } else {
3289                 ret = -EINVAL;
3290         }
3291         spin_unlock_irqrestore(&id_priv->lock, flags);
3292         return ret;
3293 }
3294 EXPORT_SYMBOL(rdma_set_afonly);
3295
3296 static void cma_bind_port(struct rdma_bind_list *bind_list,
3297                           struct rdma_id_private *id_priv)
3298 {
3299         struct sockaddr *addr;
3300         struct sockaddr_ib *sib;
3301         u64 sid, mask;
3302         __be16 port;
3303
3304         addr = cma_src_addr(id_priv);
3305         port = htons(bind_list->port);
3306
3307         switch (addr->sa_family) {
3308         case AF_INET:
3309                 ((struct sockaddr_in *) addr)->sin_port = port;
3310                 break;
3311         case AF_INET6:
3312                 ((struct sockaddr_in6 *) addr)->sin6_port = port;
3313                 break;
3314         case AF_IB:
3315                 sib = (struct sockaddr_ib *) addr;
3316                 sid = be64_to_cpu(sib->sib_sid);
3317                 mask = be64_to_cpu(sib->sib_sid_mask);
3318                 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
3319                 sib->sib_sid_mask = cpu_to_be64(~0ULL);
3320                 break;
3321         }
3322         id_priv->bind_list = bind_list;
3323         hlist_add_head(&id_priv->node, &bind_list->owners);
3324 }
3325
3326 static int cma_alloc_port(enum rdma_ucm_port_space ps,
3327                           struct rdma_id_private *id_priv, unsigned short snum)
3328 {
3329         struct rdma_bind_list *bind_list;
3330         int ret;
3331
3332         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
3333         if (!bind_list)
3334                 return -ENOMEM;
3335
3336         ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list,
3337                            snum);
3338         if (ret < 0)
3339                 goto err;
3340
3341         bind_list->ps = ps;
3342         bind_list->port = snum;
3343         cma_bind_port(bind_list, id_priv);
3344         return 0;
3345 err:
3346         kfree(bind_list);
3347         return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
3348 }
3349
3350 static int cma_port_is_unique(struct rdma_bind_list *bind_list,
3351                               struct rdma_id_private *id_priv)
3352 {
3353         struct rdma_id_private *cur_id;
3354         struct sockaddr  *daddr = cma_dst_addr(id_priv);
3355         struct sockaddr  *saddr = cma_src_addr(id_priv);
3356         __be16 dport = cma_port(daddr);
3357
3358         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3359                 struct sockaddr  *cur_daddr = cma_dst_addr(cur_id);
3360                 struct sockaddr  *cur_saddr = cma_src_addr(cur_id);
3361                 __be16 cur_dport = cma_port(cur_daddr);
3362
3363                 if (id_priv == cur_id)
3364                         continue;
3365
3366                 /* different dest port -> unique */
3367                 if (!cma_any_port(daddr) &&
3368                     !cma_any_port(cur_daddr) &&
3369                     (dport != cur_dport))
3370                         continue;
3371
3372                 /* different src address -> unique */
3373                 if (!cma_any_addr(saddr) &&
3374                     !cma_any_addr(cur_saddr) &&
3375                     cma_addr_cmp(saddr, cur_saddr))
3376                         continue;
3377
3378                 /* different dst address -> unique */
3379                 if (!cma_any_addr(daddr) &&
3380                     !cma_any_addr(cur_daddr) &&
3381                     cma_addr_cmp(daddr, cur_daddr))
3382                         continue;
3383
3384                 return -EADDRNOTAVAIL;
3385         }
3386         return 0;
3387 }
3388
3389 static int cma_alloc_any_port(enum rdma_ucm_port_space ps,
3390                               struct rdma_id_private *id_priv)
3391 {
3392         static unsigned int last_used_port;
3393         int low, high, remaining;
3394         unsigned int rover;
3395         struct net *net = id_priv->id.route.addr.dev_addr.net;
3396
3397         inet_get_local_port_range(net, &low, &high);
3398         remaining = (high - low) + 1;
3399         rover = prandom_u32() % remaining + low;
3400 retry:
3401         if (last_used_port != rover) {
3402                 struct rdma_bind_list *bind_list;
3403                 int ret;
3404
3405                 bind_list = cma_ps_find(net, ps, (unsigned short)rover);
3406
3407                 if (!bind_list) {
3408                         ret = cma_alloc_port(ps, id_priv, rover);
3409                 } else {
3410                         ret = cma_port_is_unique(bind_list, id_priv);
3411                         if (!ret)
3412                                 cma_bind_port(bind_list, id_priv);
3413                 }
3414                 /*
3415                  * Remember previously used port number in order to avoid
3416                  * re-using same port immediately after it is closed.
3417                  */
3418                 if (!ret)
3419                         last_used_port = rover;
3420                 if (ret != -EADDRNOTAVAIL)
3421                         return ret;
3422         }
3423         if (--remaining) {
3424                 rover++;
3425                 if ((rover < low) || (rover > high))
3426                         rover = low;
3427                 goto retry;
3428         }
3429         return -EADDRNOTAVAIL;
3430 }
3431
3432 /*
3433  * Check that the requested port is available.  This is called when trying to
3434  * bind to a specific port, or when trying to listen on a bound port.  In
3435  * the latter case, the provided id_priv may already be on the bind_list, but
3436  * we still need to check that it's okay to start listening.
3437  */
3438 static int cma_check_port(struct rdma_bind_list *bind_list,
3439                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
3440 {
3441         struct rdma_id_private *cur_id;
3442         struct sockaddr *addr, *cur_addr;
3443
3444         addr = cma_src_addr(id_priv);
3445         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3446                 if (id_priv == cur_id)
3447                         continue;
3448
3449                 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
3450                     cur_id->reuseaddr)
3451                         continue;
3452
3453                 cur_addr = cma_src_addr(cur_id);
3454                 if (id_priv->afonly && cur_id->afonly &&
3455                     (addr->sa_family != cur_addr->sa_family))
3456                         continue;
3457
3458                 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
3459                         return -EADDRNOTAVAIL;
3460
3461                 if (!cma_addr_cmp(addr, cur_addr))
3462                         return -EADDRINUSE;
3463         }
3464         return 0;
3465 }
3466
3467 static int cma_use_port(enum rdma_ucm_port_space ps,
3468                         struct rdma_id_private *id_priv)
3469 {
3470         struct rdma_bind_list *bind_list;
3471         unsigned short snum;
3472         int ret;
3473
3474         snum = ntohs(cma_port(cma_src_addr(id_priv)));
3475         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
3476                 return -EACCES;
3477
3478         bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum);
3479         if (!bind_list) {
3480                 ret = cma_alloc_port(ps, id_priv, snum);
3481         } else {
3482                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
3483                 if (!ret)
3484                         cma_bind_port(bind_list, id_priv);
3485         }
3486         return ret;
3487 }
3488
3489 static int cma_bind_listen(struct rdma_id_private *id_priv)
3490 {
3491         struct rdma_bind_list *bind_list = id_priv->bind_list;
3492         int ret = 0;
3493
3494         mutex_lock(&lock);
3495         if (bind_list->owners.first->next)
3496                 ret = cma_check_port(bind_list, id_priv, 0);
3497         mutex_unlock(&lock);
3498         return ret;
3499 }
3500
3501 static enum rdma_ucm_port_space
3502 cma_select_inet_ps(struct rdma_id_private *id_priv)
3503 {
3504         switch (id_priv->id.ps) {
3505         case RDMA_PS_TCP:
3506         case RDMA_PS_UDP:
3507         case RDMA_PS_IPOIB:
3508         case RDMA_PS_IB:
3509                 return id_priv->id.ps;
3510         default:
3511
3512                 return 0;
3513         }
3514 }
3515
3516 static enum rdma_ucm_port_space
3517 cma_select_ib_ps(struct rdma_id_private *id_priv)
3518 {
3519         enum rdma_ucm_port_space ps = 0;
3520         struct sockaddr_ib *sib;
3521         u64 sid_ps, mask, sid;
3522
3523         sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
3524         mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
3525         sid = be64_to_cpu(sib->sib_sid) & mask;
3526
3527         if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
3528                 sid_ps = RDMA_IB_IP_PS_IB;
3529                 ps = RDMA_PS_IB;
3530         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
3531                    (sid == (RDMA_IB_IP_PS_TCP & mask))) {
3532                 sid_ps = RDMA_IB_IP_PS_TCP;
3533                 ps = RDMA_PS_TCP;
3534         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
3535                    (sid == (RDMA_IB_IP_PS_UDP & mask))) {
3536                 sid_ps = RDMA_IB_IP_PS_UDP;
3537                 ps = RDMA_PS_UDP;
3538         }
3539
3540         if (ps) {
3541                 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
3542                 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
3543                                                 be64_to_cpu(sib->sib_sid_mask));
3544         }
3545         return ps;
3546 }
3547
3548 static int cma_get_port(struct rdma_id_private *id_priv)
3549 {
3550         enum rdma_ucm_port_space ps;
3551         int ret;
3552
3553         if (cma_family(id_priv) != AF_IB)
3554                 ps = cma_select_inet_ps(id_priv);
3555         else
3556                 ps = cma_select_ib_ps(id_priv);
3557         if (!ps)
3558                 return -EPROTONOSUPPORT;
3559
3560         mutex_lock(&lock);
3561         if (cma_any_port(cma_src_addr(id_priv)))
3562                 ret = cma_alloc_any_port(ps, id_priv);
3563         else
3564                 ret = cma_use_port(ps, id_priv);
3565         mutex_unlock(&lock);
3566
3567         return ret;
3568 }
3569
3570 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
3571                                struct sockaddr *addr)
3572 {
3573 #if IS_ENABLED(CONFIG_IPV6)
3574         struct sockaddr_in6 *sin6;
3575
3576         if (addr->sa_family != AF_INET6)
3577                 return 0;
3578
3579         sin6 = (struct sockaddr_in6 *) addr;
3580
3581         if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL))
3582                 return 0;
3583
3584         if (!sin6->sin6_scope_id)
3585                         return -EINVAL;
3586
3587         dev_addr->bound_dev_if = sin6->sin6_scope_id;
3588 #endif
3589         return 0;
3590 }
3591
3592 int rdma_listen(struct rdma_cm_id *id, int backlog)
3593 {
3594         struct rdma_id_private *id_priv;
3595         int ret;
3596
3597         id_priv = container_of(id, struct rdma_id_private, id);
3598         if (id_priv->state == RDMA_CM_IDLE) {
3599                 id->route.addr.src_addr.ss_family = AF_INET;
3600                 ret = rdma_bind_addr(id, cma_src_addr(id_priv));
3601                 if (ret)
3602                         return ret;
3603         }
3604
3605         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
3606                 return -EINVAL;
3607
3608         if (id_priv->reuseaddr) {
3609                 ret = cma_bind_listen(id_priv);
3610                 if (ret)
3611                         goto err;
3612         }
3613
3614         id_priv->backlog = backlog;
3615         if (id->device) {
3616                 if (rdma_cap_ib_cm(id->device, 1)) {
3617                         ret = cma_ib_listen(id_priv);
3618                         if (ret)
3619                                 goto err;
3620                 } else if (rdma_cap_iw_cm(id->device, 1)) {
3621                         ret = cma_iw_listen(id_priv, backlog);
3622                         if (ret)
3623                                 goto err;
3624                 } else {
3625                         ret = -ENOSYS;
3626                         goto err;
3627                 }
3628         } else
3629                 cma_listen_on_all(id_priv);
3630
3631         return 0;
3632 err:
3633         id_priv->backlog = 0;
3634         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
3635         return ret;
3636 }
3637 EXPORT_SYMBOL(rdma_listen);
3638
3639 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
3640 {
3641         struct rdma_id_private *id_priv;
3642         int ret;
3643         struct sockaddr  *daddr;
3644
3645         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
3646             addr->sa_family != AF_IB)
3647                 return -EAFNOSUPPORT;
3648
3649         id_priv = container_of(id, struct rdma_id_private, id);
3650         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
3651                 return -EINVAL;
3652
3653         ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
3654         if (ret)
3655                 goto err1;
3656
3657         memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
3658         if (!cma_any_addr(addr)) {
3659                 ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
3660                 if (ret)
3661                         goto err1;
3662
3663                 ret = cma_acquire_dev_by_src_ip(id_priv);
3664                 if (ret)
3665                         goto err1;
3666         }
3667
3668         if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
3669                 if (addr->sa_family == AF_INET)
3670                         id_priv->afonly = 1;
3671 #if IS_ENABLED(CONFIG_IPV6)
3672                 else if (addr->sa_family == AF_INET6) {
3673                         struct net *net = id_priv->id.route.addr.dev_addr.net;
3674
3675                         id_priv->afonly = net->ipv6.sysctl.bindv6only;
3676                 }
3677 #endif
3678         }
3679         daddr = cma_dst_addr(id_priv);
3680         daddr->sa_family = addr->sa_family;
3681
3682         ret = cma_get_port(id_priv);
3683         if (ret)
3684                 goto err2;
3685
3686         return 0;
3687 err2:
3688         rdma_restrack_del(&id_priv->res);
3689         if (id_priv->cma_dev)
3690                 cma_release_dev(id_priv);
3691 err1:
3692         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
3693         return ret;
3694 }
3695 EXPORT_SYMBOL(rdma_bind_addr);
3696
3697 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
3698 {
3699         struct cma_hdr *cma_hdr;
3700
3701         cma_hdr = hdr;
3702         cma_hdr->cma_version = CMA_VERSION;
3703         if (cma_family(id_priv) == AF_INET) {
3704                 struct sockaddr_in *src4, *dst4;
3705
3706                 src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
3707                 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
3708
3709                 cma_set_ip_ver(cma_hdr, 4);
3710                 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
3711                 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
3712                 cma_hdr->port = src4->sin_port;
3713         } else if (cma_family(id_priv) == AF_INET6) {
3714                 struct sockaddr_in6 *src6, *dst6;
3715
3716                 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
3717                 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
3718
3719                 cma_set_ip_ver(cma_hdr, 6);
3720                 cma_hdr->src_addr.ip6 = src6->sin6_addr;
3721                 cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
3722                 cma_hdr->port = src6->sin6_port;
3723         }
3724         return 0;
3725 }
3726
3727 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
3728                                 const struct ib_cm_event *ib_event)
3729 {
3730         struct rdma_id_private *id_priv = cm_id->context;
3731         struct rdma_cm_event event = {};
3732         const struct ib_cm_sidr_rep_event_param *rep =
3733                                 &ib_event->param.sidr_rep_rcvd;
3734         int ret = 0;
3735
3736         mutex_lock(&id_priv->handler_mutex);
3737         if (id_priv->state != RDMA_CM_CONNECT)
3738                 goto out;
3739
3740         switch (ib_event->event) {
3741         case IB_CM_SIDR_REQ_ERROR:
3742                 event.event = RDMA_CM_EVENT_UNREACHABLE;
3743                 event.status = -ETIMEDOUT;
3744                 break;
3745         case IB_CM_SIDR_REP_RECEIVED:
3746                 event.param.ud.private_data = ib_event->private_data;
3747                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
3748                 if (rep->status != IB_SIDR_SUCCESS) {
3749                         event.event = RDMA_CM_EVENT_UNREACHABLE;
3750                         event.status = ib_event->param.sidr_rep_rcvd.status;
3751                         pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n",
3752                                              event.status);
3753                         break;
3754                 }
3755                 ret = cma_set_qkey(id_priv, rep->qkey);
3756                 if (ret) {
3757                         pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret);
3758                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
3759                         event.status = ret;
3760                         break;
3761                 }
3762                 ib_init_ah_attr_from_path(id_priv->id.device,
3763                                           id_priv->id.port_num,
3764                                           id_priv->id.route.path_rec,
3765                                           &event.param.ud.ah_attr,
3766                                           rep->sgid_attr);
3767                 event.param.ud.qp_num = rep->qpn;
3768                 event.param.ud.qkey = rep->qkey;
3769                 event.event = RDMA_CM_EVENT_ESTABLISHED;
3770                 event.status = 0;
3771                 break;
3772         default:
3773                 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
3774                        ib_event->event);
3775                 goto out;
3776         }
3777
3778         ret = cma_cm_event_handler(id_priv, &event);
3779
3780         rdma_destroy_ah_attr(&event.param.ud.ah_attr);
3781         if (ret) {
3782                 /* Destroy the CM ID by returning a non-zero value. */
3783                 id_priv->cm_id.ib = NULL;
3784                 cma_exch(id_priv, RDMA_CM_DESTROYING);
3785                 mutex_unlock(&id_priv->handler_mutex);
3786                 rdma_destroy_id(&id_priv->id);
3787                 return ret;
3788         }
3789 out:
3790         mutex_unlock(&id_priv->handler_mutex);
3791         return ret;
3792 }
3793
3794 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
3795                               struct rdma_conn_param *conn_param)
3796 {
3797         struct ib_cm_sidr_req_param req;
3798         struct ib_cm_id *id;
3799         void *private_data;
3800         u8 offset;
3801         int ret;
3802
3803         memset(&req, 0, sizeof req);
3804         offset = cma_user_data_offset(id_priv);
3805         req.private_data_len = offset + conn_param->private_data_len;
3806         if (req.private_data_len < conn_param->private_data_len)
3807                 return -EINVAL;
3808
3809         if (req.private_data_len) {
3810                 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
3811                 if (!private_data)
3812                         return -ENOMEM;
3813         } else {
3814                 private_data = NULL;
3815         }
3816
3817         if (conn_param->private_data && conn_param->private_data_len)
3818                 memcpy(private_data + offset, conn_param->private_data,
3819                        conn_param->private_data_len);
3820
3821         if (private_data) {
3822                 ret = cma_format_hdr(private_data, id_priv);
3823                 if (ret)
3824                         goto out;
3825                 req.private_data = private_data;
3826         }
3827
3828         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
3829                              id_priv);
3830         if (IS_ERR(id)) {
3831                 ret = PTR_ERR(id);
3832                 goto out;
3833         }
3834         id_priv->cm_id.ib = id;
3835
3836         req.path = id_priv->id.route.path_rec;
3837         req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
3838         req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
3839         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
3840         req.max_cm_retries = CMA_MAX_CM_RETRIES;
3841
3842         trace_cm_send_sidr_req(id_priv);
3843         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
3844         if (ret) {
3845                 ib_destroy_cm_id(id_priv->cm_id.ib);
3846                 id_priv->cm_id.ib = NULL;
3847         }
3848 out:
3849         kfree(private_data);
3850         return ret;
3851 }
3852
3853 static int cma_connect_ib(struct rdma_id_private *id_priv,
3854                           struct rdma_conn_param *conn_param)
3855 {
3856         struct ib_cm_req_param req;
3857         struct rdma_route *route;
3858         void *private_data;
3859         struct ib_cm_id *id;
3860         u8 offset;
3861         int ret;
3862
3863         memset(&req, 0, sizeof req);
3864         offset = cma_user_data_offset(id_priv);
3865         req.private_data_len = offset + conn_param->private_data_len;
3866         if (req.private_data_len < conn_param->private_data_len)
3867                 return -EINVAL;
3868
3869         if (req.private_data_len) {
3870                 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
3871                 if (!private_data)
3872                         return -ENOMEM;
3873         } else {
3874                 private_data = NULL;
3875         }
3876
3877         if (conn_param->private_data && conn_param->private_data_len)
3878                 memcpy(private_data + offset, conn_param->private_data,
3879                        conn_param->private_data_len);
3880
3881         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
3882         if (IS_ERR(id)) {
3883                 ret = PTR_ERR(id);
3884                 goto out;
3885         }
3886         id_priv->cm_id.ib = id;
3887
3888         route = &id_priv->id.route;
3889         if (private_data) {
3890                 ret = cma_format_hdr(private_data, id_priv);
3891                 if (ret)
3892                         goto out;
3893                 req.private_data = private_data;
3894         }
3895
3896         req.primary_path = &route->path_rec[0];
3897         if (route->num_paths == 2)
3898                 req.alternate_path = &route->path_rec[1];
3899
3900         req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
3901         /* Alternate path SGID attribute currently unsupported */
3902         req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
3903         req.qp_num = id_priv->qp_num;
3904         req.qp_type = id_priv->id.qp_type;
3905         req.starting_psn = id_priv->seq_num;
3906         req.responder_resources = conn_param->responder_resources;
3907         req.initiator_depth = conn_param->initiator_depth;
3908         req.flow_control = conn_param->flow_control;
3909         req.retry_count = min_t(u8, 7, conn_param->retry_count);
3910         req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
3911         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
3912         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
3913         req.max_cm_retries = CMA_MAX_CM_RETRIES;
3914         req.srq = id_priv->srq ? 1 : 0;
3915
3916         trace_cm_send_req(id_priv);
3917         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
3918 out:
3919         if (ret && !IS_ERR(id)) {
3920                 ib_destroy_cm_id(id);
3921                 id_priv->cm_id.ib = NULL;
3922         }
3923
3924         kfree(private_data);
3925         return ret;
3926 }
3927
3928 static int cma_connect_iw(struct rdma_id_private *id_priv,
3929                           struct rdma_conn_param *conn_param)
3930 {
3931         struct iw_cm_id *cm_id;
3932         int ret;
3933         struct iw_cm_conn_param iw_param;
3934
3935         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
3936         if (IS_ERR(cm_id))
3937                 return PTR_ERR(cm_id);
3938
3939         cm_id->tos = id_priv->tos;
3940         cm_id->tos_set = id_priv->tos_set;
3941         id_priv->cm_id.iw = cm_id;
3942
3943         memcpy(&cm_id->local_addr, cma_src_addr(id_priv),
3944                rdma_addr_size(cma_src_addr(id_priv)));
3945         memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv),
3946                rdma_addr_size(cma_dst_addr(id_priv)));
3947
3948         ret = cma_modify_qp_rtr(id_priv, conn_param);
3949         if (ret)
3950                 goto out;
3951
3952         if (conn_param) {
3953                 iw_param.ord = conn_param->initiator_depth;
3954                 iw_param.ird = conn_param->responder_resources;
3955                 iw_param.private_data = conn_param->private_data;
3956                 iw_param.private_data_len = conn_param->private_data_len;
3957                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
3958         } else {
3959                 memset(&iw_param, 0, sizeof iw_param);
3960                 iw_param.qpn = id_priv->qp_num;
3961         }
3962         ret = iw_cm_connect(cm_id, &iw_param);
3963 out:
3964         if (ret) {
3965                 iw_destroy_cm_id(cm_id);
3966                 id_priv->cm_id.iw = NULL;
3967         }
3968         return ret;
3969 }
3970
3971 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
3972 {
3973         struct rdma_id_private *id_priv;
3974         int ret;
3975
3976         id_priv = container_of(id, struct rdma_id_private, id);
3977         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
3978                 return -EINVAL;
3979
3980         if (!id->qp) {
3981                 id_priv->qp_num = conn_param->qp_num;
3982                 id_priv->srq = conn_param->srq;
3983         }
3984
3985         if (rdma_cap_ib_cm(id->device, id->port_num)) {
3986                 if (id->qp_type == IB_QPT_UD)
3987                         ret = cma_resolve_ib_udp(id_priv, conn_param);
3988                 else
3989                         ret = cma_connect_ib(id_priv, conn_param);
3990         } else if (rdma_cap_iw_cm(id->device, id->port_num))
3991                 ret = cma_connect_iw(id_priv, conn_param);
3992         else
3993                 ret = -ENOSYS;
3994         if (ret)
3995                 goto err;
3996
3997         return 0;
3998 err:
3999         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
4000         return ret;
4001 }
4002 EXPORT_SYMBOL(rdma_connect);
4003
4004 static int cma_accept_ib(struct rdma_id_private *id_priv,
4005                          struct rdma_conn_param *conn_param)
4006 {
4007         struct ib_cm_rep_param rep;
4008         int ret;
4009
4010         ret = cma_modify_qp_rtr(id_priv, conn_param);
4011         if (ret)
4012                 goto out;
4013
4014         ret = cma_modify_qp_rts(id_priv, conn_param);
4015         if (ret)
4016                 goto out;
4017
4018         memset(&rep, 0, sizeof rep);
4019         rep.qp_num = id_priv->qp_num;
4020         rep.starting_psn = id_priv->seq_num;
4021         rep.private_data = conn_param->private_data;
4022         rep.private_data_len = conn_param->private_data_len;
4023         rep.responder_resources = conn_param->responder_resources;
4024         rep.initiator_depth = conn_param->initiator_depth;
4025         rep.failover_accepted = 0;
4026         rep.flow_control = conn_param->flow_control;
4027         rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4028         rep.srq = id_priv->srq ? 1 : 0;
4029
4030         trace_cm_send_rep(id_priv);
4031         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
4032 out:
4033         return ret;
4034 }
4035
4036 static int cma_accept_iw(struct rdma_id_private *id_priv,
4037                   struct rdma_conn_param *conn_param)
4038 {
4039         struct iw_cm_conn_param iw_param;
4040         int ret;
4041
4042         if (!conn_param)
4043                 return -EINVAL;
4044
4045         ret = cma_modify_qp_rtr(id_priv, conn_param);
4046         if (ret)
4047                 return ret;
4048
4049         iw_param.ord = conn_param->initiator_depth;
4050         iw_param.ird = conn_param->responder_resources;
4051         iw_param.private_data = conn_param->private_data;
4052         iw_param.private_data_len = conn_param->private_data_len;
4053         if (id_priv->id.qp) {
4054                 iw_param.qpn = id_priv->qp_num;
4055         } else
4056                 iw_param.qpn = conn_param->qp_num;
4057
4058         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
4059 }
4060
4061 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
4062                              enum ib_cm_sidr_status status, u32 qkey,
4063                              const void *private_data, int private_data_len)
4064 {
4065         struct ib_cm_sidr_rep_param rep;
4066         int ret;
4067
4068         memset(&rep, 0, sizeof rep);
4069         rep.status = status;
4070         if (status == IB_SIDR_SUCCESS) {
4071                 ret = cma_set_qkey(id_priv, qkey);
4072                 if (ret)
4073                         return ret;
4074                 rep.qp_num = id_priv->qp_num;
4075                 rep.qkey = id_priv->qkey;
4076         }
4077         rep.private_data = private_data;
4078         rep.private_data_len = private_data_len;
4079
4080         trace_cm_send_sidr_rep(id_priv);
4081         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
4082 }
4083
4084 int __rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4085                   const char *caller)
4086 {
4087         struct rdma_id_private *id_priv;
4088         int ret;
4089
4090         id_priv = container_of(id, struct rdma_id_private, id);
4091
4092         rdma_restrack_set_task(&id_priv->res, caller);
4093
4094         if (!cma_comp(id_priv, RDMA_CM_CONNECT))
4095                 return -EINVAL;
4096
4097         if (!id->qp && conn_param) {
4098                 id_priv->qp_num = conn_param->qp_num;
4099                 id_priv->srq = conn_param->srq;
4100         }
4101
4102         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4103                 if (id->qp_type == IB_QPT_UD) {
4104                         if (conn_param)
4105                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4106                                                         conn_param->qkey,
4107                                                         conn_param->private_data,
4108                                                         conn_param->private_data_len);
4109                         else
4110                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4111                                                         0, NULL, 0);
4112                 } else {
4113                         if (conn_param)
4114                                 ret = cma_accept_ib(id_priv, conn_param);
4115                         else
4116                                 ret = cma_rep_recv(id_priv);
4117                 }
4118         } else if (rdma_cap_iw_cm(id->device, id->port_num))
4119                 ret = cma_accept_iw(id_priv, conn_param);
4120         else
4121                 ret = -ENOSYS;
4122
4123         if (ret)
4124                 goto reject;
4125
4126         return 0;
4127 reject:
4128         cma_modify_qp_err(id_priv);
4129         rdma_reject(id, NULL, 0);
4130         return ret;
4131 }
4132 EXPORT_SYMBOL(__rdma_accept);
4133
4134 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
4135 {
4136         struct rdma_id_private *id_priv;
4137         int ret;
4138
4139         id_priv = container_of(id, struct rdma_id_private, id);
4140         if (!id_priv->cm_id.ib)
4141                 return -EINVAL;
4142
4143         switch (id->device->node_type) {
4144         case RDMA_NODE_IB_CA:
4145                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
4146                 break;
4147         default:
4148                 ret = 0;
4149                 break;
4150         }
4151         return ret;
4152 }
4153 EXPORT_SYMBOL(rdma_notify);
4154
4155 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
4156                 u8 private_data_len)
4157 {
4158         struct rdma_id_private *id_priv;
4159         int ret;
4160
4161         id_priv = container_of(id, struct rdma_id_private, id);
4162         if (!id_priv->cm_id.ib)
4163                 return -EINVAL;
4164
4165         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4166                 if (id->qp_type == IB_QPT_UD) {
4167                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,
4168                                                 private_data, private_data_len);
4169                 } else {
4170                         trace_cm_send_rej(id_priv);
4171                         ret = ib_send_cm_rej(id_priv->cm_id.ib,
4172                                              IB_CM_REJ_CONSUMER_DEFINED, NULL,
4173                                              0, private_data, private_data_len);
4174                 }
4175         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4176                 ret = iw_cm_reject(id_priv->cm_id.iw,
4177                                    private_data, private_data_len);
4178         } else
4179                 ret = -ENOSYS;
4180
4181         return ret;
4182 }
4183 EXPORT_SYMBOL(rdma_reject);
4184
4185 int rdma_disconnect(struct rdma_cm_id *id)
4186 {
4187         struct rdma_id_private *id_priv;
4188         int ret;
4189
4190         id_priv = container_of(id, struct rdma_id_private, id);
4191         if (!id_priv->cm_id.ib)
4192                 return -EINVAL;
4193
4194         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4195                 ret = cma_modify_qp_err(id_priv);
4196                 if (ret)
4197                         goto out;
4198                 /* Initiate or respond to a disconnect. */
4199                 trace_cm_disconnect(id_priv);
4200                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
4201                         if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
4202                                 trace_cm_sent_drep(id_priv);
4203                 } else {
4204                         trace_cm_sent_dreq(id_priv);
4205                 }
4206         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4207                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
4208         } else
4209                 ret = -EINVAL;
4210
4211 out:
4212         return ret;
4213 }
4214 EXPORT_SYMBOL(rdma_disconnect);
4215
4216 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
4217 {
4218         struct rdma_id_private *id_priv;
4219         struct cma_multicast *mc = multicast->context;
4220         struct rdma_cm_event event = {};
4221         int ret = 0;
4222
4223         id_priv = mc->id_priv;
4224         mutex_lock(&id_priv->handler_mutex);
4225         if (id_priv->state != RDMA_CM_ADDR_BOUND &&
4226             id_priv->state != RDMA_CM_ADDR_RESOLVED)
4227                 goto out;
4228
4229         if (!status)
4230                 status = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey));
4231         else
4232                 pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",
4233                                      status);
4234         mutex_lock(&id_priv->qp_mutex);
4235         if (!status && id_priv->id.qp) {
4236                 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
4237                                          be16_to_cpu(multicast->rec.mlid));
4238                 if (status)
4239                         pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to attach QP. status %d\n",
4240                                              status);
4241         }
4242         mutex_unlock(&id_priv->qp_mutex);
4243
4244         event.status = status;
4245         event.param.ud.private_data = mc->context;
4246         if (!status) {
4247                 struct rdma_dev_addr *dev_addr =
4248                         &id_priv->id.route.addr.dev_addr;
4249                 struct net_device *ndev =
4250                         dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4251                 enum ib_gid_type gid_type =
4252                         id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
4253                         rdma_start_port(id_priv->cma_dev->device)];
4254
4255                 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
4256                 ret = ib_init_ah_from_mcmember(id_priv->id.device,
4257                                                id_priv->id.port_num,
4258                                                &multicast->rec,
4259                                                ndev, gid_type,
4260                                                &event.param.ud.ah_attr);
4261                 if (ret)
4262                         event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
4263
4264                 event.param.ud.qp_num = 0xFFFFFF;
4265                 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
4266                 if (ndev)
4267                         dev_put(ndev);
4268         } else
4269                 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
4270
4271         ret = cma_cm_event_handler(id_priv, &event);
4272
4273         rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4274         if (ret) {
4275                 cma_exch(id_priv, RDMA_CM_DESTROYING);
4276                 mutex_unlock(&id_priv->handler_mutex);
4277                 rdma_destroy_id(&id_priv->id);
4278                 return 0;
4279         }
4280
4281 out:
4282         mutex_unlock(&id_priv->handler_mutex);
4283         return 0;
4284 }
4285
4286 static void cma_set_mgid(struct rdma_id_private *id_priv,
4287                          struct sockaddr *addr, union ib_gid *mgid)
4288 {
4289         unsigned char mc_map[MAX_ADDR_LEN];
4290         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4291         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
4292         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
4293
4294         if (cma_any_addr(addr)) {
4295                 memset(mgid, 0, sizeof *mgid);
4296         } else if ((addr->sa_family == AF_INET6) &&
4297                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
4298                                                                  0xFF10A01B)) {
4299                 /* IPv6 address is an SA assigned MGID. */
4300                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4301         } else if (addr->sa_family == AF_IB) {
4302                 memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid);
4303         } else if (addr->sa_family == AF_INET6) {
4304                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
4305                 if (id_priv->id.ps == RDMA_PS_UDP)
4306                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
4307                 *mgid = *(union ib_gid *) (mc_map + 4);
4308         } else {
4309                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
4310                 if (id_priv->id.ps == RDMA_PS_UDP)
4311                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
4312                 *mgid = *(union ib_gid *) (mc_map + 4);
4313         }
4314 }
4315
4316 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
4317                                  struct cma_multicast *mc)
4318 {
4319         struct ib_sa_mcmember_rec rec;
4320         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4321         ib_sa_comp_mask comp_mask;
4322         int ret;
4323
4324         ib_addr_get_mgid(dev_addr, &rec.mgid);
4325         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
4326                                      &rec.mgid, &rec);
4327         if (ret)
4328                 return ret;
4329
4330         ret = cma_set_qkey(id_priv, 0);
4331         if (ret)
4332                 return ret;
4333
4334         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
4335         rec.qkey = cpu_to_be32(id_priv->qkey);
4336         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
4337         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
4338         rec.join_state = mc->join_state;
4339
4340         if ((rec.join_state == BIT(SENDONLY_FULLMEMBER_JOIN)) &&
4341             (!ib_sa_sendonly_fullmem_support(&sa_client,
4342                                              id_priv->id.device,
4343                                              id_priv->id.port_num))) {
4344                 dev_warn(
4345                         &id_priv->id.device->dev,
4346                         "RDMA CM: port %u Unable to multicast join: SM doesn't support Send Only Full Member option\n",
4347                         id_priv->id.port_num);
4348                 return -EOPNOTSUPP;
4349         }
4350
4351         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
4352                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
4353                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
4354                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
4355                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
4356
4357         if (id_priv->id.ps == RDMA_PS_IPOIB)
4358                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
4359                              IB_SA_MCMEMBER_REC_RATE_SELECTOR |
4360                              IB_SA_MCMEMBER_REC_MTU_SELECTOR |
4361                              IB_SA_MCMEMBER_REC_MTU |
4362                              IB_SA_MCMEMBER_REC_HOP_LIMIT;
4363
4364         mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
4365                                                 id_priv->id.port_num, &rec,
4366                                                 comp_mask, GFP_KERNEL,
4367                                                 cma_ib_mc_handler, mc);
4368         return PTR_ERR_OR_ZERO(mc->multicast.ib);
4369 }
4370
4371 static void iboe_mcast_work_handler(struct work_struct *work)
4372 {
4373         struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
4374         struct cma_multicast *mc = mw->mc;
4375         struct ib_sa_multicast *m = mc->multicast.ib;
4376
4377         mc->multicast.ib->context = mc;
4378         cma_ib_mc_handler(0, m);
4379         kref_put(&mc->mcref, release_mc);
4380         kfree(mw);
4381 }
4382
4383 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
4384                               enum ib_gid_type gid_type)
4385 {
4386         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
4387         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
4388
4389         if (cma_any_addr(addr)) {
4390                 memset(mgid, 0, sizeof *mgid);
4391         } else if (addr->sa_family == AF_INET6) {
4392                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4393         } else {
4394                 mgid->raw[0] =
4395                         (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff;
4396                 mgid->raw[1] =
4397                         (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e;
4398                 mgid->raw[2] = 0;
4399                 mgid->raw[3] = 0;
4400                 mgid->raw[4] = 0;
4401                 mgid->raw[5] = 0;
4402                 mgid->raw[6] = 0;
4403                 mgid->raw[7] = 0;
4404                 mgid->raw[8] = 0;
4405                 mgid->raw[9] = 0;
4406                 mgid->raw[10] = 0xff;
4407                 mgid->raw[11] = 0xff;
4408                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
4409         }
4410 }
4411
4412 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
4413                                    struct cma_multicast *mc)
4414 {
4415         struct iboe_mcast_work *work;
4416         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4417         int err = 0;
4418         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
4419         struct net_device *ndev = NULL;
4420         enum ib_gid_type gid_type;
4421         bool send_only;
4422
4423         send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
4424
4425         if (cma_zero_addr((struct sockaddr *)&mc->addr))
4426                 return -EINVAL;
4427
4428         work = kzalloc(sizeof *work, GFP_KERNEL);
4429         if (!work)
4430                 return -ENOMEM;
4431
4432         mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
4433         if (!mc->multicast.ib) {
4434                 err = -ENOMEM;
4435                 goto out1;
4436         }
4437
4438         gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
4439                    rdma_start_port(id_priv->cma_dev->device)];
4440         cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid, gid_type);
4441
4442         mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
4443         if (id_priv->id.ps == RDMA_PS_UDP)
4444                 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
4445
4446         if (dev_addr->bound_dev_if)
4447                 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4448         if (!ndev) {
4449                 err = -ENODEV;
4450                 goto out2;
4451         }
4452         mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
4453         mc->multicast.ib->rec.hop_limit = 1;
4454         mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
4455
4456         if (addr->sa_family == AF_INET) {
4457                 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
4458                         mc->multicast.ib->rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
4459                         if (!send_only) {
4460                                 err = cma_igmp_send(ndev, &mc->multicast.ib->rec.mgid,
4461                                                     true);
4462                         }
4463                 }
4464         } else {
4465                 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4466                         err = -ENOTSUPP;
4467         }
4468         dev_put(ndev);
4469         if (err || !mc->multicast.ib->rec.mtu) {
4470                 if (!err)
4471                         err = -EINVAL;
4472                 goto out2;
4473         }
4474         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
4475                     &mc->multicast.ib->rec.port_gid);
4476         work->id = id_priv;
4477         work->mc = mc;
4478         INIT_WORK(&work->work, iboe_mcast_work_handler);
4479         kref_get(&mc->mcref);
4480         queue_work(cma_wq, &work->work);
4481
4482         return 0;
4483
4484 out2:
4485         kfree(mc->multicast.ib);
4486 out1:
4487         kfree(work);
4488         return err;
4489 }
4490
4491 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
4492                         u8 join_state, void *context)
4493 {
4494         struct rdma_id_private *id_priv;
4495         struct cma_multicast *mc;
4496         int ret;
4497
4498         if (!id->device)
4499                 return -EINVAL;
4500
4501         id_priv = container_of(id, struct rdma_id_private, id);
4502         if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
4503             !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
4504                 return -EINVAL;
4505
4506         mc = kmalloc(sizeof *mc, GFP_KERNEL);
4507         if (!mc)
4508                 return -ENOMEM;
4509
4510         memcpy(&mc->addr, addr, rdma_addr_size(addr));
4511         mc->context = context;
4512         mc->id_priv = id_priv;
4513         mc->join_state = join_state;
4514
4515         if (rdma_protocol_roce(id->device, id->port_num)) {
4516                 kref_init(&mc->mcref);
4517                 ret = cma_iboe_join_multicast(id_priv, mc);
4518                 if (ret)
4519                         goto out_err;
4520         } else if (rdma_cap_ib_mcast(id->device, id->port_num)) {
4521                 ret = cma_join_ib_multicast(id_priv, mc);
4522                 if (ret)
4523                         goto out_err;
4524         } else {
4525                 ret = -ENOSYS;
4526                 goto out_err;
4527         }
4528
4529         spin_lock(&id_priv->lock);
4530         list_add(&mc->list, &id_priv->mc_list);
4531         spin_unlock(&id_priv->lock);
4532
4533         return 0;
4534 out_err:
4535         kfree(mc);
4536         return ret;
4537 }
4538 EXPORT_SYMBOL(rdma_join_multicast);
4539
4540 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
4541 {
4542         struct rdma_id_private *id_priv;
4543         struct cma_multicast *mc;
4544
4545         id_priv = container_of(id, struct rdma_id_private, id);
4546         spin_lock_irq(&id_priv->lock);
4547         list_for_each_entry(mc, &id_priv->mc_list, list) {
4548                 if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) {
4549                         list_del(&mc->list);
4550                         spin_unlock_irq(&id_priv->lock);
4551
4552                         if (id->qp)
4553                                 ib_detach_mcast(id->qp,
4554                                                 &mc->multicast.ib->rec.mgid,
4555                                                 be16_to_cpu(mc->multicast.ib->rec.mlid));
4556
4557                         BUG_ON(id_priv->cma_dev->device != id->device);
4558
4559                         if (rdma_cap_ib_mcast(id->device, id->port_num)) {
4560                                 ib_sa_free_multicast(mc->multicast.ib);
4561                                 kfree(mc);
4562                         } else if (rdma_protocol_roce(id->device, id->port_num)) {
4563                                 cma_leave_roce_mc_group(id_priv, mc);
4564                         }
4565                         return;
4566                 }
4567         }
4568         spin_unlock_irq(&id_priv->lock);
4569 }
4570 EXPORT_SYMBOL(rdma_leave_multicast);
4571
4572 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
4573 {
4574         struct rdma_dev_addr *dev_addr;
4575         struct cma_ndev_work *work;
4576
4577         dev_addr = &id_priv->id.route.addr.dev_addr;
4578
4579         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
4580             (net_eq(dev_net(ndev), dev_addr->net)) &&
4581             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
4582                 pr_info("RDMA CM addr change for ndev %s used by id %p\n",
4583                         ndev->name, &id_priv->id);
4584                 work = kzalloc(sizeof *work, GFP_KERNEL);
4585                 if (!work)
4586                         return -ENOMEM;
4587
4588                 INIT_WORK(&work->work, cma_ndev_work_handler);
4589                 work->id = id_priv;
4590                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
4591                 atomic_inc(&id_priv->refcount);
4592                 queue_work(cma_wq, &work->work);
4593         }
4594
4595         return 0;
4596 }
4597
4598 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
4599                                void *ptr)
4600 {
4601         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
4602         struct cma_device *cma_dev;
4603         struct rdma_id_private *id_priv;
4604         int ret = NOTIFY_DONE;
4605
4606         if (event != NETDEV_BONDING_FAILOVER)
4607                 return NOTIFY_DONE;
4608
4609         if (!netif_is_bond_master(ndev))
4610                 return NOTIFY_DONE;
4611
4612         mutex_lock(&lock);
4613         list_for_each_entry(cma_dev, &dev_list, list)
4614                 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
4615                         ret = cma_netdev_change(ndev, id_priv);
4616                         if (ret)
4617                                 goto out;
4618                 }
4619
4620 out:
4621         mutex_unlock(&lock);
4622         return ret;
4623 }
4624
4625 static struct notifier_block cma_nb = {
4626         .notifier_call = cma_netdev_callback
4627 };
4628
4629 static void cma_add_one(struct ib_device *device)
4630 {
4631         struct cma_device *cma_dev;
4632         struct rdma_id_private *id_priv;
4633         unsigned int i;
4634         unsigned long supported_gids = 0;
4635
4636         cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
4637         if (!cma_dev)
4638                 return;
4639
4640         cma_dev->device = device;
4641         cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
4642                                             sizeof(*cma_dev->default_gid_type),
4643                                             GFP_KERNEL);
4644         if (!cma_dev->default_gid_type)
4645                 goto free_cma_dev;
4646
4647         cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
4648                                             sizeof(*cma_dev->default_roce_tos),
4649                                             GFP_KERNEL);
4650         if (!cma_dev->default_roce_tos)
4651                 goto free_gid_type;
4652
4653         rdma_for_each_port (device, i) {
4654                 supported_gids = roce_gid_type_mask_support(device, i);
4655                 WARN_ON(!supported_gids);
4656                 if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
4657                         cma_dev->default_gid_type[i - rdma_start_port(device)] =
4658                                 CMA_PREFERRED_ROCE_GID_TYPE;
4659                 else
4660                         cma_dev->default_gid_type[i - rdma_start_port(device)] =
4661                                 find_first_bit(&supported_gids, BITS_PER_LONG);
4662                 cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
4663         }
4664
4665         init_completion(&cma_dev->comp);
4666         atomic_set(&cma_dev->refcount, 1);
4667         INIT_LIST_HEAD(&cma_dev->id_list);
4668         ib_set_client_data(device, &cma_client, cma_dev);
4669
4670         mutex_lock(&lock);
4671         list_add_tail(&cma_dev->list, &dev_list);
4672         list_for_each_entry(id_priv, &listen_any_list, list)
4673                 cma_listen_on_dev(id_priv, cma_dev);
4674         mutex_unlock(&lock);
4675
4676         trace_cm_add_one(device);
4677         return;
4678
4679 free_gid_type:
4680         kfree(cma_dev->default_gid_type);
4681
4682 free_cma_dev:
4683         kfree(cma_dev);
4684
4685         return;
4686 }
4687
4688 static int cma_remove_id_dev(struct rdma_id_private *id_priv)
4689 {
4690         struct rdma_cm_event event = {};
4691         enum rdma_cm_state state;
4692         int ret = 0;
4693
4694         /* Record that we want to remove the device */
4695         state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
4696         if (state == RDMA_CM_DESTROYING)
4697                 return 0;
4698
4699         cma_cancel_operation(id_priv, state);
4700         mutex_lock(&id_priv->handler_mutex);
4701
4702         /* Check for destruction from another callback. */
4703         if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
4704                 goto out;
4705
4706         event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
4707         ret = cma_cm_event_handler(id_priv, &event);
4708 out:
4709         mutex_unlock(&id_priv->handler_mutex);
4710         return ret;
4711 }
4712
4713 static void cma_process_remove(struct cma_device *cma_dev)
4714 {
4715         struct rdma_id_private *id_priv;
4716         int ret;
4717
4718         mutex_lock(&lock);
4719         while (!list_empty(&cma_dev->id_list)) {
4720                 id_priv = list_entry(cma_dev->id_list.next,
4721                                      struct rdma_id_private, list);
4722
4723                 list_del(&id_priv->listen_list);
4724                 list_del_init(&id_priv->list);
4725                 atomic_inc(&id_priv->refcount);
4726                 mutex_unlock(&lock);
4727
4728                 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
4729                 cma_deref_id(id_priv);
4730                 if (ret)
4731                         rdma_destroy_id(&id_priv->id);
4732
4733                 mutex_lock(&lock);
4734         }
4735         mutex_unlock(&lock);
4736
4737         cma_deref_dev(cma_dev);
4738         wait_for_completion(&cma_dev->comp);
4739 }
4740
4741 static void cma_remove_one(struct ib_device *device, void *client_data)
4742 {
4743         struct cma_device *cma_dev = client_data;
4744
4745         trace_cm_remove_one(device);
4746
4747         if (!cma_dev)
4748                 return;
4749
4750         mutex_lock(&lock);
4751         list_del(&cma_dev->list);
4752         mutex_unlock(&lock);
4753
4754         cma_process_remove(cma_dev);
4755         kfree(cma_dev->default_roce_tos);
4756         kfree(cma_dev->default_gid_type);
4757         kfree(cma_dev);
4758 }
4759
4760 static int cma_init_net(struct net *net)
4761 {
4762         struct cma_pernet *pernet = cma_pernet(net);
4763
4764         xa_init(&pernet->tcp_ps);
4765         xa_init(&pernet->udp_ps);
4766         xa_init(&pernet->ipoib_ps);
4767         xa_init(&pernet->ib_ps);
4768
4769         return 0;
4770 }
4771
4772 static void cma_exit_net(struct net *net)
4773 {
4774         struct cma_pernet *pernet = cma_pernet(net);
4775
4776         WARN_ON(!xa_empty(&pernet->tcp_ps));
4777         WARN_ON(!xa_empty(&pernet->udp_ps));
4778         WARN_ON(!xa_empty(&pernet->ipoib_ps));
4779         WARN_ON(!xa_empty(&pernet->ib_ps));
4780 }
4781
4782 static struct pernet_operations cma_pernet_operations = {
4783         .init = cma_init_net,
4784         .exit = cma_exit_net,
4785         .id = &cma_pernet_id,
4786         .size = sizeof(struct cma_pernet),
4787 };
4788
4789 static int __init cma_init(void)
4790 {
4791         int ret;
4792
4793         cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
4794         if (!cma_wq)
4795                 return -ENOMEM;
4796
4797         ret = register_pernet_subsys(&cma_pernet_operations);
4798         if (ret)
4799                 goto err_wq;
4800
4801         ib_sa_register_client(&sa_client);
4802         register_netdevice_notifier(&cma_nb);
4803
4804         ret = ib_register_client(&cma_client);
4805         if (ret)
4806                 goto err;
4807
4808         ret = cma_configfs_init();
4809         if (ret)
4810                 goto err_ib;
4811
4812         return 0;
4813
4814 err_ib:
4815         ib_unregister_client(&cma_client);
4816 err:
4817         unregister_netdevice_notifier(&cma_nb);
4818         ib_sa_unregister_client(&sa_client);
4819         unregister_pernet_subsys(&cma_pernet_operations);
4820 err_wq:
4821         destroy_workqueue(cma_wq);
4822         return ret;
4823 }
4824
4825 static void __exit cma_cleanup(void)
4826 {
4827         cma_configfs_exit();
4828         ib_unregister_client(&cma_client);
4829         unregister_netdevice_notifier(&cma_nb);
4830         ib_sa_unregister_client(&sa_client);
4831         unregister_pernet_subsys(&cma_pernet_operations);
4832         destroy_workqueue(cma_wq);
4833 }
4834
4835 module_init(cma_init);
4836 module_exit(cma_cleanup);