drivers/infiniband/core/cma.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2005 Voltaire Inc.  All rights reserved.
   4  * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
   5  * Copyright (c) 1999-2019, Mellanox Technologies, Inc. All rights reserved.
   6  * Copyright (c) 2005-2006 Intel Corporation.  All rights reserved.
   7  */
   8
   9 #include <linux/completion.h>
  10 #include <linux/in.h>
  11 #include <linux/in6.h>
  12 #include <linux/mutex.h>
  13 #include <linux/random.h>
  14 #include <linux/rbtree.h>
  15 #include <linux/igmp.h>
  16 #include <linux/xarray.h>
  17 #include <linux/inetdevice.h>
  18 #include <linux/slab.h>
  19 #include <linux/module.h>
  20 #include <net/route.h>
  21
  22 #include <net/net_namespace.h>
  23 #include <net/netns/generic.h>
  24 #include <net/netevent.h>
  25 #include <net/tcp.h>
  26 #include <net/ipv6.h>
  27 #include <net/ip_fib.h>
  28 #include <net/ip6_route.h>
  29
  30 #include <rdma/rdma_cm.h>
  31 #include <rdma/rdma_cm_ib.h>
  32 #include <rdma/rdma_netlink.h>
  33 #include <rdma/ib.h>
  34 #include <rdma/ib_cache.h>
  35 #include <rdma/ib_cm.h>
  36 #include <rdma/ib_sa.h>
  37 #include <rdma/iw_cm.h>
  38
  39 #include "core_priv.h"
  40 #include "cma_priv.h"
  41 #include "cma_trace.h"
  42
  43 MODULE_AUTHOR("Sean Hefty");
  44 MODULE_DESCRIPTION("Generic RDMA CM Agent");
  45 MODULE_LICENSE("Dual BSD/GPL");
  46
  47 #define CMA_CM_RESPONSE_TIMEOUT 20
  48 #define CMA_MAX_CM_RETRIES 15
  49 #define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
  50 #define CMA_IBOE_PACKET_LIFETIME 16
  51 #define CMA_PREFERRED_ROCE_GID_TYPE IB_GID_TYPE_ROCE_UDP_ENCAP
  52
  53 static const char * const cma_events[] = {
  54         [RDMA_CM_EVENT_ADDR_RESOLVED]    = "address resolved",
  55         [RDMA_CM_EVENT_ADDR_ERROR]       = "address error",
  56         [RDMA_CM_EVENT_ROUTE_RESOLVED]   = "route resolved ",
  57         [RDMA_CM_EVENT_ROUTE_ERROR]      = "route error",
  58         [RDMA_CM_EVENT_CONNECT_REQUEST]  = "connect request",
  59         [RDMA_CM_EVENT_CONNECT_RESPONSE] = "connect response",
  60         [RDMA_CM_EVENT_CONNECT_ERROR]    = "connect error",
  61         [RDMA_CM_EVENT_UNREACHABLE]      = "unreachable",
  62         [RDMA_CM_EVENT_REJECTED]         = "rejected",
  63         [RDMA_CM_EVENT_ESTABLISHED]      = "established",
  64         [RDMA_CM_EVENT_DISCONNECTED]     = "disconnected",
  65         [RDMA_CM_EVENT_DEVICE_REMOVAL]   = "device removal",
  66         [RDMA_CM_EVENT_MULTICAST_JOIN]   = "multicast join",
  67         [RDMA_CM_EVENT_MULTICAST_ERROR]  = "multicast error",
  68         [RDMA_CM_EVENT_ADDR_CHANGE]      = "address change",
  69         [RDMA_CM_EVENT_TIMEWAIT_EXIT]    = "timewait exit",
  70 };
  71
  72 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
  73                               enum ib_gid_type gid_type);
  74
  75 const char *__attribute_const__ rdma_event_msg(enum rdma_cm_event_type event)
  76 {
  77         size_t index = event;
  78
  79         return (index < ARRAY_SIZE(cma_events) && cma_events[index]) ?
  80                         cma_events[index] : "unrecognized event";
  81 }
  82 EXPORT_SYMBOL(rdma_event_msg);
  83
  84 const char *__attribute_const__ rdma_reject_msg(struct rdma_cm_id *id,
  85                                                 int reason)
  86 {
  87         if (rdma_ib_or_roce(id->device, id->port_num))
  88                 return ibcm_reject_msg(reason);
  89
  90         if (rdma_protocol_iwarp(id->device, id->port_num))
  91                 return iwcm_reject_msg(reason);
  92
  93         WARN_ON_ONCE(1);
  94         return "unrecognized transport";
  95 }
  96 EXPORT_SYMBOL(rdma_reject_msg);
  97
  98 /**
  99  * rdma_is_consumer_reject - return true if the consumer rejected the connect
 100  *                           request.
 101  * @id: Communication identifier that received the REJECT event.
 102  * @reason: Value returned in the REJECT event status field.
 103  */
 104 static bool rdma_is_consumer_reject(struct rdma_cm_id *id, int reason)
 105 {
 106         if (rdma_ib_or_roce(id->device, id->port_num))
 107                 return reason == IB_CM_REJ_CONSUMER_DEFINED;
 108
 109         if (rdma_protocol_iwarp(id->device, id->port_num))
 110                 return reason == -ECONNREFUSED;
 111
 112         WARN_ON_ONCE(1);
 113         return false;
 114 }
 115
 116 const void *rdma_consumer_reject_data(struct rdma_cm_id *id,
 117                                       struct rdma_cm_event *ev, u8 *data_len)
 118 {
 119         const void *p;
 120
 121         if (rdma_is_consumer_reject(id, ev->status)) {
 122                 *data_len = ev->param.conn.private_data_len;
 123                 p = ev->param.conn.private_data;
 124         } else {
 125                 *data_len = 0;
 126                 p = NULL;
 127         }
 128         return p;
 129 }
 130 EXPORT_SYMBOL(rdma_consumer_reject_data);
 131
 132 /**
 133  * rdma_iw_cm_id() - return the iw_cm_id pointer for this cm_id.
 134  * @id: Communication Identifier
 135  */
 136 struct iw_cm_id *rdma_iw_cm_id(struct rdma_cm_id *id)
 137 {
 138         struct rdma_id_private *id_priv;
 139
 140         id_priv = container_of(id, struct rdma_id_private, id);
 141         if (id->device->node_type == RDMA_NODE_RNIC)
 142                 return id_priv->cm_id.iw;
 143         return NULL;
 144 }
 145 EXPORT_SYMBOL(rdma_iw_cm_id);
 146
 147 /**
 148  * rdma_res_to_id() - return the rdma_cm_id pointer for this restrack.
 149  * @res: rdma resource tracking entry pointer
 150  */
 151 struct rdma_cm_id *rdma_res_to_id(struct rdma_restrack_entry *res)
 152 {
 153         struct rdma_id_private *id_priv =
 154                 container_of(res, struct rdma_id_private, res);
 155
 156         return &id_priv->id;
 157 }
 158 EXPORT_SYMBOL(rdma_res_to_id);
 159
 160 static int cma_add_one(struct ib_device *device);
 161 static void cma_remove_one(struct ib_device *device, void *client_data);
 162
 163 static struct ib_client cma_client = {
 164         .name   = "cma",
 165         .add    = cma_add_one,
 166         .remove = cma_remove_one
 167 };
 168
 169 static struct ib_sa_client sa_client;
 170 static LIST_HEAD(dev_list);
 171 static LIST_HEAD(listen_any_list);
 172 static DEFINE_MUTEX(lock);
 173 static struct rb_root id_table = RB_ROOT;
 174 /* Serialize operations of id_table tree */
 175 static DEFINE_SPINLOCK(id_table_lock);
 176 static struct workqueue_struct *cma_wq;
 177 static unsigned int cma_pernet_id;
 178
 179 struct cma_pernet {
 180         struct xarray tcp_ps;
 181         struct xarray udp_ps;
 182         struct xarray ipoib_ps;
 183         struct xarray ib_ps;
 184 };
 185
 186 static struct cma_pernet *cma_pernet(struct net *net)
 187 {
 188         return net_generic(net, cma_pernet_id);
 189 }
 190
 191 static
 192 struct xarray *cma_pernet_xa(struct net *net, enum rdma_ucm_port_space ps)
 193 {
 194         struct cma_pernet *pernet = cma_pernet(net);
 195
 196         switch (ps) {
 197         case RDMA_PS_TCP:
 198                 return &pernet->tcp_ps;
 199         case RDMA_PS_UDP:
 200                 return &pernet->udp_ps;
 201         case RDMA_PS_IPOIB:
 202                 return &pernet->ipoib_ps;
 203         case RDMA_PS_IB:
 204                 return &pernet->ib_ps;
 205         default:
 206                 return NULL;
 207         }
 208 }
 209
 210 struct id_table_entry {
 211         struct list_head id_list;
 212         struct rb_node rb_node;
 213 };
 214
 215 struct cma_device {
 216         struct list_head        list;
 217         struct ib_device        *device;
 218         struct completion       comp;
 219         refcount_t refcount;
 220         struct list_head        id_list;
 221         enum ib_gid_type        *default_gid_type;
 222         u8                      *default_roce_tos;
 223 };
 224
 225 struct rdma_bind_list {
 226         enum rdma_ucm_port_space ps;
 227         struct hlist_head       owners;
 228         unsigned short          port;
 229 };
 230
 231 static int cma_ps_alloc(struct net *net, enum rdma_ucm_port_space ps,
 232                         struct rdma_bind_list *bind_list, int snum)
 233 {
 234         struct xarray *xa = cma_pernet_xa(net, ps);
 235
 236         return xa_insert(xa, snum, bind_list, GFP_KERNEL);
 237 }
 238
 239 static struct rdma_bind_list *cma_ps_find(struct net *net,
 240                                           enum rdma_ucm_port_space ps, int snum)
 241 {
 242         struct xarray *xa = cma_pernet_xa(net, ps);
 243
 244         return xa_load(xa, snum);
 245 }
 246
 247 static void cma_ps_remove(struct net *net, enum rdma_ucm_port_space ps,
 248                           int snum)
 249 {
 250         struct xarray *xa = cma_pernet_xa(net, ps);
 251
 252         xa_erase(xa, snum);
 253 }
 254
 255 enum {
 256         CMA_OPTION_AFONLY,
 257 };
 258
 259 void cma_dev_get(struct cma_device *cma_dev)
 260 {
 261         refcount_inc(&cma_dev->refcount);
 262 }
 263
 264 void cma_dev_put(struct cma_device *cma_dev)
 265 {
 266         if (refcount_dec_and_test(&cma_dev->refcount))
 267                 complete(&cma_dev->comp);
 268 }
 269
 270 struct cma_device *cma_enum_devices_by_ibdev(cma_device_filter  filter,
 271                                              void               *cookie)
 272 {
 273         struct cma_device *cma_dev;
 274         struct cma_device *found_cma_dev = NULL;
 275
 276         mutex_lock(&lock);
 277
 278         list_for_each_entry(cma_dev, &dev_list, list)
 279                 if (filter(cma_dev->device, cookie)) {
 280                         found_cma_dev = cma_dev;
 281                         break;
 282                 }
 283
 284         if (found_cma_dev)
 285                 cma_dev_get(found_cma_dev);
 286         mutex_unlock(&lock);
 287         return found_cma_dev;
 288 }
 289
 290 int cma_get_default_gid_type(struct cma_device *cma_dev,
 291                              u32 port)
 292 {
 293         if (!rdma_is_port_valid(cma_dev->device, port))
 294                 return -EINVAL;
 295
 296         return cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)];
 297 }
 298
 299 int cma_set_default_gid_type(struct cma_device *cma_dev,
 300                              u32 port,
 301                              enum ib_gid_type default_gid_type)
 302 {
 303         unsigned long supported_gids;
 304
 305         if (!rdma_is_port_valid(cma_dev->device, port))
 306                 return -EINVAL;
 307
 308         if (default_gid_type == IB_GID_TYPE_IB &&
 309             rdma_protocol_roce_eth_encap(cma_dev->device, port))
 310                 default_gid_type = IB_GID_TYPE_ROCE;
 311
 312         supported_gids = roce_gid_type_mask_support(cma_dev->device, port);
 313
 314         if (!(supported_gids & 1 << default_gid_type))
 315                 return -EINVAL;
 316
 317         cma_dev->default_gid_type[port - rdma_start_port(cma_dev->device)] =
 318                 default_gid_type;
 319
 320         return 0;
 321 }
 322
 323 int cma_get_default_roce_tos(struct cma_device *cma_dev, u32 port)
 324 {
 325         if (!rdma_is_port_valid(cma_dev->device, port))
 326                 return -EINVAL;
 327
 328         return cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)];
 329 }
 330
 331 int cma_set_default_roce_tos(struct cma_device *cma_dev, u32 port,
 332                              u8 default_roce_tos)
 333 {
 334         if (!rdma_is_port_valid(cma_dev->device, port))
 335                 return -EINVAL;
 336
 337         cma_dev->default_roce_tos[port - rdma_start_port(cma_dev->device)] =
 338                  default_roce_tos;
 339
 340         return 0;
 341 }
 342 struct ib_device *cma_get_ib_dev(struct cma_device *cma_dev)
 343 {
 344         return cma_dev->device;
 345 }
 346
 347 /*
 348  * Device removal can occur at anytime, so we need extra handling to
 349  * serialize notifying the user of device removal with other callbacks.
 350  * We do this by disabling removal notification while a callback is in process,
 351  * and reporting it after the callback completes.
 352  */
 353
 354 struct cma_multicast {
 355         struct rdma_id_private *id_priv;
 356         union {
 357                 struct ib_sa_multicast *sa_mc;
 358                 struct {
 359                         struct work_struct work;
 360                         struct rdma_cm_event event;
 361                 } iboe_join;
 362         };
 363         struct list_head        list;
 364         void                    *context;
 365         struct sockaddr_storage addr;
 366         u8                      join_state;
 367 };
 368
 369 struct cma_work {
 370         struct work_struct      work;
 371         struct rdma_id_private  *id;
 372         enum rdma_cm_state      old_state;
 373         enum rdma_cm_state      new_state;
 374         struct rdma_cm_event    event;
 375 };
 376
 377 union cma_ip_addr {
 378         struct in6_addr ip6;
 379         struct {
 380                 __be32 pad[3];
 381                 __be32 addr;
 382         } ip4;
 383 };
 384
 385 struct cma_hdr {
 386         u8 cma_version;
 387         u8 ip_version;  /* IP version: 7:4 */
 388         __be16 port;
 389         union cma_ip_addr src_addr;
 390         union cma_ip_addr dst_addr;
 391 };
 392
 393 #define CMA_VERSION 0x00
 394
 395 struct cma_req_info {
 396         struct sockaddr_storage listen_addr_storage;
 397         struct sockaddr_storage src_addr_storage;
 398         struct ib_device *device;
 399         union ib_gid local_gid;
 400         __be64 service_id;
 401         int port;
 402         bool has_gid;
 403         u16 pkey;
 404 };
 405
 406 static int cma_comp_exch(struct rdma_id_private *id_priv,
 407                          enum rdma_cm_state comp, enum rdma_cm_state exch)
 408 {
 409         unsigned long flags;
 410         int ret;
 411
 412         /*
 413          * The FSM uses a funny double locking where state is protected by both
 414          * the handler_mutex and the spinlock. State is not allowed to change
 415          * to/from a handler_mutex protected value without also holding
 416          * handler_mutex.
 417          */
 418         if (comp == RDMA_CM_CONNECT || exch == RDMA_CM_CONNECT)
 419                 lockdep_assert_held(&id_priv->handler_mutex);
 420
 421         spin_lock_irqsave(&id_priv->lock, flags);
 422         if ((ret = (id_priv->state == comp)))
 423                 id_priv->state = exch;
 424         spin_unlock_irqrestore(&id_priv->lock, flags);
 425         return ret;
 426 }
 427
 428 static inline u8 cma_get_ip_ver(const struct cma_hdr *hdr)
 429 {
 430         return hdr->ip_version >> 4;
 431 }
 432
 433 static void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
 434 {
 435         hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
 436 }
 437
 438 static struct sockaddr *cma_src_addr(struct rdma_id_private *id_priv)
 439 {
 440         return (struct sockaddr *)&id_priv->id.route.addr.src_addr;
 441 }
 442
 443 static inline struct sockaddr *cma_dst_addr(struct rdma_id_private *id_priv)
 444 {
 445         return (struct sockaddr *)&id_priv->id.route.addr.dst_addr;
 446 }
 447
 448 static int cma_igmp_send(struct net_device *ndev, union ib_gid *mgid, bool join)
 449 {
 450         struct in_device *in_dev = NULL;
 451
 452         if (ndev) {
 453                 rtnl_lock();
 454                 in_dev = __in_dev_get_rtnl(ndev);
 455                 if (in_dev) {
 456                         if (join)
 457                                 ip_mc_inc_group(in_dev,
 458                                                 *(__be32 *)(mgid->raw + 12));
 459                         else
 460                                 ip_mc_dec_group(in_dev,
 461                                                 *(__be32 *)(mgid->raw + 12));
 462                 }
 463                 rtnl_unlock();
 464         }
 465         return (in_dev) ? 0 : -ENODEV;
 466 }
 467
 468 static int compare_netdev_and_ip(int ifindex_a, struct sockaddr *sa,
 469                                  struct id_table_entry *entry_b)
 470 {
 471         struct rdma_id_private *id_priv = list_first_entry(
 472                 &entry_b->id_list, struct rdma_id_private, id_list_entry);
 473         int ifindex_b = id_priv->id.route.addr.dev_addr.bound_dev_if;
 474         struct sockaddr *sb = cma_dst_addr(id_priv);
 475
 476         if (ifindex_a != ifindex_b)
 477                 return (ifindex_a > ifindex_b) ? 1 : -1;
 478
 479         if (sa->sa_family != sb->sa_family)
 480                 return sa->sa_family - sb->sa_family;
 481
 482         if (sa->sa_family == AF_INET &&
 483             __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in)) {
 484                 return memcmp(&((struct sockaddr_in *)sa)->sin_addr,
 485                               &((struct sockaddr_in *)sb)->sin_addr,
 486                               sizeof(((struct sockaddr_in *)sa)->sin_addr));
 487         }
 488
 489         if (sa->sa_family == AF_INET6 &&
 490             __builtin_object_size(sa, 0) >= sizeof(struct sockaddr_in6)) {
 491                 return ipv6_addr_cmp(&((struct sockaddr_in6 *)sa)->sin6_addr,
 492                                      &((struct sockaddr_in6 *)sb)->sin6_addr);
 493         }
 494
 495         return -1;
 496 }
 497
 498 static int cma_add_id_to_tree(struct rdma_id_private *node_id_priv)
 499 {
 500         struct rb_node **new, *parent = NULL;
 501         struct id_table_entry *this, *node;
 502         unsigned long flags;
 503         int result;
 504
 505         node = kzalloc(sizeof(*node), GFP_KERNEL);
 506         if (!node)
 507                 return -ENOMEM;
 508
 509         spin_lock_irqsave(&id_table_lock, flags);
 510         new = &id_table.rb_node;
 511         while (*new) {
 512                 this = container_of(*new, struct id_table_entry, rb_node);
 513                 result = compare_netdev_and_ip(
 514                         node_id_priv->id.route.addr.dev_addr.bound_dev_if,
 515                         cma_dst_addr(node_id_priv), this);
 516
 517                 parent = *new;
 518                 if (result < 0)
 519                         new = &((*new)->rb_left);
 520                 else if (result > 0)
 521                         new = &((*new)->rb_right);
 522                 else {
 523                         list_add_tail(&node_id_priv->id_list_entry,
 524                                       &this->id_list);
 525                         kfree(node);
 526                         goto unlock;
 527                 }
 528         }
 529
 530         INIT_LIST_HEAD(&node->id_list);
 531         list_add_tail(&node_id_priv->id_list_entry, &node->id_list);
 532
 533         rb_link_node(&node->rb_node, parent, new);
 534         rb_insert_color(&node->rb_node, &id_table);
 535
 536 unlock:
 537         spin_unlock_irqrestore(&id_table_lock, flags);
 538         return 0;
 539 }
 540
 541 static struct id_table_entry *
 542 node_from_ndev_ip(struct rb_root *root, int ifindex, struct sockaddr *sa)
 543 {
 544         struct rb_node *node = root->rb_node;
 545         struct id_table_entry *data;
 546         int result;
 547
 548         while (node) {
 549                 data = container_of(node, struct id_table_entry, rb_node);
 550                 result = compare_netdev_and_ip(ifindex, sa, data);
 551                 if (result < 0)
 552                         node = node->rb_left;
 553                 else if (result > 0)
 554                         node = node->rb_right;
 555                 else
 556                         return data;
 557         }
 558
 559         return NULL;
 560 }
 561
 562 static void cma_remove_id_from_tree(struct rdma_id_private *id_priv)
 563 {
 564         struct id_table_entry *data;
 565         unsigned long flags;
 566
 567         spin_lock_irqsave(&id_table_lock, flags);
 568         if (list_empty(&id_priv->id_list_entry))
 569                 goto out;
 570
 571         data = node_from_ndev_ip(&id_table,
 572                                  id_priv->id.route.addr.dev_addr.bound_dev_if,
 573                                  cma_dst_addr(id_priv));
 574         if (!data)
 575                 goto out;
 576
 577         list_del_init(&id_priv->id_list_entry);
 578         if (list_empty(&data->id_list)) {
 579                 rb_erase(&data->rb_node, &id_table);
 580                 kfree(data);
 581         }
 582 out:
 583         spin_unlock_irqrestore(&id_table_lock, flags);
 584 }
 585
 586 static void _cma_attach_to_dev(struct rdma_id_private *id_priv,
 587                                struct cma_device *cma_dev)
 588 {
 589         cma_dev_get(cma_dev);
 590         id_priv->cma_dev = cma_dev;
 591         id_priv->id.device = cma_dev->device;
 592         id_priv->id.route.addr.dev_addr.transport =
 593                 rdma_node_get_transport(cma_dev->device->node_type);
 594         list_add_tail(&id_priv->device_item, &cma_dev->id_list);
 595
 596         trace_cm_id_attach(id_priv, cma_dev->device);
 597 }
 598
 599 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 600                               struct cma_device *cma_dev)
 601 {
 602         _cma_attach_to_dev(id_priv, cma_dev);
 603         id_priv->gid_type =
 604                 cma_dev->default_gid_type[id_priv->id.port_num -
 605                                           rdma_start_port(cma_dev->device)];
 606 }
 607
 608 static void cma_release_dev(struct rdma_id_private *id_priv)
 609 {
 610         mutex_lock(&lock);
 611         list_del_init(&id_priv->device_item);
 612         cma_dev_put(id_priv->cma_dev);
 613         id_priv->cma_dev = NULL;
 614         id_priv->id.device = NULL;
 615         if (id_priv->id.route.addr.dev_addr.sgid_attr) {
 616                 rdma_put_gid_attr(id_priv->id.route.addr.dev_addr.sgid_attr);
 617                 id_priv->id.route.addr.dev_addr.sgid_attr = NULL;
 618         }
 619         mutex_unlock(&lock);
 620 }
 621
 622 static inline unsigned short cma_family(struct rdma_id_private *id_priv)
 623 {
 624         return id_priv->id.route.addr.src_addr.ss_family;
 625 }
 626
 627 static int cma_set_default_qkey(struct rdma_id_private *id_priv)
 628 {
 629         struct ib_sa_mcmember_rec rec;
 630         int ret = 0;
 631
 632         switch (id_priv->id.ps) {
 633         case RDMA_PS_UDP:
 634         case RDMA_PS_IB:
 635                 id_priv->qkey = RDMA_UDP_QKEY;
 636                 break;
 637         case RDMA_PS_IPOIB:
 638                 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
 639                 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
 640                                              id_priv->id.port_num, &rec.mgid,
 641                                              &rec);
 642                 if (!ret)
 643                         id_priv->qkey = be32_to_cpu(rec.qkey);
 644                 break;
 645         default:
 646                 break;
 647         }
 648         return ret;
 649 }
 650
 651 static int cma_set_qkey(struct rdma_id_private *id_priv, u32 qkey)
 652 {
 653         if (!qkey ||
 654             (id_priv->qkey && (id_priv->qkey != qkey)))
 655                 return -EINVAL;
 656
 657         id_priv->qkey = qkey;
 658         return 0;
 659 }
 660
 661 static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
 662 {
 663         dev_addr->dev_type = ARPHRD_INFINIBAND;
 664         rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
 665         ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
 666 }
 667
 668 static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 669 {
 670         int ret;
 671
 672         if (addr->sa_family != AF_IB) {
 673                 ret = rdma_translate_ip(addr, dev_addr);
 674         } else {
 675                 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
 676                 ret = 0;
 677         }
 678
 679         return ret;
 680 }
 681
 682 static const struct ib_gid_attr *
 683 cma_validate_port(struct ib_device *device, u32 port,
 684                   enum ib_gid_type gid_type,
 685                   union ib_gid *gid,
 686                   struct rdma_id_private *id_priv)
 687 {
 688         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 689         int bound_if_index = dev_addr->bound_dev_if;
 690         const struct ib_gid_attr *sgid_attr;
 691         int dev_type = dev_addr->dev_type;
 692         struct net_device *ndev = NULL;
 693
 694         if (!rdma_dev_access_netns(device, id_priv->id.route.addr.dev_addr.net))
 695                 return ERR_PTR(-ENODEV);
 696
 697         if ((dev_type == ARPHRD_INFINIBAND) && !rdma_protocol_ib(device, port))
 698                 return ERR_PTR(-ENODEV);
 699
 700         if ((dev_type != ARPHRD_INFINIBAND) && rdma_protocol_ib(device, port))
 701                 return ERR_PTR(-ENODEV);
 702
 703         if (dev_type == ARPHRD_ETHER && rdma_protocol_roce(device, port)) {
 704                 ndev = dev_get_by_index(dev_addr->net, bound_if_index);
 705                 if (!ndev)
 706                         return ERR_PTR(-ENODEV);
 707         } else {
 708                 gid_type = IB_GID_TYPE_IB;
 709         }
 710
 711         sgid_attr = rdma_find_gid_by_port(device, gid, gid_type, port, ndev);
 712         dev_put(ndev);
 713         return sgid_attr;
 714 }
 715
 716 static void cma_bind_sgid_attr(struct rdma_id_private *id_priv,
 717                                const struct ib_gid_attr *sgid_attr)
 718 {
 719         WARN_ON(id_priv->id.route.addr.dev_addr.sgid_attr);
 720         id_priv->id.route.addr.dev_addr.sgid_attr = sgid_attr;
 721 }
 722
 723 /**
 724  * cma_acquire_dev_by_src_ip - Acquire cma device, port, gid attribute
 725  * based on source ip address.
 726  * @id_priv:    cm_id which should be bound to cma device
 727  *
 728  * cma_acquire_dev_by_src_ip() binds cm id to cma device, port and GID attribute
 729  * based on source IP address. It returns 0 on success or error code otherwise.
 730  * It is applicable to active and passive side cm_id.
 731  */
 732 static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)
 733 {
 734         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 735         const struct ib_gid_attr *sgid_attr;
 736         union ib_gid gid, iboe_gid, *gidp;
 737         struct cma_device *cma_dev;
 738         enum ib_gid_type gid_type;
 739         int ret = -ENODEV;
 740         u32 port;
 741
 742         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 743             id_priv->id.ps == RDMA_PS_IPOIB)
 744                 return -EINVAL;
 745
 746         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 747                     &iboe_gid);
 748
 749         memcpy(&gid, dev_addr->src_dev_addr +
 750                rdma_addr_gid_offset(dev_addr), sizeof(gid));
 751
 752         mutex_lock(&lock);
 753         list_for_each_entry(cma_dev, &dev_list, list) {
 754                 rdma_for_each_port (cma_dev->device, port) {
 755                         gidp = rdma_protocol_roce(cma_dev->device, port) ?
 756                                &iboe_gid : &gid;
 757                         gid_type = cma_dev->default_gid_type[port - 1];
 758                         sgid_attr = cma_validate_port(cma_dev->device, port,
 759                                                       gid_type, gidp, id_priv);
 760                         if (!IS_ERR(sgid_attr)) {
 761                                 id_priv->id.port_num = port;
 762                                 cma_bind_sgid_attr(id_priv, sgid_attr);
 763                                 cma_attach_to_dev(id_priv, cma_dev);
 764                                 ret = 0;
 765                                 goto out;
 766                         }
 767                 }
 768         }
 769 out:
 770         mutex_unlock(&lock);
 771         return ret;
 772 }
 773
 774 /**
 775  * cma_ib_acquire_dev - Acquire cma device, port and SGID attribute
 776  * @id_priv:            cm id to bind to cma device
 777  * @listen_id_priv:     listener cm id to match against
 778  * @req:                Pointer to req structure containaining incoming
 779  *                      request information
 780  * cma_ib_acquire_dev() acquires cma device, port and SGID attribute when
 781  * rdma device matches for listen_id and incoming request. It also verifies
 782  * that a GID table entry is present for the source address.
 783  * Returns 0 on success, or returns error code otherwise.
 784  */
 785 static int cma_ib_acquire_dev(struct rdma_id_private *id_priv,
 786                               const struct rdma_id_private *listen_id_priv,
 787                               struct cma_req_info *req)
 788 {
 789         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 790         const struct ib_gid_attr *sgid_attr;
 791         enum ib_gid_type gid_type;
 792         union ib_gid gid;
 793
 794         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 795             id_priv->id.ps == RDMA_PS_IPOIB)
 796                 return -EINVAL;
 797
 798         if (rdma_protocol_roce(req->device, req->port))
 799                 rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
 800                             &gid);
 801         else
 802                 memcpy(&gid, dev_addr->src_dev_addr +
 803                        rdma_addr_gid_offset(dev_addr), sizeof(gid));
 804
 805         gid_type = listen_id_priv->cma_dev->default_gid_type[req->port - 1];
 806         sgid_attr = cma_validate_port(req->device, req->port,
 807                                       gid_type, &gid, id_priv);
 808         if (IS_ERR(sgid_attr))
 809                 return PTR_ERR(sgid_attr);
 810
 811         id_priv->id.port_num = req->port;
 812         cma_bind_sgid_attr(id_priv, sgid_attr);
 813         /* Need to acquire lock to protect against reader
 814          * of cma_dev->id_list such as cma_netdev_callback() and
 815          * cma_process_remove().
 816          */
 817         mutex_lock(&lock);
 818         cma_attach_to_dev(id_priv, listen_id_priv->cma_dev);
 819         mutex_unlock(&lock);
 820         rdma_restrack_add(&id_priv->res);
 821         return 0;
 822 }
 823
 824 static int cma_iw_acquire_dev(struct rdma_id_private *id_priv,
 825                               const struct rdma_id_private *listen_id_priv)
 826 {
 827         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
 828         const struct ib_gid_attr *sgid_attr;
 829         struct cma_device *cma_dev;
 830         enum ib_gid_type gid_type;
 831         int ret = -ENODEV;
 832         union ib_gid gid;
 833         u32 port;
 834
 835         if (dev_addr->dev_type != ARPHRD_INFINIBAND &&
 836             id_priv->id.ps == RDMA_PS_IPOIB)
 837                 return -EINVAL;
 838
 839         memcpy(&gid, dev_addr->src_dev_addr +
 840                rdma_addr_gid_offset(dev_addr), sizeof(gid));
 841
 842         mutex_lock(&lock);
 843
 844         cma_dev = listen_id_priv->cma_dev;
 845         port = listen_id_priv->id.port_num;
 846         gid_type = listen_id_priv->gid_type;
 847         sgid_attr = cma_validate_port(cma_dev->device, port,
 848                                       gid_type, &gid, id_priv);
 849         if (!IS_ERR(sgid_attr)) {
 850                 id_priv->id.port_num = port;
 851                 cma_bind_sgid_attr(id_priv, sgid_attr);
 852                 ret = 0;
 853                 goto out;
 854         }
 855
 856         list_for_each_entry(cma_dev, &dev_list, list) {
 857                 rdma_for_each_port (cma_dev->device, port) {
 858                         if (listen_id_priv->cma_dev == cma_dev &&
 859                             listen_id_priv->id.port_num == port)
 860                                 continue;
 861
 862                         gid_type = cma_dev->default_gid_type[port - 1];
 863                         sgid_attr = cma_validate_port(cma_dev->device, port,
 864                                                       gid_type, &gid, id_priv);
 865                         if (!IS_ERR(sgid_attr)) {
 866                                 id_priv->id.port_num = port;
 867                                 cma_bind_sgid_attr(id_priv, sgid_attr);
 868                                 ret = 0;
 869                                 goto out;
 870                         }
 871                 }
 872         }
 873
 874 out:
 875         if (!ret) {
 876                 cma_attach_to_dev(id_priv, cma_dev);
 877                 rdma_restrack_add(&id_priv->res);
 878         }
 879
 880         mutex_unlock(&lock);
 881         return ret;
 882 }
 883
 884 /*
 885  * Select the source IB device and address to reach the destination IB address.
 886  */
 887 static int cma_resolve_ib_dev(struct rdma_id_private *id_priv)
 888 {
 889         struct cma_device *cma_dev, *cur_dev;
 890         struct sockaddr_ib *addr;
 891         union ib_gid gid, sgid, *dgid;
 892         unsigned int p;
 893         u16 pkey, index;
 894         enum ib_port_state port_state;
 895         int ret;
 896         int i;
 897
 898         cma_dev = NULL;
 899         addr = (struct sockaddr_ib *) cma_dst_addr(id_priv);
 900         dgid = (union ib_gid *) &addr->sib_addr;
 901         pkey = ntohs(addr->sib_pkey);
 902
 903         mutex_lock(&lock);
 904         list_for_each_entry(cur_dev, &dev_list, list) {
 905                 rdma_for_each_port (cur_dev->device, p) {
 906                         if (!rdma_cap_af_ib(cur_dev->device, p))
 907                                 continue;
 908
 909                         if (ib_find_cached_pkey(cur_dev->device, p, pkey, &index))
 910                                 continue;
 911
 912                         if (ib_get_cached_port_state(cur_dev->device, p, &port_state))
 913                                 continue;
 914
 915                         for (i = 0; i < cur_dev->device->port_data[p].immutable.gid_tbl_len;
 916                              ++i) {
 917                                 ret = rdma_query_gid(cur_dev->device, p, i,
 918                                                      &gid);
 919                                 if (ret)
 920                                         continue;
 921
 922                                 if (!memcmp(&gid, dgid, sizeof(gid))) {
 923                                         cma_dev = cur_dev;
 924                                         sgid = gid;
 925                                         id_priv->id.port_num = p;
 926                                         goto found;
 927                                 }
 928
 929                                 if (!cma_dev && (gid.global.subnet_prefix ==
 930                                     dgid->global.subnet_prefix) &&
 931                                     port_state == IB_PORT_ACTIVE) {
 932                                         cma_dev = cur_dev;
 933                                         sgid = gid;
 934                                         id_priv->id.port_num = p;
 935                                         goto found;
 936                                 }
 937                         }
 938                 }
 939         }
 940         mutex_unlock(&lock);
 941         return -ENODEV;
 942
 943 found:
 944         cma_attach_to_dev(id_priv, cma_dev);
 945         rdma_restrack_add(&id_priv->res);
 946         mutex_unlock(&lock);
 947         addr = (struct sockaddr_ib *)cma_src_addr(id_priv);
 948         memcpy(&addr->sib_addr, &sgid, sizeof(sgid));
 949         cma_translate_ib(addr, &id_priv->id.route.addr.dev_addr);
 950         return 0;
 951 }
 952
 953 static void cma_id_get(struct rdma_id_private *id_priv)
 954 {
 955         refcount_inc(&id_priv->refcount);
 956 }
 957
 958 static void cma_id_put(struct rdma_id_private *id_priv)
 959 {
 960         if (refcount_dec_and_test(&id_priv->refcount))
 961                 complete(&id_priv->comp);
 962 }
 963
 964 static struct rdma_id_private *
 965 __rdma_create_id(struct net *net, rdma_cm_event_handler event_handler,
 966                  void *context, enum rdma_ucm_port_space ps,
 967                  enum ib_qp_type qp_type, const struct rdma_id_private *parent)
 968 {
 969         struct rdma_id_private *id_priv;
 970
 971         id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
 972         if (!id_priv)
 973                 return ERR_PTR(-ENOMEM);
 974
 975         id_priv->state = RDMA_CM_IDLE;
 976         id_priv->id.context = context;
 977         id_priv->id.event_handler = event_handler;
 978         id_priv->id.ps = ps;
 979         id_priv->id.qp_type = qp_type;
 980         id_priv->tos_set = false;
 981         id_priv->timeout_set = false;
 982         id_priv->min_rnr_timer_set = false;
 983         id_priv->gid_type = IB_GID_TYPE_IB;
 984         spin_lock_init(&id_priv->lock);
 985         mutex_init(&id_priv->qp_mutex);
 986         init_completion(&id_priv->comp);
 987         refcount_set(&id_priv->refcount, 1);
 988         mutex_init(&id_priv->handler_mutex);
 989         INIT_LIST_HEAD(&id_priv->device_item);
 990         INIT_LIST_HEAD(&id_priv->id_list_entry);
 991         INIT_LIST_HEAD(&id_priv->listen_list);
 992         INIT_LIST_HEAD(&id_priv->mc_list);
 993         get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
 994         id_priv->id.route.addr.dev_addr.net = get_net(net);
 995         id_priv->seq_num &= 0x00ffffff;
 996
 997         rdma_restrack_new(&id_priv->res, RDMA_RESTRACK_CM_ID);
 998         if (parent)
 999                 rdma_restrack_parent_name(&id_priv->res, &parent->res);
1000
1001         return id_priv;
1002 }
1003
1004 struct rdma_cm_id *
1005 __rdma_create_kernel_id(struct net *net, rdma_cm_event_handler event_handler,
1006                         void *context, enum rdma_ucm_port_space ps,
1007                         enum ib_qp_type qp_type, const char *caller)
1008 {
1009         struct rdma_id_private *ret;
1010
1011         ret = __rdma_create_id(net, event_handler, context, ps, qp_type, NULL);
1012         if (IS_ERR(ret))
1013                 return ERR_CAST(ret);
1014
1015         rdma_restrack_set_name(&ret->res, caller);
1016         return &ret->id;
1017 }
1018 EXPORT_SYMBOL(__rdma_create_kernel_id);
1019
1020 struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler,
1021                                        void *context,
1022                                        enum rdma_ucm_port_space ps,
1023                                        enum ib_qp_type qp_type)
1024 {
1025         struct rdma_id_private *ret;
1026
1027         ret = __rdma_create_id(current->nsproxy->net_ns, event_handler, context,
1028                                ps, qp_type, NULL);
1029         if (IS_ERR(ret))
1030                 return ERR_CAST(ret);
1031
1032         rdma_restrack_set_name(&ret->res, NULL);
1033         return &ret->id;
1034 }
1035 EXPORT_SYMBOL(rdma_create_user_id);
1036
1037 static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
1038 {
1039         struct ib_qp_attr qp_attr;
1040         int qp_attr_mask, ret;
1041
1042         qp_attr.qp_state = IB_QPS_INIT;
1043         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1044         if (ret)
1045                 return ret;
1046
1047         ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
1048         if (ret)
1049                 return ret;
1050
1051         qp_attr.qp_state = IB_QPS_RTR;
1052         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
1053         if (ret)
1054                 return ret;
1055
1056         qp_attr.qp_state = IB_QPS_RTS;
1057         qp_attr.sq_psn = 0;
1058         ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
1059
1060         return ret;
1061 }
1062
1063 static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
1064 {
1065         struct ib_qp_attr qp_attr;
1066         int qp_attr_mask, ret;
1067
1068         qp_attr.qp_state = IB_QPS_INIT;
1069         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1070         if (ret)
1071                 return ret;
1072
1073         return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
1074 }
1075
1076 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
1077                    struct ib_qp_init_attr *qp_init_attr)
1078 {
1079         struct rdma_id_private *id_priv;
1080         struct ib_qp *qp;
1081         int ret;
1082
1083         id_priv = container_of(id, struct rdma_id_private, id);
1084         if (id->device != pd->device) {
1085                 ret = -EINVAL;
1086                 goto out_err;
1087         }
1088
1089         qp_init_attr->port_num = id->port_num;
1090         qp = ib_create_qp(pd, qp_init_attr);
1091         if (IS_ERR(qp)) {
1092                 ret = PTR_ERR(qp);
1093                 goto out_err;
1094         }
1095
1096         if (id->qp_type == IB_QPT_UD)
1097                 ret = cma_init_ud_qp(id_priv, qp);
1098         else
1099                 ret = cma_init_conn_qp(id_priv, qp);
1100         if (ret)
1101                 goto out_destroy;
1102
1103         id->qp = qp;
1104         id_priv->qp_num = qp->qp_num;
1105         id_priv->srq = (qp->srq != NULL);
1106         trace_cm_qp_create(id_priv, pd, qp_init_attr, 0);
1107         return 0;
1108 out_destroy:
1109         ib_destroy_qp(qp);
1110 out_err:
1111         trace_cm_qp_create(id_priv, pd, qp_init_attr, ret);
1112         return ret;
1113 }
1114 EXPORT_SYMBOL(rdma_create_qp);
1115
1116 void rdma_destroy_qp(struct rdma_cm_id *id)
1117 {
1118         struct rdma_id_private *id_priv;
1119
1120         id_priv = container_of(id, struct rdma_id_private, id);
1121         trace_cm_qp_destroy(id_priv);
1122         mutex_lock(&id_priv->qp_mutex);
1123         ib_destroy_qp(id_priv->id.qp);
1124         id_priv->id.qp = NULL;
1125         mutex_unlock(&id_priv->qp_mutex);
1126 }
1127 EXPORT_SYMBOL(rdma_destroy_qp);
1128
1129 static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
1130                              struct rdma_conn_param *conn_param)
1131 {
1132         struct ib_qp_attr qp_attr;
1133         int qp_attr_mask, ret;
1134
1135         mutex_lock(&id_priv->qp_mutex);
1136         if (!id_priv->id.qp) {
1137                 ret = 0;
1138                 goto out;
1139         }
1140
1141         /* Need to update QP attributes from default values. */
1142         qp_attr.qp_state = IB_QPS_INIT;
1143         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1144         if (ret)
1145                 goto out;
1146
1147         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1148         if (ret)
1149                 goto out;
1150
1151         qp_attr.qp_state = IB_QPS_RTR;
1152         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1153         if (ret)
1154                 goto out;
1155
1156         BUG_ON(id_priv->cma_dev->device != id_priv->id.device);
1157
1158         if (conn_param)
1159                 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
1160         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1161 out:
1162         mutex_unlock(&id_priv->qp_mutex);
1163         return ret;
1164 }
1165
1166 static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
1167                              struct rdma_conn_param *conn_param)
1168 {
1169         struct ib_qp_attr qp_attr;
1170         int qp_attr_mask, ret;
1171
1172         mutex_lock(&id_priv->qp_mutex);
1173         if (!id_priv->id.qp) {
1174                 ret = 0;
1175                 goto out;
1176         }
1177
1178         qp_attr.qp_state = IB_QPS_RTS;
1179         ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
1180         if (ret)
1181                 goto out;
1182
1183         if (conn_param)
1184                 qp_attr.max_rd_atomic = conn_param->initiator_depth;
1185         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
1186 out:
1187         mutex_unlock(&id_priv->qp_mutex);
1188         return ret;
1189 }
1190
1191 static int cma_modify_qp_err(struct rdma_id_private *id_priv)
1192 {
1193         struct ib_qp_attr qp_attr;
1194         int ret;
1195
1196         mutex_lock(&id_priv->qp_mutex);
1197         if (!id_priv->id.qp) {
1198                 ret = 0;
1199                 goto out;
1200         }
1201
1202         qp_attr.qp_state = IB_QPS_ERR;
1203         ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
1204 out:
1205         mutex_unlock(&id_priv->qp_mutex);
1206         return ret;
1207 }
1208
1209 static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
1210                                struct ib_qp_attr *qp_attr, int *qp_attr_mask)
1211 {
1212         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
1213         int ret;
1214         u16 pkey;
1215
1216         if (rdma_cap_eth_ah(id_priv->id.device, id_priv->id.port_num))
1217                 pkey = 0xffff;
1218         else
1219                 pkey = ib_addr_get_pkey(dev_addr);
1220
1221         ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
1222                                   pkey, &qp_attr->pkey_index);
1223         if (ret)
1224                 return ret;
1225
1226         qp_attr->port_num = id_priv->id.port_num;
1227         *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
1228
1229         if (id_priv->id.qp_type == IB_QPT_UD) {
1230                 ret = cma_set_default_qkey(id_priv);
1231                 if (ret)
1232                         return ret;
1233
1234                 qp_attr->qkey = id_priv->qkey;
1235                 *qp_attr_mask |= IB_QP_QKEY;
1236         } else {
1237                 qp_attr->qp_access_flags = 0;
1238                 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
1239         }
1240         return 0;
1241 }
1242
1243 int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
1244                        int *qp_attr_mask)
1245 {
1246         struct rdma_id_private *id_priv;
1247         int ret = 0;
1248
1249         id_priv = container_of(id, struct rdma_id_private, id);
1250         if (rdma_cap_ib_cm(id->device, id->port_num)) {
1251                 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
1252                         ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
1253                 else
1254                         ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
1255                                                  qp_attr_mask);
1256
1257                 if (qp_attr->qp_state == IB_QPS_RTR)
1258                         qp_attr->rq_psn = id_priv->seq_num;
1259         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
1260                 if (!id_priv->cm_id.iw) {
1261                         qp_attr->qp_access_flags = 0;
1262                         *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
1263                 } else
1264                         ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
1265                                                  qp_attr_mask);
1266                 qp_attr->port_num = id_priv->id.port_num;
1267                 *qp_attr_mask |= IB_QP_PORT;
1268         } else {
1269                 ret = -ENOSYS;
1270         }
1271
1272         if ((*qp_attr_mask & IB_QP_TIMEOUT) && id_priv->timeout_set)
1273                 qp_attr->timeout = id_priv->timeout;
1274
1275         if ((*qp_attr_mask & IB_QP_MIN_RNR_TIMER) && id_priv->min_rnr_timer_set)
1276                 qp_attr->min_rnr_timer = id_priv->min_rnr_timer;
1277
1278         return ret;
1279 }
1280 EXPORT_SYMBOL(rdma_init_qp_attr);
1281
1282 static inline bool cma_zero_addr(const struct sockaddr *addr)
1283 {
1284         switch (addr->sa_family) {
1285         case AF_INET:
1286                 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
1287         case AF_INET6:
1288                 return ipv6_addr_any(&((struct sockaddr_in6 *)addr)->sin6_addr);
1289         case AF_IB:
1290                 return ib_addr_any(&((struct sockaddr_ib *)addr)->sib_addr);
1291         default:
1292                 return false;
1293         }
1294 }
1295
1296 static inline bool cma_loopback_addr(const struct sockaddr *addr)
1297 {
1298         switch (addr->sa_family) {
1299         case AF_INET:
1300                 return ipv4_is_loopback(
1301                         ((struct sockaddr_in *)addr)->sin_addr.s_addr);
1302         case AF_INET6:
1303                 return ipv6_addr_loopback(
1304                         &((struct sockaddr_in6 *)addr)->sin6_addr);
1305         case AF_IB:
1306                 return ib_addr_loopback(
1307                         &((struct sockaddr_ib *)addr)->sib_addr);
1308         default:
1309                 return false;
1310         }
1311 }
1312
1313 static inline bool cma_any_addr(const struct sockaddr *addr)
1314 {
1315         return cma_zero_addr(addr) || cma_loopback_addr(addr);
1316 }
1317
1318 static int cma_addr_cmp(const struct sockaddr *src, const struct sockaddr *dst)
1319 {
1320         if (src->sa_family != dst->sa_family)
1321                 return -1;
1322
1323         switch (src->sa_family) {
1324         case AF_INET:
1325                 return ((struct sockaddr_in *)src)->sin_addr.s_addr !=
1326                        ((struct sockaddr_in *)dst)->sin_addr.s_addr;
1327         case AF_INET6: {
1328                 struct sockaddr_in6 *src_addr6 = (struct sockaddr_in6 *)src;
1329                 struct sockaddr_in6 *dst_addr6 = (struct sockaddr_in6 *)dst;
1330                 bool link_local;
1331
1332                 if (ipv6_addr_cmp(&src_addr6->sin6_addr,
1333                                           &dst_addr6->sin6_addr))
1334                         return 1;
1335                 link_local = ipv6_addr_type(&dst_addr6->sin6_addr) &
1336                              IPV6_ADDR_LINKLOCAL;
1337                 /* Link local must match their scope_ids */
1338                 return link_local ? (src_addr6->sin6_scope_id !=
1339                                      dst_addr6->sin6_scope_id) :
1340                                     0;
1341         }
1342
1343         default:
1344                 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
1345                                    &((struct sockaddr_ib *) dst)->sib_addr);
1346         }
1347 }
1348
1349 static __be16 cma_port(const struct sockaddr *addr)
1350 {
1351         struct sockaddr_ib *sib;
1352
1353         switch (addr->sa_family) {
1354         case AF_INET:
1355                 return ((struct sockaddr_in *) addr)->sin_port;
1356         case AF_INET6:
1357                 return ((struct sockaddr_in6 *) addr)->sin6_port;
1358         case AF_IB:
1359                 sib = (struct sockaddr_ib *) addr;
1360                 return htons((u16) (be64_to_cpu(sib->sib_sid) &
1361                                     be64_to_cpu(sib->sib_sid_mask)));
1362         default:
1363                 return 0;
1364         }
1365 }
1366
1367 static inline int cma_any_port(const struct sockaddr *addr)
1368 {
1369         return !cma_port(addr);
1370 }
1371
1372 static void cma_save_ib_info(struct sockaddr *src_addr,
1373                              struct sockaddr *dst_addr,
1374                              const struct rdma_cm_id *listen_id,
1375                              const struct sa_path_rec *path)
1376 {
1377         struct sockaddr_ib *listen_ib, *ib;
1378
1379         listen_ib = (struct sockaddr_ib *) &listen_id->route.addr.src_addr;
1380         if (src_addr) {
1381                 ib = (struct sockaddr_ib *)src_addr;
1382                 ib->sib_family = AF_IB;
1383                 if (path) {
1384                         ib->sib_pkey = path->pkey;
1385                         ib->sib_flowinfo = path->flow_label;
1386                         memcpy(&ib->sib_addr, &path->sgid, 16);
1387                         ib->sib_sid = path->service_id;
1388                         ib->sib_scope_id = 0;
1389                 } else {
1390                         ib->sib_pkey = listen_ib->sib_pkey;
1391                         ib->sib_flowinfo = listen_ib->sib_flowinfo;
1392                         ib->sib_addr = listen_ib->sib_addr;
1393                         ib->sib_sid = listen_ib->sib_sid;
1394                         ib->sib_scope_id = listen_ib->sib_scope_id;
1395                 }
1396                 ib->sib_sid_mask = cpu_to_be64(0xffffffffffffffffULL);
1397         }
1398         if (dst_addr) {
1399                 ib = (struct sockaddr_ib *)dst_addr;
1400                 ib->sib_family = AF_IB;
1401                 if (path) {
1402                         ib->sib_pkey = path->pkey;
1403                         ib->sib_flowinfo = path->flow_label;
1404                         memcpy(&ib->sib_addr, &path->dgid, 16);
1405                 }
1406         }
1407 }
1408
1409 static void cma_save_ip4_info(struct sockaddr_in *src_addr,
1410                               struct sockaddr_in *dst_addr,
1411                               struct cma_hdr *hdr,
1412                               __be16 local_port)
1413 {
1414         if (src_addr) {
1415                 *src_addr = (struct sockaddr_in) {
1416                         .sin_family = AF_INET,
1417                         .sin_addr.s_addr = hdr->dst_addr.ip4.addr,
1418                         .sin_port = local_port,
1419                 };
1420         }
1421
1422         if (dst_addr) {
1423                 *dst_addr = (struct sockaddr_in) {
1424                         .sin_family = AF_INET,
1425                         .sin_addr.s_addr = hdr->src_addr.ip4.addr,
1426                         .sin_port = hdr->port,
1427                 };
1428         }
1429 }
1430
1431 static void cma_save_ip6_info(struct sockaddr_in6 *src_addr,
1432                               struct sockaddr_in6 *dst_addr,
1433                               struct cma_hdr *hdr,
1434                               __be16 local_port)
1435 {
1436         if (src_addr) {
1437                 *src_addr = (struct sockaddr_in6) {
1438                         .sin6_family = AF_INET6,
1439                         .sin6_addr = hdr->dst_addr.ip6,
1440                         .sin6_port = local_port,
1441                 };
1442         }
1443
1444         if (dst_addr) {
1445                 *dst_addr = (struct sockaddr_in6) {
1446                         .sin6_family = AF_INET6,
1447                         .sin6_addr = hdr->src_addr.ip6,
1448                         .sin6_port = hdr->port,
1449                 };
1450         }
1451 }
1452
1453 static u16 cma_port_from_service_id(__be64 service_id)
1454 {
1455         return (u16)be64_to_cpu(service_id);
1456 }
1457
1458 static int cma_save_ip_info(struct sockaddr *src_addr,
1459                             struct sockaddr *dst_addr,
1460                             const struct ib_cm_event *ib_event,
1461                             __be64 service_id)
1462 {
1463         struct cma_hdr *hdr;
1464         __be16 port;
1465
1466         hdr = ib_event->private_data;
1467         if (hdr->cma_version != CMA_VERSION)
1468                 return -EINVAL;
1469
1470         port = htons(cma_port_from_service_id(service_id));
1471
1472         switch (cma_get_ip_ver(hdr)) {
1473         case 4:
1474                 cma_save_ip4_info((struct sockaddr_in *)src_addr,
1475                                   (struct sockaddr_in *)dst_addr, hdr, port);
1476                 break;
1477         case 6:
1478                 cma_save_ip6_info((struct sockaddr_in6 *)src_addr,
1479                                   (struct sockaddr_in6 *)dst_addr, hdr, port);
1480                 break;
1481         default:
1482                 return -EAFNOSUPPORT;
1483         }
1484
1485         return 0;
1486 }
1487
1488 static int cma_save_net_info(struct sockaddr *src_addr,
1489                              struct sockaddr *dst_addr,
1490                              const struct rdma_cm_id *listen_id,
1491                              const struct ib_cm_event *ib_event,
1492                              sa_family_t sa_family, __be64 service_id)
1493 {
1494         if (sa_family == AF_IB) {
1495                 if (ib_event->event == IB_CM_REQ_RECEIVED)
1496                         cma_save_ib_info(src_addr, dst_addr, listen_id,
1497                                          ib_event->param.req_rcvd.primary_path);
1498                 else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1499                         cma_save_ib_info(src_addr, dst_addr, listen_id, NULL);
1500                 return 0;
1501         }
1502
1503         return cma_save_ip_info(src_addr, dst_addr, ib_event, service_id);
1504 }
1505
1506 static int cma_save_req_info(const struct ib_cm_event *ib_event,
1507                              struct cma_req_info *req)
1508 {
1509         const struct ib_cm_req_event_param *req_param =
1510                 &ib_event->param.req_rcvd;
1511         const struct ib_cm_sidr_req_event_param *sidr_param =
1512                 &ib_event->param.sidr_req_rcvd;
1513
1514         switch (ib_event->event) {
1515         case IB_CM_REQ_RECEIVED:
1516                 req->device     = req_param->listen_id->device;
1517                 req->port       = req_param->port;
1518                 memcpy(&req->local_gid, &req_param->primary_path->sgid,
1519                        sizeof(req->local_gid));
1520                 req->has_gid    = true;
1521                 req->service_id = req_param->primary_path->service_id;
1522                 req->pkey       = be16_to_cpu(req_param->primary_path->pkey);
1523                 if (req->pkey != req_param->bth_pkey)
1524                         pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and primary path P_Key (0x%x)\n"
1525                                             "RDMA CMA: in the future this may cause the request to be dropped\n",
1526                                             req_param->bth_pkey, req->pkey);
1527                 break;
1528         case IB_CM_SIDR_REQ_RECEIVED:
1529                 req->device     = sidr_param->listen_id->device;
1530                 req->port       = sidr_param->port;
1531                 req->has_gid    = false;
1532                 req->service_id = sidr_param->service_id;
1533                 req->pkey       = sidr_param->pkey;
1534                 if (req->pkey != sidr_param->bth_pkey)
1535                         pr_warn_ratelimited("RDMA CMA: got different BTH P_Key (0x%x) and SIDR request payload P_Key (0x%x)\n"
1536                                             "RDMA CMA: in the future this may cause the request to be dropped\n",
1537                                             sidr_param->bth_pkey, req->pkey);
1538                 break;
1539         default:
1540                 return -EINVAL;
1541         }
1542
1543         return 0;
1544 }
1545
1546 static bool validate_ipv4_net_dev(struct net_device *net_dev,
1547                                   const struct sockaddr_in *dst_addr,
1548                                   const struct sockaddr_in *src_addr)
1549 {
1550         __be32 daddr = dst_addr->sin_addr.s_addr,
1551                saddr = src_addr->sin_addr.s_addr;
1552         struct fib_result res;
1553         struct flowi4 fl4;
1554         int err;
1555         bool ret;
1556
1557         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1558             ipv4_is_lbcast(daddr) || ipv4_is_zeronet(saddr) ||
1559             ipv4_is_zeronet(daddr) || ipv4_is_loopback(daddr) ||
1560             ipv4_is_loopback(saddr))
1561                 return false;
1562
1563         memset(&fl4, 0, sizeof(fl4));
1564         fl4.flowi4_oif = net_dev->ifindex;
1565         fl4.daddr = daddr;
1566         fl4.saddr = saddr;
1567
1568         rcu_read_lock();
1569         err = fib_lookup(dev_net(net_dev), &fl4, &res, 0);
1570         ret = err == 0 && FIB_RES_DEV(res) == net_dev;
1571         rcu_read_unlock();
1572
1573         return ret;
1574 }
1575
1576 static bool validate_ipv6_net_dev(struct net_device *net_dev,
1577                                   const struct sockaddr_in6 *dst_addr,
1578                                   const struct sockaddr_in6 *src_addr)
1579 {
1580 #if IS_ENABLED(CONFIG_IPV6)
1581         const int strict = ipv6_addr_type(&dst_addr->sin6_addr) &
1582                            IPV6_ADDR_LINKLOCAL;
1583         struct rt6_info *rt = rt6_lookup(dev_net(net_dev), &dst_addr->sin6_addr,
1584                                          &src_addr->sin6_addr, net_dev->ifindex,
1585                                          NULL, strict);
1586         bool ret;
1587
1588         if (!rt)
1589                 return false;
1590
1591         ret = rt->rt6i_idev->dev == net_dev;
1592         ip6_rt_put(rt);
1593
1594         return ret;
1595 #else
1596         return false;
1597 #endif
1598 }
1599
1600 static bool validate_net_dev(struct net_device *net_dev,
1601                              const struct sockaddr *daddr,
1602                              const struct sockaddr *saddr)
1603 {
1604         const struct sockaddr_in *daddr4 = (const struct sockaddr_in *)daddr;
1605         const struct sockaddr_in *saddr4 = (const struct sockaddr_in *)saddr;
1606         const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1607         const struct sockaddr_in6 *saddr6 = (const struct sockaddr_in6 *)saddr;
1608
1609         switch (daddr->sa_family) {
1610         case AF_INET:
1611                 return saddr->sa_family == AF_INET &&
1612                        validate_ipv4_net_dev(net_dev, daddr4, saddr4);
1613
1614         case AF_INET6:
1615                 return saddr->sa_family == AF_INET6 &&
1616                        validate_ipv6_net_dev(net_dev, daddr6, saddr6);
1617
1618         default:
1619                 return false;
1620         }
1621 }
1622
1623 static struct net_device *
1624 roce_get_net_dev_by_cm_event(const struct ib_cm_event *ib_event)
1625 {
1626         const struct ib_gid_attr *sgid_attr = NULL;
1627         struct net_device *ndev;
1628
1629         if (ib_event->event == IB_CM_REQ_RECEIVED)
1630                 sgid_attr = ib_event->param.req_rcvd.ppath_sgid_attr;
1631         else if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED)
1632                 sgid_attr = ib_event->param.sidr_req_rcvd.sgid_attr;
1633
1634         if (!sgid_attr)
1635                 return NULL;
1636
1637         rcu_read_lock();
1638         ndev = rdma_read_gid_attr_ndev_rcu(sgid_attr);
1639         if (IS_ERR(ndev))
1640                 ndev = NULL;
1641         else
1642                 dev_hold(ndev);
1643         rcu_read_unlock();
1644         return ndev;
1645 }
1646
1647 static struct net_device *cma_get_net_dev(const struct ib_cm_event *ib_event,
1648                                           struct cma_req_info *req)
1649 {
1650         struct sockaddr *listen_addr =
1651                         (struct sockaddr *)&req->listen_addr_storage;
1652         struct sockaddr *src_addr = (struct sockaddr *)&req->src_addr_storage;
1653         struct net_device *net_dev;
1654         const union ib_gid *gid = req->has_gid ? &req->local_gid : NULL;
1655         int err;
1656
1657         err = cma_save_ip_info(listen_addr, src_addr, ib_event,
1658                                req->service_id);
1659         if (err)
1660                 return ERR_PTR(err);
1661
1662         if (rdma_protocol_roce(req->device, req->port))
1663                 net_dev = roce_get_net_dev_by_cm_event(ib_event);
1664         else
1665                 net_dev = ib_get_net_dev_by_params(req->device, req->port,
1666                                                    req->pkey,
1667                                                    gid, listen_addr);
1668         if (!net_dev)
1669                 return ERR_PTR(-ENODEV);
1670
1671         return net_dev;
1672 }
1673
1674 static enum rdma_ucm_port_space rdma_ps_from_service_id(__be64 service_id)
1675 {
1676         return (be64_to_cpu(service_id) >> 16) & 0xffff;
1677 }
1678
1679 static bool cma_match_private_data(struct rdma_id_private *id_priv,
1680                                    const struct cma_hdr *hdr)
1681 {
1682         struct sockaddr *addr = cma_src_addr(id_priv);
1683         __be32 ip4_addr;
1684         struct in6_addr ip6_addr;
1685
1686         if (cma_any_addr(addr) && !id_priv->afonly)
1687                 return true;
1688
1689         switch (addr->sa_family) {
1690         case AF_INET:
1691                 ip4_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
1692                 if (cma_get_ip_ver(hdr) != 4)
1693                         return false;
1694                 if (!cma_any_addr(addr) &&
1695                     hdr->dst_addr.ip4.addr != ip4_addr)
1696                         return false;
1697                 break;
1698         case AF_INET6:
1699                 ip6_addr = ((struct sockaddr_in6 *)addr)->sin6_addr;
1700                 if (cma_get_ip_ver(hdr) != 6)
1701                         return false;
1702                 if (!cma_any_addr(addr) &&
1703                     memcmp(&hdr->dst_addr.ip6, &ip6_addr, sizeof(ip6_addr)))
1704                         return false;
1705                 break;
1706         case AF_IB:
1707                 return true;
1708         default:
1709                 return false;
1710         }
1711
1712         return true;
1713 }
1714
1715 static bool cma_protocol_roce(const struct rdma_cm_id *id)
1716 {
1717         struct ib_device *device = id->device;
1718         const u32 port_num = id->port_num ?: rdma_start_port(device);
1719
1720         return rdma_protocol_roce(device, port_num);
1721 }
1722
1723 static bool cma_is_req_ipv6_ll(const struct cma_req_info *req)
1724 {
1725         const struct sockaddr *daddr =
1726                         (const struct sockaddr *)&req->listen_addr_storage;
1727         const struct sockaddr_in6 *daddr6 = (const struct sockaddr_in6 *)daddr;
1728
1729         /* Returns true if the req is for IPv6 link local */
1730         return (daddr->sa_family == AF_INET6 &&
1731                 (ipv6_addr_type(&daddr6->sin6_addr) & IPV6_ADDR_LINKLOCAL));
1732 }
1733
1734 static bool cma_match_net_dev(const struct rdma_cm_id *id,
1735                               const struct net_device *net_dev,
1736                               const struct cma_req_info *req)
1737 {
1738         const struct rdma_addr *addr = &id->route.addr;
1739
1740         if (!net_dev)
1741                 /* This request is an AF_IB request */
1742                 return (!id->port_num || id->port_num == req->port) &&
1743                        (addr->src_addr.ss_family == AF_IB);
1744
1745         /*
1746          * If the request is not for IPv6 link local, allow matching
1747          * request to any netdevice of the one or multiport rdma device.
1748          */
1749         if (!cma_is_req_ipv6_ll(req))
1750                 return true;
1751         /*
1752          * Net namespaces must match, and if the listner is listening
1753          * on a specific netdevice than netdevice must match as well.
1754          */
1755         if (net_eq(dev_net(net_dev), addr->dev_addr.net) &&
1756             (!!addr->dev_addr.bound_dev_if ==
1757              (addr->dev_addr.bound_dev_if == net_dev->ifindex)))
1758                 return true;
1759         else
1760                 return false;
1761 }
1762
1763 static struct rdma_id_private *cma_find_listener(
1764                 const struct rdma_bind_list *bind_list,
1765                 const struct ib_cm_id *cm_id,
1766                 const struct ib_cm_event *ib_event,
1767                 const struct cma_req_info *req,
1768                 const struct net_device *net_dev)
1769 {
1770         struct rdma_id_private *id_priv, *id_priv_dev;
1771
1772         lockdep_assert_held(&lock);
1773
1774         if (!bind_list)
1775                 return ERR_PTR(-EINVAL);
1776
1777         hlist_for_each_entry(id_priv, &bind_list->owners, node) {
1778                 if (cma_match_private_data(id_priv, ib_event->private_data)) {
1779                         if (id_priv->id.device == cm_id->device &&
1780                             cma_match_net_dev(&id_priv->id, net_dev, req))
1781                                 return id_priv;
1782                         list_for_each_entry(id_priv_dev,
1783                                             &id_priv->listen_list,
1784                                             listen_item) {
1785                                 if (id_priv_dev->id.device == cm_id->device &&
1786                                     cma_match_net_dev(&id_priv_dev->id,
1787                                                       net_dev, req))
1788                                         return id_priv_dev;
1789                         }
1790                 }
1791         }
1792
1793         return ERR_PTR(-EINVAL);
1794 }
1795
1796 static struct rdma_id_private *
1797 cma_ib_id_from_event(struct ib_cm_id *cm_id,
1798                      const struct ib_cm_event *ib_event,
1799                      struct cma_req_info *req,
1800                      struct net_device **net_dev)
1801 {
1802         struct rdma_bind_list *bind_list;
1803         struct rdma_id_private *id_priv;
1804         int err;
1805
1806         err = cma_save_req_info(ib_event, req);
1807         if (err)
1808                 return ERR_PTR(err);
1809
1810         *net_dev = cma_get_net_dev(ib_event, req);
1811         if (IS_ERR(*net_dev)) {
1812                 if (PTR_ERR(*net_dev) == -EAFNOSUPPORT) {
1813                         /* Assuming the protocol is AF_IB */
1814                         *net_dev = NULL;
1815                 } else {
1816                         return ERR_CAST(*net_dev);
1817                 }
1818         }
1819
1820         mutex_lock(&lock);
1821         /*
1822          * Net namespace might be getting deleted while route lookup,
1823          * cm_id lookup is in progress. Therefore, perform netdevice
1824          * validation, cm_id lookup under rcu lock.
1825          * RCU lock along with netdevice state check, synchronizes with
1826          * netdevice migrating to different net namespace and also avoids
1827          * case where net namespace doesn't get deleted while lookup is in
1828          * progress.
1829          * If the device state is not IFF_UP, its properties such as ifindex
1830          * and nd_net cannot be trusted to remain valid without rcu lock.
1831          * net/core/dev.c change_net_namespace() ensures to synchronize with
1832          * ongoing operations on net device after device is closed using
1833          * synchronize_net().
1834          */
1835         rcu_read_lock();
1836         if (*net_dev) {
1837                 /*
1838                  * If netdevice is down, it is likely that it is administratively
1839                  * down or it might be migrating to different namespace.
1840                  * In that case avoid further processing, as the net namespace
1841                  * or ifindex may change.
1842                  */
1843                 if (((*net_dev)->flags & IFF_UP) == 0) {
1844                         id_priv = ERR_PTR(-EHOSTUNREACH);
1845                         goto err;
1846                 }
1847
1848                 if (!validate_net_dev(*net_dev,
1849                                  (struct sockaddr *)&req->src_addr_storage,
1850                                  (struct sockaddr *)&req->listen_addr_storage)) {
1851                         id_priv = ERR_PTR(-EHOSTUNREACH);
1852                         goto err;
1853                 }
1854         }
1855
1856         bind_list = cma_ps_find(*net_dev ? dev_net(*net_dev) : &init_net,
1857                                 rdma_ps_from_service_id(req->service_id),
1858                                 cma_port_from_service_id(req->service_id));
1859         id_priv = cma_find_listener(bind_list, cm_id, ib_event, req, *net_dev);
1860 err:
1861         rcu_read_unlock();
1862         mutex_unlock(&lock);
1863         if (IS_ERR(id_priv) && *net_dev) {
1864                 dev_put(*net_dev);
1865                 *net_dev = NULL;
1866         }
1867         return id_priv;
1868 }
1869
1870 static inline u8 cma_user_data_offset(struct rdma_id_private *id_priv)
1871 {
1872         return cma_family(id_priv) == AF_IB ? 0 : sizeof(struct cma_hdr);
1873 }
1874
1875 static void cma_cancel_route(struct rdma_id_private *id_priv)
1876 {
1877         if (rdma_cap_ib_sa(id_priv->id.device, id_priv->id.port_num)) {
1878                 if (id_priv->query)
1879                         ib_sa_cancel_query(id_priv->query_id, id_priv->query);
1880         }
1881 }
1882
1883 static void _cma_cancel_listens(struct rdma_id_private *id_priv)
1884 {
1885         struct rdma_id_private *dev_id_priv;
1886
1887         lockdep_assert_held(&lock);
1888
1889         /*
1890          * Remove from listen_any_list to prevent added devices from spawning
1891          * additional listen requests.
1892          */
1893         list_del_init(&id_priv->listen_any_item);
1894
1895         while (!list_empty(&id_priv->listen_list)) {
1896                 dev_id_priv =
1897                         list_first_entry(&id_priv->listen_list,
1898                                          struct rdma_id_private, listen_item);
1899                 /* sync with device removal to avoid duplicate destruction */
1900                 list_del_init(&dev_id_priv->device_item);
1901                 list_del_init(&dev_id_priv->listen_item);
1902                 mutex_unlock(&lock);
1903
1904                 rdma_destroy_id(&dev_id_priv->id);
1905                 mutex_lock(&lock);
1906         }
1907 }
1908
1909 static void cma_cancel_listens(struct rdma_id_private *id_priv)
1910 {
1911         mutex_lock(&lock);
1912         _cma_cancel_listens(id_priv);
1913         mutex_unlock(&lock);
1914 }
1915
1916 static void cma_cancel_operation(struct rdma_id_private *id_priv,
1917                                  enum rdma_cm_state state)
1918 {
1919         switch (state) {
1920         case RDMA_CM_ADDR_QUERY:
1921                 /*
1922                  * We can avoid doing the rdma_addr_cancel() based on state,
1923                  * only RDMA_CM_ADDR_QUERY has a work that could still execute.
1924                  * Notice that the addr_handler work could still be exiting
1925                  * outside this state, however due to the interaction with the
1926                  * handler_mutex the work is guaranteed not to touch id_priv
1927                  * during exit.
1928                  */
1929                 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
1930                 break;
1931         case RDMA_CM_ROUTE_QUERY:
1932                 cma_cancel_route(id_priv);
1933                 break;
1934         case RDMA_CM_LISTEN:
1935                 if (cma_any_addr(cma_src_addr(id_priv)) && !id_priv->cma_dev)
1936                         cma_cancel_listens(id_priv);
1937                 break;
1938         default:
1939                 break;
1940         }
1941 }
1942
1943 static void cma_release_port(struct rdma_id_private *id_priv)
1944 {
1945         struct rdma_bind_list *bind_list = id_priv->bind_list;
1946         struct net *net = id_priv->id.route.addr.dev_addr.net;
1947
1948         if (!bind_list)
1949                 return;
1950
1951         mutex_lock(&lock);
1952         hlist_del(&id_priv->node);
1953         if (hlist_empty(&bind_list->owners)) {
1954                 cma_ps_remove(net, bind_list->ps, bind_list->port);
1955                 kfree(bind_list);
1956         }
1957         mutex_unlock(&lock);
1958 }
1959
1960 static void destroy_mc(struct rdma_id_private *id_priv,
1961                        struct cma_multicast *mc)
1962 {
1963         bool send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
1964
1965         if (rdma_cap_ib_mcast(id_priv->id.device, id_priv->id.port_num))
1966                 ib_sa_free_multicast(mc->sa_mc);
1967
1968         if (rdma_protocol_roce(id_priv->id.device, id_priv->id.port_num)) {
1969                 struct rdma_dev_addr *dev_addr =
1970                         &id_priv->id.route.addr.dev_addr;
1971                 struct net_device *ndev = NULL;
1972
1973                 if (dev_addr->bound_dev_if)
1974                         ndev = dev_get_by_index(dev_addr->net,
1975                                                 dev_addr->bound_dev_if);
1976                 if (ndev && !send_only) {
1977                         enum ib_gid_type gid_type;
1978                         union ib_gid mgid;
1979
1980                         gid_type = id_priv->cma_dev->default_gid_type
1981                                            [id_priv->id.port_num -
1982                                             rdma_start_port(
1983                                                     id_priv->cma_dev->device)];
1984                         cma_iboe_set_mgid((struct sockaddr *)&mc->addr, &mgid,
1985                                           gid_type);
1986                         cma_igmp_send(ndev, &mgid, false);
1987                 }
1988                 dev_put(ndev);
1989
1990                 cancel_work_sync(&mc->iboe_join.work);
1991         }
1992         kfree(mc);
1993 }
1994
1995 static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
1996 {
1997         struct cma_multicast *mc;
1998
1999         while (!list_empty(&id_priv->mc_list)) {
2000                 mc = list_first_entry(&id_priv->mc_list, struct cma_multicast,
2001                                       list);
2002                 list_del(&mc->list);
2003                 destroy_mc(id_priv, mc);
2004         }
2005 }
2006
2007 static void _destroy_id(struct rdma_id_private *id_priv,
2008                         enum rdma_cm_state state)
2009 {
2010         cma_cancel_operation(id_priv, state);
2011
2012         rdma_restrack_del(&id_priv->res);
2013         cma_remove_id_from_tree(id_priv);
2014         if (id_priv->cma_dev) {
2015                 if (rdma_cap_ib_cm(id_priv->id.device, 1)) {
2016                         if (id_priv->cm_id.ib)
2017                                 ib_destroy_cm_id(id_priv->cm_id.ib);
2018                 } else if (rdma_cap_iw_cm(id_priv->id.device, 1)) {
2019                         if (id_priv->cm_id.iw)
2020                                 iw_destroy_cm_id(id_priv->cm_id.iw);
2021                 }
2022                 cma_leave_mc_groups(id_priv);
2023                 cma_release_dev(id_priv);
2024         }
2025
2026         cma_release_port(id_priv);
2027         cma_id_put(id_priv);
2028         wait_for_completion(&id_priv->comp);
2029
2030         if (id_priv->internal_id)
2031                 cma_id_put(id_priv->id.context);
2032
2033         kfree(id_priv->id.route.path_rec);
2034         kfree(id_priv->id.route.path_rec_inbound);
2035         kfree(id_priv->id.route.path_rec_outbound);
2036
2037         put_net(id_priv->id.route.addr.dev_addr.net);
2038         kfree(id_priv);
2039 }
2040
2041 /*
2042  * destroy an ID from within the handler_mutex. This ensures that no other
2043  * handlers can start running concurrently.
2044  */
2045 static void destroy_id_handler_unlock(struct rdma_id_private *id_priv)
2046         __releases(&idprv->handler_mutex)
2047 {
2048         enum rdma_cm_state state;
2049         unsigned long flags;
2050
2051         trace_cm_id_destroy(id_priv);
2052
2053         /*
2054          * Setting the state to destroyed under the handler mutex provides a
2055          * fence against calling handler callbacks. If this is invoked due to
2056          * the failure of a handler callback then it guarentees that no future
2057          * handlers will be called.
2058          */
2059         lockdep_assert_held(&id_priv->handler_mutex);
2060         spin_lock_irqsave(&id_priv->lock, flags);
2061         state = id_priv->state;
2062         id_priv->state = RDMA_CM_DESTROYING;
2063         spin_unlock_irqrestore(&id_priv->lock, flags);
2064         mutex_unlock(&id_priv->handler_mutex);
2065         _destroy_id(id_priv, state);
2066 }
2067
2068 void rdma_destroy_id(struct rdma_cm_id *id)
2069 {
2070         struct rdma_id_private *id_priv =
2071                 container_of(id, struct rdma_id_private, id);
2072
2073         mutex_lock(&id_priv->handler_mutex);
2074         destroy_id_handler_unlock(id_priv);
2075 }
2076 EXPORT_SYMBOL(rdma_destroy_id);
2077
2078 static int cma_rep_recv(struct rdma_id_private *id_priv)
2079 {
2080         int ret;
2081
2082         ret = cma_modify_qp_rtr(id_priv, NULL);
2083         if (ret)
2084                 goto reject;
2085
2086         ret = cma_modify_qp_rts(id_priv, NULL);
2087         if (ret)
2088                 goto reject;
2089
2090         trace_cm_send_rtu(id_priv);
2091         ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
2092         if (ret)
2093                 goto reject;
2094
2095         return 0;
2096 reject:
2097         pr_debug_ratelimited("RDMA CM: CONNECT_ERROR: failed to handle reply. status %d\n", ret);
2098         cma_modify_qp_err(id_priv);
2099         trace_cm_send_rej(id_priv);
2100         ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
2101                        NULL, 0, NULL, 0);
2102         return ret;
2103 }
2104
2105 static void cma_set_rep_event_data(struct rdma_cm_event *event,
2106                                    const struct ib_cm_rep_event_param *rep_data,
2107                                    void *private_data)
2108 {
2109         event->param.conn.private_data = private_data;
2110         event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
2111         event->param.conn.responder_resources = rep_data->responder_resources;
2112         event->param.conn.initiator_depth = rep_data->initiator_depth;
2113         event->param.conn.flow_control = rep_data->flow_control;
2114         event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
2115         event->param.conn.srq = rep_data->srq;
2116         event->param.conn.qp_num = rep_data->remote_qpn;
2117
2118         event->ece.vendor_id = rep_data->ece.vendor_id;
2119         event->ece.attr_mod = rep_data->ece.attr_mod;
2120 }
2121
2122 static int cma_cm_event_handler(struct rdma_id_private *id_priv,
2123                                 struct rdma_cm_event *event)
2124 {
2125         int ret;
2126
2127         lockdep_assert_held(&id_priv->handler_mutex);
2128
2129         trace_cm_event_handler(id_priv, event);
2130         ret = id_priv->id.event_handler(&id_priv->id, event);
2131         trace_cm_event_done(id_priv, event, ret);
2132         return ret;
2133 }
2134
2135 static int cma_ib_handler(struct ib_cm_id *cm_id,
2136                           const struct ib_cm_event *ib_event)
2137 {
2138         struct rdma_id_private *id_priv = cm_id->context;
2139         struct rdma_cm_event event = {};
2140         enum rdma_cm_state state;
2141         int ret;
2142
2143         mutex_lock(&id_priv->handler_mutex);
2144         state = READ_ONCE(id_priv->state);
2145         if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
2146              state != RDMA_CM_CONNECT) ||
2147             (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
2148              state != RDMA_CM_DISCONNECT))
2149                 goto out;
2150
2151         switch (ib_event->event) {
2152         case IB_CM_REQ_ERROR:
2153         case IB_CM_REP_ERROR:
2154                 event.event = RDMA_CM_EVENT_UNREACHABLE;
2155                 event.status = -ETIMEDOUT;
2156                 break;
2157         case IB_CM_REP_RECEIVED:
2158                 if (state == RDMA_CM_CONNECT &&
2159                     (id_priv->id.qp_type != IB_QPT_UD)) {
2160                         trace_cm_send_mra(id_priv);
2161                         ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2162                 }
2163                 if (id_priv->id.qp) {
2164                         event.status = cma_rep_recv(id_priv);
2165                         event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
2166                                                      RDMA_CM_EVENT_ESTABLISHED;
2167                 } else {
2168                         event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
2169                 }
2170                 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
2171                                        ib_event->private_data);
2172                 break;
2173         case IB_CM_RTU_RECEIVED:
2174         case IB_CM_USER_ESTABLISHED:
2175                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2176                 break;
2177         case IB_CM_DREQ_ERROR:
2178                 event.status = -ETIMEDOUT;
2179                 fallthrough;
2180         case IB_CM_DREQ_RECEIVED:
2181         case IB_CM_DREP_RECEIVED:
2182                 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
2183                                    RDMA_CM_DISCONNECT))
2184                         goto out;
2185                 event.event = RDMA_CM_EVENT_DISCONNECTED;
2186                 break;
2187         case IB_CM_TIMEWAIT_EXIT:
2188                 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
2189                 break;
2190         case IB_CM_MRA_RECEIVED:
2191                 /* ignore event */
2192                 goto out;
2193         case IB_CM_REJ_RECEIVED:
2194                 pr_debug_ratelimited("RDMA CM: REJECTED: %s\n", rdma_reject_msg(&id_priv->id,
2195                                                                                 ib_event->param.rej_rcvd.reason));
2196                 cma_modify_qp_err(id_priv);
2197                 event.status = ib_event->param.rej_rcvd.reason;
2198                 event.event = RDMA_CM_EVENT_REJECTED;
2199                 event.param.conn.private_data = ib_event->private_data;
2200                 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
2201                 break;
2202         default:
2203                 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
2204                        ib_event->event);
2205                 goto out;
2206         }
2207
2208         ret = cma_cm_event_handler(id_priv, &event);
2209         if (ret) {
2210                 /* Destroy the CM ID by returning a non-zero value. */
2211                 id_priv->cm_id.ib = NULL;
2212                 destroy_id_handler_unlock(id_priv);
2213                 return ret;
2214         }
2215 out:
2216         mutex_unlock(&id_priv->handler_mutex);
2217         return 0;
2218 }
2219
2220 static struct rdma_id_private *
2221 cma_ib_new_conn_id(const struct rdma_cm_id *listen_id,
2222                    const struct ib_cm_event *ib_event,
2223                    struct net_device *net_dev)
2224 {
2225         struct rdma_id_private *listen_id_priv;
2226         struct rdma_id_private *id_priv;
2227         struct rdma_cm_id *id;
2228         struct rdma_route *rt;
2229         const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2230         struct sa_path_rec *path = ib_event->param.req_rcvd.primary_path;
2231         const __be64 service_id =
2232                 ib_event->param.req_rcvd.primary_path->service_id;
2233         int ret;
2234
2235         listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2236         id_priv = __rdma_create_id(listen_id->route.addr.dev_addr.net,
2237                                    listen_id->event_handler, listen_id->context,
2238                                    listen_id->ps,
2239                                    ib_event->param.req_rcvd.qp_type,
2240                                    listen_id_priv);
2241         if (IS_ERR(id_priv))
2242                 return NULL;
2243
2244         id = &id_priv->id;
2245         if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2246                               (struct sockaddr *)&id->route.addr.dst_addr,
2247                               listen_id, ib_event, ss_family, service_id))
2248                 goto err;
2249
2250         rt = &id->route;
2251         rt->num_pri_alt_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
2252         rt->path_rec = kmalloc_array(rt->num_pri_alt_paths,
2253                                      sizeof(*rt->path_rec), GFP_KERNEL);
2254         if (!rt->path_rec)
2255                 goto err;
2256
2257         rt->path_rec[0] = *path;
2258         if (rt->num_pri_alt_paths == 2)
2259                 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
2260
2261         if (net_dev) {
2262                 rdma_copy_src_l2_addr(&rt->addr.dev_addr, net_dev);
2263         } else {
2264                 if (!cma_protocol_roce(listen_id) &&
2265                     cma_any_addr(cma_src_addr(id_priv))) {
2266                         rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
2267                         rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
2268                         ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
2269                 } else if (!cma_any_addr(cma_src_addr(id_priv))) {
2270                         ret = cma_translate_addr(cma_src_addr(id_priv), &rt->addr.dev_addr);
2271                         if (ret)
2272                                 goto err;
2273                 }
2274         }
2275         rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
2276
2277         id_priv->state = RDMA_CM_CONNECT;
2278         return id_priv;
2279
2280 err:
2281         rdma_destroy_id(id);
2282         return NULL;
2283 }
2284
2285 static struct rdma_id_private *
2286 cma_ib_new_udp_id(const struct rdma_cm_id *listen_id,
2287                   const struct ib_cm_event *ib_event,
2288                   struct net_device *net_dev)
2289 {
2290         const struct rdma_id_private *listen_id_priv;
2291         struct rdma_id_private *id_priv;
2292         struct rdma_cm_id *id;
2293         const sa_family_t ss_family = listen_id->route.addr.src_addr.ss_family;
2294         struct net *net = listen_id->route.addr.dev_addr.net;
2295         int ret;
2296
2297         listen_id_priv = container_of(listen_id, struct rdma_id_private, id);
2298         id_priv = __rdma_create_id(net, listen_id->event_handler,
2299                                    listen_id->context, listen_id->ps, IB_QPT_UD,
2300                                    listen_id_priv);
2301         if (IS_ERR(id_priv))
2302                 return NULL;
2303
2304         id = &id_priv->id;
2305         if (cma_save_net_info((struct sockaddr *)&id->route.addr.src_addr,
2306                               (struct sockaddr *)&id->route.addr.dst_addr,
2307                               listen_id, ib_event, ss_family,
2308                               ib_event->param.sidr_req_rcvd.service_id))
2309                 goto err;
2310
2311         if (net_dev) {
2312                 rdma_copy_src_l2_addr(&id->route.addr.dev_addr, net_dev);
2313         } else {
2314                 if (!cma_any_addr(cma_src_addr(id_priv))) {
2315                         ret = cma_translate_addr(cma_src_addr(id_priv),
2316                                                  &id->route.addr.dev_addr);
2317                         if (ret)
2318                                 goto err;
2319                 }
2320         }
2321
2322         id_priv->state = RDMA_CM_CONNECT;
2323         return id_priv;
2324 err:
2325         rdma_destroy_id(id);
2326         return NULL;
2327 }
2328
2329 static void cma_set_req_event_data(struct rdma_cm_event *event,
2330                                    const struct ib_cm_req_event_param *req_data,
2331                                    void *private_data, int offset)
2332 {
2333         event->param.conn.private_data = private_data + offset;
2334         event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
2335         event->param.conn.responder_resources = req_data->responder_resources;
2336         event->param.conn.initiator_depth = req_data->initiator_depth;
2337         event->param.conn.flow_control = req_data->flow_control;
2338         event->param.conn.retry_count = req_data->retry_count;
2339         event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
2340         event->param.conn.srq = req_data->srq;
2341         event->param.conn.qp_num = req_data->remote_qpn;
2342
2343         event->ece.vendor_id = req_data->ece.vendor_id;
2344         event->ece.attr_mod = req_data->ece.attr_mod;
2345 }
2346
2347 static int cma_ib_check_req_qp_type(const struct rdma_cm_id *id,
2348                                     const struct ib_cm_event *ib_event)
2349 {
2350         return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
2351                  (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
2352                 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
2353                  (id->qp_type == IB_QPT_UD)) ||
2354                 (!id->qp_type));
2355 }
2356
2357 static int cma_ib_req_handler(struct ib_cm_id *cm_id,
2358                               const struct ib_cm_event *ib_event)
2359 {
2360         struct rdma_id_private *listen_id, *conn_id = NULL;
2361         struct rdma_cm_event event = {};
2362         struct cma_req_info req = {};
2363         struct net_device *net_dev;
2364         u8 offset;
2365         int ret;
2366
2367         listen_id = cma_ib_id_from_event(cm_id, ib_event, &req, &net_dev);
2368         if (IS_ERR(listen_id))
2369                 return PTR_ERR(listen_id);
2370
2371         trace_cm_req_handler(listen_id, ib_event->event);
2372         if (!cma_ib_check_req_qp_type(&listen_id->id, ib_event)) {
2373                 ret = -EINVAL;
2374                 goto net_dev_put;
2375         }
2376
2377         mutex_lock(&listen_id->handler_mutex);
2378         if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN) {
2379                 ret = -ECONNABORTED;
2380                 goto err_unlock;
2381         }
2382
2383         offset = cma_user_data_offset(listen_id);
2384         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2385         if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
2386                 conn_id = cma_ib_new_udp_id(&listen_id->id, ib_event, net_dev);
2387                 event.param.ud.private_data = ib_event->private_data + offset;
2388                 event.param.ud.private_data_len =
2389                                 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
2390         } else {
2391                 conn_id = cma_ib_new_conn_id(&listen_id->id, ib_event, net_dev);
2392                 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
2393                                        ib_event->private_data, offset);
2394         }
2395         if (!conn_id) {
2396                 ret = -ENOMEM;
2397                 goto err_unlock;
2398         }
2399
2400         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2401         ret = cma_ib_acquire_dev(conn_id, listen_id, &req);
2402         if (ret) {
2403                 destroy_id_handler_unlock(conn_id);
2404                 goto err_unlock;
2405         }
2406
2407         conn_id->cm_id.ib = cm_id;
2408         cm_id->context = conn_id;
2409         cm_id->cm_handler = cma_ib_handler;
2410
2411         ret = cma_cm_event_handler(conn_id, &event);
2412         if (ret) {
2413                 /* Destroy the CM ID by returning a non-zero value. */
2414                 conn_id->cm_id.ib = NULL;
2415                 mutex_unlock(&listen_id->handler_mutex);
2416                 destroy_id_handler_unlock(conn_id);
2417                 goto net_dev_put;
2418         }
2419
2420         if (READ_ONCE(conn_id->state) == RDMA_CM_CONNECT &&
2421             conn_id->id.qp_type != IB_QPT_UD) {
2422                 trace_cm_send_mra(cm_id->context);
2423                 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
2424         }
2425         mutex_unlock(&conn_id->handler_mutex);
2426
2427 err_unlock:
2428         mutex_unlock(&listen_id->handler_mutex);
2429
2430 net_dev_put:
2431         dev_put(net_dev);
2432
2433         return ret;
2434 }
2435
2436 __be64 rdma_get_service_id(struct rdma_cm_id *id, struct sockaddr *addr)
2437 {
2438         if (addr->sa_family == AF_IB)
2439                 return ((struct sockaddr_ib *) addr)->sib_sid;
2440
2441         return cpu_to_be64(((u64)id->ps << 16) + be16_to_cpu(cma_port(addr)));
2442 }
2443 EXPORT_SYMBOL(rdma_get_service_id);
2444
2445 void rdma_read_gids(struct rdma_cm_id *cm_id, union ib_gid *sgid,
2446                     union ib_gid *dgid)
2447 {
2448         struct rdma_addr *addr = &cm_id->route.addr;
2449
2450         if (!cm_id->device) {
2451                 if (sgid)
2452                         memset(sgid, 0, sizeof(*sgid));
2453                 if (dgid)
2454                         memset(dgid, 0, sizeof(*dgid));
2455                 return;
2456         }
2457
2458         if (rdma_protocol_roce(cm_id->device, cm_id->port_num)) {
2459                 if (sgid)
2460                         rdma_ip2gid((struct sockaddr *)&addr->src_addr, sgid);
2461                 if (dgid)
2462                         rdma_ip2gid((struct sockaddr *)&addr->dst_addr, dgid);
2463         } else {
2464                 if (sgid)
2465                         rdma_addr_get_sgid(&addr->dev_addr, sgid);
2466                 if (dgid)
2467                         rdma_addr_get_dgid(&addr->dev_addr, dgid);
2468         }
2469 }
2470 EXPORT_SYMBOL(rdma_read_gids);
2471
2472 static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
2473 {
2474         struct rdma_id_private *id_priv = iw_id->context;
2475         struct rdma_cm_event event = {};
2476         int ret = 0;
2477         struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2478         struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2479
2480         mutex_lock(&id_priv->handler_mutex);
2481         if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
2482                 goto out;
2483
2484         switch (iw_event->event) {
2485         case IW_CM_EVENT_CLOSE:
2486                 event.event = RDMA_CM_EVENT_DISCONNECTED;
2487                 break;
2488         case IW_CM_EVENT_CONNECT_REPLY:
2489                 memcpy(cma_src_addr(id_priv), laddr,
2490                        rdma_addr_size(laddr));
2491                 memcpy(cma_dst_addr(id_priv), raddr,
2492                        rdma_addr_size(raddr));
2493                 switch (iw_event->status) {
2494                 case 0:
2495                         event.event = RDMA_CM_EVENT_ESTABLISHED;
2496                         event.param.conn.initiator_depth = iw_event->ird;
2497                         event.param.conn.responder_resources = iw_event->ord;
2498                         break;
2499                 case -ECONNRESET:
2500                 case -ECONNREFUSED:
2501                         event.event = RDMA_CM_EVENT_REJECTED;
2502                         break;
2503                 case -ETIMEDOUT:
2504                         event.event = RDMA_CM_EVENT_UNREACHABLE;
2505                         break;
2506                 default:
2507                         event.event = RDMA_CM_EVENT_CONNECT_ERROR;
2508                         break;
2509                 }
2510                 break;
2511         case IW_CM_EVENT_ESTABLISHED:
2512                 event.event = RDMA_CM_EVENT_ESTABLISHED;
2513                 event.param.conn.initiator_depth = iw_event->ird;
2514                 event.param.conn.responder_resources = iw_event->ord;
2515                 break;
2516         default:
2517                 goto out;
2518         }
2519
2520         event.status = iw_event->status;
2521         event.param.conn.private_data = iw_event->private_data;
2522         event.param.conn.private_data_len = iw_event->private_data_len;
2523         ret = cma_cm_event_handler(id_priv, &event);
2524         if (ret) {
2525                 /* Destroy the CM ID by returning a non-zero value. */
2526                 id_priv->cm_id.iw = NULL;
2527                 destroy_id_handler_unlock(id_priv);
2528                 return ret;
2529         }
2530
2531 out:
2532         mutex_unlock(&id_priv->handler_mutex);
2533         return ret;
2534 }
2535
2536 static int iw_conn_req_handler(struct iw_cm_id *cm_id,
2537                                struct iw_cm_event *iw_event)
2538 {
2539         struct rdma_id_private *listen_id, *conn_id;
2540         struct rdma_cm_event event = {};
2541         int ret = -ECONNABORTED;
2542         struct sockaddr *laddr = (struct sockaddr *)&iw_event->local_addr;
2543         struct sockaddr *raddr = (struct sockaddr *)&iw_event->remote_addr;
2544
2545         event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
2546         event.param.conn.private_data = iw_event->private_data;
2547         event.param.conn.private_data_len = iw_event->private_data_len;
2548         event.param.conn.initiator_depth = iw_event->ird;
2549         event.param.conn.responder_resources = iw_event->ord;
2550
2551         listen_id = cm_id->context;
2552
2553         mutex_lock(&listen_id->handler_mutex);
2554         if (READ_ONCE(listen_id->state) != RDMA_CM_LISTEN)
2555                 goto out;
2556
2557         /* Create a new RDMA id for the new IW CM ID */
2558         conn_id = __rdma_create_id(listen_id->id.route.addr.dev_addr.net,
2559                                    listen_id->id.event_handler,
2560                                    listen_id->id.context, RDMA_PS_TCP,
2561                                    IB_QPT_RC, listen_id);
2562         if (IS_ERR(conn_id)) {
2563                 ret = -ENOMEM;
2564                 goto out;
2565         }
2566         mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
2567         conn_id->state = RDMA_CM_CONNECT;
2568
2569         ret = rdma_translate_ip(laddr, &conn_id->id.route.addr.dev_addr);
2570         if (ret) {
2571                 mutex_unlock(&listen_id->handler_mutex);
2572                 destroy_id_handler_unlock(conn_id);
2573                 return ret;
2574         }
2575
2576         ret = cma_iw_acquire_dev(conn_id, listen_id);
2577         if (ret) {
2578                 mutex_unlock(&listen_id->handler_mutex);
2579                 destroy_id_handler_unlock(conn_id);
2580                 return ret;
2581         }
2582
2583         conn_id->cm_id.iw = cm_id;
2584         cm_id->context = conn_id;
2585         cm_id->cm_handler = cma_iw_handler;
2586
2587         memcpy(cma_src_addr(conn_id), laddr, rdma_addr_size(laddr));
2588         memcpy(cma_dst_addr(conn_id), raddr, rdma_addr_size(raddr));
2589
2590         ret = cma_cm_event_handler(conn_id, &event);
2591         if (ret) {
2592                 /* User wants to destroy the CM ID */
2593                 conn_id->cm_id.iw = NULL;
2594                 mutex_unlock(&listen_id->handler_mutex);
2595                 destroy_id_handler_unlock(conn_id);
2596                 return ret;
2597         }
2598
2599         mutex_unlock(&conn_id->handler_mutex);
2600
2601 out:
2602         mutex_unlock(&listen_id->handler_mutex);
2603         return ret;
2604 }
2605
2606 static int cma_ib_listen(struct rdma_id_private *id_priv)
2607 {
2608         struct sockaddr *addr;
2609         struct ib_cm_id *id;
2610         __be64 svc_id;
2611
2612         addr = cma_src_addr(id_priv);
2613         svc_id = rdma_get_service_id(&id_priv->id, addr);
2614         id = ib_cm_insert_listen(id_priv->id.device,
2615                                  cma_ib_req_handler, svc_id);
2616         if (IS_ERR(id))
2617                 return PTR_ERR(id);
2618         id_priv->cm_id.ib = id;
2619
2620         return 0;
2621 }
2622
2623 static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
2624 {
2625         int ret;
2626         struct iw_cm_id *id;
2627
2628         id = iw_create_cm_id(id_priv->id.device,
2629                              iw_conn_req_handler,
2630                              id_priv);
2631         if (IS_ERR(id))
2632                 return PTR_ERR(id);
2633
2634         mutex_lock(&id_priv->qp_mutex);
2635         id->tos = id_priv->tos;
2636         id->tos_set = id_priv->tos_set;
2637         mutex_unlock(&id_priv->qp_mutex);
2638         id->afonly = id_priv->afonly;
2639         id_priv->cm_id.iw = id;
2640
2641         memcpy(&id_priv->cm_id.iw->local_addr, cma_src_addr(id_priv),
2642                rdma_addr_size(cma_src_addr(id_priv)));
2643
2644         ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
2645
2646         if (ret) {
2647                 iw_destroy_cm_id(id_priv->cm_id.iw);
2648                 id_priv->cm_id.iw = NULL;
2649         }
2650
2651         return ret;
2652 }
2653
2654 static int cma_listen_handler(struct rdma_cm_id *id,
2655                               struct rdma_cm_event *event)
2656 {
2657         struct rdma_id_private *id_priv = id->context;
2658
2659         /* Listening IDs are always destroyed on removal */
2660         if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
2661                 return -1;
2662
2663         id->context = id_priv->id.context;
2664         id->event_handler = id_priv->id.event_handler;
2665         trace_cm_event_handler(id_priv, event);
2666         return id_priv->id.event_handler(id, event);
2667 }
2668
2669 static int cma_listen_on_dev(struct rdma_id_private *id_priv,
2670                              struct cma_device *cma_dev,
2671                              struct rdma_id_private **to_destroy)
2672 {
2673         struct rdma_id_private *dev_id_priv;
2674         struct net *net = id_priv->id.route.addr.dev_addr.net;
2675         int ret;
2676
2677         lockdep_assert_held(&lock);
2678
2679         *to_destroy = NULL;
2680         if (cma_family(id_priv) == AF_IB && !rdma_cap_ib_cm(cma_dev->device, 1))
2681                 return 0;
2682
2683         dev_id_priv =
2684                 __rdma_create_id(net, cma_listen_handler, id_priv,
2685                                  id_priv->id.ps, id_priv->id.qp_type, id_priv);
2686         if (IS_ERR(dev_id_priv))
2687                 return PTR_ERR(dev_id_priv);
2688
2689         dev_id_priv->state = RDMA_CM_ADDR_BOUND;
2690         memcpy(cma_src_addr(dev_id_priv), cma_src_addr(id_priv),
2691                rdma_addr_size(cma_src_addr(id_priv)));
2692
2693         _cma_attach_to_dev(dev_id_priv, cma_dev);
2694         rdma_restrack_add(&dev_id_priv->res);
2695         cma_id_get(id_priv);
2696         dev_id_priv->internal_id = 1;
2697         dev_id_priv->afonly = id_priv->afonly;
2698         mutex_lock(&id_priv->qp_mutex);
2699         dev_id_priv->tos_set = id_priv->tos_set;
2700         dev_id_priv->tos = id_priv->tos;
2701         mutex_unlock(&id_priv->qp_mutex);
2702
2703         ret = rdma_listen(&dev_id_priv->id, id_priv->backlog);
2704         if (ret)
2705                 goto err_listen;
2706         list_add_tail(&dev_id_priv->listen_item, &id_priv->listen_list);
2707         return 0;
2708 err_listen:
2709         /* Caller must destroy this after releasing lock */
2710         *to_destroy = dev_id_priv;
2711         dev_warn(&cma_dev->device->dev, "RDMA CMA: %s, error %d\n", __func__, ret);
2712         return ret;
2713 }
2714
2715 static int cma_listen_on_all(struct rdma_id_private *id_priv)
2716 {
2717         struct rdma_id_private *to_destroy;
2718         struct cma_device *cma_dev;
2719         int ret;
2720
2721         mutex_lock(&lock);
2722         list_add_tail(&id_priv->listen_any_item, &listen_any_list);
2723         list_for_each_entry(cma_dev, &dev_list, list) {
2724                 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
2725                 if (ret) {
2726                         /* Prevent racing with cma_process_remove() */
2727                         if (to_destroy)
2728                                 list_del_init(&to_destroy->device_item);
2729                         goto err_listen;
2730                 }
2731         }
2732         mutex_unlock(&lock);
2733         return 0;
2734
2735 err_listen:
2736         _cma_cancel_listens(id_priv);
2737         mutex_unlock(&lock);
2738         if (to_destroy)
2739                 rdma_destroy_id(&to_destroy->id);
2740         return ret;
2741 }
2742
2743 void rdma_set_service_type(struct rdma_cm_id *id, int tos)
2744 {
2745         struct rdma_id_private *id_priv;
2746
2747         id_priv = container_of(id, struct rdma_id_private, id);
2748         mutex_lock(&id_priv->qp_mutex);
2749         id_priv->tos = (u8) tos;
2750         id_priv->tos_set = true;
2751         mutex_unlock(&id_priv->qp_mutex);
2752 }
2753 EXPORT_SYMBOL(rdma_set_service_type);
2754
2755 /**
2756  * rdma_set_ack_timeout() - Set the ack timeout of QP associated
2757  *                          with a connection identifier.
2758  * @id: Communication identifier to associated with service type.
2759  * @timeout: Ack timeout to set a QP, expressed as 4.096 * 2^(timeout) usec.
2760  *
2761  * This function should be called before rdma_connect() on active side,
2762  * and on passive side before rdma_accept(). It is applicable to primary
2763  * path only. The timeout will affect the local side of the QP, it is not
2764  * negotiated with remote side and zero disables the timer. In case it is
2765  * set before rdma_resolve_route, the value will also be used to determine
2766  * PacketLifeTime for RoCE.
2767  *
2768  * Return: 0 for success
2769  */
2770 int rdma_set_ack_timeout(struct rdma_cm_id *id, u8 timeout)
2771 {
2772         struct rdma_id_private *id_priv;
2773
2774         if (id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_INI)
2775                 return -EINVAL;
2776
2777         id_priv = container_of(id, struct rdma_id_private, id);
2778         mutex_lock(&id_priv->qp_mutex);
2779         id_priv->timeout = timeout;
2780         id_priv->timeout_set = true;
2781         mutex_unlock(&id_priv->qp_mutex);
2782
2783         return 0;
2784 }
2785 EXPORT_SYMBOL(rdma_set_ack_timeout);
2786
2787 /**
2788  * rdma_set_min_rnr_timer() - Set the minimum RNR Retry timer of the
2789  *                            QP associated with a connection identifier.
2790  * @id: Communication identifier to associated with service type.
2791  * @min_rnr_timer: 5-bit value encoded as Table 45: "Encoding for RNR NAK
2792  *                 Timer Field" in the IBTA specification.
2793  *
2794  * This function should be called before rdma_connect() on active
2795  * side, and on passive side before rdma_accept(). The timer value
2796  * will be associated with the local QP. When it receives a send it is
2797  * not read to handle, typically if the receive queue is empty, an RNR
2798  * Retry NAK is returned to the requester with the min_rnr_timer
2799  * encoded. The requester will then wait at least the time specified
2800  * in the NAK before retrying. The default is zero, which translates
2801  * to a minimum RNR Timer value of 655 ms.
2802  *
2803  * Return: 0 for success
2804  */
2805 int rdma_set_min_rnr_timer(struct rdma_cm_id *id, u8 min_rnr_timer)
2806 {
2807         struct rdma_id_private *id_priv;
2808
2809         /* It is a five-bit value */
2810         if (min_rnr_timer & 0xe0)
2811                 return -EINVAL;
2812
2813         if (WARN_ON(id->qp_type != IB_QPT_RC && id->qp_type != IB_QPT_XRC_TGT))
2814                 return -EINVAL;
2815
2816         id_priv = container_of(id, struct rdma_id_private, id);
2817         mutex_lock(&id_priv->qp_mutex);
2818         id_priv->min_rnr_timer = min_rnr_timer;
2819         id_priv->min_rnr_timer_set = true;
2820         mutex_unlock(&id_priv->qp_mutex);
2821
2822         return 0;
2823 }
2824 EXPORT_SYMBOL(rdma_set_min_rnr_timer);
2825
2826 static int route_set_path_rec_inbound(struct cma_work *work,
2827                                       struct sa_path_rec *path_rec)
2828 {
2829         struct rdma_route *route = &work->id->id.route;
2830
2831         if (!route->path_rec_inbound) {
2832                 route->path_rec_inbound =
2833                         kzalloc(sizeof(*route->path_rec_inbound), GFP_KERNEL);
2834                 if (!route->path_rec_inbound)
2835                         return -ENOMEM;
2836         }
2837
2838         *route->path_rec_inbound = *path_rec;
2839         return 0;
2840 }
2841
2842 static int route_set_path_rec_outbound(struct cma_work *work,
2843                                        struct sa_path_rec *path_rec)
2844 {
2845         struct rdma_route *route = &work->id->id.route;
2846
2847         if (!route->path_rec_outbound) {
2848                 route->path_rec_outbound =
2849                         kzalloc(sizeof(*route->path_rec_outbound), GFP_KERNEL);
2850                 if (!route->path_rec_outbound)
2851                         return -ENOMEM;
2852         }
2853
2854         *route->path_rec_outbound = *path_rec;
2855         return 0;
2856 }
2857
2858 static void cma_query_handler(int status, struct sa_path_rec *path_rec,
2859                               unsigned int num_prs, void *context)
2860 {
2861         struct cma_work *work = context;
2862         struct rdma_route *route;
2863         int i;
2864
2865         route = &work->id->id.route;
2866
2867         if (status)
2868                 goto fail;
2869
2870         for (i = 0; i < num_prs; i++) {
2871                 if (!path_rec[i].flags || (path_rec[i].flags & IB_PATH_GMP))
2872                         *route->path_rec = path_rec[i];
2873                 else if (path_rec[i].flags & IB_PATH_INBOUND)
2874                         status = route_set_path_rec_inbound(work, &path_rec[i]);
2875                 else if (path_rec[i].flags & IB_PATH_OUTBOUND)
2876                         status = route_set_path_rec_outbound(work,
2877                                                              &path_rec[i]);
2878                 else
2879                         status = -EINVAL;
2880
2881                 if (status)
2882                         goto fail;
2883         }
2884
2885         route->num_pri_alt_paths = 1;
2886         queue_work(cma_wq, &work->work);
2887         return;
2888
2889 fail:
2890         work->old_state = RDMA_CM_ROUTE_QUERY;
2891         work->new_state = RDMA_CM_ADDR_RESOLVED;
2892         work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
2893         work->event.status = status;
2894         pr_debug_ratelimited("RDMA CM: ROUTE_ERROR: failed to query path. status %d\n",
2895                              status);
2896         queue_work(cma_wq, &work->work);
2897 }
2898
2899 static int cma_query_ib_route(struct rdma_id_private *id_priv,
2900                               unsigned long timeout_ms, struct cma_work *work)
2901 {
2902         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
2903         struct sa_path_rec path_rec;
2904         ib_sa_comp_mask comp_mask;
2905         struct sockaddr_in6 *sin6;
2906         struct sockaddr_ib *sib;
2907
2908         memset(&path_rec, 0, sizeof path_rec);
2909
2910         if (rdma_cap_opa_ah(id_priv->id.device, id_priv->id.port_num))
2911                 path_rec.rec_type = SA_PATH_REC_TYPE_OPA;
2912         else
2913                 path_rec.rec_type = SA_PATH_REC_TYPE_IB;
2914         rdma_addr_get_sgid(dev_addr, &path_rec.sgid);
2915         rdma_addr_get_dgid(dev_addr, &path_rec.dgid);
2916         path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
2917         path_rec.numb_path = 1;
2918         path_rec.reversible = 1;
2919         path_rec.service_id = rdma_get_service_id(&id_priv->id,
2920                                                   cma_dst_addr(id_priv));
2921
2922         comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
2923                     IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
2924                     IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
2925
2926         switch (cma_family(id_priv)) {
2927         case AF_INET:
2928                 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
2929                 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
2930                 break;
2931         case AF_INET6:
2932                 sin6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
2933                 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
2934                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2935                 break;
2936         case AF_IB:
2937                 sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
2938                 path_rec.traffic_class = (u8) (be32_to_cpu(sib->sib_flowinfo) >> 20);
2939                 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
2940                 break;
2941         }
2942
2943         id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
2944                                                id_priv->id.port_num, &path_rec,
2945                                                comp_mask, timeout_ms,
2946                                                GFP_KERNEL, cma_query_handler,
2947                                                work, &id_priv->query);
2948
2949         return (id_priv->query_id < 0) ? id_priv->query_id : 0;
2950 }
2951
2952 static void cma_iboe_join_work_handler(struct work_struct *work)
2953 {
2954         struct cma_multicast *mc =
2955                 container_of(work, struct cma_multicast, iboe_join.work);
2956         struct rdma_cm_event *event = &mc->iboe_join.event;
2957         struct rdma_id_private *id_priv = mc->id_priv;
2958         int ret;
2959
2960         mutex_lock(&id_priv->handler_mutex);
2961         if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
2962             READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
2963                 goto out_unlock;
2964
2965         ret = cma_cm_event_handler(id_priv, event);
2966         WARN_ON(ret);
2967
2968 out_unlock:
2969         mutex_unlock(&id_priv->handler_mutex);
2970         if (event->event == RDMA_CM_EVENT_MULTICAST_JOIN)
2971                 rdma_destroy_ah_attr(&event->param.ud.ah_attr);
2972 }
2973
2974 static void cma_work_handler(struct work_struct *_work)
2975 {
2976         struct cma_work *work = container_of(_work, struct cma_work, work);
2977         struct rdma_id_private *id_priv = work->id;
2978
2979         mutex_lock(&id_priv->handler_mutex);
2980         if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
2981             READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
2982                 goto out_unlock;
2983         if (work->old_state != 0 || work->new_state != 0) {
2984                 if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
2985                         goto out_unlock;
2986         }
2987
2988         if (cma_cm_event_handler(id_priv, &work->event)) {
2989                 cma_id_put(id_priv);
2990                 destroy_id_handler_unlock(id_priv);
2991                 goto out_free;
2992         }
2993
2994 out_unlock:
2995         mutex_unlock(&id_priv->handler_mutex);
2996         cma_id_put(id_priv);
2997 out_free:
2998         if (work->event.event == RDMA_CM_EVENT_MULTICAST_JOIN)
2999                 rdma_destroy_ah_attr(&work->event.param.ud.ah_attr);
3000         kfree(work);
3001 }
3002
3003 static void cma_init_resolve_route_work(struct cma_work *work,
3004                                         struct rdma_id_private *id_priv)
3005 {
3006         work->id = id_priv;
3007         INIT_WORK(&work->work, cma_work_handler);
3008         work->old_state = RDMA_CM_ROUTE_QUERY;
3009         work->new_state = RDMA_CM_ROUTE_RESOLVED;
3010         work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
3011 }
3012
3013 static void enqueue_resolve_addr_work(struct cma_work *work,
3014                                       struct rdma_id_private *id_priv)
3015 {
3016         /* Balances with cma_id_put() in cma_work_handler */
3017         cma_id_get(id_priv);
3018
3019         work->id = id_priv;
3020         INIT_WORK(&work->work, cma_work_handler);
3021         work->old_state = RDMA_CM_ADDR_QUERY;
3022         work->new_state = RDMA_CM_ADDR_RESOLVED;
3023         work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3024
3025         queue_work(cma_wq, &work->work);
3026 }
3027
3028 static int cma_resolve_ib_route(struct rdma_id_private *id_priv,
3029                                 unsigned long timeout_ms)
3030 {
3031         struct rdma_route *route = &id_priv->id.route;
3032         struct cma_work *work;
3033         int ret;
3034
3035         work = kzalloc(sizeof *work, GFP_KERNEL);
3036         if (!work)
3037                 return -ENOMEM;
3038
3039         cma_init_resolve_route_work(work, id_priv);
3040
3041         if (!route->path_rec)
3042                 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
3043         if (!route->path_rec) {
3044                 ret = -ENOMEM;
3045                 goto err1;
3046         }
3047
3048         ret = cma_query_ib_route(id_priv, timeout_ms, work);
3049         if (ret)
3050                 goto err2;
3051
3052         return 0;
3053 err2:
3054         kfree(route->path_rec);
3055         route->path_rec = NULL;
3056 err1:
3057         kfree(work);
3058         return ret;
3059 }
3060
3061 static enum ib_gid_type cma_route_gid_type(enum rdma_network_type network_type,
3062                                            unsigned long supported_gids,
3063                                            enum ib_gid_type default_gid)
3064 {
3065         if ((network_type == RDMA_NETWORK_IPV4 ||
3066              network_type == RDMA_NETWORK_IPV6) &&
3067             test_bit(IB_GID_TYPE_ROCE_UDP_ENCAP, &supported_gids))
3068                 return IB_GID_TYPE_ROCE_UDP_ENCAP;
3069
3070         return default_gid;
3071 }
3072
3073 /*
3074  * cma_iboe_set_path_rec_l2_fields() is helper function which sets
3075  * path record type based on GID type.
3076  * It also sets up other L2 fields which includes destination mac address
3077  * netdev ifindex, of the path record.
3078  * It returns the netdev of the bound interface for this path record entry.
3079  */
3080 static struct net_device *
3081 cma_iboe_set_path_rec_l2_fields(struct rdma_id_private *id_priv)
3082 {
3083         struct rdma_route *route = &id_priv->id.route;
3084         enum ib_gid_type gid_type = IB_GID_TYPE_ROCE;
3085         struct rdma_addr *addr = &route->addr;
3086         unsigned long supported_gids;
3087         struct net_device *ndev;
3088
3089         if (!addr->dev_addr.bound_dev_if)
3090                 return NULL;
3091
3092         ndev = dev_get_by_index(addr->dev_addr.net,
3093                                 addr->dev_addr.bound_dev_if);
3094         if (!ndev)
3095                 return NULL;
3096
3097         supported_gids = roce_gid_type_mask_support(id_priv->id.device,
3098                                                     id_priv->id.port_num);
3099         gid_type = cma_route_gid_type(addr->dev_addr.network,
3100                                       supported_gids,
3101                                       id_priv->gid_type);
3102         /* Use the hint from IP Stack to select GID Type */
3103         if (gid_type < ib_network_to_gid_type(addr->dev_addr.network))
3104                 gid_type = ib_network_to_gid_type(addr->dev_addr.network);
3105         route->path_rec->rec_type = sa_conv_gid_to_pathrec_type(gid_type);
3106
3107         route->path_rec->roce.route_resolved = true;
3108         sa_path_set_dmac(route->path_rec, addr->dev_addr.dst_dev_addr);
3109         return ndev;
3110 }
3111
3112 int rdma_set_ib_path(struct rdma_cm_id *id,
3113                      struct sa_path_rec *path_rec)
3114 {
3115         struct rdma_id_private *id_priv;
3116         struct net_device *ndev;
3117         int ret;
3118
3119         id_priv = container_of(id, struct rdma_id_private, id);
3120         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3121                            RDMA_CM_ROUTE_RESOLVED))
3122                 return -EINVAL;
3123
3124         id->route.path_rec = kmemdup(path_rec, sizeof(*path_rec),
3125                                      GFP_KERNEL);
3126         if (!id->route.path_rec) {
3127                 ret = -ENOMEM;
3128                 goto err;
3129         }
3130
3131         if (rdma_protocol_roce(id->device, id->port_num)) {
3132                 ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
3133                 if (!ndev) {
3134                         ret = -ENODEV;
3135                         goto err_free;
3136                 }
3137                 dev_put(ndev);
3138         }
3139
3140         id->route.num_pri_alt_paths = 1;
3141         return 0;
3142
3143 err_free:
3144         kfree(id->route.path_rec);
3145         id->route.path_rec = NULL;
3146 err:
3147         cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
3148         return ret;
3149 }
3150 EXPORT_SYMBOL(rdma_set_ib_path);
3151
3152 static int cma_resolve_iw_route(struct rdma_id_private *id_priv)
3153 {
3154         struct cma_work *work;
3155
3156         work = kzalloc(sizeof *work, GFP_KERNEL);
3157         if (!work)
3158                 return -ENOMEM;
3159
3160         cma_init_resolve_route_work(work, id_priv);
3161         queue_work(cma_wq, &work->work);
3162         return 0;
3163 }
3164
3165 static int get_vlan_ndev_tc(struct net_device *vlan_ndev, int prio)
3166 {
3167         struct net_device *dev;
3168
3169         dev = vlan_dev_real_dev(vlan_ndev);
3170         if (dev->num_tc)
3171                 return netdev_get_prio_tc_map(dev, prio);
3172
3173         return (vlan_dev_get_egress_qos_mask(vlan_ndev, prio) &
3174                 VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
3175 }
3176
3177 struct iboe_prio_tc_map {
3178         int input_prio;
3179         int output_tc;
3180         bool found;
3181 };
3182
3183 static int get_lower_vlan_dev_tc(struct net_device *dev,
3184                                  struct netdev_nested_priv *priv)
3185 {
3186         struct iboe_prio_tc_map *map = (struct iboe_prio_tc_map *)priv->data;
3187
3188         if (is_vlan_dev(dev))
3189                 map->output_tc = get_vlan_ndev_tc(dev, map->input_prio);
3190         else if (dev->num_tc)
3191                 map->output_tc = netdev_get_prio_tc_map(dev, map->input_prio);
3192         else
3193                 map->output_tc = 0;
3194         /* We are interested only in first level VLAN device, so always
3195          * return 1 to stop iterating over next level devices.
3196          */
3197         map->found = true;
3198         return 1;
3199 }
3200
3201 static int iboe_tos_to_sl(struct net_device *ndev, int tos)
3202 {
3203         struct iboe_prio_tc_map prio_tc_map = {};
3204         int prio = rt_tos2priority(tos);
3205         struct netdev_nested_priv priv;
3206
3207         /* If VLAN device, get it directly from the VLAN netdev */
3208         if (is_vlan_dev(ndev))
3209                 return get_vlan_ndev_tc(ndev, prio);
3210
3211         prio_tc_map.input_prio = prio;
3212         priv.data = (void *)&prio_tc_map;
3213         rcu_read_lock();
3214         netdev_walk_all_lower_dev_rcu(ndev,
3215                                       get_lower_vlan_dev_tc,
3216                                       &priv);
3217         rcu_read_unlock();
3218         /* If map is found from lower device, use it; Otherwise
3219          * continue with the current netdevice to get priority to tc map.
3220          */
3221         if (prio_tc_map.found)
3222                 return prio_tc_map.output_tc;
3223         else if (ndev->num_tc)
3224                 return netdev_get_prio_tc_map(ndev, prio);
3225         else
3226                 return 0;
3227 }
3228
3229 static __be32 cma_get_roce_udp_flow_label(struct rdma_id_private *id_priv)
3230 {
3231         struct sockaddr_in6 *addr6;
3232         u16 dport, sport;
3233         u32 hash, fl;
3234
3235         addr6 = (struct sockaddr_in6 *)cma_src_addr(id_priv);
3236         fl = be32_to_cpu(addr6->sin6_flowinfo) & IB_GRH_FLOWLABEL_MASK;
3237         if ((cma_family(id_priv) != AF_INET6) || !fl) {
3238                 dport = be16_to_cpu(cma_port(cma_dst_addr(id_priv)));
3239                 sport = be16_to_cpu(cma_port(cma_src_addr(id_priv)));
3240                 hash = (u32)sport * 31 + dport;
3241                 fl = hash & IB_GRH_FLOWLABEL_MASK;
3242         }
3243
3244         return cpu_to_be32(fl);
3245 }
3246
3247 static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
3248 {
3249         struct rdma_route *route = &id_priv->id.route;
3250         struct rdma_addr *addr = &route->addr;
3251         struct cma_work *work;
3252         int ret;
3253         struct net_device *ndev;
3254
3255         u8 default_roce_tos = id_priv->cma_dev->default_roce_tos[id_priv->id.port_num -
3256                                         rdma_start_port(id_priv->cma_dev->device)];
3257         u8 tos;
3258
3259         mutex_lock(&id_priv->qp_mutex);
3260         tos = id_priv->tos_set ? id_priv->tos : default_roce_tos;
3261         mutex_unlock(&id_priv->qp_mutex);
3262
3263         work = kzalloc(sizeof *work, GFP_KERNEL);
3264         if (!work)
3265                 return -ENOMEM;
3266
3267         route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
3268         if (!route->path_rec) {
3269                 ret = -ENOMEM;
3270                 goto err1;
3271         }
3272
3273         route->num_pri_alt_paths = 1;
3274
3275         ndev = cma_iboe_set_path_rec_l2_fields(id_priv);
3276         if (!ndev) {
3277                 ret = -ENODEV;
3278                 goto err2;
3279         }
3280
3281         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
3282                     &route->path_rec->sgid);
3283         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.dst_addr,
3284                     &route->path_rec->dgid);
3285
3286         if (((struct sockaddr *)&id_priv->id.route.addr.dst_addr)->sa_family != AF_IB)
3287                 /* TODO: get the hoplimit from the inet/inet6 device */
3288                 route->path_rec->hop_limit = addr->dev_addr.hoplimit;
3289         else
3290                 route->path_rec->hop_limit = 1;
3291         route->path_rec->reversible = 1;
3292         route->path_rec->pkey = cpu_to_be16(0xffff);
3293         route->path_rec->mtu_selector = IB_SA_EQ;
3294         route->path_rec->sl = iboe_tos_to_sl(ndev, tos);
3295         route->path_rec->traffic_class = tos;
3296         route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
3297         route->path_rec->rate_selector = IB_SA_EQ;
3298         route->path_rec->rate = IB_RATE_PORT_CURRENT;
3299         dev_put(ndev);
3300         route->path_rec->packet_life_time_selector = IB_SA_EQ;
3301         /* In case ACK timeout is set, use this value to calculate
3302          * PacketLifeTime.  As per IBTA 12.7.34,
3303          * local ACK timeout = (2 * PacketLifeTime + Local CA’s ACK delay).
3304          * Assuming a negligible local ACK delay, we can use
3305          * PacketLifeTime = local ACK timeout/2
3306          * as a reasonable approximation for RoCE networks.
3307          */
3308         mutex_lock(&id_priv->qp_mutex);
3309         if (id_priv->timeout_set && id_priv->timeout)
3310                 route->path_rec->packet_life_time = id_priv->timeout - 1;
3311         else
3312                 route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
3313         mutex_unlock(&id_priv->qp_mutex);
3314
3315         if (!route->path_rec->mtu) {
3316                 ret = -EINVAL;
3317                 goto err2;
3318         }
3319
3320         if (rdma_protocol_roce_udp_encap(id_priv->id.device,
3321                                          id_priv->id.port_num))
3322                 route->path_rec->flow_label =
3323                         cma_get_roce_udp_flow_label(id_priv);
3324
3325         cma_init_resolve_route_work(work, id_priv);
3326         queue_work(cma_wq, &work->work);
3327
3328         return 0;
3329
3330 err2:
3331         kfree(route->path_rec);
3332         route->path_rec = NULL;
3333         route->num_pri_alt_paths = 0;
3334 err1:
3335         kfree(work);
3336         return ret;
3337 }
3338
3339 int rdma_resolve_route(struct rdma_cm_id *id, unsigned long timeout_ms)
3340 {
3341         struct rdma_id_private *id_priv;
3342         int ret;
3343
3344         if (!timeout_ms)
3345                 return -EINVAL;
3346
3347         id_priv = container_of(id, struct rdma_id_private, id);
3348         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
3349                 return -EINVAL;
3350
3351         cma_id_get(id_priv);
3352         if (rdma_cap_ib_sa(id->device, id->port_num))
3353                 ret = cma_resolve_ib_route(id_priv, timeout_ms);
3354         else if (rdma_protocol_roce(id->device, id->port_num)) {
3355                 ret = cma_resolve_iboe_route(id_priv);
3356                 if (!ret)
3357                         cma_add_id_to_tree(id_priv);
3358         }
3359         else if (rdma_protocol_iwarp(id->device, id->port_num))
3360                 ret = cma_resolve_iw_route(id_priv);
3361         else
3362                 ret = -ENOSYS;
3363
3364         if (ret)
3365                 goto err;
3366
3367         return 0;
3368 err:
3369         cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
3370         cma_id_put(id_priv);
3371         return ret;
3372 }
3373 EXPORT_SYMBOL(rdma_resolve_route);
3374
3375 static void cma_set_loopback(struct sockaddr *addr)
3376 {
3377         switch (addr->sa_family) {
3378         case AF_INET:
3379                 ((struct sockaddr_in *) addr)->sin_addr.s_addr = htonl(INADDR_LOOPBACK);
3380                 break;
3381         case AF_INET6:
3382                 ipv6_addr_set(&((struct sockaddr_in6 *) addr)->sin6_addr,
3383                               0, 0, 0, htonl(1));
3384                 break;
3385         default:
3386                 ib_addr_set(&((struct sockaddr_ib *) addr)->sib_addr,
3387                             0, 0, 0, htonl(1));
3388                 break;
3389         }
3390 }
3391
3392 static int cma_bind_loopback(struct rdma_id_private *id_priv)
3393 {
3394         struct cma_device *cma_dev, *cur_dev;
3395         union ib_gid gid;
3396         enum ib_port_state port_state;
3397         unsigned int p;
3398         u16 pkey;
3399         int ret;
3400
3401         cma_dev = NULL;
3402         mutex_lock(&lock);
3403         list_for_each_entry(cur_dev, &dev_list, list) {
3404                 if (cma_family(id_priv) == AF_IB &&
3405                     !rdma_cap_ib_cm(cur_dev->device, 1))
3406                         continue;
3407
3408                 if (!cma_dev)
3409                         cma_dev = cur_dev;
3410
3411                 rdma_for_each_port (cur_dev->device, p) {
3412                         if (!ib_get_cached_port_state(cur_dev->device, p, &port_state) &&
3413                             port_state == IB_PORT_ACTIVE) {
3414                                 cma_dev = cur_dev;
3415                                 goto port_found;
3416                         }
3417                 }
3418         }
3419
3420         if (!cma_dev) {
3421                 ret = -ENODEV;
3422                 goto out;
3423         }
3424
3425         p = 1;
3426
3427 port_found:
3428         ret = rdma_query_gid(cma_dev->device, p, 0, &gid);
3429         if (ret)
3430                 goto out;
3431
3432         ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
3433         if (ret)
3434                 goto out;
3435
3436         id_priv->id.route.addr.dev_addr.dev_type =
3437                 (rdma_protocol_ib(cma_dev->device, p)) ?
3438                 ARPHRD_INFINIBAND : ARPHRD_ETHER;
3439
3440         rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3441         ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
3442         id_priv->id.port_num = p;
3443         cma_attach_to_dev(id_priv, cma_dev);
3444         rdma_restrack_add(&id_priv->res);
3445         cma_set_loopback(cma_src_addr(id_priv));
3446 out:
3447         mutex_unlock(&lock);
3448         return ret;
3449 }
3450
3451 static void addr_handler(int status, struct sockaddr *src_addr,
3452                          struct rdma_dev_addr *dev_addr, void *context)
3453 {
3454         struct rdma_id_private *id_priv = context;
3455         struct rdma_cm_event event = {};
3456         struct sockaddr *addr;
3457         struct sockaddr_storage old_addr;
3458
3459         mutex_lock(&id_priv->handler_mutex);
3460         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
3461                            RDMA_CM_ADDR_RESOLVED))
3462                 goto out;
3463
3464         /*
3465          * Store the previous src address, so that if we fail to acquire
3466          * matching rdma device, old address can be restored back, which helps
3467          * to cancel the cma listen operation correctly.
3468          */
3469         addr = cma_src_addr(id_priv);
3470         memcpy(&old_addr, addr, rdma_addr_size(addr));
3471         memcpy(addr, src_addr, rdma_addr_size(src_addr));
3472         if (!status && !id_priv->cma_dev) {
3473                 status = cma_acquire_dev_by_src_ip(id_priv);
3474                 if (status)
3475                         pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to acquire device. status %d\n",
3476                                              status);
3477                 rdma_restrack_add(&id_priv->res);
3478         } else if (status) {
3479                 pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to resolve IP. status %d\n", status);
3480         }
3481
3482         if (status) {
3483                 memcpy(addr, &old_addr,
3484                        rdma_addr_size((struct sockaddr *)&old_addr));
3485                 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
3486                                    RDMA_CM_ADDR_BOUND))
3487                         goto out;
3488                 event.event = RDMA_CM_EVENT_ADDR_ERROR;
3489                 event.status = status;
3490         } else
3491                 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
3492
3493         if (cma_cm_event_handler(id_priv, &event)) {
3494                 destroy_id_handler_unlock(id_priv);
3495                 return;
3496         }
3497 out:
3498         mutex_unlock(&id_priv->handler_mutex);
3499 }
3500
3501 static int cma_resolve_loopback(struct rdma_id_private *id_priv)
3502 {
3503         struct cma_work *work;
3504         union ib_gid gid;
3505         int ret;
3506
3507         work = kzalloc(sizeof *work, GFP_KERNEL);
3508         if (!work)
3509                 return -ENOMEM;
3510
3511         if (!id_priv->cma_dev) {
3512                 ret = cma_bind_loopback(id_priv);
3513                 if (ret)
3514                         goto err;
3515         }
3516
3517         rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
3518         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
3519
3520         enqueue_resolve_addr_work(work, id_priv);
3521         return 0;
3522 err:
3523         kfree(work);
3524         return ret;
3525 }
3526
3527 static int cma_resolve_ib_addr(struct rdma_id_private *id_priv)
3528 {
3529         struct cma_work *work;
3530         int ret;
3531
3532         work = kzalloc(sizeof *work, GFP_KERNEL);
3533         if (!work)
3534                 return -ENOMEM;
3535
3536         if (!id_priv->cma_dev) {
3537                 ret = cma_resolve_ib_dev(id_priv);
3538                 if (ret)
3539                         goto err;
3540         }
3541
3542         rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, (union ib_gid *)
3543                 &(((struct sockaddr_ib *) &id_priv->id.route.addr.dst_addr)->sib_addr));
3544
3545         enqueue_resolve_addr_work(work, id_priv);
3546         return 0;
3547 err:
3548         kfree(work);
3549         return ret;
3550 }
3551
3552 int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
3553 {
3554         struct rdma_id_private *id_priv;
3555         unsigned long flags;
3556         int ret;
3557
3558         id_priv = container_of(id, struct rdma_id_private, id);
3559         spin_lock_irqsave(&id_priv->lock, flags);
3560         if ((reuse && id_priv->state != RDMA_CM_LISTEN) ||
3561             id_priv->state == RDMA_CM_IDLE) {
3562                 id_priv->reuseaddr = reuse;
3563                 ret = 0;
3564         } else {
3565                 ret = -EINVAL;
3566         }
3567         spin_unlock_irqrestore(&id_priv->lock, flags);
3568         return ret;
3569 }
3570 EXPORT_SYMBOL(rdma_set_reuseaddr);
3571
3572 int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
3573 {
3574         struct rdma_id_private *id_priv;
3575         unsigned long flags;
3576         int ret;
3577
3578         id_priv = container_of(id, struct rdma_id_private, id);
3579         spin_lock_irqsave(&id_priv->lock, flags);
3580         if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
3581                 id_priv->options |= (1 << CMA_OPTION_AFONLY);
3582                 id_priv->afonly = afonly;
3583                 ret = 0;
3584         } else {
3585                 ret = -EINVAL;
3586         }
3587         spin_unlock_irqrestore(&id_priv->lock, flags);
3588         return ret;
3589 }
3590 EXPORT_SYMBOL(rdma_set_afonly);
3591
3592 static void cma_bind_port(struct rdma_bind_list *bind_list,
3593                           struct rdma_id_private *id_priv)
3594 {
3595         struct sockaddr *addr;
3596         struct sockaddr_ib *sib;
3597         u64 sid, mask;
3598         __be16 port;
3599
3600         lockdep_assert_held(&lock);
3601
3602         addr = cma_src_addr(id_priv);
3603         port = htons(bind_list->port);
3604
3605         switch (addr->sa_family) {
3606         case AF_INET:
3607                 ((struct sockaddr_in *) addr)->sin_port = port;
3608                 break;
3609         case AF_INET6:
3610                 ((struct sockaddr_in6 *) addr)->sin6_port = port;
3611                 break;
3612         case AF_IB:
3613                 sib = (struct sockaddr_ib *) addr;
3614                 sid = be64_to_cpu(sib->sib_sid);
3615                 mask = be64_to_cpu(sib->sib_sid_mask);
3616                 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
3617                 sib->sib_sid_mask = cpu_to_be64(~0ULL);
3618                 break;
3619         }
3620         id_priv->bind_list = bind_list;
3621         hlist_add_head(&id_priv->node, &bind_list->owners);
3622 }
3623
3624 static int cma_alloc_port(enum rdma_ucm_port_space ps,
3625                           struct rdma_id_private *id_priv, unsigned short snum)
3626 {
3627         struct rdma_bind_list *bind_list;
3628         int ret;
3629
3630         lockdep_assert_held(&lock);
3631
3632         bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
3633         if (!bind_list)
3634                 return -ENOMEM;
3635
3636         ret = cma_ps_alloc(id_priv->id.route.addr.dev_addr.net, ps, bind_list,
3637                            snum);
3638         if (ret < 0)
3639                 goto err;
3640
3641         bind_list->ps = ps;
3642         bind_list->port = snum;
3643         cma_bind_port(bind_list, id_priv);
3644         return 0;
3645 err:
3646         kfree(bind_list);
3647         return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
3648 }
3649
3650 static int cma_port_is_unique(struct rdma_bind_list *bind_list,
3651                               struct rdma_id_private *id_priv)
3652 {
3653         struct rdma_id_private *cur_id;
3654         struct sockaddr  *daddr = cma_dst_addr(id_priv);
3655         struct sockaddr  *saddr = cma_src_addr(id_priv);
3656         __be16 dport = cma_port(daddr);
3657
3658         lockdep_assert_held(&lock);
3659
3660         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3661                 struct sockaddr  *cur_daddr = cma_dst_addr(cur_id);
3662                 struct sockaddr  *cur_saddr = cma_src_addr(cur_id);
3663                 __be16 cur_dport = cma_port(cur_daddr);
3664
3665                 if (id_priv == cur_id)
3666                         continue;
3667
3668                 /* different dest port -> unique */
3669                 if (!cma_any_port(daddr) &&
3670                     !cma_any_port(cur_daddr) &&
3671                     (dport != cur_dport))
3672                         continue;
3673
3674                 /* different src address -> unique */
3675                 if (!cma_any_addr(saddr) &&
3676                     !cma_any_addr(cur_saddr) &&
3677                     cma_addr_cmp(saddr, cur_saddr))
3678                         continue;
3679
3680                 /* different dst address -> unique */
3681                 if (!cma_any_addr(daddr) &&
3682                     !cma_any_addr(cur_daddr) &&
3683                     cma_addr_cmp(daddr, cur_daddr))
3684                         continue;
3685
3686                 return -EADDRNOTAVAIL;
3687         }
3688         return 0;
3689 }
3690
3691 static int cma_alloc_any_port(enum rdma_ucm_port_space ps,
3692                               struct rdma_id_private *id_priv)
3693 {
3694         static unsigned int last_used_port;
3695         int low, high, remaining;
3696         unsigned int rover;
3697         struct net *net = id_priv->id.route.addr.dev_addr.net;
3698
3699         lockdep_assert_held(&lock);
3700
3701         inet_get_local_port_range(net, &low, &high);
3702         remaining = (high - low) + 1;
3703         rover = get_random_u32_inclusive(low, remaining + low - 1);
3704 retry:
3705         if (last_used_port != rover) {
3706                 struct rdma_bind_list *bind_list;
3707                 int ret;
3708
3709                 bind_list = cma_ps_find(net, ps, (unsigned short)rover);
3710
3711                 if (!bind_list) {
3712                         ret = cma_alloc_port(ps, id_priv, rover);
3713                 } else {
3714                         ret = cma_port_is_unique(bind_list, id_priv);
3715                         if (!ret)
3716                                 cma_bind_port(bind_list, id_priv);
3717                 }
3718                 /*
3719                  * Remember previously used port number in order to avoid
3720                  * re-using same port immediately after it is closed.
3721                  */
3722                 if (!ret)
3723                         last_used_port = rover;
3724                 if (ret != -EADDRNOTAVAIL)
3725                         return ret;
3726         }
3727         if (--remaining) {
3728                 rover++;
3729                 if ((rover < low) || (rover > high))
3730                         rover = low;
3731                 goto retry;
3732         }
3733         return -EADDRNOTAVAIL;
3734 }
3735
3736 /*
3737  * Check that the requested port is available.  This is called when trying to
3738  * bind to a specific port, or when trying to listen on a bound port.  In
3739  * the latter case, the provided id_priv may already be on the bind_list, but
3740  * we still need to check that it's okay to start listening.
3741  */
3742 static int cma_check_port(struct rdma_bind_list *bind_list,
3743                           struct rdma_id_private *id_priv, uint8_t reuseaddr)
3744 {
3745         struct rdma_id_private *cur_id;
3746         struct sockaddr *addr, *cur_addr;
3747
3748         lockdep_assert_held(&lock);
3749
3750         addr = cma_src_addr(id_priv);
3751         hlist_for_each_entry(cur_id, &bind_list->owners, node) {
3752                 if (id_priv == cur_id)
3753                         continue;
3754
3755                 if (reuseaddr && cur_id->reuseaddr)
3756                         continue;
3757
3758                 cur_addr = cma_src_addr(cur_id);
3759                 if (id_priv->afonly && cur_id->afonly &&
3760                     (addr->sa_family != cur_addr->sa_family))
3761                         continue;
3762
3763                 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
3764                         return -EADDRNOTAVAIL;
3765
3766                 if (!cma_addr_cmp(addr, cur_addr))
3767                         return -EADDRINUSE;
3768         }
3769         return 0;
3770 }
3771
3772 static int cma_use_port(enum rdma_ucm_port_space ps,
3773                         struct rdma_id_private *id_priv)
3774 {
3775         struct rdma_bind_list *bind_list;
3776         unsigned short snum;
3777         int ret;
3778
3779         lockdep_assert_held(&lock);
3780
3781         snum = ntohs(cma_port(cma_src_addr(id_priv)));
3782         if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
3783                 return -EACCES;
3784
3785         bind_list = cma_ps_find(id_priv->id.route.addr.dev_addr.net, ps, snum);
3786         if (!bind_list) {
3787                 ret = cma_alloc_port(ps, id_priv, snum);
3788         } else {
3789                 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
3790                 if (!ret)
3791                         cma_bind_port(bind_list, id_priv);
3792         }
3793         return ret;
3794 }
3795
3796 static enum rdma_ucm_port_space
3797 cma_select_inet_ps(struct rdma_id_private *id_priv)
3798 {
3799         switch (id_priv->id.ps) {
3800         case RDMA_PS_TCP:
3801         case RDMA_PS_UDP:
3802         case RDMA_PS_IPOIB:
3803         case RDMA_PS_IB:
3804                 return id_priv->id.ps;
3805         default:
3806
3807                 return 0;
3808         }
3809 }
3810
3811 static enum rdma_ucm_port_space
3812 cma_select_ib_ps(struct rdma_id_private *id_priv)
3813 {
3814         enum rdma_ucm_port_space ps = 0;
3815         struct sockaddr_ib *sib;
3816         u64 sid_ps, mask, sid;
3817
3818         sib = (struct sockaddr_ib *) cma_src_addr(id_priv);
3819         mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
3820         sid = be64_to_cpu(sib->sib_sid) & mask;
3821
3822         if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
3823                 sid_ps = RDMA_IB_IP_PS_IB;
3824                 ps = RDMA_PS_IB;
3825         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
3826                    (sid == (RDMA_IB_IP_PS_TCP & mask))) {
3827                 sid_ps = RDMA_IB_IP_PS_TCP;
3828                 ps = RDMA_PS_TCP;
3829         } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
3830                    (sid == (RDMA_IB_IP_PS_UDP & mask))) {
3831                 sid_ps = RDMA_IB_IP_PS_UDP;
3832                 ps = RDMA_PS_UDP;
3833         }
3834
3835         if (ps) {
3836                 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
3837                 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
3838                                                 be64_to_cpu(sib->sib_sid_mask));
3839         }
3840         return ps;
3841 }
3842
3843 static int cma_get_port(struct rdma_id_private *id_priv)
3844 {
3845         enum rdma_ucm_port_space ps;
3846         int ret;
3847
3848         if (cma_family(id_priv) != AF_IB)
3849                 ps = cma_select_inet_ps(id_priv);
3850         else
3851                 ps = cma_select_ib_ps(id_priv);
3852         if (!ps)
3853                 return -EPROTONOSUPPORT;
3854
3855         mutex_lock(&lock);
3856         if (cma_any_port(cma_src_addr(id_priv)))
3857                 ret = cma_alloc_any_port(ps, id_priv);
3858         else
3859                 ret = cma_use_port(ps, id_priv);
3860         mutex_unlock(&lock);
3861
3862         return ret;
3863 }
3864
3865 static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
3866                                struct sockaddr *addr)
3867 {
3868 #if IS_ENABLED(CONFIG_IPV6)
3869         struct sockaddr_in6 *sin6;
3870
3871         if (addr->sa_family != AF_INET6)
3872                 return 0;
3873
3874         sin6 = (struct sockaddr_in6 *) addr;
3875
3876         if (!(ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL))
3877                 return 0;
3878
3879         if (!sin6->sin6_scope_id)
3880                         return -EINVAL;
3881
3882         dev_addr->bound_dev_if = sin6->sin6_scope_id;
3883 #endif
3884         return 0;
3885 }
3886
3887 int rdma_listen(struct rdma_cm_id *id, int backlog)
3888 {
3889         struct rdma_id_private *id_priv =
3890                 container_of(id, struct rdma_id_private, id);
3891         int ret;
3892
3893         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN)) {
3894                 struct sockaddr_in any_in = {
3895                         .sin_family = AF_INET,
3896                         .sin_addr.s_addr = htonl(INADDR_ANY),
3897                 };
3898
3899                 /* For a well behaved ULP state will be RDMA_CM_IDLE */
3900                 ret = rdma_bind_addr(id, (struct sockaddr *)&any_in);
3901                 if (ret)
3902                         return ret;
3903                 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
3904                                            RDMA_CM_LISTEN)))
3905                         return -EINVAL;
3906         }
3907
3908         /*
3909          * Once the ID reaches RDMA_CM_LISTEN it is not allowed to be reusable
3910          * any more, and has to be unique in the bind list.
3911          */
3912         if (id_priv->reuseaddr) {
3913                 mutex_lock(&lock);
3914                 ret = cma_check_port(id_priv->bind_list, id_priv, 0);
3915                 if (!ret)
3916                         id_priv->reuseaddr = 0;
3917                 mutex_unlock(&lock);
3918                 if (ret)
3919                         goto err;
3920         }
3921
3922         id_priv->backlog = backlog;
3923         if (id_priv->cma_dev) {
3924                 if (rdma_cap_ib_cm(id->device, 1)) {
3925                         ret = cma_ib_listen(id_priv);
3926                         if (ret)
3927                                 goto err;
3928                 } else if (rdma_cap_iw_cm(id->device, 1)) {
3929                         ret = cma_iw_listen(id_priv, backlog);
3930                         if (ret)
3931                                 goto err;
3932                 } else {
3933                         ret = -ENOSYS;
3934                         goto err;
3935                 }
3936         } else {
3937                 ret = cma_listen_on_all(id_priv);
3938                 if (ret)
3939                         goto err;
3940         }
3941
3942         return 0;
3943 err:
3944         id_priv->backlog = 0;
3945         /*
3946          * All the failure paths that lead here will not allow the req_handler's
3947          * to have run.
3948          */
3949         cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
3950         return ret;
3951 }
3952 EXPORT_SYMBOL(rdma_listen);
3953
3954 static int rdma_bind_addr_dst(struct rdma_id_private *id_priv,
3955                               struct sockaddr *addr, const struct sockaddr *daddr)
3956 {
3957         struct sockaddr *id_daddr;
3958         int ret;
3959
3960         if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
3961             addr->sa_family != AF_IB)
3962                 return -EAFNOSUPPORT;
3963
3964         if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
3965                 return -EINVAL;
3966
3967         ret = cma_check_linklocal(&id_priv->id.route.addr.dev_addr, addr);
3968         if (ret)
3969                 goto err1;
3970
3971         memcpy(cma_src_addr(id_priv), addr, rdma_addr_size(addr));
3972         if (!cma_any_addr(addr)) {
3973                 ret = cma_translate_addr(addr, &id_priv->id.route.addr.dev_addr);
3974                 if (ret)
3975                         goto err1;
3976
3977                 ret = cma_acquire_dev_by_src_ip(id_priv);
3978                 if (ret)
3979                         goto err1;
3980         }
3981
3982         if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
3983                 if (addr->sa_family == AF_INET)
3984                         id_priv->afonly = 1;
3985 #if IS_ENABLED(CONFIG_IPV6)
3986                 else if (addr->sa_family == AF_INET6) {
3987                         struct net *net = id_priv->id.route.addr.dev_addr.net;
3988
3989                         id_priv->afonly = net->ipv6.sysctl.bindv6only;
3990                 }
3991 #endif
3992         }
3993         id_daddr = cma_dst_addr(id_priv);
3994         if (daddr != id_daddr)
3995                 memcpy(id_daddr, daddr, rdma_addr_size(addr));
3996         id_daddr->sa_family = addr->sa_family;
3997
3998         ret = cma_get_port(id_priv);
3999         if (ret)
4000                 goto err2;
4001
4002         if (!cma_any_addr(addr))
4003                 rdma_restrack_add(&id_priv->res);
4004         return 0;
4005 err2:
4006         if (id_priv->cma_dev)
4007                 cma_release_dev(id_priv);
4008 err1:
4009         cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
4010         return ret;
4011 }
4012
4013 static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4014                          const struct sockaddr *dst_addr)
4015 {
4016         struct rdma_id_private *id_priv =
4017                 container_of(id, struct rdma_id_private, id);
4018         struct sockaddr_storage zero_sock = {};
4019
4020         if (src_addr && src_addr->sa_family)
4021                 return rdma_bind_addr_dst(id_priv, src_addr, dst_addr);
4022
4023         /*
4024          * When the src_addr is not specified, automatically supply an any addr
4025          */
4026         zero_sock.ss_family = dst_addr->sa_family;
4027         if (IS_ENABLED(CONFIG_IPV6) && dst_addr->sa_family == AF_INET6) {
4028                 struct sockaddr_in6 *src_addr6 =
4029                         (struct sockaddr_in6 *)&zero_sock;
4030                 struct sockaddr_in6 *dst_addr6 =
4031                         (struct sockaddr_in6 *)dst_addr;
4032
4033                 src_addr6->sin6_scope_id = dst_addr6->sin6_scope_id;
4034                 if (ipv6_addr_type(&dst_addr6->sin6_addr) & IPV6_ADDR_LINKLOCAL)
4035                         id->route.addr.dev_addr.bound_dev_if =
4036                                 dst_addr6->sin6_scope_id;
4037         } else if (dst_addr->sa_family == AF_IB) {
4038                 ((struct sockaddr_ib *)&zero_sock)->sib_pkey =
4039                         ((struct sockaddr_ib *)dst_addr)->sib_pkey;
4040         }
4041         return rdma_bind_addr_dst(id_priv, (struct sockaddr *)&zero_sock, dst_addr);
4042 }
4043
4044 /*
4045  * If required, resolve the source address for bind and leave the id_priv in
4046  * state RDMA_CM_ADDR_BOUND. This oddly uses the state to determine the prior
4047  * calls made by ULP, a previously bound ID will not be re-bound and src_addr is
4048  * ignored.
4049  */
4050 static int resolve_prepare_src(struct rdma_id_private *id_priv,
4051                                struct sockaddr *src_addr,
4052                                const struct sockaddr *dst_addr)
4053 {
4054         int ret;
4055
4056         if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY)) {
4057                 /* For a well behaved ULP state will be RDMA_CM_IDLE */
4058                 ret = cma_bind_addr(&id_priv->id, src_addr, dst_addr);
4059                 if (ret)
4060                         return ret;
4061                 if (WARN_ON(!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND,
4062                                            RDMA_CM_ADDR_QUERY)))
4063                         return -EINVAL;
4064
4065         }
4066
4067         if (cma_family(id_priv) != dst_addr->sa_family) {
4068                 ret = -EINVAL;
4069                 goto err_state;
4070         }
4071         return 0;
4072
4073 err_state:
4074         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4075         return ret;
4076 }
4077
4078 int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
4079                       const struct sockaddr *dst_addr, unsigned long timeout_ms)
4080 {
4081         struct rdma_id_private *id_priv =
4082                 container_of(id, struct rdma_id_private, id);
4083         int ret;
4084
4085         ret = resolve_prepare_src(id_priv, src_addr, dst_addr);
4086         if (ret)
4087                 return ret;
4088
4089         if (cma_any_addr(dst_addr)) {
4090                 ret = cma_resolve_loopback(id_priv);
4091         } else {
4092                 if (dst_addr->sa_family == AF_IB) {
4093                         ret = cma_resolve_ib_addr(id_priv);
4094                 } else {
4095                         /*
4096                          * The FSM can return back to RDMA_CM_ADDR_BOUND after
4097                          * rdma_resolve_ip() is called, eg through the error
4098                          * path in addr_handler(). If this happens the existing
4099                          * request must be canceled before issuing a new one.
4100                          * Since canceling a request is a bit slow and this
4101                          * oddball path is rare, keep track once a request has
4102                          * been issued. The track turns out to be a permanent
4103                          * state since this is the only cancel as it is
4104                          * immediately before rdma_resolve_ip().
4105                          */
4106                         if (id_priv->used_resolve_ip)
4107                                 rdma_addr_cancel(&id->route.addr.dev_addr);
4108                         else
4109                                 id_priv->used_resolve_ip = 1;
4110                         ret = rdma_resolve_ip(cma_src_addr(id_priv), dst_addr,
4111                                               &id->route.addr.dev_addr,
4112                                               timeout_ms, addr_handler,
4113                                               false, id_priv);
4114                 }
4115         }
4116         if (ret)
4117                 goto err;
4118
4119         return 0;
4120 err:
4121         cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
4122         return ret;
4123 }
4124 EXPORT_SYMBOL(rdma_resolve_addr);
4125
4126 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
4127 {
4128         struct rdma_id_private *id_priv =
4129                 container_of(id, struct rdma_id_private, id);
4130
4131         return rdma_bind_addr_dst(id_priv, addr, cma_dst_addr(id_priv));
4132 }
4133 EXPORT_SYMBOL(rdma_bind_addr);
4134
4135 static int cma_format_hdr(void *hdr, struct rdma_id_private *id_priv)
4136 {
4137         struct cma_hdr *cma_hdr;
4138
4139         cma_hdr = hdr;
4140         cma_hdr->cma_version = CMA_VERSION;
4141         if (cma_family(id_priv) == AF_INET) {
4142                 struct sockaddr_in *src4, *dst4;
4143
4144                 src4 = (struct sockaddr_in *) cma_src_addr(id_priv);
4145                 dst4 = (struct sockaddr_in *) cma_dst_addr(id_priv);
4146
4147                 cma_set_ip_ver(cma_hdr, 4);
4148                 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
4149                 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
4150                 cma_hdr->port = src4->sin_port;
4151         } else if (cma_family(id_priv) == AF_INET6) {
4152                 struct sockaddr_in6 *src6, *dst6;
4153
4154                 src6 = (struct sockaddr_in6 *) cma_src_addr(id_priv);
4155                 dst6 = (struct sockaddr_in6 *) cma_dst_addr(id_priv);
4156
4157                 cma_set_ip_ver(cma_hdr, 6);
4158                 cma_hdr->src_addr.ip6 = src6->sin6_addr;
4159                 cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
4160                 cma_hdr->port = src6->sin6_port;
4161         }
4162         return 0;
4163 }
4164
4165 static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
4166                                 const struct ib_cm_event *ib_event)
4167 {
4168         struct rdma_id_private *id_priv = cm_id->context;
4169         struct rdma_cm_event event = {};
4170         const struct ib_cm_sidr_rep_event_param *rep =
4171                                 &ib_event->param.sidr_rep_rcvd;
4172         int ret;
4173
4174         mutex_lock(&id_priv->handler_mutex);
4175         if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
4176                 goto out;
4177
4178         switch (ib_event->event) {
4179         case IB_CM_SIDR_REQ_ERROR:
4180                 event.event = RDMA_CM_EVENT_UNREACHABLE;
4181                 event.status = -ETIMEDOUT;
4182                 break;
4183         case IB_CM_SIDR_REP_RECEIVED:
4184                 event.param.ud.private_data = ib_event->private_data;
4185                 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
4186                 if (rep->status != IB_SIDR_SUCCESS) {
4187                         event.event = RDMA_CM_EVENT_UNREACHABLE;
4188                         event.status = ib_event->param.sidr_rep_rcvd.status;
4189                         pr_debug_ratelimited("RDMA CM: UNREACHABLE: bad SIDR reply. status %d\n",
4190                                              event.status);
4191                         break;
4192                 }
4193                 ret = cma_set_qkey(id_priv, rep->qkey);
4194                 if (ret) {
4195                         pr_debug_ratelimited("RDMA CM: ADDR_ERROR: failed to set qkey. status %d\n", ret);
4196                         event.event = RDMA_CM_EVENT_ADDR_ERROR;
4197                         event.status = ret;
4198                         break;
4199                 }
4200                 ib_init_ah_attr_from_path(id_priv->id.device,
4201                                           id_priv->id.port_num,
4202                                           id_priv->id.route.path_rec,
4203                                           &event.param.ud.ah_attr,
4204                                           rep->sgid_attr);
4205                 event.param.ud.qp_num = rep->qpn;
4206                 event.param.ud.qkey = rep->qkey;
4207                 event.event = RDMA_CM_EVENT_ESTABLISHED;
4208                 event.status = 0;
4209                 break;
4210         default:
4211                 pr_err("RDMA CMA: unexpected IB CM event: %d\n",
4212                        ib_event->event);
4213                 goto out;
4214         }
4215
4216         ret = cma_cm_event_handler(id_priv, &event);
4217
4218         rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4219         if (ret) {
4220                 /* Destroy the CM ID by returning a non-zero value. */
4221                 id_priv->cm_id.ib = NULL;
4222                 destroy_id_handler_unlock(id_priv);
4223                 return ret;
4224         }
4225 out:
4226         mutex_unlock(&id_priv->handler_mutex);
4227         return 0;
4228 }
4229
4230 static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
4231                               struct rdma_conn_param *conn_param)
4232 {
4233         struct ib_cm_sidr_req_param req;
4234         struct ib_cm_id *id;
4235         void *private_data;
4236         u8 offset;
4237         int ret;
4238
4239         memset(&req, 0, sizeof req);
4240         offset = cma_user_data_offset(id_priv);
4241         if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4242                 return -EINVAL;
4243
4244         if (req.private_data_len) {
4245                 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
4246                 if (!private_data)
4247                         return -ENOMEM;
4248         } else {
4249                 private_data = NULL;
4250         }
4251
4252         if (conn_param->private_data && conn_param->private_data_len)
4253                 memcpy(private_data + offset, conn_param->private_data,
4254                        conn_param->private_data_len);
4255
4256         if (private_data) {
4257                 ret = cma_format_hdr(private_data, id_priv);
4258                 if (ret)
4259                         goto out;
4260                 req.private_data = private_data;
4261         }
4262
4263         id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
4264                              id_priv);
4265         if (IS_ERR(id)) {
4266                 ret = PTR_ERR(id);
4267                 goto out;
4268         }
4269         id_priv->cm_id.ib = id;
4270
4271         req.path = id_priv->id.route.path_rec;
4272         req.sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
4273         req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
4274         req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
4275         req.max_cm_retries = CMA_MAX_CM_RETRIES;
4276
4277         trace_cm_send_sidr_req(id_priv);
4278         ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
4279         if (ret) {
4280                 ib_destroy_cm_id(id_priv->cm_id.ib);
4281                 id_priv->cm_id.ib = NULL;
4282         }
4283 out:
4284         kfree(private_data);
4285         return ret;
4286 }
4287
4288 static int cma_connect_ib(struct rdma_id_private *id_priv,
4289                           struct rdma_conn_param *conn_param)
4290 {
4291         struct ib_cm_req_param req;
4292         struct rdma_route *route;
4293         void *private_data;
4294         struct ib_cm_id *id;
4295         u8 offset;
4296         int ret;
4297
4298         memset(&req, 0, sizeof req);
4299         offset = cma_user_data_offset(id_priv);
4300         if (check_add_overflow(offset, conn_param->private_data_len, &req.private_data_len))
4301                 return -EINVAL;
4302
4303         if (req.private_data_len) {
4304                 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
4305                 if (!private_data)
4306                         return -ENOMEM;
4307         } else {
4308                 private_data = NULL;
4309         }
4310
4311         if (conn_param->private_data && conn_param->private_data_len)
4312                 memcpy(private_data + offset, conn_param->private_data,
4313                        conn_param->private_data_len);
4314
4315         id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
4316         if (IS_ERR(id)) {
4317                 ret = PTR_ERR(id);
4318                 goto out;
4319         }
4320         id_priv->cm_id.ib = id;
4321
4322         route = &id_priv->id.route;
4323         if (private_data) {
4324                 ret = cma_format_hdr(private_data, id_priv);
4325                 if (ret)
4326                         goto out;
4327                 req.private_data = private_data;
4328         }
4329
4330         req.primary_path = &route->path_rec[0];
4331         req.primary_path_inbound = route->path_rec_inbound;
4332         req.primary_path_outbound = route->path_rec_outbound;
4333         if (route->num_pri_alt_paths == 2)
4334                 req.alternate_path = &route->path_rec[1];
4335
4336         req.ppath_sgid_attr = id_priv->id.route.addr.dev_addr.sgid_attr;
4337         /* Alternate path SGID attribute currently unsupported */
4338         req.service_id = rdma_get_service_id(&id_priv->id, cma_dst_addr(id_priv));
4339         req.qp_num = id_priv->qp_num;
4340         req.qp_type = id_priv->id.qp_type;
4341         req.starting_psn = id_priv->seq_num;
4342         req.responder_resources = conn_param->responder_resources;
4343         req.initiator_depth = conn_param->initiator_depth;
4344         req.flow_control = conn_param->flow_control;
4345         req.retry_count = min_t(u8, 7, conn_param->retry_count);
4346         req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4347         req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4348         req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
4349         req.max_cm_retries = CMA_MAX_CM_RETRIES;
4350         req.srq = id_priv->srq ? 1 : 0;
4351         req.ece.vendor_id = id_priv->ece.vendor_id;
4352         req.ece.attr_mod = id_priv->ece.attr_mod;
4353
4354         trace_cm_send_req(id_priv);
4355         ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
4356 out:
4357         if (ret && !IS_ERR(id)) {
4358                 ib_destroy_cm_id(id);
4359                 id_priv->cm_id.ib = NULL;
4360         }
4361
4362         kfree(private_data);
4363         return ret;
4364 }
4365
4366 static int cma_connect_iw(struct rdma_id_private *id_priv,
4367                           struct rdma_conn_param *conn_param)
4368 {
4369         struct iw_cm_id *cm_id;
4370         int ret;
4371         struct iw_cm_conn_param iw_param;
4372
4373         cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
4374         if (IS_ERR(cm_id))
4375                 return PTR_ERR(cm_id);
4376
4377         mutex_lock(&id_priv->qp_mutex);
4378         cm_id->tos = id_priv->tos;
4379         cm_id->tos_set = id_priv->tos_set;
4380         mutex_unlock(&id_priv->qp_mutex);
4381
4382         id_priv->cm_id.iw = cm_id;
4383
4384         memcpy(&cm_id->local_addr, cma_src_addr(id_priv),
4385                rdma_addr_size(cma_src_addr(id_priv)));
4386         memcpy(&cm_id->remote_addr, cma_dst_addr(id_priv),
4387                rdma_addr_size(cma_dst_addr(id_priv)));
4388
4389         ret = cma_modify_qp_rtr(id_priv, conn_param);
4390         if (ret)
4391                 goto out;
4392
4393         if (conn_param) {
4394                 iw_param.ord = conn_param->initiator_depth;
4395                 iw_param.ird = conn_param->responder_resources;
4396                 iw_param.private_data = conn_param->private_data;
4397                 iw_param.private_data_len = conn_param->private_data_len;
4398                 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
4399         } else {
4400                 memset(&iw_param, 0, sizeof iw_param);
4401                 iw_param.qpn = id_priv->qp_num;
4402         }
4403         ret = iw_cm_connect(cm_id, &iw_param);
4404 out:
4405         if (ret) {
4406                 iw_destroy_cm_id(cm_id);
4407                 id_priv->cm_id.iw = NULL;
4408         }
4409         return ret;
4410 }
4411
4412 /**
4413  * rdma_connect_locked - Initiate an active connection request.
4414  * @id: Connection identifier to connect.
4415  * @conn_param: Connection information used for connected QPs.
4416  *
4417  * Same as rdma_connect() but can only be called from the
4418  * RDMA_CM_EVENT_ROUTE_RESOLVED handler callback.
4419  */
4420 int rdma_connect_locked(struct rdma_cm_id *id,
4421                         struct rdma_conn_param *conn_param)
4422 {
4423         struct rdma_id_private *id_priv =
4424                 container_of(id, struct rdma_id_private, id);
4425         int ret;
4426
4427         if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
4428                 return -EINVAL;
4429
4430         if (!id->qp) {
4431                 id_priv->qp_num = conn_param->qp_num;
4432                 id_priv->srq = conn_param->srq;
4433         }
4434
4435         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4436                 if (id->qp_type == IB_QPT_UD)
4437                         ret = cma_resolve_ib_udp(id_priv, conn_param);
4438                 else
4439                         ret = cma_connect_ib(id_priv, conn_param);
4440         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4441                 ret = cma_connect_iw(id_priv, conn_param);
4442         } else {
4443                 ret = -ENOSYS;
4444         }
4445         if (ret)
4446                 goto err_state;
4447         return 0;
4448 err_state:
4449         cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
4450         return ret;
4451 }
4452 EXPORT_SYMBOL(rdma_connect_locked);
4453
4454 /**
4455  * rdma_connect - Initiate an active connection request.
4456  * @id: Connection identifier to connect.
4457  * @conn_param: Connection information used for connected QPs.
4458  *
4459  * Users must have resolved a route for the rdma_cm_id to connect with by having
4460  * called rdma_resolve_route before calling this routine.
4461  *
4462  * This call will either connect to a remote QP or obtain remote QP information
4463  * for unconnected rdma_cm_id's.  The actual operation is based on the
4464  * rdma_cm_id's port space.
4465  */
4466 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4467 {
4468         struct rdma_id_private *id_priv =
4469                 container_of(id, struct rdma_id_private, id);
4470         int ret;
4471
4472         mutex_lock(&id_priv->handler_mutex);
4473         ret = rdma_connect_locked(id, conn_param);
4474         mutex_unlock(&id_priv->handler_mutex);
4475         return ret;
4476 }
4477 EXPORT_SYMBOL(rdma_connect);
4478
4479 /**
4480  * rdma_connect_ece - Initiate an active connection request with ECE data.
4481  * @id: Connection identifier to connect.
4482  * @conn_param: Connection information used for connected QPs.
4483  * @ece: ECE parameters
4484  *
4485  * See rdma_connect() explanation.
4486  */
4487 int rdma_connect_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4488                      struct rdma_ucm_ece *ece)
4489 {
4490         struct rdma_id_private *id_priv =
4491                 container_of(id, struct rdma_id_private, id);
4492
4493         id_priv->ece.vendor_id = ece->vendor_id;
4494         id_priv->ece.attr_mod = ece->attr_mod;
4495
4496         return rdma_connect(id, conn_param);
4497 }
4498 EXPORT_SYMBOL(rdma_connect_ece);
4499
4500 static int cma_accept_ib(struct rdma_id_private *id_priv,
4501                          struct rdma_conn_param *conn_param)
4502 {
4503         struct ib_cm_rep_param rep;
4504         int ret;
4505
4506         ret = cma_modify_qp_rtr(id_priv, conn_param);
4507         if (ret)
4508                 goto out;
4509
4510         ret = cma_modify_qp_rts(id_priv, conn_param);
4511         if (ret)
4512                 goto out;
4513
4514         memset(&rep, 0, sizeof rep);
4515         rep.qp_num = id_priv->qp_num;
4516         rep.starting_psn = id_priv->seq_num;
4517         rep.private_data = conn_param->private_data;
4518         rep.private_data_len = conn_param->private_data_len;
4519         rep.responder_resources = conn_param->responder_resources;
4520         rep.initiator_depth = conn_param->initiator_depth;
4521         rep.failover_accepted = 0;
4522         rep.flow_control = conn_param->flow_control;
4523         rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
4524         rep.srq = id_priv->srq ? 1 : 0;
4525         rep.ece.vendor_id = id_priv->ece.vendor_id;
4526         rep.ece.attr_mod = id_priv->ece.attr_mod;
4527
4528         trace_cm_send_rep(id_priv);
4529         ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
4530 out:
4531         return ret;
4532 }
4533
4534 static int cma_accept_iw(struct rdma_id_private *id_priv,
4535                   struct rdma_conn_param *conn_param)
4536 {
4537         struct iw_cm_conn_param iw_param;
4538         int ret;
4539
4540         if (!conn_param)
4541                 return -EINVAL;
4542
4543         ret = cma_modify_qp_rtr(id_priv, conn_param);
4544         if (ret)
4545                 return ret;
4546
4547         iw_param.ord = conn_param->initiator_depth;
4548         iw_param.ird = conn_param->responder_resources;
4549         iw_param.private_data = conn_param->private_data;
4550         iw_param.private_data_len = conn_param->private_data_len;
4551         if (id_priv->id.qp)
4552                 iw_param.qpn = id_priv->qp_num;
4553         else
4554                 iw_param.qpn = conn_param->qp_num;
4555
4556         return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
4557 }
4558
4559 static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
4560                              enum ib_cm_sidr_status status, u32 qkey,
4561                              const void *private_data, int private_data_len)
4562 {
4563         struct ib_cm_sidr_rep_param rep;
4564         int ret;
4565
4566         memset(&rep, 0, sizeof rep);
4567         rep.status = status;
4568         if (status == IB_SIDR_SUCCESS) {
4569                 if (qkey)
4570                         ret = cma_set_qkey(id_priv, qkey);
4571                 else
4572                         ret = cma_set_default_qkey(id_priv);
4573                 if (ret)
4574                         return ret;
4575                 rep.qp_num = id_priv->qp_num;
4576                 rep.qkey = id_priv->qkey;
4577
4578                 rep.ece.vendor_id = id_priv->ece.vendor_id;
4579                 rep.ece.attr_mod = id_priv->ece.attr_mod;
4580         }
4581
4582         rep.private_data = private_data;
4583         rep.private_data_len = private_data_len;
4584
4585         trace_cm_send_sidr_rep(id_priv);
4586         return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
4587 }
4588
4589 /**
4590  * rdma_accept - Called to accept a connection request or response.
4591  * @id: Connection identifier associated with the request.
4592  * @conn_param: Information needed to establish the connection.  This must be
4593  *   provided if accepting a connection request.  If accepting a connection
4594  *   response, this parameter must be NULL.
4595  *
4596  * Typically, this routine is only called by the listener to accept a connection
4597  * request.  It must also be called on the active side of a connection if the
4598  * user is performing their own QP transitions.
4599  *
4600  * In the case of error, a reject message is sent to the remote side and the
4601  * state of the qp associated with the id is modified to error, such that any
4602  * previously posted receive buffers would be flushed.
4603  *
4604  * This function is for use by kernel ULPs and must be called from under the
4605  * handler callback.
4606  */
4607 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
4608 {
4609         struct rdma_id_private *id_priv =
4610                 container_of(id, struct rdma_id_private, id);
4611         int ret;
4612
4613         lockdep_assert_held(&id_priv->handler_mutex);
4614
4615         if (READ_ONCE(id_priv->state) != RDMA_CM_CONNECT)
4616                 return -EINVAL;
4617
4618         if (!id->qp && conn_param) {
4619                 id_priv->qp_num = conn_param->qp_num;
4620                 id_priv->srq = conn_param->srq;
4621         }
4622
4623         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4624                 if (id->qp_type == IB_QPT_UD) {
4625                         if (conn_param)
4626                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4627                                                         conn_param->qkey,
4628                                                         conn_param->private_data,
4629                                                         conn_param->private_data_len);
4630                         else
4631                                 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
4632                                                         0, NULL, 0);
4633                 } else {
4634                         if (conn_param)
4635                                 ret = cma_accept_ib(id_priv, conn_param);
4636                         else
4637                                 ret = cma_rep_recv(id_priv);
4638                 }
4639         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4640                 ret = cma_accept_iw(id_priv, conn_param);
4641         } else {
4642                 ret = -ENOSYS;
4643         }
4644         if (ret)
4645                 goto reject;
4646
4647         return 0;
4648 reject:
4649         cma_modify_qp_err(id_priv);
4650         rdma_reject(id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED);
4651         return ret;
4652 }
4653 EXPORT_SYMBOL(rdma_accept);
4654
4655 int rdma_accept_ece(struct rdma_cm_id *id, struct rdma_conn_param *conn_param,
4656                     struct rdma_ucm_ece *ece)
4657 {
4658         struct rdma_id_private *id_priv =
4659                 container_of(id, struct rdma_id_private, id);
4660
4661         id_priv->ece.vendor_id = ece->vendor_id;
4662         id_priv->ece.attr_mod = ece->attr_mod;
4663
4664         return rdma_accept(id, conn_param);
4665 }
4666 EXPORT_SYMBOL(rdma_accept_ece);
4667
4668 void rdma_lock_handler(struct rdma_cm_id *id)
4669 {
4670         struct rdma_id_private *id_priv =
4671                 container_of(id, struct rdma_id_private, id);
4672
4673         mutex_lock(&id_priv->handler_mutex);
4674 }
4675 EXPORT_SYMBOL(rdma_lock_handler);
4676
4677 void rdma_unlock_handler(struct rdma_cm_id *id)
4678 {
4679         struct rdma_id_private *id_priv =
4680                 container_of(id, struct rdma_id_private, id);
4681
4682         mutex_unlock(&id_priv->handler_mutex);
4683 }
4684 EXPORT_SYMBOL(rdma_unlock_handler);
4685
4686 int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
4687 {
4688         struct rdma_id_private *id_priv;
4689         int ret;
4690
4691         id_priv = container_of(id, struct rdma_id_private, id);
4692         if (!id_priv->cm_id.ib)
4693                 return -EINVAL;
4694
4695         switch (id->device->node_type) {
4696         case RDMA_NODE_IB_CA:
4697                 ret = ib_cm_notify(id_priv->cm_id.ib, event);
4698                 break;
4699         default:
4700                 ret = 0;
4701                 break;
4702         }
4703         return ret;
4704 }
4705 EXPORT_SYMBOL(rdma_notify);
4706
4707 int rdma_reject(struct rdma_cm_id *id, const void *private_data,
4708                 u8 private_data_len, u8 reason)
4709 {
4710         struct rdma_id_private *id_priv;
4711         int ret;
4712
4713         id_priv = container_of(id, struct rdma_id_private, id);
4714         if (!id_priv->cm_id.ib)
4715                 return -EINVAL;
4716
4717         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4718                 if (id->qp_type == IB_QPT_UD) {
4719                         ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT, 0,
4720                                                 private_data, private_data_len);
4721                 } else {
4722                         trace_cm_send_rej(id_priv);
4723                         ret = ib_send_cm_rej(id_priv->cm_id.ib, reason, NULL, 0,
4724                                              private_data, private_data_len);
4725                 }
4726         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4727                 ret = iw_cm_reject(id_priv->cm_id.iw,
4728                                    private_data, private_data_len);
4729         } else {
4730                 ret = -ENOSYS;
4731         }
4732
4733         return ret;
4734 }
4735 EXPORT_SYMBOL(rdma_reject);
4736
4737 int rdma_disconnect(struct rdma_cm_id *id)
4738 {
4739         struct rdma_id_private *id_priv;
4740         int ret;
4741
4742         id_priv = container_of(id, struct rdma_id_private, id);
4743         if (!id_priv->cm_id.ib)
4744                 return -EINVAL;
4745
4746         if (rdma_cap_ib_cm(id->device, id->port_num)) {
4747                 ret = cma_modify_qp_err(id_priv);
4748                 if (ret)
4749                         goto out;
4750                 /* Initiate or respond to a disconnect. */
4751                 trace_cm_disconnect(id_priv);
4752                 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0)) {
4753                         if (!ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0))
4754                                 trace_cm_sent_drep(id_priv);
4755                 } else {
4756                         trace_cm_sent_dreq(id_priv);
4757                 }
4758         } else if (rdma_cap_iw_cm(id->device, id->port_num)) {
4759                 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
4760         } else
4761                 ret = -EINVAL;
4762
4763 out:
4764         return ret;
4765 }
4766 EXPORT_SYMBOL(rdma_disconnect);
4767
4768 static void cma_make_mc_event(int status, struct rdma_id_private *id_priv,
4769                               struct ib_sa_multicast *multicast,
4770                               struct rdma_cm_event *event,
4771                               struct cma_multicast *mc)
4772 {
4773         struct rdma_dev_addr *dev_addr;
4774         enum ib_gid_type gid_type;
4775         struct net_device *ndev;
4776
4777         if (status)
4778                 pr_debug_ratelimited("RDMA CM: MULTICAST_ERROR: failed to join multicast. status %d\n",
4779                                      status);
4780
4781         event->status = status;
4782         event->param.ud.private_data = mc->context;
4783         if (status) {
4784                 event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4785                 return;
4786         }
4787
4788         dev_addr = &id_priv->id.route.addr.dev_addr;
4789         ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4790         gid_type =
4791                 id_priv->cma_dev
4792                         ->default_gid_type[id_priv->id.port_num -
4793                                            rdma_start_port(
4794                                                    id_priv->cma_dev->device)];
4795
4796         event->event = RDMA_CM_EVENT_MULTICAST_JOIN;
4797         if (ib_init_ah_from_mcmember(id_priv->id.device, id_priv->id.port_num,
4798                                      &multicast->rec, ndev, gid_type,
4799                                      &event->param.ud.ah_attr)) {
4800                 event->event = RDMA_CM_EVENT_MULTICAST_ERROR;
4801                 goto out;
4802         }
4803
4804         event->param.ud.qp_num = 0xFFFFFF;
4805         event->param.ud.qkey = id_priv->qkey;
4806
4807 out:
4808         dev_put(ndev);
4809 }
4810
4811 static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
4812 {
4813         struct cma_multicast *mc = multicast->context;
4814         struct rdma_id_private *id_priv = mc->id_priv;
4815         struct rdma_cm_event event = {};
4816         int ret = 0;
4817
4818         mutex_lock(&id_priv->handler_mutex);
4819         if (READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL ||
4820             READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING)
4821                 goto out;
4822
4823         ret = cma_set_qkey(id_priv, be32_to_cpu(multicast->rec.qkey));
4824         if (!ret) {
4825                 cma_make_mc_event(status, id_priv, multicast, &event, mc);
4826                 ret = cma_cm_event_handler(id_priv, &event);
4827         }
4828         rdma_destroy_ah_attr(&event.param.ud.ah_attr);
4829         WARN_ON(ret);
4830
4831 out:
4832         mutex_unlock(&id_priv->handler_mutex);
4833         return 0;
4834 }
4835
4836 static void cma_set_mgid(struct rdma_id_private *id_priv,
4837                          struct sockaddr *addr, union ib_gid *mgid)
4838 {
4839         unsigned char mc_map[MAX_ADDR_LEN];
4840         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4841         struct sockaddr_in *sin = (struct sockaddr_in *) addr;
4842         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
4843
4844         if (cma_any_addr(addr)) {
4845                 memset(mgid, 0, sizeof *mgid);
4846         } else if ((addr->sa_family == AF_INET6) &&
4847                    ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
4848                                                                  0xFF10A01B)) {
4849                 /* IPv6 address is an SA assigned MGID. */
4850                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4851         } else if (addr->sa_family == AF_IB) {
4852                 memcpy(mgid, &((struct sockaddr_ib *) addr)->sib_addr, sizeof *mgid);
4853         } else if (addr->sa_family == AF_INET6) {
4854                 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
4855                 if (id_priv->id.ps == RDMA_PS_UDP)
4856                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
4857                 *mgid = *(union ib_gid *) (mc_map + 4);
4858         } else {
4859                 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
4860                 if (id_priv->id.ps == RDMA_PS_UDP)
4861                         mc_map[7] = 0x01;       /* Use RDMA CM signature */
4862                 *mgid = *(union ib_gid *) (mc_map + 4);
4863         }
4864 }
4865
4866 static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
4867                                  struct cma_multicast *mc)
4868 {
4869         struct ib_sa_mcmember_rec rec;
4870         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4871         ib_sa_comp_mask comp_mask;
4872         int ret;
4873
4874         ib_addr_get_mgid(dev_addr, &rec.mgid);
4875         ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
4876                                      &rec.mgid, &rec);
4877         if (ret)
4878                 return ret;
4879
4880         if (!id_priv->qkey) {
4881                 ret = cma_set_default_qkey(id_priv);
4882                 if (ret)
4883                         return ret;
4884         }
4885
4886         cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
4887         rec.qkey = cpu_to_be32(id_priv->qkey);
4888         rdma_addr_get_sgid(dev_addr, &rec.port_gid);
4889         rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
4890         rec.join_state = mc->join_state;
4891
4892         comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
4893                     IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
4894                     IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
4895                     IB_SA_MCMEMBER_REC_FLOW_LABEL |
4896                     IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
4897
4898         if (id_priv->id.ps == RDMA_PS_IPOIB)
4899                 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
4900                              IB_SA_MCMEMBER_REC_RATE_SELECTOR |
4901                              IB_SA_MCMEMBER_REC_MTU_SELECTOR |
4902                              IB_SA_MCMEMBER_REC_MTU |
4903                              IB_SA_MCMEMBER_REC_HOP_LIMIT;
4904
4905         mc->sa_mc = ib_sa_join_multicast(&sa_client, id_priv->id.device,
4906                                          id_priv->id.port_num, &rec, comp_mask,
4907                                          GFP_KERNEL, cma_ib_mc_handler, mc);
4908         return PTR_ERR_OR_ZERO(mc->sa_mc);
4909 }
4910
4911 static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid,
4912                               enum ib_gid_type gid_type)
4913 {
4914         struct sockaddr_in *sin = (struct sockaddr_in *)addr;
4915         struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
4916
4917         if (cma_any_addr(addr)) {
4918                 memset(mgid, 0, sizeof *mgid);
4919         } else if (addr->sa_family == AF_INET6) {
4920                 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
4921         } else {
4922                 mgid->raw[0] =
4923                         (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0xff;
4924                 mgid->raw[1] =
4925                         (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ? 0 : 0x0e;
4926                 mgid->raw[2] = 0;
4927                 mgid->raw[3] = 0;
4928                 mgid->raw[4] = 0;
4929                 mgid->raw[5] = 0;
4930                 mgid->raw[6] = 0;
4931                 mgid->raw[7] = 0;
4932                 mgid->raw[8] = 0;
4933                 mgid->raw[9] = 0;
4934                 mgid->raw[10] = 0xff;
4935                 mgid->raw[11] = 0xff;
4936                 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
4937         }
4938 }
4939
4940 static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
4941                                    struct cma_multicast *mc)
4942 {
4943         struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
4944         int err = 0;
4945         struct sockaddr *addr = (struct sockaddr *)&mc->addr;
4946         struct net_device *ndev = NULL;
4947         struct ib_sa_multicast ib;
4948         enum ib_gid_type gid_type;
4949         bool send_only;
4950
4951         send_only = mc->join_state == BIT(SENDONLY_FULLMEMBER_JOIN);
4952
4953         if (cma_zero_addr(addr))
4954                 return -EINVAL;
4955
4956         gid_type = id_priv->cma_dev->default_gid_type[id_priv->id.port_num -
4957                    rdma_start_port(id_priv->cma_dev->device)];
4958         cma_iboe_set_mgid(addr, &ib.rec.mgid, gid_type);
4959
4960         ib.rec.pkey = cpu_to_be16(0xffff);
4961         if (dev_addr->bound_dev_if)
4962                 ndev = dev_get_by_index(dev_addr->net, dev_addr->bound_dev_if);
4963         if (!ndev)
4964                 return -ENODEV;
4965
4966         ib.rec.rate = IB_RATE_PORT_CURRENT;
4967         ib.rec.hop_limit = 1;
4968         ib.rec.mtu = iboe_get_mtu(ndev->mtu);
4969
4970         if (addr->sa_family == AF_INET) {
4971                 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) {
4972                         ib.rec.hop_limit = IPV6_DEFAULT_HOPLIMIT;
4973                         if (!send_only) {
4974                                 err = cma_igmp_send(ndev, &ib.rec.mgid,
4975                                                     true);
4976                         }
4977                 }
4978         } else {
4979                 if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
4980                         err = -ENOTSUPP;
4981         }
4982         dev_put(ndev);
4983         if (err || !ib.rec.mtu)
4984                 return err ?: -EINVAL;
4985
4986         if (!id_priv->qkey)
4987                 cma_set_default_qkey(id_priv);
4988
4989         rdma_ip2gid((struct sockaddr *)&id_priv->id.route.addr.src_addr,
4990                     &ib.rec.port_gid);
4991         INIT_WORK(&mc->iboe_join.work, cma_iboe_join_work_handler);
4992         cma_make_mc_event(0, id_priv, &ib, &mc->iboe_join.event, mc);
4993         queue_work(cma_wq, &mc->iboe_join.work);
4994         return 0;
4995 }
4996
4997 int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
4998                         u8 join_state, void *context)
4999 {
5000         struct rdma_id_private *id_priv =
5001                 container_of(id, struct rdma_id_private, id);
5002         struct cma_multicast *mc;
5003         int ret;
5004
5005         /* Not supported for kernel QPs */
5006         if (WARN_ON(id->qp))
5007                 return -EINVAL;
5008
5009         /* ULP is calling this wrong. */
5010         if (!id->device || (READ_ONCE(id_priv->state) != RDMA_CM_ADDR_BOUND &&
5011                             READ_ONCE(id_priv->state) != RDMA_CM_ADDR_RESOLVED))
5012                 return -EINVAL;
5013
5014         if (id_priv->id.qp_type != IB_QPT_UD)
5015                 return -EINVAL;
5016
5017         mc = kzalloc(sizeof(*mc), GFP_KERNEL);
5018         if (!mc)
5019                 return -ENOMEM;
5020
5021         memcpy(&mc->addr, addr, rdma_addr_size(addr));
5022         mc->context = context;
5023         mc->id_priv = id_priv;
5024         mc->join_state = join_state;
5025
5026         if (rdma_protocol_roce(id->device, id->port_num)) {
5027                 ret = cma_iboe_join_multicast(id_priv, mc);
5028                 if (ret)
5029                         goto out_err;
5030         } else if (rdma_cap_ib_mcast(id->device, id->port_num)) {
5031                 ret = cma_join_ib_multicast(id_priv, mc);
5032                 if (ret)
5033                         goto out_err;
5034         } else {
5035                 ret = -ENOSYS;
5036                 goto out_err;
5037         }
5038
5039         spin_lock(&id_priv->lock);
5040         list_add(&mc->list, &id_priv->mc_list);
5041         spin_unlock(&id_priv->lock);
5042
5043         return 0;
5044 out_err:
5045         kfree(mc);
5046         return ret;
5047 }
5048 EXPORT_SYMBOL(rdma_join_multicast);
5049
5050 void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
5051 {
5052         struct rdma_id_private *id_priv;
5053         struct cma_multicast *mc;
5054
5055         id_priv = container_of(id, struct rdma_id_private, id);
5056         spin_lock_irq(&id_priv->lock);
5057         list_for_each_entry(mc, &id_priv->mc_list, list) {
5058                 if (memcmp(&mc->addr, addr, rdma_addr_size(addr)) != 0)
5059                         continue;
5060                 list_del(&mc->list);
5061                 spin_unlock_irq(&id_priv->lock);
5062
5063                 WARN_ON(id_priv->cma_dev->device != id->device);
5064                 destroy_mc(id_priv, mc);
5065                 return;
5066         }
5067         spin_unlock_irq(&id_priv->lock);
5068 }
5069 EXPORT_SYMBOL(rdma_leave_multicast);
5070
5071 static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
5072 {
5073         struct rdma_dev_addr *dev_addr;
5074         struct cma_work *work;
5075
5076         dev_addr = &id_priv->id.route.addr.dev_addr;
5077
5078         if ((dev_addr->bound_dev_if == ndev->ifindex) &&
5079             (net_eq(dev_net(ndev), dev_addr->net)) &&
5080             memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
5081                 pr_info("RDMA CM addr change for ndev %s used by id %p\n",
5082                         ndev->name, &id_priv->id);
5083                 work = kzalloc(sizeof *work, GFP_KERNEL);
5084                 if (!work)
5085                         return -ENOMEM;
5086
5087                 INIT_WORK(&work->work, cma_work_handler);
5088                 work->id = id_priv;
5089                 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
5090                 cma_id_get(id_priv);
5091                 queue_work(cma_wq, &work->work);
5092         }
5093
5094         return 0;
5095 }
5096
5097 static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
5098                                void *ptr)
5099 {
5100         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
5101         struct cma_device *cma_dev;
5102         struct rdma_id_private *id_priv;
5103         int ret = NOTIFY_DONE;
5104
5105         if (event != NETDEV_BONDING_FAILOVER)
5106                 return NOTIFY_DONE;
5107
5108         if (!netif_is_bond_master(ndev))
5109                 return NOTIFY_DONE;
5110
5111         mutex_lock(&lock);
5112         list_for_each_entry(cma_dev, &dev_list, list)
5113                 list_for_each_entry(id_priv, &cma_dev->id_list, device_item) {
5114                         ret = cma_netdev_change(ndev, id_priv);
5115                         if (ret)
5116                                 goto out;
5117                 }
5118
5119 out:
5120         mutex_unlock(&lock);
5121         return ret;
5122 }
5123
5124 static void cma_netevent_work_handler(struct work_struct *_work)
5125 {
5126         struct rdma_id_private *id_priv =
5127                 container_of(_work, struct rdma_id_private, id.net_work);
5128         struct rdma_cm_event event = {};
5129
5130         mutex_lock(&id_priv->handler_mutex);
5131
5132         if (READ_ONCE(id_priv->state) == RDMA_CM_DESTROYING ||
5133             READ_ONCE(id_priv->state) == RDMA_CM_DEVICE_REMOVAL)
5134                 goto out_unlock;
5135
5136         event.event = RDMA_CM_EVENT_UNREACHABLE;
5137         event.status = -ETIMEDOUT;
5138
5139         if (cma_cm_event_handler(id_priv, &event)) {
5140                 __acquire(&id_priv->handler_mutex);
5141                 id_priv->cm_id.ib = NULL;
5142                 cma_id_put(id_priv);
5143                 destroy_id_handler_unlock(id_priv);
5144                 return;
5145         }
5146
5147 out_unlock:
5148         mutex_unlock(&id_priv->handler_mutex);
5149         cma_id_put(id_priv);
5150 }
5151
5152 static int cma_netevent_callback(struct notifier_block *self,
5153                                  unsigned long event, void *ctx)
5154 {
5155         struct id_table_entry *ips_node = NULL;
5156         struct rdma_id_private *current_id;
5157         struct neighbour *neigh = ctx;
5158         unsigned long flags;
5159
5160         if (event != NETEVENT_NEIGH_UPDATE)
5161                 return NOTIFY_DONE;
5162
5163         spin_lock_irqsave(&id_table_lock, flags);
5164         if (neigh->tbl->family == AF_INET6) {
5165                 struct sockaddr_in6 neigh_sock_6;
5166
5167                 neigh_sock_6.sin6_family = AF_INET6;
5168                 neigh_sock_6.sin6_addr = *(struct in6_addr *)neigh->primary_key;
5169                 ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex,
5170                                              (struct sockaddr *)&neigh_sock_6);
5171         } else if (neigh->tbl->family == AF_INET) {
5172                 struct sockaddr_in neigh_sock_4;
5173
5174                 neigh_sock_4.sin_family = AF_INET;
5175                 neigh_sock_4.sin_addr.s_addr = *(__be32 *)(neigh->primary_key);
5176                 ips_node = node_from_ndev_ip(&id_table, neigh->dev->ifindex,
5177                                              (struct sockaddr *)&neigh_sock_4);
5178         } else
5179                 goto out;
5180
5181         if (!ips_node)
5182                 goto out;
5183
5184         list_for_each_entry(current_id, &ips_node->id_list, id_list_entry) {
5185                 if (!memcmp(current_id->id.route.addr.dev_addr.dst_dev_addr,
5186                            neigh->ha, ETH_ALEN))
5187                         continue;
5188                 INIT_WORK(&current_id->id.net_work, cma_netevent_work_handler);
5189                 cma_id_get(current_id);
5190                 queue_work(cma_wq, &current_id->id.net_work);
5191         }
5192 out:
5193         spin_unlock_irqrestore(&id_table_lock, flags);
5194         return NOTIFY_DONE;
5195 }
5196
5197 static struct notifier_block cma_nb = {
5198         .notifier_call = cma_netdev_callback
5199 };
5200
5201 static struct notifier_block cma_netevent_cb = {
5202         .notifier_call = cma_netevent_callback
5203 };
5204
5205 static void cma_send_device_removal_put(struct rdma_id_private *id_priv)
5206 {
5207         struct rdma_cm_event event = { .event = RDMA_CM_EVENT_DEVICE_REMOVAL };
5208         enum rdma_cm_state state;
5209         unsigned long flags;
5210
5211         mutex_lock(&id_priv->handler_mutex);
5212         /* Record that we want to remove the device */
5213         spin_lock_irqsave(&id_priv->lock, flags);
5214         state = id_priv->state;
5215         if (state == RDMA_CM_DESTROYING || state == RDMA_CM_DEVICE_REMOVAL) {
5216                 spin_unlock_irqrestore(&id_priv->lock, flags);
5217                 mutex_unlock(&id_priv->handler_mutex);
5218                 cma_id_put(id_priv);
5219                 return;
5220         }
5221         id_priv->state = RDMA_CM_DEVICE_REMOVAL;
5222         spin_unlock_irqrestore(&id_priv->lock, flags);
5223
5224         if (cma_cm_event_handler(id_priv, &event)) {
5225                 /*
5226                  * At this point the ULP promises it won't call
5227                  * rdma_destroy_id() concurrently
5228                  */
5229                 cma_id_put(id_priv);
5230                 mutex_unlock(&id_priv->handler_mutex);
5231                 trace_cm_id_destroy(id_priv);
5232                 _destroy_id(id_priv, state);
5233                 return;
5234         }
5235         mutex_unlock(&id_priv->handler_mutex);
5236
5237         /*
5238          * If this races with destroy then the thread that first assigns state
5239          * to a destroying does the cancel.
5240          */
5241         cma_cancel_operation(id_priv, state);
5242         cma_id_put(id_priv);
5243 }
5244
5245 static void cma_process_remove(struct cma_device *cma_dev)
5246 {
5247         mutex_lock(&lock);
5248         while (!list_empty(&cma_dev->id_list)) {
5249                 struct rdma_id_private *id_priv = list_first_entry(
5250                         &cma_dev->id_list, struct rdma_id_private, device_item);
5251
5252                 list_del_init(&id_priv->listen_item);
5253                 list_del_init(&id_priv->device_item);
5254                 cma_id_get(id_priv);
5255                 mutex_unlock(&lock);
5256
5257                 cma_send_device_removal_put(id_priv);
5258
5259                 mutex_lock(&lock);
5260         }
5261         mutex_unlock(&lock);
5262
5263         cma_dev_put(cma_dev);
5264         wait_for_completion(&cma_dev->comp);
5265 }
5266
5267 static bool cma_supported(struct ib_device *device)
5268 {
5269         u32 i;
5270
5271         rdma_for_each_port(device, i) {
5272                 if (rdma_cap_ib_cm(device, i) || rdma_cap_iw_cm(device, i))
5273                         return true;
5274         }
5275         return false;
5276 }
5277
5278 static int cma_add_one(struct ib_device *device)
5279 {
5280         struct rdma_id_private *to_destroy;
5281         struct cma_device *cma_dev;
5282         struct rdma_id_private *id_priv;
5283         unsigned long supported_gids = 0;
5284         int ret;
5285         u32 i;
5286
5287         if (!cma_supported(device))
5288                 return -EOPNOTSUPP;
5289
5290         cma_dev = kmalloc(sizeof(*cma_dev), GFP_KERNEL);
5291         if (!cma_dev)
5292                 return -ENOMEM;
5293
5294         cma_dev->device = device;
5295         cma_dev->default_gid_type = kcalloc(device->phys_port_cnt,
5296                                             sizeof(*cma_dev->default_gid_type),
5297                                             GFP_KERNEL);
5298         if (!cma_dev->default_gid_type) {
5299                 ret = -ENOMEM;
5300                 goto free_cma_dev;
5301         }
5302
5303         cma_dev->default_roce_tos = kcalloc(device->phys_port_cnt,
5304                                             sizeof(*cma_dev->default_roce_tos),
5305                                             GFP_KERNEL);
5306         if (!cma_dev->default_roce_tos) {
5307                 ret = -ENOMEM;
5308                 goto free_gid_type;
5309         }
5310
5311         rdma_for_each_port (device, i) {
5312                 supported_gids = roce_gid_type_mask_support(device, i);
5313                 WARN_ON(!supported_gids);
5314                 if (supported_gids & (1 << CMA_PREFERRED_ROCE_GID_TYPE))
5315                         cma_dev->default_gid_type[i - rdma_start_port(device)] =
5316                                 CMA_PREFERRED_ROCE_GID_TYPE;
5317                 else
5318                         cma_dev->default_gid_type[i - rdma_start_port(device)] =
5319                                 find_first_bit(&supported_gids, BITS_PER_LONG);
5320                 cma_dev->default_roce_tos[i - rdma_start_port(device)] = 0;
5321         }
5322
5323         init_completion(&cma_dev->comp);
5324         refcount_set(&cma_dev->refcount, 1);
5325         INIT_LIST_HEAD(&cma_dev->id_list);
5326         ib_set_client_data(device, &cma_client, cma_dev);
5327
5328         mutex_lock(&lock);
5329         list_add_tail(&cma_dev->list, &dev_list);
5330         list_for_each_entry(id_priv, &listen_any_list, listen_any_item) {
5331                 ret = cma_listen_on_dev(id_priv, cma_dev, &to_destroy);
5332                 if (ret)
5333                         goto free_listen;
5334         }
5335         mutex_unlock(&lock);
5336
5337         trace_cm_add_one(device);
5338         return 0;
5339
5340 free_listen:
5341         list_del(&cma_dev->list);
5342         mutex_unlock(&lock);
5343
5344         /* cma_process_remove() will delete to_destroy */
5345         cma_process_remove(cma_dev);
5346         kfree(cma_dev->default_roce_tos);
5347 free_gid_type:
5348         kfree(cma_dev->default_gid_type);
5349
5350 free_cma_dev:
5351         kfree(cma_dev);
5352         return ret;
5353 }
5354
5355 static void cma_remove_one(struct ib_device *device, void *client_data)
5356 {
5357         struct cma_device *cma_dev = client_data;
5358
5359         trace_cm_remove_one(device);
5360
5361         mutex_lock(&lock);
5362         list_del(&cma_dev->list);
5363         mutex_unlock(&lock);
5364
5365         cma_process_remove(cma_dev);
5366         kfree(cma_dev->default_roce_tos);
5367         kfree(cma_dev->default_gid_type);
5368         kfree(cma_dev);
5369 }
5370
5371 static int cma_init_net(struct net *net)
5372 {
5373         struct cma_pernet *pernet = cma_pernet(net);
5374
5375         xa_init(&pernet->tcp_ps);
5376         xa_init(&pernet->udp_ps);
5377         xa_init(&pernet->ipoib_ps);
5378         xa_init(&pernet->ib_ps);
5379
5380         return 0;
5381 }
5382
5383 static void cma_exit_net(struct net *net)
5384 {
5385         struct cma_pernet *pernet = cma_pernet(net);
5386
5387         WARN_ON(!xa_empty(&pernet->tcp_ps));
5388         WARN_ON(!xa_empty(&pernet->udp_ps));
5389         WARN_ON(!xa_empty(&pernet->ipoib_ps));
5390         WARN_ON(!xa_empty(&pernet->ib_ps));
5391 }
5392
5393 static struct pernet_operations cma_pernet_operations = {
5394         .init = cma_init_net,
5395         .exit = cma_exit_net,
5396         .id = &cma_pernet_id,
5397         .size = sizeof(struct cma_pernet),
5398 };
5399
5400 static int __init cma_init(void)
5401 {
5402         int ret;
5403
5404         /*
5405          * There is a rare lock ordering dependency in cma_netdev_callback()
5406          * that only happens when bonding is enabled. Teach lockdep that rtnl
5407          * must never be nested under lock so it can find these without having
5408          * to test with bonding.
5409          */
5410         if (IS_ENABLED(CONFIG_LOCKDEP)) {
5411                 rtnl_lock();
5412                 mutex_lock(&lock);
5413                 mutex_unlock(&lock);
5414                 rtnl_unlock();
5415         }
5416
5417         cma_wq = alloc_ordered_workqueue("rdma_cm", WQ_MEM_RECLAIM);
5418         if (!cma_wq)
5419                 return -ENOMEM;
5420
5421         ret = register_pernet_subsys(&cma_pernet_operations);
5422         if (ret)
5423                 goto err_wq;
5424
5425         ib_sa_register_client(&sa_client);
5426         register_netdevice_notifier(&cma_nb);
5427         register_netevent_notifier(&cma_netevent_cb);
5428
5429         ret = ib_register_client(&cma_client);
5430         if (ret)
5431                 goto err;
5432
5433         ret = cma_configfs_init();
5434         if (ret)
5435                 goto err_ib;
5436
5437         return 0;
5438
5439 err_ib:
5440         ib_unregister_client(&cma_client);
5441 err:
5442         unregister_netevent_notifier(&cma_netevent_cb);
5443         unregister_netdevice_notifier(&cma_nb);
5444         ib_sa_unregister_client(&sa_client);
5445         unregister_pernet_subsys(&cma_pernet_operations);
5446 err_wq:
5447         destroy_workqueue(cma_wq);
5448         return ret;
5449 }
5450
5451 static void __exit cma_cleanup(void)
5452 {
5453         cma_configfs_exit();
5454         ib_unregister_client(&cma_client);
5455         unregister_netevent_notifier(&cma_netevent_cb);
5456         unregister_netdevice_notifier(&cma_nb);
5457         ib_sa_unregister_client(&sa_client);
5458         unregister_pernet_subsys(&cma_pernet_operations);
5459         destroy_workqueue(cma_wq);
5460 }
5461
5462 module_init(cma_init);
5463 module_exit(cma_cleanup);