net/smc/smc_ib.c

   1 /*
   2  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   3  *
   4  *  IB infrastructure:
   5  *  Establish SMC-R as an Infiniband Client to be notified about added and
   6  *  removed IB devices of type RDMA.
   7  *  Determine device and port characteristics for these IB devices.
   8  *
   9  *  Copyright IBM Corp. 2016
  10  *
  11  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  12  */
  13
  14 #include <linux/random.h>
  15 #include <rdma/ib_verbs.h>
  16
  17 #include "smc_pnet.h"
  18 #include "smc_ib.h"
  19 #include "smc.h"
  20
  21 struct smc_ib_devices smc_ib_devices = {        /* smc-registered ib devices */
  22         .lock = __SPIN_LOCK_UNLOCKED(smc_ib_devices.lock),
  23         .list = LIST_HEAD_INIT(smc_ib_devices.list),
  24 };
  25
  26 #define SMC_LOCAL_SYSTEMID_RESET        "%%%%%%%"
  27
  28 u8 local_systemid[SMC_SYSTEMID_LEN] = SMC_LOCAL_SYSTEMID_RESET; /* unique system
  29                                                                  * identifier
  30                                                                  */
  31
  32 static int smc_ib_fill_gid_and_mac(struct smc_ib_device *smcibdev, u8 ibport)
  33 {
  34         struct net_device *ndev;
  35         int rc;
  36
  37         rc = ib_query_gid(smcibdev->ibdev, ibport, 0,
  38                           &smcibdev->gid[ibport - 1], NULL);
  39         /* the SMC protocol requires specification of the roce MAC address;
  40          * if net_device cannot be determined, it can be derived from gid 0
  41          */
  42         ndev = smcibdev->ibdev->get_netdev(smcibdev->ibdev, ibport);
  43         if (ndev) {
  44                 memcpy(&smcibdev->mac, ndev->dev_addr, ETH_ALEN);
  45         } else if (!rc) {
  46                 memcpy(&smcibdev->mac[ibport - 1][0],
  47                        &smcibdev->gid[ibport - 1].raw[8], 3);
  48                 memcpy(&smcibdev->mac[ibport - 1][3],
  49                        &smcibdev->gid[ibport - 1].raw[13], 3);
  50                 smcibdev->mac[ibport - 1][0] &= ~0x02;
  51         }
  52         return rc;
  53 }
  54
  55 /* Create an identifier unique for this instance of SMC-R.
  56  * The MAC-address of the first active registered IB device
  57  * plus a random 2-byte number is used to create this identifier.
  58  * This name is delivered to the peer during connection initialization.
  59  */
  60 static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev,
  61                                                 u8 ibport)
  62 {
  63         memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1],
  64                sizeof(smcibdev->mac[ibport - 1]));
  65         get_random_bytes(&local_systemid[0], 2);
  66 }
  67
  68 bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport)
  69 {
  70         return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE;
  71 }
  72
  73 int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
  74 {
  75         int rc;
  76
  77         memset(&smcibdev->pattr[ibport - 1], 0,
  78                sizeof(smcibdev->pattr[ibport - 1]));
  79         rc = ib_query_port(smcibdev->ibdev, ibport,
  80                            &smcibdev->pattr[ibport - 1]);
  81         if (rc)
  82                 goto out;
  83         rc = smc_ib_fill_gid_and_mac(smcibdev, ibport);
  84         if (rc)
  85                 goto out;
  86         if (!strncmp(local_systemid, SMC_LOCAL_SYSTEMID_RESET,
  87                      sizeof(local_systemid)) &&
  88             smc_ib_port_active(smcibdev, ibport))
  89                 /* create unique system identifier */
  90                 smc_ib_define_local_systemid(smcibdev, ibport);
  91 out:
  92         return rc;
  93 }
  94
  95 static struct ib_client smc_ib_client;
  96
  97 /* callback function for ib_register_client() */
  98 static void smc_ib_add_dev(struct ib_device *ibdev)
  99 {
 100         struct smc_ib_device *smcibdev;
 101
 102         if (ibdev->node_type != RDMA_NODE_IB_CA)
 103                 return;
 104
 105         smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL);
 106         if (!smcibdev)
 107                 return;
 108
 109         smcibdev->ibdev = ibdev;
 110
 111         spin_lock(&smc_ib_devices.lock);
 112         list_add_tail(&smcibdev->list, &smc_ib_devices.list);
 113         spin_unlock(&smc_ib_devices.lock);
 114         ib_set_client_data(ibdev, &smc_ib_client, smcibdev);
 115 }
 116
 117 /* callback function for ib_register_client() */
 118 static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data)
 119 {
 120         struct smc_ib_device *smcibdev;
 121
 122         smcibdev = ib_get_client_data(ibdev, &smc_ib_client);
 123         ib_set_client_data(ibdev, &smc_ib_client, NULL);
 124         spin_lock(&smc_ib_devices.lock);
 125         list_del_init(&smcibdev->list); /* remove from smc_ib_devices */
 126         spin_unlock(&smc_ib_devices.lock);
 127         smc_pnet_remove_by_ibdev(smcibdev);
 128         kfree(smcibdev);
 129 }
 130
 131 static struct ib_client smc_ib_client = {
 132         .name   = "smc_ib",
 133         .add    = smc_ib_add_dev,
 134         .remove = smc_ib_remove_dev,
 135 };
 136
 137 int __init smc_ib_register_client(void)
 138 {
 139         return ib_register_client(&smc_ib_client);
 140 }
 141
 142 void smc_ib_unregister_client(void)
 143 {
 144         ib_unregister_client(&smc_ib_client);
 145 }