net/smc/smc_ib.c

   1 /*
   2  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   3  *
   4  *  IB infrastructure:
   5  *  Establish SMC-R as an Infiniband Client to be notified about added and
   6  *  removed IB devices of type RDMA.
   7  *  Determine device and port characteristics for these IB devices.
   8  *
   9  *  Copyright IBM Corp. 2016
  10  *
  11  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  12  */
  13
  14 #include <linux/random.h>
  15 #include <rdma/ib_verbs.h>
  16
  17 #include "smc_pnet.h"
  18 #include "smc_ib.h"
  19 #include "smc_core.h"
  20 #include "smc.h"
  21
  22 struct smc_ib_devices smc_ib_devices = {        /* smc-registered ib devices */
  23         .lock = __SPIN_LOCK_UNLOCKED(smc_ib_devices.lock),
  24         .list = LIST_HEAD_INIT(smc_ib_devices.list),
  25 };
  26
  27 #define SMC_LOCAL_SYSTEMID_RESET        "%%%%%%%"
  28
  29 u8 local_systemid[SMC_SYSTEMID_LEN] = SMC_LOCAL_SYSTEMID_RESET; /* unique system
  30                                                                  * identifier
  31                                                                  */
  32
  33 /* map a new TX or RX buffer to DMA */
  34 int smc_ib_buf_map(struct smc_ib_device *smcibdev, int buf_size,
  35                    struct smc_buf_desc *buf_slot,
  36                    enum dma_data_direction data_direction)
  37 {
  38         int rc = 0;
  39
  40         if (buf_slot->dma_addr[SMC_SINGLE_LINK])
  41                 return rc; /* already mapped */
  42         buf_slot->dma_addr[SMC_SINGLE_LINK] =
  43                 ib_dma_map_single(smcibdev->ibdev, buf_slot->cpu_addr,
  44                                   buf_size, data_direction);
  45         if (ib_dma_mapping_error(smcibdev->ibdev,
  46                                  buf_slot->dma_addr[SMC_SINGLE_LINK]))
  47                 rc = -EIO;
  48         return rc;
  49 }
  50
  51 static int smc_ib_fill_gid_and_mac(struct smc_ib_device *smcibdev, u8 ibport)
  52 {
  53         struct net_device *ndev;
  54         int rc;
  55
  56         rc = ib_query_gid(smcibdev->ibdev, ibport, 0,
  57                           &smcibdev->gid[ibport - 1], NULL);
  58         /* the SMC protocol requires specification of the roce MAC address;
  59          * if net_device cannot be determined, it can be derived from gid 0
  60          */
  61         ndev = smcibdev->ibdev->get_netdev(smcibdev->ibdev, ibport);
  62         if (ndev) {
  63                 memcpy(&smcibdev->mac, ndev->dev_addr, ETH_ALEN);
  64         } else if (!rc) {
  65                 memcpy(&smcibdev->mac[ibport - 1][0],
  66                        &smcibdev->gid[ibport - 1].raw[8], 3);
  67                 memcpy(&smcibdev->mac[ibport - 1][3],
  68                        &smcibdev->gid[ibport - 1].raw[13], 3);
  69                 smcibdev->mac[ibport - 1][0] &= ~0x02;
  70         }
  71         return rc;
  72 }
  73
  74 /* Create an identifier unique for this instance of SMC-R.
  75  * The MAC-address of the first active registered IB device
  76  * plus a random 2-byte number is used to create this identifier.
  77  * This name is delivered to the peer during connection initialization.
  78  */
  79 static inline void smc_ib_define_local_systemid(struct smc_ib_device *smcibdev,
  80                                                 u8 ibport)
  81 {
  82         memcpy(&local_systemid[2], &smcibdev->mac[ibport - 1],
  83                sizeof(smcibdev->mac[ibport - 1]));
  84         get_random_bytes(&local_systemid[0], 2);
  85 }
  86
  87 bool smc_ib_port_active(struct smc_ib_device *smcibdev, u8 ibport)
  88 {
  89         return smcibdev->pattr[ibport - 1].state == IB_PORT_ACTIVE;
  90 }
  91
  92 int smc_ib_remember_port_attr(struct smc_ib_device *smcibdev, u8 ibport)
  93 {
  94         int rc;
  95
  96         memset(&smcibdev->pattr[ibport - 1], 0,
  97                sizeof(smcibdev->pattr[ibport - 1]));
  98         rc = ib_query_port(smcibdev->ibdev, ibport,
  99                            &smcibdev->pattr[ibport - 1]);
 100         if (rc)
 101                 goto out;
 102         rc = smc_ib_fill_gid_and_mac(smcibdev, ibport);
 103         if (rc)
 104                 goto out;
 105         if (!strncmp(local_systemid, SMC_LOCAL_SYSTEMID_RESET,
 106                      sizeof(local_systemid)) &&
 107             smc_ib_port_active(smcibdev, ibport))
 108                 /* create unique system identifier */
 109                 smc_ib_define_local_systemid(smcibdev, ibport);
 110 out:
 111         return rc;
 112 }
 113
 114 static struct ib_client smc_ib_client;
 115
 116 /* callback function for ib_register_client() */
 117 static void smc_ib_add_dev(struct ib_device *ibdev)
 118 {
 119         struct smc_ib_device *smcibdev;
 120
 121         if (ibdev->node_type != RDMA_NODE_IB_CA)
 122                 return;
 123
 124         smcibdev = kzalloc(sizeof(*smcibdev), GFP_KERNEL);
 125         if (!smcibdev)
 126                 return;
 127
 128         smcibdev->ibdev = ibdev;
 129
 130         spin_lock(&smc_ib_devices.lock);
 131         list_add_tail(&smcibdev->list, &smc_ib_devices.list);
 132         spin_unlock(&smc_ib_devices.lock);
 133         ib_set_client_data(ibdev, &smc_ib_client, smcibdev);
 134 }
 135
 136 /* callback function for ib_register_client() */
 137 static void smc_ib_remove_dev(struct ib_device *ibdev, void *client_data)
 138 {
 139         struct smc_ib_device *smcibdev;
 140
 141         smcibdev = ib_get_client_data(ibdev, &smc_ib_client);
 142         ib_set_client_data(ibdev, &smc_ib_client, NULL);
 143         spin_lock(&smc_ib_devices.lock);
 144         list_del_init(&smcibdev->list); /* remove from smc_ib_devices */
 145         spin_unlock(&smc_ib_devices.lock);
 146         smc_pnet_remove_by_ibdev(smcibdev);
 147         kfree(smcibdev);
 148 }
 149
 150 static struct ib_client smc_ib_client = {
 151         .name   = "smc_ib",
 152         .add    = smc_ib_add_dev,
 153         .remove = smc_ib_remove_dev,
 154 };
 155
 156 int __init smc_ib_register_client(void)
 157 {
 158         return ib_register_client(&smc_ib_client);
 159 }
 160
 161 void smc_ib_unregister_client(void)
 162 {
 163         ib_unregister_client(&smc_ib_client);
 164 }