net/smc/smc_llc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Link Layer Control (LLC)
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Klaus Wacker <Klaus.Wacker@de.ibm.com>
  10  *              Ursula Braun <ubraun@linux.vnet.ibm.com>
  11  */
  12
  13 #include <net/tcp.h>
  14 #include <rdma/ib_verbs.h>
  15
  16 #include "smc.h"
  17 #include "smc_core.h"
  18 #include "smc_clc.h"
  19 #include "smc_llc.h"
  20 #include "smc_pnet.h"
  21
  22 #define SMC_LLC_DATA_LEN                40
  23
  24 struct smc_llc_hdr {
  25         struct smc_wr_rx_hdr common;
  26         u8 length;      /* 44 */
  27 #if defined(__BIG_ENDIAN_BITFIELD)
  28         u8 reserved:4,
  29            add_link_rej_rsn:4;
  30 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  31         u8 add_link_rej_rsn:4,
  32            reserved:4;
  33 #endif
  34         u8 flags;
  35 };
  36
  37 #define SMC_LLC_FLAG_NO_RMBE_EYEC       0x03
  38
  39 struct smc_llc_msg_confirm_link {       /* type 0x01 */
  40         struct smc_llc_hdr hd;
  41         u8 sender_mac[ETH_ALEN];
  42         u8 sender_gid[SMC_GID_SIZE];
  43         u8 sender_qp_num[3];
  44         u8 link_num;
  45         u8 link_uid[SMC_LGR_ID_SIZE];
  46         u8 max_links;
  47         u8 reserved[9];
  48 };
  49
  50 #define SMC_LLC_FLAG_ADD_LNK_REJ        0x40
  51 #define SMC_LLC_REJ_RSN_NO_ALT_PATH     1
  52
  53 #define SMC_LLC_ADD_LNK_MAX_LINKS       2
  54
  55 struct smc_llc_msg_add_link {           /* type 0x02 */
  56         struct smc_llc_hdr hd;
  57         u8 sender_mac[ETH_ALEN];
  58         u8 reserved2[2];
  59         u8 sender_gid[SMC_GID_SIZE];
  60         u8 sender_qp_num[3];
  61         u8 link_num;
  62 #if defined(__BIG_ENDIAN_BITFIELD)
  63         u8 reserved3 : 4,
  64            qp_mtu   : 4;
  65 #elif defined(__LITTLE_ENDIAN_BITFIELD)
  66         u8 qp_mtu   : 4,
  67            reserved3 : 4;
  68 #endif
  69         u8 initial_psn[3];
  70         u8 reserved[8];
  71 };
  72
  73 struct smc_llc_msg_add_link_cont_rt {
  74         __be32 rmb_key;
  75         __be32 rmb_key_new;
  76         __be64 rmb_vaddr_new;
  77 };
  78
  79 #define SMC_LLC_RKEYS_PER_CONT_MSG      2
  80
  81 struct smc_llc_msg_add_link_cont {      /* type 0x03 */
  82         struct smc_llc_hdr hd;
  83         u8 link_num;
  84         u8 num_rkeys;
  85         u8 reserved2[2];
  86         struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG];
  87         u8 reserved[4];
  88 } __packed;                     /* format defined in RFC7609 */
  89
  90 #define SMC_LLC_FLAG_DEL_LINK_ALL       0x40
  91 #define SMC_LLC_FLAG_DEL_LINK_ORDERLY   0x20
  92
  93 struct smc_llc_msg_del_link {           /* type 0x04 */
  94         struct smc_llc_hdr hd;
  95         u8 link_num;
  96         __be32 reason;
  97         u8 reserved[35];
  98 } __packed;                     /* format defined in RFC7609 */
  99
 100 struct smc_llc_msg_test_link {          /* type 0x07 */
 101         struct smc_llc_hdr hd;
 102         u8 user_data[16];
 103         u8 reserved[24];
 104 };
 105
 106 struct smc_rmb_rtoken {
 107         union {
 108                 u8 num_rkeys;   /* first rtoken byte of CONFIRM LINK msg */
 109                                 /* is actually the num of rtokens, first */
 110                                 /* rtoken is always for the current link */
 111                 u8 link_id;     /* link id of the rtoken */
 112         };
 113         __be32 rmb_key;
 114         __be64 rmb_vaddr;
 115 } __packed;                     /* format defined in RFC7609 */
 116
 117 #define SMC_LLC_RKEYS_PER_MSG   3
 118
 119 struct smc_llc_msg_confirm_rkey {       /* type 0x06 */
 120         struct smc_llc_hdr hd;
 121         struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG];
 122         u8 reserved;
 123 };
 124
 125 #define SMC_LLC_DEL_RKEY_MAX    8
 126 #define SMC_LLC_FLAG_RKEY_RETRY 0x10
 127 #define SMC_LLC_FLAG_RKEY_NEG   0x20
 128
 129 struct smc_llc_msg_delete_rkey {        /* type 0x09 */
 130         struct smc_llc_hdr hd;
 131         u8 num_rkeys;
 132         u8 err_mask;
 133         u8 reserved[2];
 134         __be32 rkey[8];
 135         u8 reserved2[4];
 136 };
 137
 138 union smc_llc_msg {
 139         struct smc_llc_msg_confirm_link confirm_link;
 140         struct smc_llc_msg_add_link add_link;
 141         struct smc_llc_msg_add_link_cont add_link_cont;
 142         struct smc_llc_msg_del_link delete_link;
 143
 144         struct smc_llc_msg_confirm_rkey confirm_rkey;
 145         struct smc_llc_msg_delete_rkey delete_rkey;
 146
 147         struct smc_llc_msg_test_link test_link;
 148         struct {
 149                 struct smc_llc_hdr hdr;
 150                 u8 data[SMC_LLC_DATA_LEN];
 151         } raw;
 152 };
 153
 154 #define SMC_LLC_FLAG_RESP               0x80
 155
 156 struct smc_llc_qentry {
 157         struct list_head list;
 158         struct smc_link *link;
 159         union smc_llc_msg msg;
 160 };
 161
 162 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc);
 163
 164 struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow)
 165 {
 166         struct smc_llc_qentry *qentry = flow->qentry;
 167
 168         flow->qentry = NULL;
 169         return qentry;
 170 }
 171
 172 void smc_llc_flow_qentry_del(struct smc_llc_flow *flow)
 173 {
 174         struct smc_llc_qentry *qentry;
 175
 176         if (flow->qentry) {
 177                 qentry = flow->qentry;
 178                 flow->qentry = NULL;
 179                 kfree(qentry);
 180         }
 181 }
 182
 183 static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow,
 184                                            struct smc_llc_qentry *qentry)
 185 {
 186         flow->qentry = qentry;
 187 }
 188
 189 /* try to start a new llc flow, initiated by an incoming llc msg */
 190 static bool smc_llc_flow_start(struct smc_llc_flow *flow,
 191                                struct smc_llc_qentry *qentry)
 192 {
 193         struct smc_link_group *lgr = qentry->link->lgr;
 194
 195         spin_lock_bh(&lgr->llc_flow_lock);
 196         if (flow->type) {
 197                 /* a flow is already active */
 198                 if ((qentry->msg.raw.hdr.common.type == SMC_LLC_ADD_LINK ||
 199                      qentry->msg.raw.hdr.common.type == SMC_LLC_DELETE_LINK) &&
 200                     !lgr->delayed_event) {
 201                         lgr->delayed_event = qentry;
 202                 } else {
 203                         /* forget this llc request */
 204                         kfree(qentry);
 205                 }
 206                 spin_unlock_bh(&lgr->llc_flow_lock);
 207                 return false;
 208         }
 209         switch (qentry->msg.raw.hdr.common.type) {
 210         case SMC_LLC_ADD_LINK:
 211                 flow->type = SMC_LLC_FLOW_ADD_LINK;
 212                 break;
 213         case SMC_LLC_DELETE_LINK:
 214                 flow->type = SMC_LLC_FLOW_DEL_LINK;
 215                 break;
 216         case SMC_LLC_CONFIRM_RKEY:
 217         case SMC_LLC_DELETE_RKEY:
 218                 flow->type = SMC_LLC_FLOW_RKEY;
 219                 break;
 220         default:
 221                 flow->type = SMC_LLC_FLOW_NONE;
 222         }
 223         if (qentry == lgr->delayed_event)
 224                 lgr->delayed_event = NULL;
 225         spin_unlock_bh(&lgr->llc_flow_lock);
 226         smc_llc_flow_qentry_set(flow, qentry);
 227         return true;
 228 }
 229
 230 /* start a new local llc flow, wait till current flow finished */
 231 int smc_llc_flow_initiate(struct smc_link_group *lgr,
 232                           enum smc_llc_flowtype type)
 233 {
 234         enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE;
 235         int rc;
 236
 237         /* all flows except confirm_rkey and delete_rkey are exclusive,
 238          * confirm/delete rkey flows can run concurrently (local and remote)
 239          */
 240         if (type == SMC_LLC_FLOW_RKEY)
 241                 allowed_remote = SMC_LLC_FLOW_RKEY;
 242 again:
 243         if (list_empty(&lgr->list))
 244                 return -ENODEV;
 245         spin_lock_bh(&lgr->llc_flow_lock);
 246         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 247             (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 248              lgr->llc_flow_rmt.type == allowed_remote)) {
 249                 lgr->llc_flow_lcl.type = type;
 250                 spin_unlock_bh(&lgr->llc_flow_lock);
 251                 return 0;
 252         }
 253         spin_unlock_bh(&lgr->llc_flow_lock);
 254         rc = wait_event_interruptible_timeout(lgr->llc_waiter,
 255                         (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE &&
 256                          (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE ||
 257                           lgr->llc_flow_rmt.type == allowed_remote)),
 258                         SMC_LLC_WAIT_TIME);
 259         if (!rc)
 260                 return -ETIMEDOUT;
 261         goto again;
 262 }
 263
 264 /* finish the current llc flow */
 265 void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow)
 266 {
 267         spin_lock_bh(&lgr->llc_flow_lock);
 268         memset(flow, 0, sizeof(*flow));
 269         flow->type = SMC_LLC_FLOW_NONE;
 270         spin_unlock_bh(&lgr->llc_flow_lock);
 271         if (!list_empty(&lgr->list) && lgr->delayed_event &&
 272             flow == &lgr->llc_flow_lcl)
 273                 schedule_work(&lgr->llc_event_work);
 274         else
 275                 wake_up_interruptible(&lgr->llc_waiter);
 276 }
 277
 278 /* lnk is optional and used for early wakeup when link goes down, useful in
 279  * cases where we wait for a response on the link after we sent a request
 280  */
 281 struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr,
 282                                     struct smc_link *lnk,
 283                                     int time_out, u8 exp_msg)
 284 {
 285         struct smc_llc_flow *flow = &lgr->llc_flow_lcl;
 286
 287         wait_event_interruptible_timeout(lgr->llc_waiter,
 288                                          (flow->qentry ||
 289                                           (lnk && !smc_link_usable(lnk)) ||
 290                                           list_empty(&lgr->list)),
 291                                          time_out);
 292         if (!flow->qentry ||
 293             (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) {
 294                 smc_llc_flow_qentry_del(flow);
 295                 goto out;
 296         }
 297         if (exp_msg && flow->qentry->msg.raw.hdr.common.type != exp_msg) {
 298                 if (exp_msg == SMC_LLC_ADD_LINK &&
 299                     flow->qentry->msg.raw.hdr.common.type ==
 300                     SMC_LLC_DELETE_LINK) {
 301                         /* flow_start will delay the unexpected msg */
 302                         smc_llc_flow_start(&lgr->llc_flow_lcl,
 303                                            smc_llc_flow_qentry_clr(flow));
 304                         return NULL;
 305                 }
 306                 smc_llc_flow_qentry_del(flow);
 307         }
 308 out:
 309         return flow->qentry;
 310 }
 311
 312 /********************************** send *************************************/
 313
 314 struct smc_llc_tx_pend {
 315 };
 316
 317 /* handler for send/transmission completion of an LLC msg */
 318 static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend,
 319                                struct smc_link *link,
 320                                enum ib_wc_status wc_status)
 321 {
 322         /* future work: handle wc_status error for recovery and failover */
 323 }
 324
 325 /**
 326  * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits
 327  * @link: Pointer to SMC link used for sending LLC control message.
 328  * @wr_buf: Out variable returning pointer to work request payload buffer.
 329  * @pend: Out variable returning pointer to private pending WR tracking.
 330  *        It's the context the transmit complete handler will get.
 331  *
 332  * Reserves and pre-fills an entry for a pending work request send/tx.
 333  * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx.
 334  * Can sleep due to smc_get_ctrl_buf (if not in softirq context).
 335  *
 336  * Return: 0 on success, otherwise an error value.
 337  */
 338 static int smc_llc_add_pending_send(struct smc_link *link,
 339                                     struct smc_wr_buf **wr_buf,
 340                                     struct smc_wr_tx_pend_priv **pend)
 341 {
 342         int rc;
 343
 344         rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL,
 345                                      pend);
 346         if (rc < 0)
 347                 return rc;
 348         BUILD_BUG_ON_MSG(
 349                 sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE,
 350                 "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)");
 351         BUILD_BUG_ON_MSG(
 352                 sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE,
 353                 "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()");
 354         BUILD_BUG_ON_MSG(
 355                 sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE,
 356                 "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)");
 357         return 0;
 358 }
 359
 360 /* high-level API to send LLC confirm link */
 361 int smc_llc_send_confirm_link(struct smc_link *link,
 362                               enum smc_llc_reqresp reqresp)
 363 {
 364         struct smc_llc_msg_confirm_link *confllc;
 365         struct smc_wr_tx_pend_priv *pend;
 366         struct smc_wr_buf *wr_buf;
 367         int rc;
 368
 369         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 370         if (rc)
 371                 return rc;
 372         confllc = (struct smc_llc_msg_confirm_link *)wr_buf;
 373         memset(confllc, 0, sizeof(*confllc));
 374         confllc->hd.common.type = SMC_LLC_CONFIRM_LINK;
 375         confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link);
 376         confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC;
 377         if (reqresp == SMC_LLC_RESP)
 378                 confllc->hd.flags |= SMC_LLC_FLAG_RESP;
 379         memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1],
 380                ETH_ALEN);
 381         memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE);
 382         hton24(confllc->sender_qp_num, link->roce_qp->qp_num);
 383         confllc->link_num = link->link_id;
 384         memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE);
 385         confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS;
 386         /* send llc message */
 387         rc = smc_wr_tx_send(link, pend);
 388         return rc;
 389 }
 390
 391 /* send LLC confirm rkey request */
 392 static int smc_llc_send_confirm_rkey(struct smc_link *send_link,
 393                                      struct smc_buf_desc *rmb_desc)
 394 {
 395         struct smc_llc_msg_confirm_rkey *rkeyllc;
 396         struct smc_wr_tx_pend_priv *pend;
 397         struct smc_wr_buf *wr_buf;
 398         struct smc_link *link;
 399         int i, rc, rtok_ix;
 400
 401         rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend);
 402         if (rc)
 403                 return rc;
 404         rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf;
 405         memset(rkeyllc, 0, sizeof(*rkeyllc));
 406         rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY;
 407         rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey);
 408
 409         rtok_ix = 1;
 410         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 411                 link = &send_link->lgr->lnk[i];
 412                 if (link->state == SMC_LNK_ACTIVE && link != send_link) {
 413                         rkeyllc->rtoken[rtok_ix].link_id = link->link_id;
 414                         rkeyllc->rtoken[rtok_ix].rmb_key =
 415                                 htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 416                         rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64(
 417                                 (u64)sg_dma_address(
 418                                         rmb_desc->sgt[link->link_idx].sgl));
 419                         rtok_ix++;
 420                 }
 421         }
 422         /* rkey of send_link is in rtoken[0] */
 423         rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1;
 424         rkeyllc->rtoken[0].rmb_key =
 425                 htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey);
 426         rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64(
 427                 (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl));
 428         /* send llc message */
 429         rc = smc_wr_tx_send(send_link, pend);
 430         return rc;
 431 }
 432
 433 /* send LLC delete rkey request */
 434 static int smc_llc_send_delete_rkey(struct smc_link *link,
 435                                     struct smc_buf_desc *rmb_desc)
 436 {
 437         struct smc_llc_msg_delete_rkey *rkeyllc;
 438         struct smc_wr_tx_pend_priv *pend;
 439         struct smc_wr_buf *wr_buf;
 440         int rc;
 441
 442         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 443         if (rc)
 444                 return rc;
 445         rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf;
 446         memset(rkeyllc, 0, sizeof(*rkeyllc));
 447         rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY;
 448         rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey);
 449         rkeyllc->num_rkeys = 1;
 450         rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey);
 451         /* send llc message */
 452         rc = smc_wr_tx_send(link, pend);
 453         return rc;
 454 }
 455
 456 /* send ADD LINK request or response */
 457 int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[],
 458                           struct smc_link *link_new,
 459                           enum smc_llc_reqresp reqresp)
 460 {
 461         struct smc_llc_msg_add_link *addllc;
 462         struct smc_wr_tx_pend_priv *pend;
 463         struct smc_wr_buf *wr_buf;
 464         int rc;
 465
 466         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 467         if (rc)
 468                 return rc;
 469         addllc = (struct smc_llc_msg_add_link *)wr_buf;
 470
 471         memset(addllc, 0, sizeof(*addllc));
 472         addllc->hd.common.type = SMC_LLC_ADD_LINK;
 473         addllc->hd.length = sizeof(struct smc_llc_msg_add_link);
 474         if (reqresp == SMC_LLC_RESP)
 475                 addllc->hd.flags |= SMC_LLC_FLAG_RESP;
 476         memcpy(addllc->sender_mac, mac, ETH_ALEN);
 477         memcpy(addllc->sender_gid, gid, SMC_GID_SIZE);
 478         if (link_new) {
 479                 addllc->link_num = link_new->link_id;
 480                 hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num);
 481                 hton24(addllc->initial_psn, link_new->psn_initial);
 482                 if (reqresp == SMC_LLC_REQ)
 483                         addllc->qp_mtu = link_new->path_mtu;
 484                 else
 485                         addllc->qp_mtu = min(link_new->path_mtu,
 486                                              link_new->peer_mtu);
 487         }
 488         /* send llc message */
 489         rc = smc_wr_tx_send(link, pend);
 490         return rc;
 491 }
 492
 493 /* send DELETE LINK request or response */
 494 int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id,
 495                              enum smc_llc_reqresp reqresp, bool orderly,
 496                              u32 reason)
 497 {
 498         struct smc_llc_msg_del_link *delllc;
 499         struct smc_wr_tx_pend_priv *pend;
 500         struct smc_wr_buf *wr_buf;
 501         int rc;
 502
 503         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 504         if (rc)
 505                 return rc;
 506         delllc = (struct smc_llc_msg_del_link *)wr_buf;
 507
 508         memset(delllc, 0, sizeof(*delllc));
 509         delllc->hd.common.type = SMC_LLC_DELETE_LINK;
 510         delllc->hd.length = sizeof(struct smc_llc_msg_del_link);
 511         if (reqresp == SMC_LLC_RESP)
 512                 delllc->hd.flags |= SMC_LLC_FLAG_RESP;
 513         if (orderly)
 514                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
 515         if (link_del_id)
 516                 delllc->link_num = link_del_id;
 517         else
 518                 delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
 519         delllc->reason = htonl(reason);
 520         /* send llc message */
 521         rc = smc_wr_tx_send(link, pend);
 522         return rc;
 523 }
 524
 525 /* send LLC test link request */
 526 static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16])
 527 {
 528         struct smc_llc_msg_test_link *testllc;
 529         struct smc_wr_tx_pend_priv *pend;
 530         struct smc_wr_buf *wr_buf;
 531         int rc;
 532
 533         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 534         if (rc)
 535                 return rc;
 536         testllc = (struct smc_llc_msg_test_link *)wr_buf;
 537         memset(testllc, 0, sizeof(*testllc));
 538         testllc->hd.common.type = SMC_LLC_TEST_LINK;
 539         testllc->hd.length = sizeof(struct smc_llc_msg_test_link);
 540         memcpy(testllc->user_data, user_data, sizeof(testllc->user_data));
 541         /* send llc message */
 542         rc = smc_wr_tx_send(link, pend);
 543         return rc;
 544 }
 545
 546 /* schedule an llc send on link, may wait for buffers */
 547 static int smc_llc_send_message(struct smc_link *link, void *llcbuf)
 548 {
 549         struct smc_wr_tx_pend_priv *pend;
 550         struct smc_wr_buf *wr_buf;
 551         int rc;
 552
 553         if (!smc_link_usable(link))
 554                 return -ENOLINK;
 555         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 556         if (rc)
 557                 return rc;
 558         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 559         return smc_wr_tx_send(link, pend);
 560 }
 561
 562 /* schedule an llc send on link, may wait for buffers,
 563  * and wait for send completion notification.
 564  * @return 0 on success
 565  */
 566 static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf)
 567 {
 568         struct smc_wr_tx_pend_priv *pend;
 569         struct smc_wr_buf *wr_buf;
 570         int rc;
 571
 572         if (!smc_link_usable(link))
 573                 return -ENOLINK;
 574         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 575         if (rc)
 576                 return rc;
 577         memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg));
 578         return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME);
 579 }
 580
 581 /********************************* receive ***********************************/
 582
 583 static int smc_llc_alloc_alt_link(struct smc_link_group *lgr,
 584                                   enum smc_lgr_type lgr_new_t)
 585 {
 586         int i;
 587
 588         if (lgr->type == SMC_LGR_SYMMETRIC ||
 589             (lgr->type != SMC_LGR_SINGLE &&
 590              (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 591               lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)))
 592                 return -EMLINK;
 593
 594         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 595             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) {
 596                 for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--)
 597                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 598                                 return i;
 599         } else {
 600                 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
 601                         if (lgr->lnk[i].state == SMC_LNK_UNUSED)
 602                                 return i;
 603         }
 604         return -EMLINK;
 605 }
 606
 607 /* return first buffer from any of the next buf lists */
 608 static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr,
 609                                                   int *buf_lst)
 610 {
 611         struct smc_buf_desc *buf_pos;
 612
 613         while (*buf_lst < SMC_RMBE_SIZES) {
 614                 buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst],
 615                                                    struct smc_buf_desc, list);
 616                 if (buf_pos)
 617                         return buf_pos;
 618                 (*buf_lst)++;
 619         }
 620         return NULL;
 621 }
 622
 623 /* return next rmb from buffer lists */
 624 static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr,
 625                                                  int *buf_lst,
 626                                                  struct smc_buf_desc *buf_pos)
 627 {
 628         struct smc_buf_desc *buf_next;
 629
 630         if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) {
 631                 (*buf_lst)++;
 632                 return _smc_llc_get_next_rmb(lgr, buf_lst);
 633         }
 634         buf_next = list_next_entry(buf_pos, list);
 635         return buf_next;
 636 }
 637
 638 static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr,
 639                                                   int *buf_lst)
 640 {
 641         *buf_lst = 0;
 642         return smc_llc_get_next_rmb(lgr, buf_lst, NULL);
 643 }
 644
 645 /* send one add_link_continue msg */
 646 static int smc_llc_add_link_cont(struct smc_link *link,
 647                                  struct smc_link *link_new, u8 *num_rkeys_todo,
 648                                  int *buf_lst, struct smc_buf_desc **buf_pos)
 649 {
 650         struct smc_llc_msg_add_link_cont *addc_llc;
 651         struct smc_link_group *lgr = link->lgr;
 652         int prim_lnk_idx, lnk_idx, i, rc;
 653         struct smc_wr_tx_pend_priv *pend;
 654         struct smc_wr_buf *wr_buf;
 655         struct smc_buf_desc *rmb;
 656         u8 n;
 657
 658         rc = smc_llc_add_pending_send(link, &wr_buf, &pend);
 659         if (rc)
 660                 return rc;
 661         addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf;
 662         memset(addc_llc, 0, sizeof(*addc_llc));
 663
 664         prim_lnk_idx = link->link_idx;
 665         lnk_idx = link_new->link_idx;
 666         addc_llc->link_num = link_new->link_id;
 667         addc_llc->num_rkeys = *num_rkeys_todo;
 668         n = *num_rkeys_todo;
 669         for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) {
 670                 if (!*buf_pos) {
 671                         addc_llc->num_rkeys = addc_llc->num_rkeys -
 672                                               *num_rkeys_todo;
 673                         *num_rkeys_todo = 0;
 674                         break;
 675                 }
 676                 rmb = *buf_pos;
 677
 678                 addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey);
 679                 addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey);
 680                 addc_llc->rt[i].rmb_vaddr_new =
 681                         cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl));
 682
 683                 (*num_rkeys_todo)--;
 684                 *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 685                 while (*buf_pos && !(*buf_pos)->used)
 686                         *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos);
 687         }
 688         addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT;
 689         addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont);
 690         if (lgr->role == SMC_CLNT)
 691                 addc_llc->hd.flags |= SMC_LLC_FLAG_RESP;
 692         return smc_wr_tx_send(link, pend);
 693 }
 694
 695 static int smc_llc_cli_rkey_exchange(struct smc_link *link,
 696                                      struct smc_link *link_new)
 697 {
 698         struct smc_llc_msg_add_link_cont *addc_llc;
 699         struct smc_link_group *lgr = link->lgr;
 700         u8 max, num_rkeys_send, num_rkeys_recv;
 701         struct smc_llc_qentry *qentry;
 702         struct smc_buf_desc *buf_pos;
 703         int buf_lst;
 704         int rc = 0;
 705         int i;
 706
 707         mutex_lock(&lgr->rmbs_lock);
 708         num_rkeys_send = lgr->conns_num;
 709         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
 710         do {
 711                 qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME,
 712                                       SMC_LLC_ADD_LINK_CONT);
 713                 if (!qentry) {
 714                         rc = -ETIMEDOUT;
 715                         break;
 716                 }
 717                 addc_llc = &qentry->msg.add_link_cont;
 718                 num_rkeys_recv = addc_llc->num_rkeys;
 719                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
 720                 for (i = 0; i < max; i++) {
 721                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
 722                                        addc_llc->rt[i].rmb_key,
 723                                        addc_llc->rt[i].rmb_vaddr_new,
 724                                        addc_llc->rt[i].rmb_key_new);
 725                         num_rkeys_recv--;
 726                 }
 727                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 728                 rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
 729                                            &buf_lst, &buf_pos);
 730                 if (rc)
 731                         break;
 732         } while (num_rkeys_send || num_rkeys_recv);
 733
 734         mutex_unlock(&lgr->rmbs_lock);
 735         return rc;
 736 }
 737
 738 /* prepare and send an add link reject response */
 739 static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry)
 740 {
 741         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 742         qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ;
 743         qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH;
 744         return smc_llc_send_message(qentry->link, &qentry->msg);
 745 }
 746
 747 static int smc_llc_cli_conf_link(struct smc_link *link,
 748                                  struct smc_init_info *ini,
 749                                  struct smc_link *link_new,
 750                                  enum smc_lgr_type lgr_new_t)
 751 {
 752         struct smc_link_group *lgr = link->lgr;
 753         struct smc_llc_qentry *qentry = NULL;
 754         int rc = 0;
 755
 756         /* receive CONFIRM LINK request over RoCE fabric */
 757         qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0);
 758         if (!qentry) {
 759                 rc = smc_llc_send_delete_link(link, link_new->link_id,
 760                                               SMC_LLC_REQ, false,
 761                                               SMC_LLC_DEL_LOST_PATH);
 762                 return -ENOLINK;
 763         }
 764         if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) {
 765                 /* received DELETE_LINK instead */
 766                 qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP;
 767                 smc_llc_send_message(link, &qentry->msg);
 768                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 769                 return -ENOLINK;
 770         }
 771         smc_llc_save_peer_uid(qentry);
 772         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 773
 774         rc = smc_ib_modify_qp_rts(link_new);
 775         if (rc) {
 776                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 777                                          false, SMC_LLC_DEL_LOST_PATH);
 778                 return -ENOLINK;
 779         }
 780         smc_wr_remember_qp_attr(link_new);
 781
 782         rc = smcr_buf_reg_lgr(link_new);
 783         if (rc) {
 784                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 785                                          false, SMC_LLC_DEL_LOST_PATH);
 786                 return -ENOLINK;
 787         }
 788
 789         /* send CONFIRM LINK response over RoCE fabric */
 790         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP);
 791         if (rc) {
 792                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
 793                                          false, SMC_LLC_DEL_LOST_PATH);
 794                 return -ENOLINK;
 795         }
 796         smc_llc_link_active(link_new);
 797         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
 798             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
 799                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
 800         else
 801                 smcr_lgr_set_type(lgr, lgr_new_t);
 802         return 0;
 803 }
 804
 805 static void smc_llc_save_add_link_info(struct smc_link *link,
 806                                        struct smc_llc_msg_add_link *add_llc)
 807 {
 808         link->peer_qpn = ntoh24(add_llc->sender_qp_num);
 809         memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE);
 810         memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN);
 811         link->peer_psn = ntoh24(add_llc->initial_psn);
 812         link->peer_mtu = add_llc->qp_mtu;
 813 }
 814
 815 /* as an SMC client, process an add link request */
 816 int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry)
 817 {
 818         struct smc_llc_msg_add_link *llc = &qentry->msg.add_link;
 819         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
 820         struct smc_link_group *lgr = smc_get_lgr(link);
 821         struct smc_link *lnk_new = NULL;
 822         struct smc_init_info ini;
 823         int lnk_idx, rc = 0;
 824
 825         ini.vlan_id = lgr->vlan_id;
 826         smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev);
 827         if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
 828             !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) {
 829                 if (!ini.ib_dev)
 830                         goto out_reject;
 831                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
 832         }
 833         if (!ini.ib_dev) {
 834                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
 835                 ini.ib_dev = link->smcibdev;
 836                 ini.ib_port = link->ibport;
 837         }
 838         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
 839         if (lnk_idx < 0)
 840                 goto out_reject;
 841         lnk_new = &lgr->lnk[lnk_idx];
 842         rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini);
 843         if (rc)
 844                 goto out_reject;
 845         smc_llc_save_add_link_info(lnk_new, llc);
 846         lnk_new->link_id = llc->link_num;       /* SMC server assigns link id */
 847         smc_llc_link_set_uid(lnk_new);
 848
 849         rc = smc_ib_ready_link(lnk_new);
 850         if (rc)
 851                 goto out_clear_lnk;
 852
 853         rc = smcr_buf_map_lgr(lnk_new);
 854         if (rc)
 855                 goto out_clear_lnk;
 856
 857         rc = smc_llc_send_add_link(link,
 858                                    lnk_new->smcibdev->mac[ini.ib_port - 1],
 859                                    lnk_new->gid, lnk_new, SMC_LLC_RESP);
 860         if (rc)
 861                 goto out_clear_lnk;
 862         rc = smc_llc_cli_rkey_exchange(link, lnk_new);
 863         if (rc) {
 864                 rc = 0;
 865                 goto out_clear_lnk;
 866         }
 867         rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t);
 868         if (!rc)
 869                 goto out;
 870 out_clear_lnk:
 871         smcr_link_clear(lnk_new, false);
 872 out_reject:
 873         smc_llc_cli_add_link_reject(qentry);
 874 out:
 875         kfree(qentry);
 876         return rc;
 877 }
 878
 879 static void smc_llc_process_cli_add_link(struct smc_link_group *lgr)
 880 {
 881         struct smc_llc_qentry *qentry;
 882
 883         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
 884
 885         mutex_lock(&lgr->llc_conf_mutex);
 886         smc_llc_cli_add_link(qentry->link, qentry);
 887         mutex_unlock(&lgr->llc_conf_mutex);
 888 }
 889
 890 static int smc_llc_active_link_count(struct smc_link_group *lgr)
 891 {
 892         int i, link_count = 0;
 893
 894         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 895                 if (!smc_link_usable(&lgr->lnk[i]))
 896                         continue;
 897                 link_count++;
 898         }
 899         return link_count;
 900 }
 901
 902 /* find the asymmetric link when 3 links are established  */
 903 static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr)
 904 {
 905         int asym_idx = -ENOENT;
 906         int i, j, k;
 907         bool found;
 908
 909         /* determine asymmetric link */
 910         found = false;
 911         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
 912                 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
 913                         if (!smc_link_usable(&lgr->lnk[i]) ||
 914                             !smc_link_usable(&lgr->lnk[j]))
 915                                 continue;
 916                         if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid,
 917                                     SMC_GID_SIZE)) {
 918                                 found = true;   /* asym_lnk is i or j */
 919                                 break;
 920                         }
 921                 }
 922                 if (found)
 923                         break;
 924         }
 925         if (!found)
 926                 goto out; /* no asymmetric link */
 927         for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) {
 928                 if (!smc_link_usable(&lgr->lnk[k]))
 929                         continue;
 930                 if (k != i &&
 931                     !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid,
 932                             SMC_GID_SIZE)) {
 933                         asym_idx = i;
 934                         break;
 935                 }
 936                 if (k != j &&
 937                     !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid,
 938                             SMC_GID_SIZE)) {
 939                         asym_idx = j;
 940                         break;
 941                 }
 942         }
 943 out:
 944         return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx];
 945 }
 946
 947 static void smc_llc_delete_asym_link(struct smc_link_group *lgr)
 948 {
 949         struct smc_link *lnk_new = NULL, *lnk_asym;
 950         struct smc_llc_qentry *qentry;
 951         int rc;
 952
 953         lnk_asym = smc_llc_find_asym_link(lgr);
 954         if (!lnk_asym)
 955                 return; /* no asymmetric link */
 956         if (!smc_link_downing(&lnk_asym->state))
 957                 return;
 958         lnk_new = smc_switch_conns(lgr, lnk_asym, false);
 959         smc_wr_tx_wait_no_pending_sends(lnk_asym);
 960         if (!lnk_new)
 961                 goto out_free;
 962         /* change flow type from ADD_LINK into DEL_LINK */
 963         lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK;
 964         rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ,
 965                                       true, SMC_LLC_DEL_NO_ASYM_NEEDED);
 966         if (rc) {
 967                 smcr_link_down_cond(lnk_new);
 968                 goto out_free;
 969         }
 970         qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME,
 971                               SMC_LLC_DELETE_LINK);
 972         if (!qentry) {
 973                 smcr_link_down_cond(lnk_new);
 974                 goto out_free;
 975         }
 976         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
 977 out_free:
 978         smcr_link_clear(lnk_asym, true);
 979 }
 980
 981 static int smc_llc_srv_rkey_exchange(struct smc_link *link,
 982                                      struct smc_link *link_new)
 983 {
 984         struct smc_llc_msg_add_link_cont *addc_llc;
 985         struct smc_link_group *lgr = link->lgr;
 986         u8 max, num_rkeys_send, num_rkeys_recv;
 987         struct smc_llc_qentry *qentry = NULL;
 988         struct smc_buf_desc *buf_pos;
 989         int buf_lst;
 990         int rc = 0;
 991         int i;
 992
 993         mutex_lock(&lgr->rmbs_lock);
 994         num_rkeys_send = lgr->conns_num;
 995         buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst);
 996         do {
 997                 smc_llc_add_link_cont(link, link_new, &num_rkeys_send,
 998                                       &buf_lst, &buf_pos);
 999                 qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME,
1000                                       SMC_LLC_ADD_LINK_CONT);
1001                 if (!qentry) {
1002                         rc = -ETIMEDOUT;
1003                         goto out;
1004                 }
1005                 addc_llc = &qentry->msg.add_link_cont;
1006                 num_rkeys_recv = addc_llc->num_rkeys;
1007                 max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG);
1008                 for (i = 0; i < max; i++) {
1009                         smc_rtoken_set(lgr, link->link_idx, link_new->link_idx,
1010                                        addc_llc->rt[i].rmb_key,
1011                                        addc_llc->rt[i].rmb_vaddr_new,
1012                                        addc_llc->rt[i].rmb_key_new);
1013                         num_rkeys_recv--;
1014                 }
1015                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1016         } while (num_rkeys_send || num_rkeys_recv);
1017 out:
1018         mutex_unlock(&lgr->rmbs_lock);
1019         return rc;
1020 }
1021
1022 static int smc_llc_srv_conf_link(struct smc_link *link,
1023                                  struct smc_link *link_new,
1024                                  enum smc_lgr_type lgr_new_t)
1025 {
1026         struct smc_link_group *lgr = link->lgr;
1027         struct smc_llc_qentry *qentry = NULL;
1028         int rc;
1029
1030         /* send CONFIRM LINK request over the RoCE fabric */
1031         rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ);
1032         if (rc)
1033                 return -ENOLINK;
1034         /* receive CONFIRM LINK response over the RoCE fabric */
1035         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME,
1036                               SMC_LLC_CONFIRM_LINK);
1037         if (!qentry) {
1038                 /* send DELETE LINK */
1039                 smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ,
1040                                          false, SMC_LLC_DEL_LOST_PATH);
1041                 return -ENOLINK;
1042         }
1043         smc_llc_save_peer_uid(qentry);
1044         smc_llc_link_active(link_new);
1045         if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL ||
1046             lgr_new_t == SMC_LGR_ASYMMETRIC_PEER)
1047                 smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx);
1048         else
1049                 smcr_lgr_set_type(lgr, lgr_new_t);
1050         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1051         return 0;
1052 }
1053
1054 int smc_llc_srv_add_link(struct smc_link *link)
1055 {
1056         enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC;
1057         struct smc_link_group *lgr = link->lgr;
1058         struct smc_llc_msg_add_link *add_llc;
1059         struct smc_llc_qentry *qentry = NULL;
1060         struct smc_link *link_new;
1061         struct smc_init_info ini;
1062         int lnk_idx, rc = 0;
1063
1064         /* ignore client add link recommendation, start new flow */
1065         ini.vlan_id = lgr->vlan_id;
1066         smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev);
1067         if (!ini.ib_dev) {
1068                 lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL;
1069                 ini.ib_dev = link->smcibdev;
1070                 ini.ib_port = link->ibport;
1071         }
1072         lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t);
1073         if (lnk_idx < 0)
1074                 return 0;
1075
1076         rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini);
1077         if (rc)
1078                 return rc;
1079         link_new = &lgr->lnk[lnk_idx];
1080         rc = smc_llc_send_add_link(link,
1081                                    link_new->smcibdev->mac[ini.ib_port - 1],
1082                                    link_new->gid, link_new, SMC_LLC_REQ);
1083         if (rc)
1084                 goto out_err;
1085         /* receive ADD LINK response over the RoCE fabric */
1086         qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK);
1087         if (!qentry) {
1088                 rc = -ETIMEDOUT;
1089                 goto out_err;
1090         }
1091         add_llc = &qentry->msg.add_link;
1092         if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) {
1093                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1094                 rc = -ENOLINK;
1095                 goto out_err;
1096         }
1097         if (lgr->type == SMC_LGR_SINGLE &&
1098             (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) &&
1099              !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) {
1100                 lgr_new_t = SMC_LGR_ASYMMETRIC_PEER;
1101         }
1102         smc_llc_save_add_link_info(link_new, add_llc);
1103         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1104
1105         rc = smc_ib_ready_link(link_new);
1106         if (rc)
1107                 goto out_err;
1108         rc = smcr_buf_map_lgr(link_new);
1109         if (rc)
1110                 goto out_err;
1111         rc = smcr_buf_reg_lgr(link_new);
1112         if (rc)
1113                 goto out_err;
1114         rc = smc_llc_srv_rkey_exchange(link, link_new);
1115         if (rc)
1116                 goto out_err;
1117         rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t);
1118         if (rc)
1119                 goto out_err;
1120         return 0;
1121 out_err:
1122         smcr_link_clear(link_new, false);
1123         return rc;
1124 }
1125
1126 static void smc_llc_process_srv_add_link(struct smc_link_group *lgr)
1127 {
1128         struct smc_link *link = lgr->llc_flow_lcl.qentry->link;
1129         int rc;
1130
1131         smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1132
1133         mutex_lock(&lgr->llc_conf_mutex);
1134         rc = smc_llc_srv_add_link(link);
1135         if (!rc && lgr->type == SMC_LGR_SYMMETRIC) {
1136                 /* delete any asymmetric link */
1137                 smc_llc_delete_asym_link(lgr);
1138         }
1139         mutex_unlock(&lgr->llc_conf_mutex);
1140 }
1141
1142 /* enqueue a local add_link req to trigger a new add_link flow, only as SERV */
1143 void smc_llc_srv_add_link_local(struct smc_link *link)
1144 {
1145         struct smc_llc_msg_add_link add_llc = {0};
1146
1147         add_llc.hd.length = sizeof(add_llc);
1148         add_llc.hd.common.type = SMC_LLC_ADD_LINK;
1149         /* no dev and port needed, we as server ignore client data anyway */
1150         smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc);
1151 }
1152
1153 /* worker to process an add link message */
1154 static void smc_llc_add_link_work(struct work_struct *work)
1155 {
1156         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1157                                                   llc_add_link_work);
1158
1159         if (list_empty(&lgr->list)) {
1160                 /* link group is terminating */
1161                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1162                 goto out;
1163         }
1164
1165         if (lgr->role == SMC_CLNT)
1166                 smc_llc_process_cli_add_link(lgr);
1167         else
1168                 smc_llc_process_srv_add_link(lgr);
1169 out:
1170         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1171 }
1172
1173 /* enqueue a local del_link msg to trigger a new del_link flow,
1174  * called only for role SMC_SERV
1175  */
1176 void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id)
1177 {
1178         struct smc_llc_msg_del_link del_llc = {0};
1179
1180         del_llc.hd.length = sizeof(del_llc);
1181         del_llc.hd.common.type = SMC_LLC_DELETE_LINK;
1182         del_llc.link_num = del_link_id;
1183         del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH);
1184         del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1185         smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc);
1186 }
1187
1188 static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr)
1189 {
1190         struct smc_link *lnk_del = NULL, *lnk_asym, *lnk;
1191         struct smc_llc_msg_del_link *del_llc;
1192         struct smc_llc_qentry *qentry;
1193         int active_links;
1194         int lnk_idx;
1195
1196         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1197         lnk = qentry->link;
1198         del_llc = &qentry->msg.delete_link;
1199
1200         if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1201                 smc_lgr_terminate_sched(lgr);
1202                 goto out;
1203         }
1204         mutex_lock(&lgr->llc_conf_mutex);
1205         /* delete single link */
1206         for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) {
1207                 if (lgr->lnk[lnk_idx].link_id != del_llc->link_num)
1208                         continue;
1209                 lnk_del = &lgr->lnk[lnk_idx];
1210                 break;
1211         }
1212         del_llc->hd.flags |= SMC_LLC_FLAG_RESP;
1213         if (!lnk_del) {
1214                 /* link was not found */
1215                 del_llc->reason = htonl(SMC_LLC_DEL_NOLNK);
1216                 smc_llc_send_message(lnk, &qentry->msg);
1217                 goto out_unlock;
1218         }
1219         lnk_asym = smc_llc_find_asym_link(lgr);
1220
1221         del_llc->reason = 0;
1222         smc_llc_send_message(lnk, &qentry->msg); /* response */
1223
1224         if (smc_link_downing(&lnk_del->state)) {
1225                 smc_switch_conns(lgr, lnk_del, false);
1226                 smc_wr_tx_wait_no_pending_sends(lnk_del);
1227         }
1228         smcr_link_clear(lnk_del, true);
1229
1230         active_links = smc_llc_active_link_count(lgr);
1231         if (lnk_del == lnk_asym) {
1232                 /* expected deletion of asym link, don't change lgr state */
1233         } else if (active_links == 1) {
1234                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1235         } else if (!active_links) {
1236                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1237                 smc_lgr_terminate_sched(lgr);
1238         }
1239 out_unlock:
1240         mutex_unlock(&lgr->llc_conf_mutex);
1241 out:
1242         kfree(qentry);
1243 }
1244
1245 /* try to send a DELETE LINK ALL request on any active link,
1246  * waiting for send completion
1247  */
1248 void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn)
1249 {
1250         struct smc_llc_msg_del_link delllc = {0};
1251         int i;
1252
1253         delllc.hd.common.type = SMC_LLC_DELETE_LINK;
1254         delllc.hd.length = sizeof(delllc);
1255         if (ord)
1256                 delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY;
1257         delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL;
1258         delllc.reason = htonl(rsn);
1259
1260         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1261                 if (!smc_link_usable(&lgr->lnk[i]))
1262                         continue;
1263                 if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc))
1264                         break;
1265         }
1266 }
1267
1268 static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr)
1269 {
1270         struct smc_llc_msg_del_link *del_llc;
1271         struct smc_link *lnk, *lnk_del;
1272         struct smc_llc_qentry *qentry;
1273         int active_links;
1274         int i;
1275
1276         mutex_lock(&lgr->llc_conf_mutex);
1277         qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl);
1278         lnk = qentry->link;
1279         del_llc = &qentry->msg.delete_link;
1280
1281         if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) {
1282                 /* delete entire lgr */
1283                 smc_llc_send_link_delete_all(lgr, true, ntohl(
1284                                               qentry->msg.delete_link.reason));
1285                 smc_lgr_terminate_sched(lgr);
1286                 goto out;
1287         }
1288         /* delete single link */
1289         lnk_del = NULL;
1290         for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1291                 if (lgr->lnk[i].link_id == del_llc->link_num) {
1292                         lnk_del = &lgr->lnk[i];
1293                         break;
1294                 }
1295         }
1296         if (!lnk_del)
1297                 goto out; /* asymmetric link already deleted */
1298
1299         if (smc_link_downing(&lnk_del->state)) {
1300                 smc_switch_conns(lgr, lnk_del, false);
1301                 smc_wr_tx_wait_no_pending_sends(lnk_del);
1302         }
1303         if (!list_empty(&lgr->list)) {
1304                 /* qentry is either a request from peer (send it back to
1305                  * initiate the DELETE_LINK processing), or a locally
1306                  * enqueued DELETE_LINK request (forward it)
1307                  */
1308                 if (!smc_llc_send_message(lnk, &qentry->msg)) {
1309                         struct smc_llc_qentry *qentry2;
1310
1311                         qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME,
1312                                                SMC_LLC_DELETE_LINK);
1313                         if (qentry2)
1314                                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1315                 }
1316         }
1317         smcr_link_clear(lnk_del, true);
1318
1319         active_links = smc_llc_active_link_count(lgr);
1320         if (active_links == 1) {
1321                 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
1322         } else if (!active_links) {
1323                 smcr_lgr_set_type(lgr, SMC_LGR_NONE);
1324                 smc_lgr_terminate_sched(lgr);
1325         }
1326
1327         if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) {
1328                 /* trigger setup of asymm alt link */
1329                 smc_llc_srv_add_link_local(lnk);
1330         }
1331 out:
1332         mutex_unlock(&lgr->llc_conf_mutex);
1333         kfree(qentry);
1334 }
1335
1336 static void smc_llc_delete_link_work(struct work_struct *work)
1337 {
1338         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1339                                                   llc_del_link_work);
1340
1341         if (list_empty(&lgr->list)) {
1342                 /* link group is terminating */
1343                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1344                 goto out;
1345         }
1346
1347         if (lgr->role == SMC_CLNT)
1348                 smc_llc_process_cli_delete_link(lgr);
1349         else
1350                 smc_llc_process_srv_delete_link(lgr);
1351 out:
1352         smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1353 }
1354
1355 /* process a confirm_rkey request from peer, remote flow */
1356 static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr)
1357 {
1358         struct smc_llc_msg_confirm_rkey *llc;
1359         struct smc_llc_qentry *qentry;
1360         struct smc_link *link;
1361         int num_entries;
1362         int rk_idx;
1363         int i;
1364
1365         qentry = lgr->llc_flow_rmt.qentry;
1366         llc = &qentry->msg.confirm_rkey;
1367         link = qentry->link;
1368
1369         num_entries = llc->rtoken[0].num_rkeys;
1370         /* first rkey entry is for receiving link */
1371         rk_idx = smc_rtoken_add(link,
1372                                 llc->rtoken[0].rmb_vaddr,
1373                                 llc->rtoken[0].rmb_key);
1374         if (rk_idx < 0)
1375                 goto out_err;
1376
1377         for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++)
1378                 smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id,
1379                                 llc->rtoken[i].rmb_vaddr,
1380                                 llc->rtoken[i].rmb_key);
1381         /* max links is 3 so there is no need to support conf_rkey_cont msgs */
1382         goto out;
1383 out_err:
1384         llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1385         llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY;
1386 out:
1387         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1388         smc_llc_send_message(link, &qentry->msg);
1389         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1390 }
1391
1392 /* process a delete_rkey request from peer, remote flow */
1393 static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr)
1394 {
1395         struct smc_llc_msg_delete_rkey *llc;
1396         struct smc_llc_qentry *qentry;
1397         struct smc_link *link;
1398         u8 err_mask = 0;
1399         int i, max;
1400
1401         qentry = lgr->llc_flow_rmt.qentry;
1402         llc = &qentry->msg.delete_rkey;
1403         link = qentry->link;
1404
1405         max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX);
1406         for (i = 0; i < max; i++) {
1407                 if (smc_rtoken_delete(link, llc->rkey[i]))
1408                         err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i);
1409         }
1410         if (err_mask) {
1411                 llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG;
1412                 llc->err_mask = err_mask;
1413         }
1414         llc->hd.flags |= SMC_LLC_FLAG_RESP;
1415         smc_llc_send_message(link, &qentry->msg);
1416         smc_llc_flow_qentry_del(&lgr->llc_flow_rmt);
1417 }
1418
1419 static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type)
1420 {
1421         pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: "
1422                             "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type);
1423         smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL);
1424         smc_lgr_terminate_sched(lgr);
1425 }
1426
1427 /* flush the llc event queue */
1428 static void smc_llc_event_flush(struct smc_link_group *lgr)
1429 {
1430         struct smc_llc_qentry *qentry, *q;
1431
1432         spin_lock_bh(&lgr->llc_event_q_lock);
1433         list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) {
1434                 list_del_init(&qentry->list);
1435                 kfree(qentry);
1436         }
1437         spin_unlock_bh(&lgr->llc_event_q_lock);
1438 }
1439
1440 static void smc_llc_event_handler(struct smc_llc_qentry *qentry)
1441 {
1442         union smc_llc_msg *llc = &qentry->msg;
1443         struct smc_link *link = qentry->link;
1444         struct smc_link_group *lgr = link->lgr;
1445
1446         if (!smc_link_usable(link))
1447                 goto out;
1448
1449         switch (llc->raw.hdr.common.type) {
1450         case SMC_LLC_TEST_LINK:
1451                 llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP;
1452                 smc_llc_send_message(link, llc);
1453                 break;
1454         case SMC_LLC_ADD_LINK:
1455                 if (list_empty(&lgr->list))
1456                         goto out;       /* lgr is terminating */
1457                 if (lgr->role == SMC_CLNT) {
1458                         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) {
1459                                 /* a flow is waiting for this message */
1460                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1461                                                         qentry);
1462                                 wake_up_interruptible(&lgr->llc_waiter);
1463                         } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1464                                                       qentry)) {
1465                                 schedule_work(&lgr->llc_add_link_work);
1466                         }
1467                 } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) {
1468                         /* as smc server, handle client suggestion */
1469                         schedule_work(&lgr->llc_add_link_work);
1470                 }
1471                 return;
1472         case SMC_LLC_CONFIRM_LINK:
1473         case SMC_LLC_ADD_LINK_CONT:
1474                 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1475                         /* a flow is waiting for this message */
1476                         smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry);
1477                         wake_up_interruptible(&lgr->llc_waiter);
1478                         return;
1479                 }
1480                 break;
1481         case SMC_LLC_DELETE_LINK:
1482                 if (lgr->role == SMC_CLNT) {
1483                         /* server requests to delete this link, send response */
1484                         if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1485                                 /* DEL LINK REQ during ADD LINK SEQ */
1486                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1487                                                         qentry);
1488                                 wake_up_interruptible(&lgr->llc_waiter);
1489                         } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1490                                                       qentry)) {
1491                                 schedule_work(&lgr->llc_del_link_work);
1492                         }
1493                 } else {
1494                         if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK &&
1495                             !lgr->llc_flow_lcl.qentry) {
1496                                 /* DEL LINK REQ during ADD LINK SEQ */
1497                                 smc_llc_flow_qentry_set(&lgr->llc_flow_lcl,
1498                                                         qentry);
1499                                 wake_up_interruptible(&lgr->llc_waiter);
1500                         } else if (smc_llc_flow_start(&lgr->llc_flow_lcl,
1501                                                       qentry)) {
1502                                 schedule_work(&lgr->llc_del_link_work);
1503                         }
1504                 }
1505                 return;
1506         case SMC_LLC_CONFIRM_RKEY:
1507                 /* new request from remote, assign to remote flow */
1508                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1509                         /* process here, does not wait for more llc msgs */
1510                         smc_llc_rmt_conf_rkey(lgr);
1511                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1512                 }
1513                 return;
1514         case SMC_LLC_CONFIRM_RKEY_CONT:
1515                 /* not used because max links is 3, and 3 rkeys fit into
1516                  * one CONFIRM_RKEY message
1517                  */
1518                 break;
1519         case SMC_LLC_DELETE_RKEY:
1520                 /* new request from remote, assign to remote flow */
1521                 if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) {
1522                         /* process here, does not wait for more llc msgs */
1523                         smc_llc_rmt_delete_rkey(lgr);
1524                         smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt);
1525                 }
1526                 return;
1527         default:
1528                 smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type);
1529                 break;
1530         }
1531 out:
1532         kfree(qentry);
1533 }
1534
1535 /* worker to process llc messages on the event queue */
1536 static void smc_llc_event_work(struct work_struct *work)
1537 {
1538         struct smc_link_group *lgr = container_of(work, struct smc_link_group,
1539                                                   llc_event_work);
1540         struct smc_llc_qentry *qentry;
1541
1542         if (!lgr->llc_flow_lcl.type && lgr->delayed_event) {
1543                 if (smc_link_usable(lgr->delayed_event->link)) {
1544                         smc_llc_event_handler(lgr->delayed_event);
1545                 } else {
1546                         qentry = lgr->delayed_event;
1547                         lgr->delayed_event = NULL;
1548                         kfree(qentry);
1549                 }
1550         }
1551
1552 again:
1553         spin_lock_bh(&lgr->llc_event_q_lock);
1554         if (!list_empty(&lgr->llc_event_q)) {
1555                 qentry = list_first_entry(&lgr->llc_event_q,
1556                                           struct smc_llc_qentry, list);
1557                 list_del_init(&qentry->list);
1558                 spin_unlock_bh(&lgr->llc_event_q_lock);
1559                 smc_llc_event_handler(qentry);
1560                 goto again;
1561         }
1562         spin_unlock_bh(&lgr->llc_event_q_lock);
1563 }
1564
1565 /* process llc responses in tasklet context */
1566 static void smc_llc_rx_response(struct smc_link *link,
1567                                 struct smc_llc_qentry *qentry)
1568 {
1569         u8 llc_type = qentry->msg.raw.hdr.common.type;
1570
1571         switch (llc_type) {
1572         case SMC_LLC_TEST_LINK:
1573                 if (link->state == SMC_LNK_ACTIVE)
1574                         complete(&link->llc_testlink_resp);
1575                 break;
1576         case SMC_LLC_ADD_LINK:
1577         case SMC_LLC_DELETE_LINK:
1578         case SMC_LLC_CONFIRM_LINK:
1579         case SMC_LLC_ADD_LINK_CONT:
1580         case SMC_LLC_CONFIRM_RKEY:
1581         case SMC_LLC_DELETE_RKEY:
1582                 /* assign responses to the local flow, we requested them */
1583                 smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry);
1584                 wake_up_interruptible(&link->lgr->llc_waiter);
1585                 return;
1586         case SMC_LLC_CONFIRM_RKEY_CONT:
1587                 /* not used because max links is 3 */
1588                 break;
1589         default:
1590                 smc_llc_protocol_violation(link->lgr, llc_type);
1591                 break;
1592         }
1593         kfree(qentry);
1594 }
1595
1596 static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc)
1597 {
1598         struct smc_link_group *lgr = link->lgr;
1599         struct smc_llc_qentry *qentry;
1600         unsigned long flags;
1601
1602         qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
1603         if (!qentry)
1604                 return;
1605         qentry->link = link;
1606         INIT_LIST_HEAD(&qentry->list);
1607         memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg));
1608
1609         /* process responses immediately */
1610         if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) {
1611                 smc_llc_rx_response(link, qentry);
1612                 return;
1613         }
1614
1615         /* add requests to event queue */
1616         spin_lock_irqsave(&lgr->llc_event_q_lock, flags);
1617         list_add_tail(&qentry->list, &lgr->llc_event_q);
1618         spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags);
1619         schedule_work(&link->lgr->llc_event_work);
1620 }
1621
1622 /* copy received msg and add it to the event queue */
1623 static void smc_llc_rx_handler(struct ib_wc *wc, void *buf)
1624 {
1625         struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
1626         union smc_llc_msg *llc = buf;
1627
1628         if (wc->byte_len < sizeof(*llc))
1629                 return; /* short message */
1630         if (llc->raw.hdr.length != sizeof(*llc))
1631                 return; /* invalid message */
1632
1633         smc_llc_enqueue(link, llc);
1634 }
1635
1636 /***************************** worker, utils *********************************/
1637
1638 static void smc_llc_testlink_work(struct work_struct *work)
1639 {
1640         struct smc_link *link = container_of(to_delayed_work(work),
1641                                              struct smc_link, llc_testlink_wrk);
1642         unsigned long next_interval;
1643         unsigned long expire_time;
1644         u8 user_data[16] = { 0 };
1645         int rc;
1646
1647         if (link->state != SMC_LNK_ACTIVE)
1648                 return;         /* don't reschedule worker */
1649         expire_time = link->wr_rx_tstamp + link->llc_testlink_time;
1650         if (time_is_after_jiffies(expire_time)) {
1651                 next_interval = expire_time - jiffies;
1652                 goto out;
1653         }
1654         reinit_completion(&link->llc_testlink_resp);
1655         smc_llc_send_test_link(link, user_data);
1656         /* receive TEST LINK response over RoCE fabric */
1657         rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp,
1658                                                        SMC_LLC_WAIT_TIME);
1659         if (link->state != SMC_LNK_ACTIVE)
1660                 return;         /* link state changed */
1661         if (rc <= 0) {
1662                 smcr_link_down_cond_sched(link);
1663                 return;
1664         }
1665         next_interval = link->llc_testlink_time;
1666 out:
1667         schedule_delayed_work(&link->llc_testlink_wrk, next_interval);
1668 }
1669
1670 void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
1671 {
1672         struct net *net = sock_net(smc->clcsock->sk);
1673
1674         INIT_WORK(&lgr->llc_event_work, smc_llc_event_work);
1675         INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work);
1676         INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work);
1677         INIT_LIST_HEAD(&lgr->llc_event_q);
1678         spin_lock_init(&lgr->llc_event_q_lock);
1679         spin_lock_init(&lgr->llc_flow_lock);
1680         init_waitqueue_head(&lgr->llc_waiter);
1681         mutex_init(&lgr->llc_conf_mutex);
1682         lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time;
1683 }
1684
1685 /* called after lgr was removed from lgr_list */
1686 void smc_llc_lgr_clear(struct smc_link_group *lgr)
1687 {
1688         smc_llc_event_flush(lgr);
1689         wake_up_interruptible_all(&lgr->llc_waiter);
1690         cancel_work_sync(&lgr->llc_event_work);
1691         cancel_work_sync(&lgr->llc_add_link_work);
1692         cancel_work_sync(&lgr->llc_del_link_work);
1693         if (lgr->delayed_event) {
1694                 kfree(lgr->delayed_event);
1695                 lgr->delayed_event = NULL;
1696         }
1697 }
1698
1699 int smc_llc_link_init(struct smc_link *link)
1700 {
1701         init_completion(&link->llc_testlink_resp);
1702         INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work);
1703         return 0;
1704 }
1705
1706 void smc_llc_link_active(struct smc_link *link)
1707 {
1708         pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, "
1709                             "peerid %*phN, ibdev %s, ibport %d\n",
1710                             SMC_LGR_ID_SIZE, &link->lgr->id,
1711                             SMC_LGR_ID_SIZE, &link->link_uid,
1712                             SMC_LGR_ID_SIZE, &link->peer_link_uid,
1713                             link->smcibdev->ibdev->name, link->ibport);
1714         link->state = SMC_LNK_ACTIVE;
1715         if (link->lgr->llc_testlink_time) {
1716                 link->llc_testlink_time = link->lgr->llc_testlink_time * HZ;
1717                 schedule_delayed_work(&link->llc_testlink_wrk,
1718                                       link->llc_testlink_time);
1719         }
1720 }
1721
1722 /* called in worker context */
1723 void smc_llc_link_clear(struct smc_link *link, bool log)
1724 {
1725         if (log)
1726                 pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN"
1727                                     ", peerid %*phN, ibdev %s, ibport %d\n",
1728                                     SMC_LGR_ID_SIZE, &link->lgr->id,
1729                                     SMC_LGR_ID_SIZE, &link->link_uid,
1730                                     SMC_LGR_ID_SIZE, &link->peer_link_uid,
1731                                     link->smcibdev->ibdev->name, link->ibport);
1732         complete(&link->llc_testlink_resp);
1733         cancel_delayed_work_sync(&link->llc_testlink_wrk);
1734         smc_wr_wakeup_reg_wait(link);
1735         smc_wr_wakeup_tx_wait(link);
1736 }
1737
1738 /* register a new rtoken at the remote peer (for all links) */
1739 int smc_llc_do_confirm_rkey(struct smc_link *send_link,
1740                             struct smc_buf_desc *rmb_desc)
1741 {
1742         struct smc_link_group *lgr = send_link->lgr;
1743         struct smc_llc_qentry *qentry = NULL;
1744         int rc = 0;
1745
1746         rc = smc_llc_send_confirm_rkey(send_link, rmb_desc);
1747         if (rc)
1748                 goto out;
1749         /* receive CONFIRM RKEY response from server over RoCE fabric */
1750         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
1751                               SMC_LLC_CONFIRM_RKEY);
1752         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
1753                 rc = -EFAULT;
1754 out:
1755         if (qentry)
1756                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1757         return rc;
1758 }
1759
1760 /* unregister an rtoken at the remote peer */
1761 int smc_llc_do_delete_rkey(struct smc_link_group *lgr,
1762                            struct smc_buf_desc *rmb_desc)
1763 {
1764         struct smc_llc_qentry *qentry = NULL;
1765         struct smc_link *send_link;
1766         int rc = 0;
1767
1768         send_link = smc_llc_usable_link(lgr);
1769         if (!send_link)
1770                 return -ENOLINK;
1771
1772         /* protected by llc_flow control */
1773         rc = smc_llc_send_delete_rkey(send_link, rmb_desc);
1774         if (rc)
1775                 goto out;
1776         /* receive DELETE RKEY response from server over RoCE fabric */
1777         qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME,
1778                               SMC_LLC_DELETE_RKEY);
1779         if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG))
1780                 rc = -EFAULT;
1781 out:
1782         if (qentry)
1783                 smc_llc_flow_qentry_del(&lgr->llc_flow_lcl);
1784         return rc;
1785 }
1786
1787 void smc_llc_link_set_uid(struct smc_link *link)
1788 {
1789         __be32 link_uid;
1790
1791         link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id);
1792         memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE);
1793 }
1794
1795 /* save peers link user id, used for debug purposes */
1796 void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry)
1797 {
1798         memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid,
1799                SMC_LGR_ID_SIZE);
1800 }
1801
1802 /* evaluate confirm link request or response */
1803 int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry,
1804                            enum smc_llc_reqresp type)
1805 {
1806         if (type == SMC_LLC_REQ) {      /* SMC server assigns link_id */
1807                 qentry->link->link_id = qentry->msg.confirm_link.link_num;
1808                 smc_llc_link_set_uid(qentry->link);
1809         }
1810         if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC))
1811                 return -ENOTSUPP;
1812         return 0;
1813 }
1814
1815 /***************************** init, exit, misc ******************************/
1816
1817 static struct smc_wr_rx_handler smc_llc_rx_handlers[] = {
1818         {
1819                 .handler        = smc_llc_rx_handler,
1820                 .type           = SMC_LLC_CONFIRM_LINK
1821         },
1822         {
1823                 .handler        = smc_llc_rx_handler,
1824                 .type           = SMC_LLC_TEST_LINK
1825         },
1826         {
1827                 .handler        = smc_llc_rx_handler,
1828                 .type           = SMC_LLC_ADD_LINK
1829         },
1830         {
1831                 .handler        = smc_llc_rx_handler,
1832                 .type           = SMC_LLC_ADD_LINK_CONT
1833         },
1834         {
1835                 .handler        = smc_llc_rx_handler,
1836                 .type           = SMC_LLC_DELETE_LINK
1837         },
1838         {
1839                 .handler        = smc_llc_rx_handler,
1840                 .type           = SMC_LLC_CONFIRM_RKEY
1841         },
1842         {
1843                 .handler        = smc_llc_rx_handler,
1844                 .type           = SMC_LLC_CONFIRM_RKEY_CONT
1845         },
1846         {
1847                 .handler        = smc_llc_rx_handler,
1848                 .type           = SMC_LLC_DELETE_RKEY
1849         },
1850         {
1851                 .handler        = NULL,
1852         }
1853 };
1854
1855 int __init smc_llc_init(void)
1856 {
1857         struct smc_wr_rx_handler *handler;
1858         int rc = 0;
1859
1860         for (handler = smc_llc_rx_handlers; handler->handler; handler++) {
1861                 INIT_HLIST_NODE(&handler->list);
1862                 rc = smc_wr_rx_register_handler(handler);
1863                 if (rc)
1864                         break;
1865         }
1866         return rc;
1867 }