drivers/nvme/host/tcp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * NVMe over Fabrics TCP host.
   4  * Copyright (c) 2018 Lightbits Labs. All rights reserved.
   5  */
   6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7 #include <linux/module.h>
   8 #include <linux/init.h>
   9 #include <linux/slab.h>
  10 #include <linux/err.h>
  11 #include <linux/nvme-tcp.h>
  12 #include <net/sock.h>
  13 #include <net/tcp.h>
  14 #include <linux/blk-mq.h>
  15 #include <crypto/hash.h>
  16 #include <net/busy_poll.h>
  17
  18 #include "nvme.h"
  19 #include "fabrics.h"
  20
  21 struct nvme_tcp_queue;
  22
  23 /* Define the socket priority to use for connections were it is desirable
  24  * that the NIC consider performing optimized packet processing or filtering.
  25  * A non-zero value being sufficient to indicate general consideration of any
  26  * possible optimization.  Making it a module param allows for alternative
  27  * values that may be unique for some NIC implementations.
  28  */
  29 static int so_priority;
  30 module_param(so_priority, int, 0644);
  31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
  32
  33 enum nvme_tcp_send_state {
  34         NVME_TCP_SEND_CMD_PDU = 0,
  35         NVME_TCP_SEND_H2C_PDU,
  36         NVME_TCP_SEND_DATA,
  37         NVME_TCP_SEND_DDGST,
  38 };
  39
  40 struct nvme_tcp_request {
  41         struct nvme_request     req;
  42         void                    *pdu;
  43         struct nvme_tcp_queue   *queue;
  44         u32                     data_len;
  45         u32                     pdu_len;
  46         u32                     pdu_sent;
  47         u16                     ttag;
  48         struct list_head        entry;
  49         struct llist_node       lentry;
  50         __le32                  ddgst;
  51
  52         struct bio              *curr_bio;
  53         struct iov_iter         iter;
  54
  55         /* send state */
  56         size_t                  offset;
  57         size_t                  data_sent;
  58         enum nvme_tcp_send_state state;
  59 };
  60
  61 enum nvme_tcp_queue_flags {
  62         NVME_TCP_Q_ALLOCATED    = 0,
  63         NVME_TCP_Q_LIVE         = 1,
  64         NVME_TCP_Q_POLLING      = 2,
  65 };
  66
  67 enum nvme_tcp_recv_state {
  68         NVME_TCP_RECV_PDU = 0,
  69         NVME_TCP_RECV_DATA,
  70         NVME_TCP_RECV_DDGST,
  71 };
  72
  73 struct nvme_tcp_ctrl;
  74 struct nvme_tcp_queue {
  75         struct socket           *sock;
  76         struct work_struct      io_work;
  77         int                     io_cpu;
  78
  79         struct mutex            send_mutex;
  80         struct llist_head       req_list;
  81         struct list_head        send_list;
  82         bool                    more_requests;
  83
  84         /* recv state */
  85         void                    *pdu;
  86         int                     pdu_remaining;
  87         int                     pdu_offset;
  88         size_t                  data_remaining;
  89         size_t                  ddgst_remaining;
  90         unsigned int            nr_cqe;
  91
  92         /* send state */
  93         struct nvme_tcp_request *request;
  94
  95         int                     queue_size;
  96         size_t                  cmnd_capsule_len;
  97         struct nvme_tcp_ctrl    *ctrl;
  98         unsigned long           flags;
  99         bool                    rd_enabled;
 100
 101         bool                    hdr_digest;
 102         bool                    data_digest;
 103         struct ahash_request    *rcv_hash;
 104         struct ahash_request    *snd_hash;
 105         __le32                  exp_ddgst;
 106         __le32                  recv_ddgst;
 107
 108         struct page_frag_cache  pf_cache;
 109
 110         void (*state_change)(struct sock *);
 111         void (*data_ready)(struct sock *);
 112         void (*write_space)(struct sock *);
 113 };
 114
 115 struct nvme_tcp_ctrl {
 116         /* read only in the hot path */
 117         struct nvme_tcp_queue   *queues;
 118         struct blk_mq_tag_set   tag_set;
 119
 120         /* other member variables */
 121         struct list_head        list;
 122         struct blk_mq_tag_set   admin_tag_set;
 123         struct sockaddr_storage addr;
 124         struct sockaddr_storage src_addr;
 125         struct nvme_ctrl        ctrl;
 126
 127         struct mutex            teardown_lock;
 128         struct work_struct      err_work;
 129         struct delayed_work     connect_work;
 130         struct nvme_tcp_request async_req;
 131         u32                     io_queues[HCTX_MAX_TYPES];
 132 };
 133
 134 static LIST_HEAD(nvme_tcp_ctrl_list);
 135 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
 136 static struct workqueue_struct *nvme_tcp_wq;
 137 static const struct blk_mq_ops nvme_tcp_mq_ops;
 138 static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
 139 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
 140
 141 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
 142 {
 143         return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
 144 }
 145
 146 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
 147 {
 148         return queue - queue->ctrl->queues;
 149 }
 150
 151 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
 152 {
 153         u32 queue_idx = nvme_tcp_queue_id(queue);
 154
 155         if (queue_idx == 0)
 156                 return queue->ctrl->admin_tag_set.tags[queue_idx];
 157         return queue->ctrl->tag_set.tags[queue_idx - 1];
 158 }
 159
 160 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
 161 {
 162         return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 163 }
 164
 165 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
 166 {
 167         return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 168 }
 169
 170 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue)
 171 {
 172         return queue->cmnd_capsule_len - sizeof(struct nvme_command);
 173 }
 174
 175 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
 176 {
 177         return req == &req->queue->ctrl->async_req;
 178 }
 179
 180 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
 181 {
 182         struct request *rq;
 183
 184         if (unlikely(nvme_tcp_async_req(req)))
 185                 return false; /* async events don't have a request */
 186
 187         rq = blk_mq_rq_from_pdu(req);
 188
 189         return rq_data_dir(rq) == WRITE && req->data_len &&
 190                 req->data_len <= nvme_tcp_inline_data_size(req->queue);
 191 }
 192
 193 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
 194 {
 195         return req->iter.bvec->bv_page;
 196 }
 197
 198 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
 199 {
 200         return req->iter.bvec->bv_offset + req->iter.iov_offset;
 201 }
 202
 203 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
 204 {
 205         return min_t(size_t, req->iter.bvec->bv_len - req->iter.iov_offset,
 206                         req->pdu_len - req->pdu_sent);
 207 }
 208
 209 static inline size_t nvme_tcp_req_offset(struct nvme_tcp_request *req)
 210 {
 211         return req->iter.iov_offset;
 212 }
 213
 214 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
 215 {
 216         return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
 217                         req->pdu_len - req->pdu_sent : 0;
 218 }
 219
 220 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
 221                 int len)
 222 {
 223         return nvme_tcp_pdu_data_left(req) <= len;
 224 }
 225
 226 static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
 227                 unsigned int dir)
 228 {
 229         struct request *rq = blk_mq_rq_from_pdu(req);
 230         struct bio_vec *vec;
 231         unsigned int size;
 232         int nsegs;
 233         size_t offset;
 234
 235         if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
 236                 vec = &rq->special_vec;
 237                 nsegs = 1;
 238                 size = blk_rq_payload_bytes(rq);
 239                 offset = 0;
 240         } else {
 241                 struct bio *bio = req->curr_bio;
 242
 243                 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
 244                 nsegs = bio_segments(bio);
 245                 size = bio->bi_iter.bi_size;
 246                 offset = bio->bi_iter.bi_bvec_done;
 247         }
 248
 249         iov_iter_bvec(&req->iter, dir, vec, nsegs, size);
 250         req->iter.iov_offset = offset;
 251 }
 252
 253 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
 254                 int len)
 255 {
 256         req->data_sent += len;
 257         req->pdu_sent += len;
 258         iov_iter_advance(&req->iter, len);
 259         if (!iov_iter_count(&req->iter) &&
 260             req->data_sent < req->data_len) {
 261                 req->curr_bio = req->curr_bio->bi_next;
 262                 nvme_tcp_init_iter(req, WRITE);
 263         }
 264 }
 265
 266 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
 267                 bool sync, bool last)
 268 {
 269         struct nvme_tcp_queue *queue = req->queue;
 270         bool empty;
 271
 272         empty = llist_add(&req->lentry, &queue->req_list) &&
 273                 list_empty(&queue->send_list) && !queue->request;
 274
 275         /*
 276          * if we're the first on the send_list and we can try to send
 277          * directly, otherwise queue io_work. Also, only do that if we
 278          * are on the same cpu, so we don't introduce contention.
 279          */
 280         if (queue->io_cpu == smp_processor_id() &&
 281             sync && empty && mutex_trylock(&queue->send_mutex)) {
 282                 queue->more_requests = !last;
 283                 nvme_tcp_try_send(queue);
 284                 queue->more_requests = false;
 285                 mutex_unlock(&queue->send_mutex);
 286         } else if (last) {
 287                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 288         }
 289 }
 290
 291 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
 292 {
 293         struct nvme_tcp_request *req;
 294         struct llist_node *node;
 295
 296         for (node = llist_del_all(&queue->req_list); node; node = node->next) {
 297                 req = llist_entry(node, struct nvme_tcp_request, lentry);
 298                 list_add(&req->entry, &queue->send_list);
 299         }
 300 }
 301
 302 static inline struct nvme_tcp_request *
 303 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
 304 {
 305         struct nvme_tcp_request *req;
 306
 307         req = list_first_entry_or_null(&queue->send_list,
 308                         struct nvme_tcp_request, entry);
 309         if (!req) {
 310                 nvme_tcp_process_req_list(queue);
 311                 req = list_first_entry_or_null(&queue->send_list,
 312                                 struct nvme_tcp_request, entry);
 313                 if (unlikely(!req))
 314                         return NULL;
 315         }
 316
 317         list_del(&req->entry);
 318         return req;
 319 }
 320
 321 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
 322                 __le32 *dgst)
 323 {
 324         ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
 325         crypto_ahash_final(hash);
 326 }
 327
 328 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
 329                 struct page *page, off_t off, size_t len)
 330 {
 331         struct scatterlist sg;
 332
 333         sg_init_marker(&sg, 1);
 334         sg_set_page(&sg, page, len, off);
 335         ahash_request_set_crypt(hash, &sg, NULL, len);
 336         crypto_ahash_update(hash);
 337 }
 338
 339 static inline void nvme_tcp_hdgst(struct ahash_request *hash,
 340                 void *pdu, size_t len)
 341 {
 342         struct scatterlist sg;
 343
 344         sg_init_one(&sg, pdu, len);
 345         ahash_request_set_crypt(hash, &sg, pdu + len, len);
 346         crypto_ahash_digest(hash);
 347 }
 348
 349 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
 350                 void *pdu, size_t pdu_len)
 351 {
 352         struct nvme_tcp_hdr *hdr = pdu;
 353         __le32 recv_digest;
 354         __le32 exp_digest;
 355
 356         if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
 357                 dev_err(queue->ctrl->ctrl.device,
 358                         "queue %d: header digest flag is cleared\n",
 359                         nvme_tcp_queue_id(queue));
 360                 return -EPROTO;
 361         }
 362
 363         recv_digest = *(__le32 *)(pdu + hdr->hlen);
 364         nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
 365         exp_digest = *(__le32 *)(pdu + hdr->hlen);
 366         if (recv_digest != exp_digest) {
 367                 dev_err(queue->ctrl->ctrl.device,
 368                         "header digest error: recv %#x expected %#x\n",
 369                         le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
 370                 return -EIO;
 371         }
 372
 373         return 0;
 374 }
 375
 376 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
 377 {
 378         struct nvme_tcp_hdr *hdr = pdu;
 379         u8 digest_len = nvme_tcp_hdgst_len(queue);
 380         u32 len;
 381
 382         len = le32_to_cpu(hdr->plen) - hdr->hlen -
 383                 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
 384
 385         if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
 386                 dev_err(queue->ctrl->ctrl.device,
 387                         "queue %d: data digest flag is cleared\n",
 388                 nvme_tcp_queue_id(queue));
 389                 return -EPROTO;
 390         }
 391         crypto_ahash_init(queue->rcv_hash);
 392
 393         return 0;
 394 }
 395
 396 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
 397                 struct request *rq, unsigned int hctx_idx)
 398 {
 399         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 400
 401         page_frag_free(req->pdu);
 402 }
 403
 404 static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
 405                 struct request *rq, unsigned int hctx_idx,
 406                 unsigned int numa_node)
 407 {
 408         struct nvme_tcp_ctrl *ctrl = set->driver_data;
 409         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 410         int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
 411         struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
 412         u8 hdgst = nvme_tcp_hdgst_len(queue);
 413
 414         req->pdu = page_frag_alloc(&queue->pf_cache,
 415                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
 416                 GFP_KERNEL | __GFP_ZERO);
 417         if (!req->pdu)
 418                 return -ENOMEM;
 419
 420         req->queue = queue;
 421         nvme_req(rq)->ctrl = &ctrl->ctrl;
 422
 423         return 0;
 424 }
 425
 426 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 427                 unsigned int hctx_idx)
 428 {
 429         struct nvme_tcp_ctrl *ctrl = data;
 430         struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
 431
 432         hctx->driver_data = queue;
 433         return 0;
 434 }
 435
 436 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 437                 unsigned int hctx_idx)
 438 {
 439         struct nvme_tcp_ctrl *ctrl = data;
 440         struct nvme_tcp_queue *queue = &ctrl->queues[0];
 441
 442         hctx->driver_data = queue;
 443         return 0;
 444 }
 445
 446 static enum nvme_tcp_recv_state
 447 nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
 448 {
 449         return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
 450                 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
 451                 NVME_TCP_RECV_DATA;
 452 }
 453
 454 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
 455 {
 456         queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
 457                                 nvme_tcp_hdgst_len(queue);
 458         queue->pdu_offset = 0;
 459         queue->data_remaining = -1;
 460         queue->ddgst_remaining = 0;
 461 }
 462
 463 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
 464 {
 465         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 466                 return;
 467
 468         dev_warn(ctrl->device, "starting error recovery\n");
 469         queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
 470 }
 471
 472 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
 473                 struct nvme_completion *cqe)
 474 {
 475         struct request *rq;
 476
 477         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), cqe->command_id);
 478         if (!rq) {
 479                 dev_err(queue->ctrl->ctrl.device,
 480                         "queue %d tag 0x%x not found\n",
 481                         nvme_tcp_queue_id(queue), cqe->command_id);
 482                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 483                 return -EINVAL;
 484         }
 485
 486         if (!nvme_try_complete_req(rq, cqe->status, cqe->result))
 487                 nvme_complete_rq(rq);
 488         queue->nr_cqe++;
 489
 490         return 0;
 491 }
 492
 493 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
 494                 struct nvme_tcp_data_pdu *pdu)
 495 {
 496         struct request *rq;
 497
 498         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
 499         if (!rq) {
 500                 dev_err(queue->ctrl->ctrl.device,
 501                         "queue %d tag %#x not found\n",
 502                         nvme_tcp_queue_id(queue), pdu->command_id);
 503                 return -ENOENT;
 504         }
 505
 506         if (!blk_rq_payload_bytes(rq)) {
 507                 dev_err(queue->ctrl->ctrl.device,
 508                         "queue %d tag %#x unexpected data\n",
 509                         nvme_tcp_queue_id(queue), rq->tag);
 510                 return -EIO;
 511         }
 512
 513         queue->data_remaining = le32_to_cpu(pdu->data_length);
 514
 515         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
 516             unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
 517                 dev_err(queue->ctrl->ctrl.device,
 518                         "queue %d tag %#x SUCCESS set but not last PDU\n",
 519                         nvme_tcp_queue_id(queue), rq->tag);
 520                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 521                 return -EPROTO;
 522         }
 523
 524         return 0;
 525 }
 526
 527 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
 528                 struct nvme_tcp_rsp_pdu *pdu)
 529 {
 530         struct nvme_completion *cqe = &pdu->cqe;
 531         int ret = 0;
 532
 533         /*
 534          * AEN requests are special as they don't time out and can
 535          * survive any kind of queue freeze and often don't respond to
 536          * aborts.  We don't even bother to allocate a struct request
 537          * for them but rather special case them here.
 538          */
 539         if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
 540                                      cqe->command_id)))
 541                 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
 542                                 &cqe->result);
 543         else
 544                 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
 545
 546         return ret;
 547 }
 548
 549 static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req,
 550                 struct nvme_tcp_r2t_pdu *pdu)
 551 {
 552         struct nvme_tcp_data_pdu *data = req->pdu;
 553         struct nvme_tcp_queue *queue = req->queue;
 554         struct request *rq = blk_mq_rq_from_pdu(req);
 555         u8 hdgst = nvme_tcp_hdgst_len(queue);
 556         u8 ddgst = nvme_tcp_ddgst_len(queue);
 557
 558         req->pdu_len = le32_to_cpu(pdu->r2t_length);
 559         req->pdu_sent = 0;
 560
 561         if (unlikely(req->data_sent + req->pdu_len > req->data_len)) {
 562                 dev_err(queue->ctrl->ctrl.device,
 563                         "req %d r2t len %u exceeded data len %u (%zu sent)\n",
 564                         rq->tag, req->pdu_len, req->data_len,
 565                         req->data_sent);
 566                 return -EPROTO;
 567         }
 568
 569         if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) {
 570                 dev_err(queue->ctrl->ctrl.device,
 571                         "req %d unexpected r2t offset %u (expected %zu)\n",
 572                         rq->tag, le32_to_cpu(pdu->r2t_offset),
 573                         req->data_sent);
 574                 return -EPROTO;
 575         }
 576
 577         memset(data, 0, sizeof(*data));
 578         data->hdr.type = nvme_tcp_h2c_data;
 579         data->hdr.flags = NVME_TCP_F_DATA_LAST;
 580         if (queue->hdr_digest)
 581                 data->hdr.flags |= NVME_TCP_F_HDGST;
 582         if (queue->data_digest)
 583                 data->hdr.flags |= NVME_TCP_F_DDGST;
 584         data->hdr.hlen = sizeof(*data);
 585         data->hdr.pdo = data->hdr.hlen + hdgst;
 586         data->hdr.plen =
 587                 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
 588         data->ttag = pdu->ttag;
 589         data->command_id = rq->tag;
 590         data->data_offset = cpu_to_le32(req->data_sent);
 591         data->data_length = cpu_to_le32(req->pdu_len);
 592         return 0;
 593 }
 594
 595 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
 596                 struct nvme_tcp_r2t_pdu *pdu)
 597 {
 598         struct nvme_tcp_request *req;
 599         struct request *rq;
 600         int ret;
 601
 602         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
 603         if (!rq) {
 604                 dev_err(queue->ctrl->ctrl.device,
 605                         "queue %d tag %#x not found\n",
 606                         nvme_tcp_queue_id(queue), pdu->command_id);
 607                 return -ENOENT;
 608         }
 609         req = blk_mq_rq_to_pdu(rq);
 610
 611         ret = nvme_tcp_setup_h2c_data_pdu(req, pdu);
 612         if (unlikely(ret))
 613                 return ret;
 614
 615         req->state = NVME_TCP_SEND_H2C_PDU;
 616         req->offset = 0;
 617
 618         nvme_tcp_queue_request(req, false, true);
 619
 620         return 0;
 621 }
 622
 623 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 624                 unsigned int *offset, size_t *len)
 625 {
 626         struct nvme_tcp_hdr *hdr;
 627         char *pdu = queue->pdu;
 628         size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
 629         int ret;
 630
 631         ret = skb_copy_bits(skb, *offset,
 632                 &pdu[queue->pdu_offset], rcv_len);
 633         if (unlikely(ret))
 634                 return ret;
 635
 636         queue->pdu_remaining -= rcv_len;
 637         queue->pdu_offset += rcv_len;
 638         *offset += rcv_len;
 639         *len -= rcv_len;
 640         if (queue->pdu_remaining)
 641                 return 0;
 642
 643         hdr = queue->pdu;
 644         if (queue->hdr_digest) {
 645                 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
 646                 if (unlikely(ret))
 647                         return ret;
 648         }
 649
 650
 651         if (queue->data_digest) {
 652                 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
 653                 if (unlikely(ret))
 654                         return ret;
 655         }
 656
 657         switch (hdr->type) {
 658         case nvme_tcp_c2h_data:
 659                 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
 660         case nvme_tcp_rsp:
 661                 nvme_tcp_init_recv_ctx(queue);
 662                 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
 663         case nvme_tcp_r2t:
 664                 nvme_tcp_init_recv_ctx(queue);
 665                 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
 666         default:
 667                 dev_err(queue->ctrl->ctrl.device,
 668                         "unsupported pdu type (%d)\n", hdr->type);
 669                 return -EINVAL;
 670         }
 671 }
 672
 673 static inline void nvme_tcp_end_request(struct request *rq, u16 status)
 674 {
 675         union nvme_result res = {};
 676
 677         if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
 678                 nvme_complete_rq(rq);
 679 }
 680
 681 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 682                               unsigned int *offset, size_t *len)
 683 {
 684         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 685         struct nvme_tcp_request *req;
 686         struct request *rq;
 687
 688         rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
 689         if (!rq) {
 690                 dev_err(queue->ctrl->ctrl.device,
 691                         "queue %d tag %#x not found\n",
 692                         nvme_tcp_queue_id(queue), pdu->command_id);
 693                 return -ENOENT;
 694         }
 695         req = blk_mq_rq_to_pdu(rq);
 696
 697         while (true) {
 698                 int recv_len, ret;
 699
 700                 recv_len = min_t(size_t, *len, queue->data_remaining);
 701                 if (!recv_len)
 702                         break;
 703
 704                 if (!iov_iter_count(&req->iter)) {
 705                         req->curr_bio = req->curr_bio->bi_next;
 706
 707                         /*
 708                          * If we don`t have any bios it means that controller
 709                          * sent more data than we requested, hence error
 710                          */
 711                         if (!req->curr_bio) {
 712                                 dev_err(queue->ctrl->ctrl.device,
 713                                         "queue %d no space in request %#x",
 714                                         nvme_tcp_queue_id(queue), rq->tag);
 715                                 nvme_tcp_init_recv_ctx(queue);
 716                                 return -EIO;
 717                         }
 718                         nvme_tcp_init_iter(req, READ);
 719                 }
 720
 721                 /* we can read only from what is left in this bio */
 722                 recv_len = min_t(size_t, recv_len,
 723                                 iov_iter_count(&req->iter));
 724
 725                 if (queue->data_digest)
 726                         ret = skb_copy_and_hash_datagram_iter(skb, *offset,
 727                                 &req->iter, recv_len, queue->rcv_hash);
 728                 else
 729                         ret = skb_copy_datagram_iter(skb, *offset,
 730                                         &req->iter, recv_len);
 731                 if (ret) {
 732                         dev_err(queue->ctrl->ctrl.device,
 733                                 "queue %d failed to copy request %#x data",
 734                                 nvme_tcp_queue_id(queue), rq->tag);
 735                         return ret;
 736                 }
 737
 738                 *len -= recv_len;
 739                 *offset += recv_len;
 740                 queue->data_remaining -= recv_len;
 741         }
 742
 743         if (!queue->data_remaining) {
 744                 if (queue->data_digest) {
 745                         nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
 746                         queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
 747                 } else {
 748                         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 749                                 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
 750                                 queue->nr_cqe++;
 751                         }
 752                         nvme_tcp_init_recv_ctx(queue);
 753                 }
 754         }
 755
 756         return 0;
 757 }
 758
 759 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
 760                 struct sk_buff *skb, unsigned int *offset, size_t *len)
 761 {
 762         struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 763         char *ddgst = (char *)&queue->recv_ddgst;
 764         size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
 765         off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
 766         int ret;
 767
 768         ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
 769         if (unlikely(ret))
 770                 return ret;
 771
 772         queue->ddgst_remaining -= recv_len;
 773         *offset += recv_len;
 774         *len -= recv_len;
 775         if (queue->ddgst_remaining)
 776                 return 0;
 777
 778         if (queue->recv_ddgst != queue->exp_ddgst) {
 779                 dev_err(queue->ctrl->ctrl.device,
 780                         "data digest error: recv %#x expected %#x\n",
 781                         le32_to_cpu(queue->recv_ddgst),
 782                         le32_to_cpu(queue->exp_ddgst));
 783                 return -EIO;
 784         }
 785
 786         if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 787                 struct request *rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue),
 788                                                 pdu->command_id);
 789
 790                 nvme_tcp_end_request(rq, NVME_SC_SUCCESS);
 791                 queue->nr_cqe++;
 792         }
 793
 794         nvme_tcp_init_recv_ctx(queue);
 795         return 0;
 796 }
 797
 798 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
 799                              unsigned int offset, size_t len)
 800 {
 801         struct nvme_tcp_queue *queue = desc->arg.data;
 802         size_t consumed = len;
 803         int result;
 804
 805         while (len) {
 806                 switch (nvme_tcp_recv_state(queue)) {
 807                 case NVME_TCP_RECV_PDU:
 808                         result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
 809                         break;
 810                 case NVME_TCP_RECV_DATA:
 811                         result = nvme_tcp_recv_data(queue, skb, &offset, &len);
 812                         break;
 813                 case NVME_TCP_RECV_DDGST:
 814                         result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
 815                         break;
 816                 default:
 817                         result = -EFAULT;
 818                 }
 819                 if (result) {
 820                         dev_err(queue->ctrl->ctrl.device,
 821                                 "receive failed:  %d\n", result);
 822                         queue->rd_enabled = false;
 823                         nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 824                         return result;
 825                 }
 826         }
 827
 828         return consumed;
 829 }
 830
 831 static void nvme_tcp_data_ready(struct sock *sk)
 832 {
 833         struct nvme_tcp_queue *queue;
 834
 835         read_lock_bh(&sk->sk_callback_lock);
 836         queue = sk->sk_user_data;
 837         if (likely(queue && queue->rd_enabled) &&
 838             !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
 839                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 840         read_unlock_bh(&sk->sk_callback_lock);
 841 }
 842
 843 static void nvme_tcp_write_space(struct sock *sk)
 844 {
 845         struct nvme_tcp_queue *queue;
 846
 847         read_lock_bh(&sk->sk_callback_lock);
 848         queue = sk->sk_user_data;
 849         if (likely(queue && sk_stream_is_writeable(sk))) {
 850                 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 851                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 852         }
 853         read_unlock_bh(&sk->sk_callback_lock);
 854 }
 855
 856 static void nvme_tcp_state_change(struct sock *sk)
 857 {
 858         struct nvme_tcp_queue *queue;
 859
 860         read_lock(&sk->sk_callback_lock);
 861         queue = sk->sk_user_data;
 862         if (!queue)
 863                 goto done;
 864
 865         switch (sk->sk_state) {
 866         case TCP_CLOSE:
 867         case TCP_CLOSE_WAIT:
 868         case TCP_LAST_ACK:
 869         case TCP_FIN_WAIT1:
 870         case TCP_FIN_WAIT2:
 871                 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 872                 break;
 873         default:
 874                 dev_info(queue->ctrl->ctrl.device,
 875                         "queue %d socket state %d\n",
 876                         nvme_tcp_queue_id(queue), sk->sk_state);
 877         }
 878
 879         queue->state_change(sk);
 880 done:
 881         read_unlock(&sk->sk_callback_lock);
 882 }
 883
 884 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
 885 {
 886         return !list_empty(&queue->send_list) ||
 887                 !llist_empty(&queue->req_list) || queue->more_requests;
 888 }
 889
 890 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
 891 {
 892         queue->request = NULL;
 893 }
 894
 895 static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
 896 {
 897         nvme_tcp_end_request(blk_mq_rq_from_pdu(req), NVME_SC_HOST_PATH_ERROR);
 898 }
 899
 900 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
 901 {
 902         struct nvme_tcp_queue *queue = req->queue;
 903
 904         while (true) {
 905                 struct page *page = nvme_tcp_req_cur_page(req);
 906                 size_t offset = nvme_tcp_req_cur_offset(req);
 907                 size_t len = nvme_tcp_req_cur_length(req);
 908                 bool last = nvme_tcp_pdu_last_send(req, len);
 909                 int ret, flags = MSG_DONTWAIT;
 910
 911                 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
 912                         flags |= MSG_EOR;
 913                 else
 914                         flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
 915
 916                 /* can't zcopy slab pages */
 917                 if (unlikely(PageSlab(page))) {
 918                         ret = sock_no_sendpage(queue->sock, page, offset, len,
 919                                         flags);
 920                 } else {
 921                         ret = kernel_sendpage(queue->sock, page, offset, len,
 922                                         flags);
 923                 }
 924                 if (ret <= 0)
 925                         return ret;
 926
 927                 nvme_tcp_advance_req(req, ret);
 928                 if (queue->data_digest)
 929                         nvme_tcp_ddgst_update(queue->snd_hash, page,
 930                                         offset, ret);
 931
 932                 /* fully successful last write*/
 933                 if (last && ret == len) {
 934                         if (queue->data_digest) {
 935                                 nvme_tcp_ddgst_final(queue->snd_hash,
 936                                         &req->ddgst);
 937                                 req->state = NVME_TCP_SEND_DDGST;
 938                                 req->offset = 0;
 939                         } else {
 940                                 nvme_tcp_done_send_req(queue);
 941                         }
 942                         return 1;
 943                 }
 944         }
 945         return -EAGAIN;
 946 }
 947
 948 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
 949 {
 950         struct nvme_tcp_queue *queue = req->queue;
 951         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
 952         bool inline_data = nvme_tcp_has_inline_data(req);
 953         u8 hdgst = nvme_tcp_hdgst_len(queue);
 954         int len = sizeof(*pdu) + hdgst - req->offset;
 955         int flags = MSG_DONTWAIT;
 956         int ret;
 957
 958         if (inline_data || nvme_tcp_queue_more(queue))
 959                 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST;
 960         else
 961                 flags |= MSG_EOR;
 962
 963         if (queue->hdr_digest && !req->offset)
 964                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
 965
 966         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
 967                         offset_in_page(pdu) + req->offset, len,  flags);
 968         if (unlikely(ret <= 0))
 969                 return ret;
 970
 971         len -= ret;
 972         if (!len) {
 973                 if (inline_data) {
 974                         req->state = NVME_TCP_SEND_DATA;
 975                         if (queue->data_digest)
 976                                 crypto_ahash_init(queue->snd_hash);
 977                         nvme_tcp_init_iter(req, WRITE);
 978                 } else {
 979                         nvme_tcp_done_send_req(queue);
 980                 }
 981                 return 1;
 982         }
 983         req->offset += ret;
 984
 985         return -EAGAIN;
 986 }
 987
 988 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
 989 {
 990         struct nvme_tcp_queue *queue = req->queue;
 991         struct nvme_tcp_data_pdu *pdu = req->pdu;
 992         u8 hdgst = nvme_tcp_hdgst_len(queue);
 993         int len = sizeof(*pdu) - req->offset + hdgst;
 994         int ret;
 995
 996         if (queue->hdr_digest && !req->offset)
 997                 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
 998
 999         ret = kernel_sendpage(queue->sock, virt_to_page(pdu),
1000                         offset_in_page(pdu) + req->offset, len,
1001                         MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST);
1002         if (unlikely(ret <= 0))
1003                 return ret;
1004
1005         len -= ret;
1006         if (!len) {
1007                 req->state = NVME_TCP_SEND_DATA;
1008                 if (queue->data_digest)
1009                         crypto_ahash_init(queue->snd_hash);
1010                 if (!req->data_sent)
1011                         nvme_tcp_init_iter(req, WRITE);
1012                 return 1;
1013         }
1014         req->offset += ret;
1015
1016         return -EAGAIN;
1017 }
1018
1019 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1020 {
1021         struct nvme_tcp_queue *queue = req->queue;
1022         int ret;
1023         struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1024         struct kvec iov = {
1025                 .iov_base = &req->ddgst + req->offset,
1026                 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1027         };
1028
1029         if (nvme_tcp_queue_more(queue))
1030                 msg.msg_flags |= MSG_MORE;
1031         else
1032                 msg.msg_flags |= MSG_EOR;
1033
1034         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1035         if (unlikely(ret <= 0))
1036                 return ret;
1037
1038         if (req->offset + ret == NVME_TCP_DIGEST_LENGTH) {
1039                 nvme_tcp_done_send_req(queue);
1040                 return 1;
1041         }
1042
1043         req->offset += ret;
1044         return -EAGAIN;
1045 }
1046
1047 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1048 {
1049         struct nvme_tcp_request *req;
1050         int ret = 1;
1051
1052         if (!queue->request) {
1053                 queue->request = nvme_tcp_fetch_request(queue);
1054                 if (!queue->request)
1055                         return 0;
1056         }
1057         req = queue->request;
1058
1059         if (req->state == NVME_TCP_SEND_CMD_PDU) {
1060                 ret = nvme_tcp_try_send_cmd_pdu(req);
1061                 if (ret <= 0)
1062                         goto done;
1063                 if (!nvme_tcp_has_inline_data(req))
1064                         return ret;
1065         }
1066
1067         if (req->state == NVME_TCP_SEND_H2C_PDU) {
1068                 ret = nvme_tcp_try_send_data_pdu(req);
1069                 if (ret <= 0)
1070                         goto done;
1071         }
1072
1073         if (req->state == NVME_TCP_SEND_DATA) {
1074                 ret = nvme_tcp_try_send_data(req);
1075                 if (ret <= 0)
1076                         goto done;
1077         }
1078
1079         if (req->state == NVME_TCP_SEND_DDGST)
1080                 ret = nvme_tcp_try_send_ddgst(req);
1081 done:
1082         if (ret == -EAGAIN) {
1083                 ret = 0;
1084         } else if (ret < 0) {
1085                 dev_err(queue->ctrl->ctrl.device,
1086                         "failed to send request %d\n", ret);
1087                 if (ret != -EPIPE && ret != -ECONNRESET)
1088                         nvme_tcp_fail_request(queue->request);
1089                 nvme_tcp_done_send_req(queue);
1090         }
1091         return ret;
1092 }
1093
1094 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1095 {
1096         struct socket *sock = queue->sock;
1097         struct sock *sk = sock->sk;
1098         read_descriptor_t rd_desc;
1099         int consumed;
1100
1101         rd_desc.arg.data = queue;
1102         rd_desc.count = 1;
1103         lock_sock(sk);
1104         queue->nr_cqe = 0;
1105         consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1106         release_sock(sk);
1107         return consumed;
1108 }
1109
1110 static void nvme_tcp_io_work(struct work_struct *w)
1111 {
1112         struct nvme_tcp_queue *queue =
1113                 container_of(w, struct nvme_tcp_queue, io_work);
1114         unsigned long deadline = jiffies + msecs_to_jiffies(1);
1115
1116         do {
1117                 bool pending = false;
1118                 int result;
1119
1120                 if (mutex_trylock(&queue->send_mutex)) {
1121                         result = nvme_tcp_try_send(queue);
1122                         mutex_unlock(&queue->send_mutex);
1123                         if (result > 0)
1124                                 pending = true;
1125                         else if (unlikely(result < 0))
1126                                 break;
1127                 }
1128
1129                 result = nvme_tcp_try_recv(queue);
1130                 if (result > 0)
1131                         pending = true;
1132                 else if (unlikely(result < 0))
1133                         return;
1134
1135                 if (!pending)
1136                         return;
1137
1138         } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1139
1140         queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1141 }
1142
1143 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1144 {
1145         struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1146
1147         ahash_request_free(queue->rcv_hash);
1148         ahash_request_free(queue->snd_hash);
1149         crypto_free_ahash(tfm);
1150 }
1151
1152 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1153 {
1154         struct crypto_ahash *tfm;
1155
1156         tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1157         if (IS_ERR(tfm))
1158                 return PTR_ERR(tfm);
1159
1160         queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1161         if (!queue->snd_hash)
1162                 goto free_tfm;
1163         ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1164
1165         queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1166         if (!queue->rcv_hash)
1167                 goto free_snd_hash;
1168         ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1169
1170         return 0;
1171 free_snd_hash:
1172         ahash_request_free(queue->snd_hash);
1173 free_tfm:
1174         crypto_free_ahash(tfm);
1175         return -ENOMEM;
1176 }
1177
1178 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1179 {
1180         struct nvme_tcp_request *async = &ctrl->async_req;
1181
1182         page_frag_free(async->pdu);
1183 }
1184
1185 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1186 {
1187         struct nvme_tcp_queue *queue = &ctrl->queues[0];
1188         struct nvme_tcp_request *async = &ctrl->async_req;
1189         u8 hdgst = nvme_tcp_hdgst_len(queue);
1190
1191         async->pdu = page_frag_alloc(&queue->pf_cache,
1192                 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1193                 GFP_KERNEL | __GFP_ZERO);
1194         if (!async->pdu)
1195                 return -ENOMEM;
1196
1197         async->queue = &ctrl->queues[0];
1198         return 0;
1199 }
1200
1201 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1202 {
1203         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1204         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1205
1206         if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1207                 return;
1208
1209         if (queue->hdr_digest || queue->data_digest)
1210                 nvme_tcp_free_crypto(queue);
1211
1212         sock_release(queue->sock);
1213         kfree(queue->pdu);
1214 }
1215
1216 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1217 {
1218         struct nvme_tcp_icreq_pdu *icreq;
1219         struct nvme_tcp_icresp_pdu *icresp;
1220         struct msghdr msg = {};
1221         struct kvec iov;
1222         bool ctrl_hdgst, ctrl_ddgst;
1223         int ret;
1224
1225         icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1226         if (!icreq)
1227                 return -ENOMEM;
1228
1229         icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1230         if (!icresp) {
1231                 ret = -ENOMEM;
1232                 goto free_icreq;
1233         }
1234
1235         icreq->hdr.type = nvme_tcp_icreq;
1236         icreq->hdr.hlen = sizeof(*icreq);
1237         icreq->hdr.pdo = 0;
1238         icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1239         icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1240         icreq->maxr2t = 0; /* single inflight r2t supported */
1241         icreq->hpda = 0; /* no alignment constraint */
1242         if (queue->hdr_digest)
1243                 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1244         if (queue->data_digest)
1245                 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1246
1247         iov.iov_base = icreq;
1248         iov.iov_len = sizeof(*icreq);
1249         ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1250         if (ret < 0)
1251                 goto free_icresp;
1252
1253         memset(&msg, 0, sizeof(msg));
1254         iov.iov_base = icresp;
1255         iov.iov_len = sizeof(*icresp);
1256         ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1257                         iov.iov_len, msg.msg_flags);
1258         if (ret < 0)
1259                 goto free_icresp;
1260
1261         ret = -EINVAL;
1262         if (icresp->hdr.type != nvme_tcp_icresp) {
1263                 pr_err("queue %d: bad type returned %d\n",
1264                         nvme_tcp_queue_id(queue), icresp->hdr.type);
1265                 goto free_icresp;
1266         }
1267
1268         if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1269                 pr_err("queue %d: bad pdu length returned %d\n",
1270                         nvme_tcp_queue_id(queue), icresp->hdr.plen);
1271                 goto free_icresp;
1272         }
1273
1274         if (icresp->pfv != NVME_TCP_PFV_1_0) {
1275                 pr_err("queue %d: bad pfv returned %d\n",
1276                         nvme_tcp_queue_id(queue), icresp->pfv);
1277                 goto free_icresp;
1278         }
1279
1280         ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1281         if ((queue->data_digest && !ctrl_ddgst) ||
1282             (!queue->data_digest && ctrl_ddgst)) {
1283                 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1284                         nvme_tcp_queue_id(queue),
1285                         queue->data_digest ? "enabled" : "disabled",
1286                         ctrl_ddgst ? "enabled" : "disabled");
1287                 goto free_icresp;
1288         }
1289
1290         ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1291         if ((queue->hdr_digest && !ctrl_hdgst) ||
1292             (!queue->hdr_digest && ctrl_hdgst)) {
1293                 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1294                         nvme_tcp_queue_id(queue),
1295                         queue->hdr_digest ? "enabled" : "disabled",
1296                         ctrl_hdgst ? "enabled" : "disabled");
1297                 goto free_icresp;
1298         }
1299
1300         if (icresp->cpda != 0) {
1301                 pr_err("queue %d: unsupported cpda returned %d\n",
1302                         nvme_tcp_queue_id(queue), icresp->cpda);
1303                 goto free_icresp;
1304         }
1305
1306         ret = 0;
1307 free_icresp:
1308         kfree(icresp);
1309 free_icreq:
1310         kfree(icreq);
1311         return ret;
1312 }
1313
1314 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1315 {
1316         return nvme_tcp_queue_id(queue) == 0;
1317 }
1318
1319 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1320 {
1321         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1322         int qid = nvme_tcp_queue_id(queue);
1323
1324         return !nvme_tcp_admin_queue(queue) &&
1325                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1326 }
1327
1328 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1329 {
1330         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1331         int qid = nvme_tcp_queue_id(queue);
1332
1333         return !nvme_tcp_admin_queue(queue) &&
1334                 !nvme_tcp_default_queue(queue) &&
1335                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1336                           ctrl->io_queues[HCTX_TYPE_READ];
1337 }
1338
1339 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1340 {
1341         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1342         int qid = nvme_tcp_queue_id(queue);
1343
1344         return !nvme_tcp_admin_queue(queue) &&
1345                 !nvme_tcp_default_queue(queue) &&
1346                 !nvme_tcp_read_queue(queue) &&
1347                 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1348                           ctrl->io_queues[HCTX_TYPE_READ] +
1349                           ctrl->io_queues[HCTX_TYPE_POLL];
1350 }
1351
1352 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1353 {
1354         struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1355         int qid = nvme_tcp_queue_id(queue);
1356         int n = 0;
1357
1358         if (nvme_tcp_default_queue(queue))
1359                 n = qid - 1;
1360         else if (nvme_tcp_read_queue(queue))
1361                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1362         else if (nvme_tcp_poll_queue(queue))
1363                 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1364                                 ctrl->io_queues[HCTX_TYPE_READ] - 1;
1365         queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1366 }
1367
1368 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl,
1369                 int qid, size_t queue_size)
1370 {
1371         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1372         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1373         int ret, rcv_pdu_size;
1374
1375         queue->ctrl = ctrl;
1376         init_llist_head(&queue->req_list);
1377         INIT_LIST_HEAD(&queue->send_list);
1378         mutex_init(&queue->send_mutex);
1379         INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1380         queue->queue_size = queue_size;
1381
1382         if (qid > 0)
1383                 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1384         else
1385                 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1386                                                 NVME_TCP_ADMIN_CCSZ;
1387
1388         ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1389                         IPPROTO_TCP, &queue->sock);
1390         if (ret) {
1391                 dev_err(nctrl->device,
1392                         "failed to create socket: %d\n", ret);
1393                 return ret;
1394         }
1395
1396         /* Single syn retry */
1397         tcp_sock_set_syncnt(queue->sock->sk, 1);
1398
1399         /* Set TCP no delay */
1400         tcp_sock_set_nodelay(queue->sock->sk);
1401
1402         /*
1403          * Cleanup whatever is sitting in the TCP transmit queue on socket
1404          * close. This is done to prevent stale data from being sent should
1405          * the network connection be restored before TCP times out.
1406          */
1407         sock_no_linger(queue->sock->sk);
1408
1409         if (so_priority > 0)
1410                 sock_set_priority(queue->sock->sk, so_priority);
1411
1412         /* Set socket type of service */
1413         if (nctrl->opts->tos >= 0)
1414                 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1415
1416         /* Set 10 seconds timeout for icresp recvmsg */
1417         queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1418
1419         queue->sock->sk->sk_allocation = GFP_ATOMIC;
1420         nvme_tcp_set_queue_io_cpu(queue);
1421         queue->request = NULL;
1422         queue->data_remaining = 0;
1423         queue->ddgst_remaining = 0;
1424         queue->pdu_remaining = 0;
1425         queue->pdu_offset = 0;
1426         sk_set_memalloc(queue->sock->sk);
1427
1428         if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1429                 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1430                         sizeof(ctrl->src_addr));
1431                 if (ret) {
1432                         dev_err(nctrl->device,
1433                                 "failed to bind queue %d socket %d\n",
1434                                 qid, ret);
1435                         goto err_sock;
1436                 }
1437         }
1438
1439         queue->hdr_digest = nctrl->opts->hdr_digest;
1440         queue->data_digest = nctrl->opts->data_digest;
1441         if (queue->hdr_digest || queue->data_digest) {
1442                 ret = nvme_tcp_alloc_crypto(queue);
1443                 if (ret) {
1444                         dev_err(nctrl->device,
1445                                 "failed to allocate queue %d crypto\n", qid);
1446                         goto err_sock;
1447                 }
1448         }
1449
1450         rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1451                         nvme_tcp_hdgst_len(queue);
1452         queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1453         if (!queue->pdu) {
1454                 ret = -ENOMEM;
1455                 goto err_crypto;
1456         }
1457
1458         dev_dbg(nctrl->device, "connecting queue %d\n",
1459                         nvme_tcp_queue_id(queue));
1460
1461         ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1462                 sizeof(ctrl->addr), 0);
1463         if (ret) {
1464                 dev_err(nctrl->device,
1465                         "failed to connect socket: %d\n", ret);
1466                 goto err_rcv_pdu;
1467         }
1468
1469         ret = nvme_tcp_init_connection(queue);
1470         if (ret)
1471                 goto err_init_connect;
1472
1473         queue->rd_enabled = true;
1474         set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1475         nvme_tcp_init_recv_ctx(queue);
1476
1477         write_lock_bh(&queue->sock->sk->sk_callback_lock);
1478         queue->sock->sk->sk_user_data = queue;
1479         queue->state_change = queue->sock->sk->sk_state_change;
1480         queue->data_ready = queue->sock->sk->sk_data_ready;
1481         queue->write_space = queue->sock->sk->sk_write_space;
1482         queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1483         queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1484         queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1485 #ifdef CONFIG_NET_RX_BUSY_POLL
1486         queue->sock->sk->sk_ll_usec = 1;
1487 #endif
1488         write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1489
1490         return 0;
1491
1492 err_init_connect:
1493         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1494 err_rcv_pdu:
1495         kfree(queue->pdu);
1496 err_crypto:
1497         if (queue->hdr_digest || queue->data_digest)
1498                 nvme_tcp_free_crypto(queue);
1499 err_sock:
1500         sock_release(queue->sock);
1501         queue->sock = NULL;
1502         return ret;
1503 }
1504
1505 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue)
1506 {
1507         struct socket *sock = queue->sock;
1508
1509         write_lock_bh(&sock->sk->sk_callback_lock);
1510         sock->sk->sk_user_data  = NULL;
1511         sock->sk->sk_data_ready = queue->data_ready;
1512         sock->sk->sk_state_change = queue->state_change;
1513         sock->sk->sk_write_space  = queue->write_space;
1514         write_unlock_bh(&sock->sk->sk_callback_lock);
1515 }
1516
1517 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1518 {
1519         kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1520         nvme_tcp_restore_sock_calls(queue);
1521         cancel_work_sync(&queue->io_work);
1522 }
1523
1524 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1525 {
1526         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1527         struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1528
1529         if (!test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1530                 return;
1531         __nvme_tcp_stop_queue(queue);
1532 }
1533
1534 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1535 {
1536         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1537         int ret;
1538
1539         if (idx)
1540                 ret = nvmf_connect_io_queue(nctrl, idx, false);
1541         else
1542                 ret = nvmf_connect_admin_queue(nctrl);
1543
1544         if (!ret) {
1545                 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags);
1546         } else {
1547                 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags))
1548                         __nvme_tcp_stop_queue(&ctrl->queues[idx]);
1549                 dev_err(nctrl->device,
1550                         "failed to connect queue: %d ret=%d\n", idx, ret);
1551         }
1552         return ret;
1553 }
1554
1555 static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl,
1556                 bool admin)
1557 {
1558         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1559         struct blk_mq_tag_set *set;
1560         int ret;
1561
1562         if (admin) {
1563                 set = &ctrl->admin_tag_set;
1564                 memset(set, 0, sizeof(*set));
1565                 set->ops = &nvme_tcp_admin_mq_ops;
1566                 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1567                 set->reserved_tags = 2; /* connect + keep-alive */
1568                 set->numa_node = nctrl->numa_node;
1569                 set->flags = BLK_MQ_F_BLOCKING;
1570                 set->cmd_size = sizeof(struct nvme_tcp_request);
1571                 set->driver_data = ctrl;
1572                 set->nr_hw_queues = 1;
1573                 set->timeout = ADMIN_TIMEOUT;
1574         } else {
1575                 set = &ctrl->tag_set;
1576                 memset(set, 0, sizeof(*set));
1577                 set->ops = &nvme_tcp_mq_ops;
1578                 set->queue_depth = nctrl->sqsize + 1;
1579                 set->reserved_tags = 1; /* fabric connect */
1580                 set->numa_node = nctrl->numa_node;
1581                 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
1582                 set->cmd_size = sizeof(struct nvme_tcp_request);
1583                 set->driver_data = ctrl;
1584                 set->nr_hw_queues = nctrl->queue_count - 1;
1585                 set->timeout = NVME_IO_TIMEOUT;
1586                 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2;
1587         }
1588
1589         ret = blk_mq_alloc_tag_set(set);
1590         if (ret)
1591                 return ERR_PTR(ret);
1592
1593         return set;
1594 }
1595
1596 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1597 {
1598         if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1599                 cancel_work_sync(&ctrl->async_event_work);
1600                 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1601                 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1602         }
1603
1604         nvme_tcp_free_queue(ctrl, 0);
1605 }
1606
1607 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1608 {
1609         int i;
1610
1611         for (i = 1; i < ctrl->queue_count; i++)
1612                 nvme_tcp_free_queue(ctrl, i);
1613 }
1614
1615 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1616 {
1617         int i;
1618
1619         for (i = 1; i < ctrl->queue_count; i++)
1620                 nvme_tcp_stop_queue(ctrl, i);
1621 }
1622
1623 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl)
1624 {
1625         int i, ret = 0;
1626
1627         for (i = 1; i < ctrl->queue_count; i++) {
1628                 ret = nvme_tcp_start_queue(ctrl, i);
1629                 if (ret)
1630                         goto out_stop_queues;
1631         }
1632
1633         return 0;
1634
1635 out_stop_queues:
1636         for (i--; i >= 1; i--)
1637                 nvme_tcp_stop_queue(ctrl, i);
1638         return ret;
1639 }
1640
1641 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1642 {
1643         int ret;
1644
1645         ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
1646         if (ret)
1647                 return ret;
1648
1649         ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1650         if (ret)
1651                 goto out_free_queue;
1652
1653         return 0;
1654
1655 out_free_queue:
1656         nvme_tcp_free_queue(ctrl, 0);
1657         return ret;
1658 }
1659
1660 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1661 {
1662         int i, ret;
1663
1664         for (i = 1; i < ctrl->queue_count; i++) {
1665                 ret = nvme_tcp_alloc_queue(ctrl, i,
1666                                 ctrl->sqsize + 1);
1667                 if (ret)
1668                         goto out_free_queues;
1669         }
1670
1671         return 0;
1672
1673 out_free_queues:
1674         for (i--; i >= 1; i--)
1675                 nvme_tcp_free_queue(ctrl, i);
1676
1677         return ret;
1678 }
1679
1680 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl)
1681 {
1682         unsigned int nr_io_queues;
1683
1684         nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus());
1685         nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus());
1686         nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus());
1687
1688         return nr_io_queues;
1689 }
1690
1691 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl,
1692                 unsigned int nr_io_queues)
1693 {
1694         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1695         struct nvmf_ctrl_options *opts = nctrl->opts;
1696
1697         if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) {
1698                 /*
1699                  * separate read/write queues
1700                  * hand out dedicated default queues only after we have
1701                  * sufficient read queues.
1702                  */
1703                 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues;
1704                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ];
1705                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1706                         min(opts->nr_write_queues, nr_io_queues);
1707                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1708         } else {
1709                 /*
1710                  * shared read/write queues
1711                  * either no write queues were requested, or we don't have
1712                  * sufficient queue count to have dedicated default queues.
1713                  */
1714                 ctrl->io_queues[HCTX_TYPE_DEFAULT] =
1715                         min(opts->nr_io_queues, nr_io_queues);
1716                 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT];
1717         }
1718
1719         if (opts->nr_poll_queues && nr_io_queues) {
1720                 /* map dedicated poll queues only if we have queues left */
1721                 ctrl->io_queues[HCTX_TYPE_POLL] =
1722                         min(opts->nr_poll_queues, nr_io_queues);
1723         }
1724 }
1725
1726 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1727 {
1728         unsigned int nr_io_queues;
1729         int ret;
1730
1731         nr_io_queues = nvme_tcp_nr_io_queues(ctrl);
1732         ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1733         if (ret)
1734                 return ret;
1735
1736         ctrl->queue_count = nr_io_queues + 1;
1737         if (ctrl->queue_count < 2)
1738                 return 0;
1739
1740         dev_info(ctrl->device,
1741                 "creating %d I/O queues.\n", nr_io_queues);
1742
1743         nvme_tcp_set_io_queues(ctrl, nr_io_queues);
1744
1745         return __nvme_tcp_alloc_io_queues(ctrl);
1746 }
1747
1748 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1749 {
1750         nvme_tcp_stop_io_queues(ctrl);
1751         if (remove) {
1752                 blk_cleanup_queue(ctrl->connect_q);
1753                 blk_mq_free_tag_set(ctrl->tagset);
1754         }
1755         nvme_tcp_free_io_queues(ctrl);
1756 }
1757
1758 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
1759 {
1760         int ret;
1761
1762         ret = nvme_tcp_alloc_io_queues(ctrl);
1763         if (ret)
1764                 return ret;
1765
1766         if (new) {
1767                 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false);
1768                 if (IS_ERR(ctrl->tagset)) {
1769                         ret = PTR_ERR(ctrl->tagset);
1770                         goto out_free_io_queues;
1771                 }
1772
1773                 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset);
1774                 if (IS_ERR(ctrl->connect_q)) {
1775                         ret = PTR_ERR(ctrl->connect_q);
1776                         goto out_free_tag_set;
1777                 }
1778         }
1779
1780         ret = nvme_tcp_start_io_queues(ctrl);
1781         if (ret)
1782                 goto out_cleanup_connect_q;
1783
1784         if (!new) {
1785                 nvme_start_queues(ctrl);
1786                 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
1787                         /*
1788                          * If we timed out waiting for freeze we are likely to
1789                          * be stuck.  Fail the controller initialization just
1790                          * to be safe.
1791                          */
1792                         ret = -ENODEV;
1793                         goto out_wait_freeze_timed_out;
1794                 }
1795                 blk_mq_update_nr_hw_queues(ctrl->tagset,
1796                         ctrl->queue_count - 1);
1797                 nvme_unfreeze(ctrl);
1798         }
1799
1800         return 0;
1801
1802 out_wait_freeze_timed_out:
1803         nvme_stop_queues(ctrl);
1804         nvme_tcp_stop_io_queues(ctrl);
1805 out_cleanup_connect_q:
1806         if (new)
1807                 blk_cleanup_queue(ctrl->connect_q);
1808 out_free_tag_set:
1809         if (new)
1810                 blk_mq_free_tag_set(ctrl->tagset);
1811 out_free_io_queues:
1812         nvme_tcp_free_io_queues(ctrl);
1813         return ret;
1814 }
1815
1816 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
1817 {
1818         nvme_tcp_stop_queue(ctrl, 0);
1819         if (remove) {
1820                 blk_cleanup_queue(ctrl->admin_q);
1821                 blk_cleanup_queue(ctrl->fabrics_q);
1822                 blk_mq_free_tag_set(ctrl->admin_tagset);
1823         }
1824         nvme_tcp_free_admin_queue(ctrl);
1825 }
1826
1827 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
1828 {
1829         int error;
1830
1831         error = nvme_tcp_alloc_admin_queue(ctrl);
1832         if (error)
1833                 return error;
1834
1835         if (new) {
1836                 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true);
1837                 if (IS_ERR(ctrl->admin_tagset)) {
1838                         error = PTR_ERR(ctrl->admin_tagset);
1839                         goto out_free_queue;
1840                 }
1841
1842                 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset);
1843                 if (IS_ERR(ctrl->fabrics_q)) {
1844                         error = PTR_ERR(ctrl->fabrics_q);
1845                         goto out_free_tagset;
1846                 }
1847
1848                 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset);
1849                 if (IS_ERR(ctrl->admin_q)) {
1850                         error = PTR_ERR(ctrl->admin_q);
1851                         goto out_cleanup_fabrics_q;
1852                 }
1853         }
1854
1855         error = nvme_tcp_start_queue(ctrl, 0);
1856         if (error)
1857                 goto out_cleanup_queue;
1858
1859         error = nvme_enable_ctrl(ctrl);
1860         if (error)
1861                 goto out_stop_queue;
1862
1863         blk_mq_unquiesce_queue(ctrl->admin_q);
1864
1865         error = nvme_init_identify(ctrl);
1866         if (error)
1867                 goto out_stop_queue;
1868
1869         return 0;
1870
1871 out_stop_queue:
1872         nvme_tcp_stop_queue(ctrl, 0);
1873 out_cleanup_queue:
1874         if (new)
1875                 blk_cleanup_queue(ctrl->admin_q);
1876 out_cleanup_fabrics_q:
1877         if (new)
1878                 blk_cleanup_queue(ctrl->fabrics_q);
1879 out_free_tagset:
1880         if (new)
1881                 blk_mq_free_tag_set(ctrl->admin_tagset);
1882 out_free_queue:
1883         nvme_tcp_free_admin_queue(ctrl);
1884         return error;
1885 }
1886
1887 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
1888                 bool remove)
1889 {
1890         mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);
1891         blk_mq_quiesce_queue(ctrl->admin_q);
1892         nvme_tcp_stop_queue(ctrl, 0);
1893         if (ctrl->admin_tagset) {
1894                 blk_mq_tagset_busy_iter(ctrl->admin_tagset,
1895                         nvme_cancel_request, ctrl);
1896                 blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
1897         }
1898         if (remove)
1899                 blk_mq_unquiesce_queue(ctrl->admin_q);
1900         nvme_tcp_destroy_admin_queue(ctrl, remove);
1901         mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);
1902 }
1903
1904 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
1905                 bool remove)
1906 {
1907         mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);
1908         if (ctrl->queue_count <= 1)
1909                 goto out;
1910         blk_mq_quiesce_queue(ctrl->admin_q);
1911         nvme_start_freeze(ctrl);
1912         nvme_stop_queues(ctrl);
1913         nvme_tcp_stop_io_queues(ctrl);
1914         if (ctrl->tagset) {
1915                 blk_mq_tagset_busy_iter(ctrl->tagset,
1916                         nvme_cancel_request, ctrl);
1917                 blk_mq_tagset_wait_completed_request(ctrl->tagset);
1918         }
1919         if (remove)
1920                 nvme_start_queues(ctrl);
1921         nvme_tcp_destroy_io_queues(ctrl, remove);
1922 out:
1923         mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);
1924 }
1925
1926 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
1927 {
1928         /* If we are resetting/deleting then do nothing */
1929         if (ctrl->state != NVME_CTRL_CONNECTING) {
1930                 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW ||
1931                         ctrl->state == NVME_CTRL_LIVE);
1932                 return;
1933         }
1934
1935         if (nvmf_should_reconnect(ctrl)) {
1936                 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
1937                         ctrl->opts->reconnect_delay);
1938                 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
1939                                 ctrl->opts->reconnect_delay * HZ);
1940         } else {
1941                 dev_info(ctrl->device, "Removing controller...\n");
1942                 nvme_delete_ctrl(ctrl);
1943         }
1944 }
1945
1946 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
1947 {
1948         struct nvmf_ctrl_options *opts = ctrl->opts;
1949         int ret;
1950
1951         ret = nvme_tcp_configure_admin_queue(ctrl, new);
1952         if (ret)
1953                 return ret;
1954
1955         if (ctrl->icdoff) {
1956                 dev_err(ctrl->device, "icdoff is not supported!\n");
1957                 goto destroy_admin;
1958         }
1959
1960         if (opts->queue_size > ctrl->sqsize + 1)
1961                 dev_warn(ctrl->device,
1962                         "queue_size %zu > ctrl sqsize %u, clamping down\n",
1963                         opts->queue_size, ctrl->sqsize + 1);
1964
1965         if (ctrl->sqsize + 1 > ctrl->maxcmd) {
1966                 dev_warn(ctrl->device,
1967                         "sqsize %u > ctrl maxcmd %u, clamping down\n",
1968                         ctrl->sqsize + 1, ctrl->maxcmd);
1969                 ctrl->sqsize = ctrl->maxcmd - 1;
1970         }
1971
1972         if (ctrl->queue_count > 1) {
1973                 ret = nvme_tcp_configure_io_queues(ctrl, new);
1974                 if (ret)
1975                         goto destroy_admin;
1976         }
1977
1978         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
1979                 /*
1980                  * state change failure is ok if we started ctrl delete,
1981                  * unless we're during creation of a new controller to
1982                  * avoid races with teardown flow.
1983                  */
1984                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
1985                              ctrl->state != NVME_CTRL_DELETING_NOIO);
1986                 WARN_ON_ONCE(new);
1987                 ret = -EINVAL;
1988                 goto destroy_io;
1989         }
1990
1991         nvme_start_ctrl(ctrl);
1992         return 0;
1993
1994 destroy_io:
1995         if (ctrl->queue_count > 1)
1996                 nvme_tcp_destroy_io_queues(ctrl, new);
1997 destroy_admin:
1998         nvme_tcp_stop_queue(ctrl, 0);
1999         nvme_tcp_destroy_admin_queue(ctrl, new);
2000         return ret;
2001 }
2002
2003 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2004 {
2005         struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2006                         struct nvme_tcp_ctrl, connect_work);
2007         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2008
2009         ++ctrl->nr_reconnects;
2010
2011         if (nvme_tcp_setup_ctrl(ctrl, false))
2012                 goto requeue;
2013
2014         dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2015                         ctrl->nr_reconnects);
2016
2017         ctrl->nr_reconnects = 0;
2018
2019         return;
2020
2021 requeue:
2022         dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2023                         ctrl->nr_reconnects);
2024         nvme_tcp_reconnect_or_remove(ctrl);
2025 }
2026
2027 static void nvme_tcp_error_recovery_work(struct work_struct *work)
2028 {
2029         struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2030                                 struct nvme_tcp_ctrl, err_work);
2031         struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2032
2033         nvme_stop_keep_alive(ctrl);
2034         nvme_tcp_teardown_io_queues(ctrl, false);
2035         /* unquiesce to fail fast pending requests */
2036         nvme_start_queues(ctrl);
2037         nvme_tcp_teardown_admin_queue(ctrl, false);
2038         blk_mq_unquiesce_queue(ctrl->admin_q);
2039
2040         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2041                 /* state change failure is ok if we started ctrl delete */
2042                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2043                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2044                 return;
2045         }
2046
2047         nvme_tcp_reconnect_or_remove(ctrl);
2048 }
2049
2050 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2051 {
2052         cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work);
2053         cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2054
2055         nvme_tcp_teardown_io_queues(ctrl, shutdown);
2056         blk_mq_quiesce_queue(ctrl->admin_q);
2057         if (shutdown)
2058                 nvme_shutdown_ctrl(ctrl);
2059         else
2060                 nvme_disable_ctrl(ctrl);
2061         nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2062 }
2063
2064 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2065 {
2066         nvme_tcp_teardown_ctrl(ctrl, true);
2067 }
2068
2069 static void nvme_reset_ctrl_work(struct work_struct *work)
2070 {
2071         struct nvme_ctrl *ctrl =
2072                 container_of(work, struct nvme_ctrl, reset_work);
2073
2074         nvme_stop_ctrl(ctrl);
2075         nvme_tcp_teardown_ctrl(ctrl, false);
2076
2077         if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2078                 /* state change failure is ok if we started ctrl delete */
2079                 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING &&
2080                              ctrl->state != NVME_CTRL_DELETING_NOIO);
2081                 return;
2082         }
2083
2084         if (nvme_tcp_setup_ctrl(ctrl, false))
2085                 goto out_fail;
2086
2087         return;
2088
2089 out_fail:
2090         ++ctrl->nr_reconnects;
2091         nvme_tcp_reconnect_or_remove(ctrl);
2092 }
2093
2094 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2095 {
2096         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2097
2098         if (list_empty(&ctrl->list))
2099                 goto free_ctrl;
2100
2101         mutex_lock(&nvme_tcp_ctrl_mutex);
2102         list_del(&ctrl->list);
2103         mutex_unlock(&nvme_tcp_ctrl_mutex);
2104
2105         nvmf_free_options(nctrl->opts);
2106 free_ctrl:
2107         kfree(ctrl->queues);
2108         kfree(ctrl);
2109 }
2110
2111 static void nvme_tcp_set_sg_null(struct nvme_command *c)
2112 {
2113         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2114
2115         sg->addr = 0;
2116         sg->length = 0;
2117         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2118                         NVME_SGL_FMT_TRANSPORT_A;
2119 }
2120
2121 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2122                 struct nvme_command *c, u32 data_len)
2123 {
2124         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2125
2126         sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2127         sg->length = cpu_to_le32(data_len);
2128         sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2129 }
2130
2131 static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2132                 u32 data_len)
2133 {
2134         struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2135
2136         sg->addr = 0;
2137         sg->length = cpu_to_le32(data_len);
2138         sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2139                         NVME_SGL_FMT_TRANSPORT_A;
2140 }
2141
2142 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2143 {
2144         struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2145         struct nvme_tcp_queue *queue = &ctrl->queues[0];
2146         struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2147         struct nvme_command *cmd = &pdu->cmd;
2148         u8 hdgst = nvme_tcp_hdgst_len(queue);
2149
2150         memset(pdu, 0, sizeof(*pdu));
2151         pdu->hdr.type = nvme_tcp_cmd;
2152         if (queue->hdr_digest)
2153                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2154         pdu->hdr.hlen = sizeof(*pdu);
2155         pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2156
2157         cmd->common.opcode = nvme_admin_async_event;
2158         cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2159         cmd->common.flags |= NVME_CMD_SGL_METABUF;
2160         nvme_tcp_set_sg_null(cmd);
2161
2162         ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2163         ctrl->async_req.offset = 0;
2164         ctrl->async_req.curr_bio = NULL;
2165         ctrl->async_req.data_len = 0;
2166
2167         nvme_tcp_queue_request(&ctrl->async_req, true, true);
2168 }
2169
2170 static void nvme_tcp_complete_timed_out(struct request *rq)
2171 {
2172         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2173         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2174
2175         /* fence other contexts that may complete the command */
2176         mutex_lock(&to_tcp_ctrl(ctrl)->teardown_lock);
2177         nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2178         if (!blk_mq_request_completed(rq)) {
2179                 nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD;
2180                 blk_mq_complete_request(rq);
2181         }
2182         mutex_unlock(&to_tcp_ctrl(ctrl)->teardown_lock);
2183 }
2184
2185 static enum blk_eh_timer_return
2186 nvme_tcp_timeout(struct request *rq, bool reserved)
2187 {
2188         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2189         struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2190         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2191
2192         dev_warn(ctrl->device,
2193                 "queue %d: timeout request %#x type %d\n",
2194                 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type);
2195
2196         if (ctrl->state != NVME_CTRL_LIVE) {
2197                 /*
2198                  * If we are resetting, connecting or deleting we should
2199                  * complete immediately because we may block controller
2200                  * teardown or setup sequence
2201                  * - ctrl disable/shutdown fabrics requests
2202                  * - connect requests
2203                  * - initialization admin requests
2204                  * - I/O requests that entered after unquiescing and
2205                  *   the controller stopped responding
2206                  *
2207                  * All other requests should be cancelled by the error
2208                  * recovery work, so it's fine that we fail it here.
2209                  */
2210                 nvme_tcp_complete_timed_out(rq);
2211                 return BLK_EH_DONE;
2212         }
2213
2214         /*
2215          * LIVE state should trigger the normal error recovery which will
2216          * handle completing this request.
2217          */
2218         nvme_tcp_error_recovery(ctrl);
2219         return BLK_EH_RESET_TIMER;
2220 }
2221
2222 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2223                         struct request *rq)
2224 {
2225         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2226         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2227         struct nvme_command *c = &pdu->cmd;
2228
2229         c->common.flags |= NVME_CMD_SGL_METABUF;
2230
2231         if (!blk_rq_nr_phys_segments(rq))
2232                 nvme_tcp_set_sg_null(c);
2233         else if (rq_data_dir(rq) == WRITE &&
2234             req->data_len <= nvme_tcp_inline_data_size(queue))
2235                 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2236         else
2237                 nvme_tcp_set_sg_host_data(c, req->data_len);
2238
2239         return 0;
2240 }
2241
2242 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2243                 struct request *rq)
2244 {
2245         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2246         struct nvme_tcp_cmd_pdu *pdu = req->pdu;
2247         struct nvme_tcp_queue *queue = req->queue;
2248         u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2249         blk_status_t ret;
2250
2251         ret = nvme_setup_cmd(ns, rq, &pdu->cmd);
2252         if (ret)
2253                 return ret;
2254
2255         req->state = NVME_TCP_SEND_CMD_PDU;
2256         req->offset = 0;
2257         req->data_sent = 0;
2258         req->pdu_len = 0;
2259         req->pdu_sent = 0;
2260         req->data_len = blk_rq_nr_phys_segments(rq) ?
2261                                 blk_rq_payload_bytes(rq) : 0;
2262         req->curr_bio = rq->bio;
2263
2264         if (rq_data_dir(rq) == WRITE &&
2265             req->data_len <= nvme_tcp_inline_data_size(queue))
2266                 req->pdu_len = req->data_len;
2267         else if (req->curr_bio)
2268                 nvme_tcp_init_iter(req, READ);
2269
2270         pdu->hdr.type = nvme_tcp_cmd;
2271         pdu->hdr.flags = 0;
2272         if (queue->hdr_digest)
2273                 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2274         if (queue->data_digest && req->pdu_len) {
2275                 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2276                 ddgst = nvme_tcp_ddgst_len(queue);
2277         }
2278         pdu->hdr.hlen = sizeof(*pdu);
2279         pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2280         pdu->hdr.plen =
2281                 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2282
2283         ret = nvme_tcp_map_data(queue, rq);
2284         if (unlikely(ret)) {
2285                 nvme_cleanup_cmd(rq);
2286                 dev_err(queue->ctrl->ctrl.device,
2287                         "Failed to map data (%d)\n", ret);
2288                 return ret;
2289         }
2290
2291         return 0;
2292 }
2293
2294 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2295 {
2296         struct nvme_tcp_queue *queue = hctx->driver_data;
2297
2298         if (!llist_empty(&queue->req_list))
2299                 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2300 }
2301
2302 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2303                 const struct blk_mq_queue_data *bd)
2304 {
2305         struct nvme_ns *ns = hctx->queue->queuedata;
2306         struct nvme_tcp_queue *queue = hctx->driver_data;
2307         struct request *rq = bd->rq;
2308         struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2309         bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2310         blk_status_t ret;
2311
2312         if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2313                 return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq);
2314
2315         ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2316         if (unlikely(ret))
2317                 return ret;
2318
2319         blk_mq_start_request(rq);
2320
2321         nvme_tcp_queue_request(req, true, bd->last);
2322
2323         return BLK_STS_OK;
2324 }
2325
2326 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2327 {
2328         struct nvme_tcp_ctrl *ctrl = set->driver_data;
2329         struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
2330
2331         if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) {
2332                 /* separate read/write queues */
2333                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2334                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2335                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2336                 set->map[HCTX_TYPE_READ].nr_queues =
2337                         ctrl->io_queues[HCTX_TYPE_READ];
2338                 set->map[HCTX_TYPE_READ].queue_offset =
2339                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2340         } else {
2341                 /* shared read/write queues */
2342                 set->map[HCTX_TYPE_DEFAULT].nr_queues =
2343                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2344                 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
2345                 set->map[HCTX_TYPE_READ].nr_queues =
2346                         ctrl->io_queues[HCTX_TYPE_DEFAULT];
2347                 set->map[HCTX_TYPE_READ].queue_offset = 0;
2348         }
2349         blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2350         blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);
2351
2352         if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) {
2353                 /* map dedicated poll queues only if we have queues left */
2354                 set->map[HCTX_TYPE_POLL].nr_queues =
2355                                 ctrl->io_queues[HCTX_TYPE_POLL];
2356                 set->map[HCTX_TYPE_POLL].queue_offset =
2357                         ctrl->io_queues[HCTX_TYPE_DEFAULT] +
2358                         ctrl->io_queues[HCTX_TYPE_READ];
2359                 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
2360         }
2361
2362         dev_info(ctrl->ctrl.device,
2363                 "mapped %d/%d/%d default/read/poll queues.\n",
2364                 ctrl->io_queues[HCTX_TYPE_DEFAULT],
2365                 ctrl->io_queues[HCTX_TYPE_READ],
2366                 ctrl->io_queues[HCTX_TYPE_POLL]);
2367
2368         return 0;
2369 }
2370
2371 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx)
2372 {
2373         struct nvme_tcp_queue *queue = hctx->driver_data;
2374         struct sock *sk = queue->sock->sk;
2375
2376         if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2377                 return 0;
2378
2379         set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2380         if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2381                 sk_busy_loop(sk, true);
2382         nvme_tcp_try_recv(queue);
2383         clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2384         return queue->nr_cqe;
2385 }
2386
2387 static const struct blk_mq_ops nvme_tcp_mq_ops = {
2388         .queue_rq       = nvme_tcp_queue_rq,
2389         .commit_rqs     = nvme_tcp_commit_rqs,
2390         .complete       = nvme_complete_rq,
2391         .init_request   = nvme_tcp_init_request,
2392         .exit_request   = nvme_tcp_exit_request,
2393         .init_hctx      = nvme_tcp_init_hctx,
2394         .timeout        = nvme_tcp_timeout,
2395         .map_queues     = nvme_tcp_map_queues,
2396         .poll           = nvme_tcp_poll,
2397 };
2398
2399 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2400         .queue_rq       = nvme_tcp_queue_rq,
2401         .complete       = nvme_complete_rq,
2402         .init_request   = nvme_tcp_init_request,
2403         .exit_request   = nvme_tcp_exit_request,
2404         .init_hctx      = nvme_tcp_init_admin_hctx,
2405         .timeout        = nvme_tcp_timeout,
2406 };
2407
2408 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2409         .name                   = "tcp",
2410         .module                 = THIS_MODULE,
2411         .flags                  = NVME_F_FABRICS,
2412         .reg_read32             = nvmf_reg_read32,
2413         .reg_read64             = nvmf_reg_read64,
2414         .reg_write32            = nvmf_reg_write32,
2415         .free_ctrl              = nvme_tcp_free_ctrl,
2416         .submit_async_event     = nvme_tcp_submit_async_event,
2417         .delete_ctrl            = nvme_tcp_delete_ctrl,
2418         .get_address            = nvmf_get_address,
2419 };
2420
2421 static bool
2422 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2423 {
2424         struct nvme_tcp_ctrl *ctrl;
2425         bool found = false;
2426
2427         mutex_lock(&nvme_tcp_ctrl_mutex);
2428         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2429                 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2430                 if (found)
2431                         break;
2432         }
2433         mutex_unlock(&nvme_tcp_ctrl_mutex);
2434
2435         return found;
2436 }
2437
2438 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2439                 struct nvmf_ctrl_options *opts)
2440 {
2441         struct nvme_tcp_ctrl *ctrl;
2442         int ret;
2443
2444         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2445         if (!ctrl)
2446                 return ERR_PTR(-ENOMEM);
2447
2448         INIT_LIST_HEAD(&ctrl->list);
2449         ctrl->ctrl.opts = opts;
2450         ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2451                                 opts->nr_poll_queues + 1;
2452         ctrl->ctrl.sqsize = opts->queue_size - 1;
2453         ctrl->ctrl.kato = opts->kato;
2454
2455         INIT_DELAYED_WORK(&ctrl->connect_work,
2456                         nvme_tcp_reconnect_ctrl_work);
2457         INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2458         INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2459         mutex_init(&ctrl->teardown_lock);
2460
2461         if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2462                 opts->trsvcid =
2463                         kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2464                 if (!opts->trsvcid) {
2465                         ret = -ENOMEM;
2466                         goto out_free_ctrl;
2467                 }
2468                 opts->mask |= NVMF_OPT_TRSVCID;
2469         }
2470
2471         ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2472                         opts->traddr, opts->trsvcid, &ctrl->addr);
2473         if (ret) {
2474                 pr_err("malformed address passed: %s:%s\n",
2475                         opts->traddr, opts->trsvcid);
2476                 goto out_free_ctrl;
2477         }
2478
2479         if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2480                 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2481                         opts->host_traddr, NULL, &ctrl->src_addr);
2482                 if (ret) {
2483                         pr_err("malformed src address passed: %s\n",
2484                                opts->host_traddr);
2485                         goto out_free_ctrl;
2486                 }
2487         }
2488
2489         if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2490                 ret = -EALREADY;
2491                 goto out_free_ctrl;
2492         }
2493
2494         ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2495                                 GFP_KERNEL);
2496         if (!ctrl->queues) {
2497                 ret = -ENOMEM;
2498                 goto out_free_ctrl;
2499         }
2500
2501         ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2502         if (ret)
2503                 goto out_kfree_queues;
2504
2505         if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2506                 WARN_ON_ONCE(1);
2507                 ret = -EINTR;
2508                 goto out_uninit_ctrl;
2509         }
2510
2511         ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2512         if (ret)
2513                 goto out_uninit_ctrl;
2514
2515         dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n",
2516                 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
2517
2518         mutex_lock(&nvme_tcp_ctrl_mutex);
2519         list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2520         mutex_unlock(&nvme_tcp_ctrl_mutex);
2521
2522         return &ctrl->ctrl;
2523
2524 out_uninit_ctrl:
2525         nvme_uninit_ctrl(&ctrl->ctrl);
2526         nvme_put_ctrl(&ctrl->ctrl);
2527         if (ret > 0)
2528                 ret = -EIO;
2529         return ERR_PTR(ret);
2530 out_kfree_queues:
2531         kfree(ctrl->queues);
2532 out_free_ctrl:
2533         kfree(ctrl);
2534         return ERR_PTR(ret);
2535 }
2536
2537 static struct nvmf_transport_ops nvme_tcp_transport = {
2538         .name           = "tcp",
2539         .module         = THIS_MODULE,
2540         .required_opts  = NVMF_OPT_TRADDR,
2541         .allowed_opts   = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2542                           NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2543                           NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2544                           NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2545                           NVMF_OPT_TOS,
2546         .create_ctrl    = nvme_tcp_create_ctrl,
2547 };
2548
2549 static int __init nvme_tcp_init_module(void)
2550 {
2551         nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2552                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2553         if (!nvme_tcp_wq)
2554                 return -ENOMEM;
2555
2556         nvmf_register_transport(&nvme_tcp_transport);
2557         return 0;
2558 }
2559
2560 static void __exit nvme_tcp_cleanup_module(void)
2561 {
2562         struct nvme_tcp_ctrl *ctrl;
2563
2564         nvmf_unregister_transport(&nvme_tcp_transport);
2565
2566         mutex_lock(&nvme_tcp_ctrl_mutex);
2567         list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2568                 nvme_delete_ctrl(&ctrl->ctrl);
2569         mutex_unlock(&nvme_tcp_ctrl_mutex);
2570         flush_workqueue(nvme_delete_wq);
2571
2572         destroy_workqueue(nvme_tcp_wq);
2573 }
2574
2575 module_init(nvme_tcp_init_module);
2576 module_exit(nvme_tcp_cleanup_module);
2577
2578 MODULE_LICENSE("GPL v2");