net/ceph/messenger_v2.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Ceph msgr2 protocol implementation
   4  *
   5  * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
   6  */
   7
   8 #include <linux/ceph/ceph_debug.h>
   9
  10 #include <crypto/aead.h>
  11 #include <crypto/algapi.h>  /* for crypto_memneq() */
  12 #include <crypto/hash.h>
  13 #include <crypto/sha2.h>
  14 #include <linux/bvec.h>
  15 #include <linux/crc32c.h>
  16 #include <linux/net.h>
  17 #include <linux/scatterlist.h>
  18 #include <linux/socket.h>
  19 #include <linux/sched/mm.h>
  20 #include <net/sock.h>
  21 #include <net/tcp.h>
  22
  23 #include <linux/ceph/ceph_features.h>
  24 #include <linux/ceph/decode.h>
  25 #include <linux/ceph/libceph.h>
  26 #include <linux/ceph/messenger.h>
  27
  28 #include "crypto.h"  /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
  29
  30 #define FRAME_TAG_HELLO                 1
  31 #define FRAME_TAG_AUTH_REQUEST          2
  32 #define FRAME_TAG_AUTH_BAD_METHOD       3
  33 #define FRAME_TAG_AUTH_REPLY_MORE       4
  34 #define FRAME_TAG_AUTH_REQUEST_MORE     5
  35 #define FRAME_TAG_AUTH_DONE             6
  36 #define FRAME_TAG_AUTH_SIGNATURE        7
  37 #define FRAME_TAG_CLIENT_IDENT          8
  38 #define FRAME_TAG_SERVER_IDENT          9
  39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10
  40 #define FRAME_TAG_SESSION_RECONNECT     11
  41 #define FRAME_TAG_SESSION_RESET         12
  42 #define FRAME_TAG_SESSION_RETRY         13
  43 #define FRAME_TAG_SESSION_RETRY_GLOBAL  14
  44 #define FRAME_TAG_SESSION_RECONNECT_OK  15
  45 #define FRAME_TAG_WAIT                  16
  46 #define FRAME_TAG_MESSAGE               17
  47 #define FRAME_TAG_KEEPALIVE2            18
  48 #define FRAME_TAG_KEEPALIVE2_ACK        19
  49 #define FRAME_TAG_ACK                   20
  50
  51 #define FRAME_LATE_STATUS_ABORTED       0x1
  52 #define FRAME_LATE_STATUS_COMPLETE      0xe
  53 #define FRAME_LATE_STATUS_ABORTED_MASK  0xf
  54
  55 #define IN_S_HANDLE_PREAMBLE            1
  56 #define IN_S_HANDLE_CONTROL             2
  57 #define IN_S_HANDLE_CONTROL_REMAINDER   3
  58 #define IN_S_PREPARE_READ_DATA          4
  59 #define IN_S_PREPARE_READ_DATA_CONT     5
  60 #define IN_S_PREPARE_READ_ENC_PAGE      6
  61 #define IN_S_HANDLE_EPILOGUE            7
  62 #define IN_S_FINISH_SKIP                8
  63
  64 #define OUT_S_QUEUE_DATA                1
  65 #define OUT_S_QUEUE_DATA_CONT           2
  66 #define OUT_S_QUEUE_ENC_PAGE            3
  67 #define OUT_S_QUEUE_ZEROS               4
  68 #define OUT_S_FINISH_MESSAGE            5
  69 #define OUT_S_GET_NEXT                  6
  70
  71 #define CTRL_BODY(p)    ((void *)(p) + CEPH_PREAMBLE_LEN)
  72 #define FRONT_PAD(p)    ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
  73 #define MIDDLE_PAD(p)   (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
  74 #define DATA_PAD(p)     (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
  75
  76 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
  77
  78 static int do_recvmsg(struct socket *sock, struct iov_iter *it)
  79 {
  80         struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
  81         int ret;
  82
  83         msg.msg_iter = *it;
  84         while (iov_iter_count(it)) {
  85                 ret = sock_recvmsg(sock, &msg, msg.msg_flags);
  86                 if (ret <= 0) {
  87                         if (ret == -EAGAIN)
  88                                 ret = 0;
  89                         return ret;
  90                 }
  91
  92                 iov_iter_advance(it, ret);
  93         }
  94
  95         WARN_ON(msg_data_left(&msg));
  96         return 1;
  97 }
  98
  99 /*
 100  * Read as much as possible.
 101  *
 102  * Return:
 103  *   1 - done, nothing (else) to read
 104  *   0 - socket is empty, need to wait
 105  *  <0 - error
 106  */
 107 static int ceph_tcp_recv(struct ceph_connection *con)
 108 {
 109         int ret;
 110
 111         dout("%s con %p %s %zu\n", __func__, con,
 112              iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
 113              iov_iter_count(&con->v2.in_iter));
 114         ret = do_recvmsg(con->sock, &con->v2.in_iter);
 115         dout("%s con %p ret %d left %zu\n", __func__, con, ret,
 116              iov_iter_count(&con->v2.in_iter));
 117         return ret;
 118 }
 119
 120 /*
 121  * Write as much as possible.  The socket is expected to be corked,
 122  * so we don't bother with MSG_MORE here.
 123  *
 124  * Return:
 125  *  >0 - done, nothing (else) to write
 126  *   0 - socket is full, need to wait
 127  *  <0 - error
 128  */
 129 static int ceph_tcp_send(struct ceph_connection *con)
 130 {
 131         struct msghdr msg = {
 132                 .msg_iter       = con->v2.out_iter,
 133                 .msg_flags      = CEPH_MSG_FLAGS,
 134         };
 135         int ret;
 136
 137         if (WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter)))
 138                 return -EINVAL;
 139
 140         if (con->v2.out_iter_sendpage)
 141                 msg.msg_flags |= MSG_SPLICE_PAGES;
 142
 143         dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
 144              iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
 145
 146         ret = sock_sendmsg(con->sock, &msg);
 147         if (ret > 0)
 148                 iov_iter_advance(&con->v2.out_iter, ret);
 149         else if (ret == -EAGAIN)
 150                 ret = 0;
 151
 152         dout("%s con %p ret %d left %zu\n", __func__, con, ret,
 153              iov_iter_count(&con->v2.out_iter));
 154         return ret;
 155 }
 156
 157 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
 158 {
 159         BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
 160         WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
 161
 162         con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
 163         con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
 164         con->v2.in_kvec_cnt++;
 165
 166         con->v2.in_iter.nr_segs++;
 167         con->v2.in_iter.count += len;
 168 }
 169
 170 static void reset_in_kvecs(struct ceph_connection *con)
 171 {
 172         WARN_ON(iov_iter_count(&con->v2.in_iter));
 173
 174         con->v2.in_kvec_cnt = 0;
 175         iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);
 176 }
 177
 178 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
 179 {
 180         WARN_ON(iov_iter_count(&con->v2.in_iter));
 181
 182         con->v2.in_bvec = *bv;
 183         iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);
 184 }
 185
 186 static void set_in_skip(struct ceph_connection *con, int len)
 187 {
 188         WARN_ON(iov_iter_count(&con->v2.in_iter));
 189
 190         dout("%s con %p len %d\n", __func__, con, len);
 191         iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);
 192 }
 193
 194 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
 195 {
 196         BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
 197         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
 198         WARN_ON(con->v2.out_zero);
 199
 200         con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
 201         con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
 202         con->v2.out_kvec_cnt++;
 203
 204         con->v2.out_iter.nr_segs++;
 205         con->v2.out_iter.count += len;
 206 }
 207
 208 static void reset_out_kvecs(struct ceph_connection *con)
 209 {
 210         WARN_ON(iov_iter_count(&con->v2.out_iter));
 211         WARN_ON(con->v2.out_zero);
 212
 213         con->v2.out_kvec_cnt = 0;
 214
 215         iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);
 216         con->v2.out_iter_sendpage = false;
 217 }
 218
 219 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
 220                          bool zerocopy)
 221 {
 222         WARN_ON(iov_iter_count(&con->v2.out_iter));
 223         WARN_ON(con->v2.out_zero);
 224
 225         con->v2.out_bvec = *bv;
 226         con->v2.out_iter_sendpage = zerocopy;
 227         iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
 228                       con->v2.out_bvec.bv_len);
 229 }
 230
 231 static void set_out_bvec_zero(struct ceph_connection *con)
 232 {
 233         WARN_ON(iov_iter_count(&con->v2.out_iter));
 234         WARN_ON(!con->v2.out_zero);
 235
 236         bvec_set_page(&con->v2.out_bvec, ceph_zero_page,
 237                       min(con->v2.out_zero, (int)PAGE_SIZE), 0);
 238         con->v2.out_iter_sendpage = true;
 239         iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
 240                       con->v2.out_bvec.bv_len);
 241 }
 242
 243 static void out_zero_add(struct ceph_connection *con, int len)
 244 {
 245         dout("%s con %p len %d\n", __func__, con, len);
 246         con->v2.out_zero += len;
 247 }
 248
 249 static void *alloc_conn_buf(struct ceph_connection *con, int len)
 250 {
 251         void *buf;
 252
 253         dout("%s con %p len %d\n", __func__, con, len);
 254
 255         if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
 256                 return NULL;
 257
 258         buf = kvmalloc(len, GFP_NOIO);
 259         if (!buf)
 260                 return NULL;
 261
 262         con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
 263         return buf;
 264 }
 265
 266 static void free_conn_bufs(struct ceph_connection *con)
 267 {
 268         while (con->v2.conn_buf_cnt)
 269                 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
 270 }
 271
 272 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
 273 {
 274         BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
 275
 276         con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
 277         con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
 278         con->v2.in_sign_kvec_cnt++;
 279 }
 280
 281 static void clear_in_sign_kvecs(struct ceph_connection *con)
 282 {
 283         con->v2.in_sign_kvec_cnt = 0;
 284 }
 285
 286 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
 287 {
 288         BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
 289
 290         con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
 291         con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
 292         con->v2.out_sign_kvec_cnt++;
 293 }
 294
 295 static void clear_out_sign_kvecs(struct ceph_connection *con)
 296 {
 297         con->v2.out_sign_kvec_cnt = 0;
 298 }
 299
 300 static bool con_secure(struct ceph_connection *con)
 301 {
 302         return con->v2.con_mode == CEPH_CON_MODE_SECURE;
 303 }
 304
 305 static int front_len(const struct ceph_msg *msg)
 306 {
 307         return le32_to_cpu(msg->hdr.front_len);
 308 }
 309
 310 static int middle_len(const struct ceph_msg *msg)
 311 {
 312         return le32_to_cpu(msg->hdr.middle_len);
 313 }
 314
 315 static int data_len(const struct ceph_msg *msg)
 316 {
 317         return le32_to_cpu(msg->hdr.data_len);
 318 }
 319
 320 static bool need_padding(int len)
 321 {
 322         return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
 323 }
 324
 325 static int padded_len(int len)
 326 {
 327         return ALIGN(len, CEPH_GCM_BLOCK_LEN);
 328 }
 329
 330 static int padding_len(int len)
 331 {
 332         return padded_len(len) - len;
 333 }
 334
 335 /* preamble + control segment */
 336 static int head_onwire_len(int ctrl_len, bool secure)
 337 {
 338         int head_len;
 339         int rem_len;
 340
 341         if (secure) {
 342                 head_len = CEPH_PREAMBLE_SECURE_LEN;
 343                 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
 344                         rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
 345                         head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
 346                 }
 347         } else {
 348                 head_len = CEPH_PREAMBLE_PLAIN_LEN;
 349                 if (ctrl_len)
 350                         head_len += ctrl_len + CEPH_CRC_LEN;
 351         }
 352         return head_len;
 353 }
 354
 355 /* front, middle and data segments + epilogue */
 356 static int __tail_onwire_len(int front_len, int middle_len, int data_len,
 357                              bool secure)
 358 {
 359         if (!front_len && !middle_len && !data_len)
 360                 return 0;
 361
 362         if (!secure)
 363                 return front_len + middle_len + data_len +
 364                        CEPH_EPILOGUE_PLAIN_LEN;
 365
 366         return padded_len(front_len) + padded_len(middle_len) +
 367                padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
 368 }
 369
 370 static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
 371 {
 372         return __tail_onwire_len(front_len(msg), middle_len(msg),
 373                                  data_len(msg), secure);
 374 }
 375
 376 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
 377 #define MESSAGE_HEAD_PLAIN_LEN  (CEPH_PREAMBLE_PLAIN_LEN +              \
 378                                  sizeof(struct ceph_msg_header2) +      \
 379                                  CEPH_CRC_LEN)
 380
 381 static const int frame_aligns[] = {
 382         sizeof(void *),
 383         sizeof(void *),
 384         sizeof(void *),
 385         PAGE_SIZE
 386 };
 387
 388 /*
 389  * Discards trailing empty segments, unless there is just one segment.
 390  * A frame always has at least one (possibly empty) segment.
 391  */
 392 static int calc_segment_count(const int *lens, int len_cnt)
 393 {
 394         int i;
 395
 396         for (i = len_cnt - 1; i >= 0; i--) {
 397                 if (lens[i])
 398                         return i + 1;
 399         }
 400
 401         return 1;
 402 }
 403
 404 static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
 405                             const int *lens, int len_cnt)
 406 {
 407         int i;
 408
 409         memset(desc, 0, sizeof(*desc));
 410
 411         desc->fd_tag = tag;
 412         desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
 413         BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
 414         for (i = 0; i < desc->fd_seg_cnt; i++) {
 415                 desc->fd_lens[i] = lens[i];
 416                 desc->fd_aligns[i] = frame_aligns[i];
 417         }
 418 }
 419
 420 /*
 421  * Preamble crc covers everything up to itself (28 bytes) and
 422  * is calculated and verified irrespective of the connection mode
 423  * (i.e. even if the frame is encrypted).
 424  */
 425 static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
 426 {
 427         void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
 428         void *start = p;
 429         int i;
 430
 431         memset(p, 0, CEPH_PREAMBLE_LEN);
 432
 433         ceph_encode_8(&p, desc->fd_tag);
 434         ceph_encode_8(&p, desc->fd_seg_cnt);
 435         for (i = 0; i < desc->fd_seg_cnt; i++) {
 436                 ceph_encode_32(&p, desc->fd_lens[i]);
 437                 ceph_encode_16(&p, desc->fd_aligns[i]);
 438         }
 439
 440         put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
 441 }
 442
 443 static int decode_preamble(void *p, struct ceph_frame_desc *desc)
 444 {
 445         void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
 446         u32 crc, expected_crc;
 447         int i;
 448
 449         crc = crc32c(0, p, crcp - p);
 450         expected_crc = get_unaligned_le32(crcp);
 451         if (crc != expected_crc) {
 452                 pr_err("bad preamble crc, calculated %u, expected %u\n",
 453                        crc, expected_crc);
 454                 return -EBADMSG;
 455         }
 456
 457         memset(desc, 0, sizeof(*desc));
 458
 459         desc->fd_tag = ceph_decode_8(&p);
 460         desc->fd_seg_cnt = ceph_decode_8(&p);
 461         if (desc->fd_seg_cnt < 1 ||
 462             desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
 463                 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
 464                 return -EINVAL;
 465         }
 466         for (i = 0; i < desc->fd_seg_cnt; i++) {
 467                 desc->fd_lens[i] = ceph_decode_32(&p);
 468                 desc->fd_aligns[i] = ceph_decode_16(&p);
 469         }
 470
 471         /*
 472          * This would fire for FRAME_TAG_WAIT (it has one empty
 473          * segment), but we should never get it as client.
 474          */
 475         if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
 476                 pr_err("last segment empty\n");
 477                 return -EINVAL;
 478         }
 479
 480         if (desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
 481                 pr_err("control segment too big %d\n", desc->fd_lens[0]);
 482                 return -EINVAL;
 483         }
 484         if (desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
 485                 pr_err("front segment too big %d\n", desc->fd_lens[1]);
 486                 return -EINVAL;
 487         }
 488         if (desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
 489                 pr_err("middle segment too big %d\n", desc->fd_lens[2]);
 490                 return -EINVAL;
 491         }
 492         if (desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
 493                 pr_err("data segment too big %d\n", desc->fd_lens[3]);
 494                 return -EINVAL;
 495         }
 496
 497         return 0;
 498 }
 499
 500 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
 501 {
 502         con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
 503                                                  FRAME_LATE_STATUS_COMPLETE;
 504         cpu_to_le32s(&con->v2.out_epil.front_crc);
 505         cpu_to_le32s(&con->v2.out_epil.middle_crc);
 506         cpu_to_le32s(&con->v2.out_epil.data_crc);
 507 }
 508
 509 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
 510 {
 511         memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
 512         con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
 513                                                  FRAME_LATE_STATUS_COMPLETE;
 514 }
 515
 516 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
 517                            u32 *data_crc)
 518 {
 519         u8 late_status;
 520
 521         late_status = ceph_decode_8(&p);
 522         if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
 523                         FRAME_LATE_STATUS_COMPLETE) {
 524                 /* we should never get an aborted message as client */
 525                 pr_err("bad late_status 0x%x\n", late_status);
 526                 return -EINVAL;
 527         }
 528
 529         if (front_crc && middle_crc && data_crc) {
 530                 *front_crc = ceph_decode_32(&p);
 531                 *middle_crc = ceph_decode_32(&p);
 532                 *data_crc = ceph_decode_32(&p);
 533         }
 534
 535         return 0;
 536 }
 537
 538 static void fill_header(struct ceph_msg_header *hdr,
 539                         const struct ceph_msg_header2 *hdr2,
 540                         int front_len, int middle_len, int data_len,
 541                         const struct ceph_entity_name *peer_name)
 542 {
 543         hdr->seq = hdr2->seq;
 544         hdr->tid = hdr2->tid;
 545         hdr->type = hdr2->type;
 546         hdr->priority = hdr2->priority;
 547         hdr->version = hdr2->version;
 548         hdr->front_len = cpu_to_le32(front_len);
 549         hdr->middle_len = cpu_to_le32(middle_len);
 550         hdr->data_len = cpu_to_le32(data_len);
 551         hdr->data_off = hdr2->data_off;
 552         hdr->src = *peer_name;
 553         hdr->compat_version = hdr2->compat_version;
 554         hdr->reserved = 0;
 555         hdr->crc = 0;
 556 }
 557
 558 static void fill_header2(struct ceph_msg_header2 *hdr2,
 559                          const struct ceph_msg_header *hdr, u64 ack_seq)
 560 {
 561         hdr2->seq = hdr->seq;
 562         hdr2->tid = hdr->tid;
 563         hdr2->type = hdr->type;
 564         hdr2->priority = hdr->priority;
 565         hdr2->version = hdr->version;
 566         hdr2->data_pre_padding_len = 0;
 567         hdr2->data_off = hdr->data_off;
 568         hdr2->ack_seq = cpu_to_le64(ack_seq);
 569         hdr2->flags = 0;
 570         hdr2->compat_version = hdr->compat_version;
 571         hdr2->reserved = 0;
 572 }
 573
 574 static int verify_control_crc(struct ceph_connection *con)
 575 {
 576         int ctrl_len = con->v2.in_desc.fd_lens[0];
 577         u32 crc, expected_crc;
 578
 579         WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
 580         WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
 581
 582         crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
 583         expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
 584         if (crc != expected_crc) {
 585                 pr_err("bad control crc, calculated %u, expected %u\n",
 586                        crc, expected_crc);
 587                 return -EBADMSG;
 588         }
 589
 590         return 0;
 591 }
 592
 593 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
 594                                 u32 middle_crc, u32 data_crc)
 595 {
 596         if (front_len(con->in_msg)) {
 597                 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
 598                                            front_len(con->in_msg));
 599         } else {
 600                 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
 601                 con->in_front_crc = -1;
 602         }
 603
 604         if (middle_len(con->in_msg))
 605                 con->in_middle_crc = crc32c(-1,
 606                                             con->in_msg->middle->vec.iov_base,
 607                                             middle_len(con->in_msg));
 608         else if (data_len(con->in_msg))
 609                 con->in_middle_crc = -1;
 610         else
 611                 con->in_middle_crc = 0;
 612
 613         if (!data_len(con->in_msg))
 614                 con->in_data_crc = 0;
 615
 616         dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
 617              con->in_front_crc, con->in_middle_crc, con->in_data_crc);
 618
 619         if (con->in_front_crc != front_crc) {
 620                 pr_err("bad front crc, calculated %u, expected %u\n",
 621                        con->in_front_crc, front_crc);
 622                 return -EBADMSG;
 623         }
 624         if (con->in_middle_crc != middle_crc) {
 625                 pr_err("bad middle crc, calculated %u, expected %u\n",
 626                        con->in_middle_crc, middle_crc);
 627                 return -EBADMSG;
 628         }
 629         if (con->in_data_crc != data_crc) {
 630                 pr_err("bad data crc, calculated %u, expected %u\n",
 631                        con->in_data_crc, data_crc);
 632                 return -EBADMSG;
 633         }
 634
 635         return 0;
 636 }
 637
 638 static int setup_crypto(struct ceph_connection *con,
 639                         const u8 *session_key, int session_key_len,
 640                         const u8 *con_secret, int con_secret_len)
 641 {
 642         unsigned int noio_flag;
 643         int ret;
 644
 645         dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
 646              __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
 647         WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);
 648
 649         if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
 650             con->v2.con_mode != CEPH_CON_MODE_SECURE) {
 651                 pr_err("bad con_mode %d\n", con->v2.con_mode);
 652                 return -EINVAL;
 653         }
 654
 655         if (!session_key_len) {
 656                 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
 657                 WARN_ON(con_secret_len);
 658                 return 0;  /* auth_none */
 659         }
 660
 661         noio_flag = memalloc_noio_save();
 662         con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
 663         memalloc_noio_restore(noio_flag);
 664         if (IS_ERR(con->v2.hmac_tfm)) {
 665                 ret = PTR_ERR(con->v2.hmac_tfm);
 666                 con->v2.hmac_tfm = NULL;
 667                 pr_err("failed to allocate hmac tfm context: %d\n", ret);
 668                 return ret;
 669         }
 670
 671         WARN_ON((unsigned long)session_key &
 672                 crypto_shash_alignmask(con->v2.hmac_tfm));
 673         ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,
 674                                   session_key_len);
 675         if (ret) {
 676                 pr_err("failed to set hmac key: %d\n", ret);
 677                 return ret;
 678         }
 679
 680         if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
 681                 WARN_ON(con_secret_len);
 682                 return 0;  /* auth_x, plain mode */
 683         }
 684
 685         if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
 686                 pr_err("con_secret too small %d\n", con_secret_len);
 687                 return -EINVAL;
 688         }
 689
 690         noio_flag = memalloc_noio_save();
 691         con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
 692         memalloc_noio_restore(noio_flag);
 693         if (IS_ERR(con->v2.gcm_tfm)) {
 694                 ret = PTR_ERR(con->v2.gcm_tfm);
 695                 con->v2.gcm_tfm = NULL;
 696                 pr_err("failed to allocate gcm tfm context: %d\n", ret);
 697                 return ret;
 698         }
 699
 700         WARN_ON((unsigned long)con_secret &
 701                 crypto_aead_alignmask(con->v2.gcm_tfm));
 702         ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);
 703         if (ret) {
 704                 pr_err("failed to set gcm key: %d\n", ret);
 705                 return ret;
 706         }
 707
 708         WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
 709         ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
 710         if (ret) {
 711                 pr_err("failed to set gcm tag size: %d\n", ret);
 712                 return ret;
 713         }
 714
 715         con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
 716         if (!con->v2.gcm_req) {
 717                 pr_err("failed to allocate gcm request\n");
 718                 return -ENOMEM;
 719         }
 720
 721         crypto_init_wait(&con->v2.gcm_wait);
 722         aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 723                                   crypto_req_done, &con->v2.gcm_wait);
 724
 725         memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
 726                CEPH_GCM_IV_LEN);
 727         memcpy(&con->v2.out_gcm_nonce,
 728                con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
 729                CEPH_GCM_IV_LEN);
 730         return 0;  /* auth_x, secure mode */
 731 }
 732
 733 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,
 734                        int kvec_cnt, u8 *hmac)
 735 {
 736         SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm);  /* tfm arg is ignored */
 737         int ret;
 738         int i;
 739
 740         dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,
 741              con->v2.hmac_tfm, kvec_cnt);
 742
 743         if (!con->v2.hmac_tfm) {
 744                 memset(hmac, 0, SHA256_DIGEST_SIZE);
 745                 return 0;  /* auth_none */
 746         }
 747
 748         desc->tfm = con->v2.hmac_tfm;
 749         ret = crypto_shash_init(desc);
 750         if (ret)
 751                 goto out;
 752
 753         for (i = 0; i < kvec_cnt; i++) {
 754                 WARN_ON((unsigned long)kvecs[i].iov_base &
 755                         crypto_shash_alignmask(con->v2.hmac_tfm));
 756                 ret = crypto_shash_update(desc, kvecs[i].iov_base,
 757                                           kvecs[i].iov_len);
 758                 if (ret)
 759                         goto out;
 760         }
 761
 762         ret = crypto_shash_final(desc, hmac);
 763
 764 out:
 765         shash_desc_zero(desc);
 766         return ret;  /* auth_x, both plain and secure modes */
 767 }
 768
 769 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
 770 {
 771         u64 counter;
 772
 773         counter = le64_to_cpu(nonce->counter);
 774         nonce->counter = cpu_to_le64(counter + 1);
 775 }
 776
 777 static int gcm_crypt(struct ceph_connection *con, bool encrypt,
 778                      struct scatterlist *src, struct scatterlist *dst,
 779                      int src_len)
 780 {
 781         struct ceph_gcm_nonce *nonce;
 782         int ret;
 783
 784         nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
 785
 786         aead_request_set_ad(con->v2.gcm_req, 0);  /* no AAD */
 787         aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
 788         ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
 789                                         crypto_aead_decrypt(con->v2.gcm_req),
 790                               &con->v2.gcm_wait);
 791         if (ret)
 792                 return ret;
 793
 794         gcm_inc_nonce(nonce);
 795         return 0;
 796 }
 797
 798 static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
 799                         struct bio_vec *bv)
 800 {
 801         struct page *page;
 802         size_t off, len;
 803
 804         WARN_ON(!cursor->total_resid);
 805
 806         /* skip zero-length data items */
 807         while (!cursor->resid)
 808                 ceph_msg_data_advance(cursor, 0);
 809
 810         /* get a piece of data, cursor isn't advanced */
 811         page = ceph_msg_data_next(cursor, &off, &len);
 812         bvec_set_page(bv, page, len, off);
 813 }
 814
 815 static int calc_sg_cnt(void *buf, int buf_len)
 816 {
 817         int sg_cnt;
 818
 819         if (!buf_len)
 820                 return 0;
 821
 822         sg_cnt = need_padding(buf_len) ? 1 : 0;
 823         if (is_vmalloc_addr(buf)) {
 824                 WARN_ON(offset_in_page(buf));
 825                 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
 826         } else {
 827                 sg_cnt++;
 828         }
 829
 830         return sg_cnt;
 831 }
 832
 833 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
 834 {
 835         int data_len = cursor->total_resid;
 836         struct bio_vec bv;
 837         int sg_cnt;
 838
 839         if (!data_len)
 840                 return 0;
 841
 842         sg_cnt = need_padding(data_len) ? 1 : 0;
 843         do {
 844                 get_bvec_at(cursor, &bv);
 845                 sg_cnt++;
 846
 847                 ceph_msg_data_advance(cursor, bv.bv_len);
 848         } while (cursor->total_resid);
 849
 850         return sg_cnt;
 851 }
 852
 853 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
 854 {
 855         void *end = buf + buf_len;
 856         struct page *page;
 857         int len;
 858         void *p;
 859
 860         if (!buf_len)
 861                 return;
 862
 863         if (is_vmalloc_addr(buf)) {
 864                 p = buf;
 865                 do {
 866                         page = vmalloc_to_page(p);
 867                         len = min_t(int, end - p, PAGE_SIZE);
 868                         WARN_ON(!page || !len || offset_in_page(p));
 869                         sg_set_page(*sg, page, len, 0);
 870                         *sg = sg_next(*sg);
 871                         p += len;
 872                 } while (p != end);
 873         } else {
 874                 sg_set_buf(*sg, buf, buf_len);
 875                 *sg = sg_next(*sg);
 876         }
 877
 878         if (need_padding(buf_len)) {
 879                 sg_set_buf(*sg, pad, padding_len(buf_len));
 880                 *sg = sg_next(*sg);
 881         }
 882 }
 883
 884 static void init_sgs_cursor(struct scatterlist **sg,
 885                             struct ceph_msg_data_cursor *cursor, u8 *pad)
 886 {
 887         int data_len = cursor->total_resid;
 888         struct bio_vec bv;
 889
 890         if (!data_len)
 891                 return;
 892
 893         do {
 894                 get_bvec_at(cursor, &bv);
 895                 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
 896                 *sg = sg_next(*sg);
 897
 898                 ceph_msg_data_advance(cursor, bv.bv_len);
 899         } while (cursor->total_resid);
 900
 901         if (need_padding(data_len)) {
 902                 sg_set_buf(*sg, pad, padding_len(data_len));
 903                 *sg = sg_next(*sg);
 904         }
 905 }
 906
 907 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
 908                              u8 *front_pad, u8 *middle_pad, u8 *data_pad,
 909                              void *epilogue, bool add_tag)
 910 {
 911         struct ceph_msg_data_cursor cursor;
 912         struct scatterlist *cur_sg;
 913         int sg_cnt;
 914         int ret;
 915
 916         if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
 917                 return 0;
 918
 919         sg_cnt = 1;  /* epilogue + [auth tag] */
 920         if (front_len(msg))
 921                 sg_cnt += calc_sg_cnt(msg->front.iov_base,
 922                                       front_len(msg));
 923         if (middle_len(msg))
 924                 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
 925                                       middle_len(msg));
 926         if (data_len(msg)) {
 927                 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
 928                 sg_cnt += calc_sg_cnt_cursor(&cursor);
 929         }
 930
 931         ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
 932         if (ret)
 933                 return ret;
 934
 935         cur_sg = sgt->sgl;
 936         if (front_len(msg))
 937                 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
 938                          front_pad);
 939         if (middle_len(msg))
 940                 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
 941                          middle_pad);
 942         if (data_len(msg)) {
 943                 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg));
 944                 init_sgs_cursor(&cur_sg, &cursor, data_pad);
 945         }
 946
 947         WARN_ON(!sg_is_last(cur_sg));
 948         sg_set_buf(cur_sg, epilogue,
 949                    CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
 950         return 0;
 951 }
 952
 953 static int decrypt_preamble(struct ceph_connection *con)
 954 {
 955         struct scatterlist sg;
 956
 957         sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
 958         return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
 959 }
 960
 961 static int decrypt_control_remainder(struct ceph_connection *con)
 962 {
 963         int ctrl_len = con->v2.in_desc.fd_lens[0];
 964         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
 965         int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
 966         struct scatterlist sgs[2];
 967
 968         WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
 969         WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
 970
 971         sg_init_table(sgs, 2);
 972         sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
 973         sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
 974
 975         return gcm_crypt(con, false, sgs, sgs,
 976                          padded_len(rem_len) + CEPH_GCM_TAG_LEN);
 977 }
 978
 979 static int decrypt_tail(struct ceph_connection *con)
 980 {
 981         struct sg_table enc_sgt = {};
 982         struct sg_table sgt = {};
 983         int tail_len;
 984         int ret;
 985
 986         tail_len = tail_onwire_len(con->in_msg, true);
 987         ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,
 988                                         con->v2.in_enc_page_cnt, 0, tail_len,
 989                                         GFP_NOIO);
 990         if (ret)
 991                 goto out;
 992
 993         ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
 994                         MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
 995                         con->v2.in_buf, true);
 996         if (ret)
 997                 goto out;
 998
 999         dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,
1000              con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);
1001         ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);
1002         if (ret)
1003                 goto out;
1004
1005         WARN_ON(!con->v2.in_enc_page_cnt);
1006         ceph_release_page_vector(con->v2.in_enc_pages,
1007                                  con->v2.in_enc_page_cnt);
1008         con->v2.in_enc_pages = NULL;
1009         con->v2.in_enc_page_cnt = 0;
1010
1011 out:
1012         sg_free_table(&sgt);
1013         sg_free_table(&enc_sgt);
1014         return ret;
1015 }
1016
1017 static int prepare_banner(struct ceph_connection *con)
1018 {
1019         int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1020         void *buf, *p;
1021
1022         buf = alloc_conn_buf(con, buf_len);
1023         if (!buf)
1024                 return -ENOMEM;
1025
1026         p = buf;
1027         ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1028         ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1029         ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1030         ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1031         WARN_ON(p != buf + buf_len);
1032
1033         add_out_kvec(con, buf, buf_len);
1034         add_out_sign_kvec(con, buf, buf_len);
1035         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1036         return 0;
1037 }
1038
1039 /*
1040  * base:
1041  *   preamble
1042  *   control body (ctrl_len bytes)
1043  *   space for control crc
1044  *
1045  * extdata (optional):
1046  *   control body (extdata_len bytes)
1047  *
1048  * Compute control crc and gather base and extdata into:
1049  *
1050  *   preamble
1051  *   control body (ctrl_len + extdata_len bytes)
1052  *   control crc
1053  *
1054  * Preamble should already be encoded at the start of base.
1055  */
1056 static void prepare_head_plain(struct ceph_connection *con, void *base,
1057                                int ctrl_len, void *extdata, int extdata_len,
1058                                bool to_be_signed)
1059 {
1060         int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1061         void *crcp = base + base_len - CEPH_CRC_LEN;
1062         u32 crc;
1063
1064         crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1065         if (extdata_len)
1066                 crc = crc32c(crc, extdata, extdata_len);
1067         put_unaligned_le32(crc, crcp);
1068
1069         if (!extdata_len) {
1070                 add_out_kvec(con, base, base_len);
1071                 if (to_be_signed)
1072                         add_out_sign_kvec(con, base, base_len);
1073                 return;
1074         }
1075
1076         add_out_kvec(con, base, crcp - base);
1077         add_out_kvec(con, extdata, extdata_len);
1078         add_out_kvec(con, crcp, CEPH_CRC_LEN);
1079         if (to_be_signed) {
1080                 add_out_sign_kvec(con, base, crcp - base);
1081                 add_out_sign_kvec(con, extdata, extdata_len);
1082                 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1083         }
1084 }
1085
1086 static int prepare_head_secure_small(struct ceph_connection *con,
1087                                      void *base, int ctrl_len)
1088 {
1089         struct scatterlist sg;
1090         int ret;
1091
1092         /* inline buffer padding? */
1093         if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1094                 memset(CTRL_BODY(base) + ctrl_len, 0,
1095                        CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1096
1097         sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1098         ret = gcm_crypt(con, true, &sg, &sg,
1099                         CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1100         if (ret)
1101                 return ret;
1102
1103         add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1104         return 0;
1105 }
1106
1107 /*
1108  * base:
1109  *   preamble
1110  *   control body (ctrl_len bytes)
1111  *   space for padding, if needed
1112  *   space for control remainder auth tag
1113  *   space for preamble auth tag
1114  *
1115  * Encrypt preamble and the inline portion, then encrypt the remainder
1116  * and gather into:
1117  *
1118  *   preamble
1119  *   control body (48 bytes)
1120  *   preamble auth tag
1121  *   control body (ctrl_len - 48 bytes)
1122  *   zero padding, if needed
1123  *   control remainder auth tag
1124  *
1125  * Preamble should already be encoded at the start of base.
1126  */
1127 static int prepare_head_secure_big(struct ceph_connection *con,
1128                                    void *base, int ctrl_len)
1129 {
1130         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1131         void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1132         void *rem_tag = rem + padded_len(rem_len);
1133         void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1134         struct scatterlist sgs[2];
1135         int ret;
1136
1137         sg_init_table(sgs, 2);
1138         sg_set_buf(&sgs[0], base, rem - base);
1139         sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1140         ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1141         if (ret)
1142                 return ret;
1143
1144         /* control remainder padding? */
1145         if (need_padding(rem_len))
1146                 memset(rem + rem_len, 0, padding_len(rem_len));
1147
1148         sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1149         ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1150         if (ret)
1151                 return ret;
1152
1153         add_out_kvec(con, base, rem - base);
1154         add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1155         add_out_kvec(con, rem, pmbl_tag - rem);
1156         return 0;
1157 }
1158
1159 static int __prepare_control(struct ceph_connection *con, int tag,
1160                              void *base, int ctrl_len, void *extdata,
1161                              int extdata_len, bool to_be_signed)
1162 {
1163         int total_len = ctrl_len + extdata_len;
1164         struct ceph_frame_desc desc;
1165         int ret;
1166
1167         dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1168              total_len, ctrl_len, extdata_len);
1169
1170         /* extdata may be vmalloc'ed but not base */
1171         if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1172                 return -EINVAL;
1173
1174         init_frame_desc(&desc, tag, &total_len, 1);
1175         encode_preamble(&desc, base);
1176
1177         if (con_secure(con)) {
1178                 if (WARN_ON(extdata_len || to_be_signed))
1179                         return -EINVAL;
1180
1181                 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1182                         /* fully inlined, inline buffer may need padding */
1183                         ret = prepare_head_secure_small(con, base, ctrl_len);
1184                 else
1185                         /* partially inlined, inline buffer is full */
1186                         ret = prepare_head_secure_big(con, base, ctrl_len);
1187                 if (ret)
1188                         return ret;
1189         } else {
1190                 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1191                                    to_be_signed);
1192         }
1193
1194         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1195         return 0;
1196 }
1197
1198 static int prepare_control(struct ceph_connection *con, int tag,
1199                            void *base, int ctrl_len)
1200 {
1201         return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1202 }
1203
1204 static int prepare_hello(struct ceph_connection *con)
1205 {
1206         void *buf, *p;
1207         int ctrl_len;
1208
1209         ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1210         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1211         if (!buf)
1212                 return -ENOMEM;
1213
1214         p = CTRL_BODY(buf);
1215         ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1216         ceph_encode_entity_addr(&p, &con->peer_addr);
1217         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1218
1219         return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1220                                  NULL, 0, true);
1221 }
1222
1223 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1224 #define AUTH_BUF_LEN    (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1225
1226 static int prepare_auth_request(struct ceph_connection *con)
1227 {
1228         void *authorizer, *authorizer_copy;
1229         int ctrl_len, authorizer_len;
1230         void *buf;
1231         int ret;
1232
1233         ctrl_len = AUTH_BUF_LEN;
1234         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1235         if (!buf)
1236                 return -ENOMEM;
1237
1238         mutex_unlock(&con->mutex);
1239         ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1240                                          &authorizer, &authorizer_len);
1241         mutex_lock(&con->mutex);
1242         if (con->state != CEPH_CON_S_V2_HELLO) {
1243                 dout("%s con %p state changed to %d\n", __func__, con,
1244                      con->state);
1245                 return -EAGAIN;
1246         }
1247
1248         dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1249         if (ret)
1250                 return ret;
1251
1252         authorizer_copy = alloc_conn_buf(con, authorizer_len);
1253         if (!authorizer_copy)
1254                 return -ENOMEM;
1255
1256         memcpy(authorizer_copy, authorizer, authorizer_len);
1257
1258         return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1259                                  authorizer_copy, authorizer_len, true);
1260 }
1261
1262 static int prepare_auth_request_more(struct ceph_connection *con,
1263                                      void *reply, int reply_len)
1264 {
1265         int ctrl_len, authorizer_len;
1266         void *authorizer;
1267         void *buf;
1268         int ret;
1269
1270         ctrl_len = AUTH_BUF_LEN;
1271         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1272         if (!buf)
1273                 return -ENOMEM;
1274
1275         mutex_unlock(&con->mutex);
1276         ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1277                                                CTRL_BODY(buf), &ctrl_len,
1278                                                &authorizer, &authorizer_len);
1279         mutex_lock(&con->mutex);
1280         if (con->state != CEPH_CON_S_V2_AUTH) {
1281                 dout("%s con %p state changed to %d\n", __func__, con,
1282                      con->state);
1283                 return -EAGAIN;
1284         }
1285
1286         dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1287         if (ret)
1288                 return ret;
1289
1290         return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1291                                  ctrl_len, authorizer, authorizer_len, true);
1292 }
1293
1294 static int prepare_auth_signature(struct ceph_connection *con)
1295 {
1296         void *buf;
1297         int ret;
1298
1299         buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,
1300                                                   con_secure(con)));
1301         if (!buf)
1302                 return -ENOMEM;
1303
1304         ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1305                           CTRL_BODY(buf));
1306         if (ret)
1307                 return ret;
1308
1309         return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1310                                SHA256_DIGEST_SIZE);
1311 }
1312
1313 static int prepare_client_ident(struct ceph_connection *con)
1314 {
1315         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1316         struct ceph_client *client = from_msgr(con->msgr);
1317         u64 global_id = ceph_client_gid(client);
1318         void *buf, *p;
1319         int ctrl_len;
1320
1321         WARN_ON(con->v2.server_cookie);
1322         WARN_ON(con->v2.connect_seq);
1323         WARN_ON(con->v2.peer_global_seq);
1324
1325         if (!con->v2.client_cookie) {
1326                 do {
1327                         get_random_bytes(&con->v2.client_cookie,
1328                                          sizeof(con->v2.client_cookie));
1329                 } while (!con->v2.client_cookie);
1330                 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1331                      con->v2.client_cookie);
1332         } else {
1333                 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1334                      con->v2.client_cookie);
1335         }
1336
1337         dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1338              __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1339              ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1340              global_id, con->v2.global_seq, client->supported_features,
1341              client->required_features, con->v2.client_cookie);
1342
1343         ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1344                    ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1345         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1346         if (!buf)
1347                 return -ENOMEM;
1348
1349         p = CTRL_BODY(buf);
1350         ceph_encode_8(&p, 2);  /* addrvec marker */
1351         ceph_encode_32(&p, 1);  /* addr_cnt */
1352         ceph_encode_entity_addr(&p, my_addr);
1353         ceph_encode_entity_addr(&p, &con->peer_addr);
1354         ceph_encode_64(&p, global_id);
1355         ceph_encode_64(&p, con->v2.global_seq);
1356         ceph_encode_64(&p, client->supported_features);
1357         ceph_encode_64(&p, client->required_features);
1358         ceph_encode_64(&p, 0);  /* flags */
1359         ceph_encode_64(&p, con->v2.client_cookie);
1360         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1361
1362         return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1363 }
1364
1365 static int prepare_session_reconnect(struct ceph_connection *con)
1366 {
1367         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1368         void *buf, *p;
1369         int ctrl_len;
1370
1371         WARN_ON(!con->v2.client_cookie);
1372         WARN_ON(!con->v2.server_cookie);
1373         WARN_ON(!con->v2.connect_seq);
1374         WARN_ON(!con->v2.peer_global_seq);
1375
1376         dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1377              __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1378              con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1379              con->v2.connect_seq, con->in_seq);
1380
1381         ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1382         buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1383         if (!buf)
1384                 return -ENOMEM;
1385
1386         p = CTRL_BODY(buf);
1387         ceph_encode_8(&p, 2);  /* entity_addrvec_t marker */
1388         ceph_encode_32(&p, 1);  /* my_addrs len */
1389         ceph_encode_entity_addr(&p, my_addr);
1390         ceph_encode_64(&p, con->v2.client_cookie);
1391         ceph_encode_64(&p, con->v2.server_cookie);
1392         ceph_encode_64(&p, con->v2.global_seq);
1393         ceph_encode_64(&p, con->v2.connect_seq);
1394         ceph_encode_64(&p, con->in_seq);
1395         WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1396
1397         return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1398 }
1399
1400 static int prepare_keepalive2(struct ceph_connection *con)
1401 {
1402         struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1403         struct timespec64 now;
1404
1405         ktime_get_real_ts64(&now);
1406         dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,
1407              now.tv_nsec);
1408
1409         ceph_encode_timespec64(ts, &now);
1410
1411         reset_out_kvecs(con);
1412         return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1413                                sizeof(struct ceph_timespec));
1414 }
1415
1416 static int prepare_ack(struct ceph_connection *con)
1417 {
1418         void *p;
1419
1420         dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1421              con->in_seq_acked, con->in_seq);
1422         con->in_seq_acked = con->in_seq;
1423
1424         p = CTRL_BODY(con->v2.out_buf);
1425         ceph_encode_64(&p, con->in_seq_acked);
1426
1427         reset_out_kvecs(con);
1428         return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1429 }
1430
1431 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)
1432 {
1433         dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1434              con->out_msg, aborted, con->v2.out_epil.front_crc,
1435              con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1436
1437         encode_epilogue_plain(con, aborted);
1438         add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1439 }
1440
1441 /*
1442  * For "used" empty segments, crc is -1.  For unused (trailing)
1443  * segments, crc is 0.
1444  */
1445 static void prepare_message_plain(struct ceph_connection *con)
1446 {
1447         struct ceph_msg *msg = con->out_msg;
1448
1449         prepare_head_plain(con, con->v2.out_buf,
1450                            sizeof(struct ceph_msg_header2), NULL, 0, false);
1451
1452         if (!front_len(msg) && !middle_len(msg)) {
1453                 if (!data_len(msg)) {
1454                         /*
1455                          * Empty message: once the head is written,
1456                          * we are done -- there is no epilogue.
1457                          */
1458                         con->v2.out_state = OUT_S_FINISH_MESSAGE;
1459                         return;
1460                 }
1461
1462                 con->v2.out_epil.front_crc = -1;
1463                 con->v2.out_epil.middle_crc = -1;
1464                 con->v2.out_state = OUT_S_QUEUE_DATA;
1465                 return;
1466         }
1467
1468         if (front_len(msg)) {
1469                 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1470                                                     front_len(msg));
1471                 add_out_kvec(con, msg->front.iov_base, front_len(msg));
1472         } else {
1473                 /* middle (at least) is there, checked above */
1474                 con->v2.out_epil.front_crc = -1;
1475         }
1476
1477         if (middle_len(msg)) {
1478                 con->v2.out_epil.middle_crc =
1479                         crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1480                 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1481         } else {
1482                 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1483         }
1484
1485         if (data_len(msg)) {
1486                 con->v2.out_state = OUT_S_QUEUE_DATA;
1487         } else {
1488                 con->v2.out_epil.data_crc = 0;
1489                 prepare_epilogue_plain(con, false);
1490                 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1491         }
1492 }
1493
1494 /*
1495  * Unfortunately the kernel crypto API doesn't support streaming
1496  * (piecewise) operation for AEAD algorithms, so we can't get away
1497  * with a fixed size buffer and a couple sgs.  Instead, we have to
1498  * allocate pages for the entire tail of the message (currently up
1499  * to ~32M) and two sgs arrays (up to ~256K each)...
1500  */
1501 static int prepare_message_secure(struct ceph_connection *con)
1502 {
1503         void *zerop = page_address(ceph_zero_page);
1504         struct sg_table enc_sgt = {};
1505         struct sg_table sgt = {};
1506         struct page **enc_pages;
1507         int enc_page_cnt;
1508         int tail_len;
1509         int ret;
1510
1511         ret = prepare_head_secure_small(con, con->v2.out_buf,
1512                                         sizeof(struct ceph_msg_header2));
1513         if (ret)
1514                 return ret;
1515
1516         tail_len = tail_onwire_len(con->out_msg, true);
1517         if (!tail_len) {
1518                 /*
1519                  * Empty message: once the head is written,
1520                  * we are done -- there is no epilogue.
1521                  */
1522                 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1523                 return 0;
1524         }
1525
1526         encode_epilogue_secure(con, false);
1527         ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,
1528                                 &con->v2.out_epil, false);
1529         if (ret)
1530                 goto out;
1531
1532         enc_page_cnt = calc_pages_for(0, tail_len);
1533         enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1534         if (IS_ERR(enc_pages)) {
1535                 ret = PTR_ERR(enc_pages);
1536                 goto out;
1537         }
1538
1539         WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1540         con->v2.out_enc_pages = enc_pages;
1541         con->v2.out_enc_page_cnt = enc_page_cnt;
1542         con->v2.out_enc_resid = tail_len;
1543         con->v2.out_enc_i = 0;
1544
1545         ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1546                                         0, tail_len, GFP_NOIO);
1547         if (ret)
1548                 goto out;
1549
1550         ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1551                         tail_len - CEPH_GCM_TAG_LEN);
1552         if (ret)
1553                 goto out;
1554
1555         dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1556              con->out_msg, sgt.orig_nents, enc_page_cnt);
1557         con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1558
1559 out:
1560         sg_free_table(&sgt);
1561         sg_free_table(&enc_sgt);
1562         return ret;
1563 }
1564
1565 static int prepare_message(struct ceph_connection *con)
1566 {
1567         int lens[] = {
1568                 sizeof(struct ceph_msg_header2),
1569                 front_len(con->out_msg),
1570                 middle_len(con->out_msg),
1571                 data_len(con->out_msg)
1572         };
1573         struct ceph_frame_desc desc;
1574         int ret;
1575
1576         dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1577              con->out_msg, lens[0], lens[1], lens[2], lens[3]);
1578
1579         if (con->in_seq > con->in_seq_acked) {
1580                 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1581                      con->in_seq_acked, con->in_seq);
1582                 con->in_seq_acked = con->in_seq;
1583         }
1584
1585         reset_out_kvecs(con);
1586         init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1587         encode_preamble(&desc, con->v2.out_buf);
1588         fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,
1589                      con->in_seq_acked);
1590
1591         if (con_secure(con)) {
1592                 ret = prepare_message_secure(con);
1593                 if (ret)
1594                         return ret;
1595         } else {
1596                 prepare_message_plain(con);
1597         }
1598
1599         ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1600         return 0;
1601 }
1602
1603 static int prepare_read_banner_prefix(struct ceph_connection *con)
1604 {
1605         void *buf;
1606
1607         buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1608         if (!buf)
1609                 return -ENOMEM;
1610
1611         reset_in_kvecs(con);
1612         add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1613         add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1614         con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1615         return 0;
1616 }
1617
1618 static int prepare_read_banner_payload(struct ceph_connection *con,
1619                                        int payload_len)
1620 {
1621         void *buf;
1622
1623         buf = alloc_conn_buf(con, payload_len);
1624         if (!buf)
1625                 return -ENOMEM;
1626
1627         reset_in_kvecs(con);
1628         add_in_kvec(con, buf, payload_len);
1629         add_in_sign_kvec(con, buf, payload_len);
1630         con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1631         return 0;
1632 }
1633
1634 static void prepare_read_preamble(struct ceph_connection *con)
1635 {
1636         reset_in_kvecs(con);
1637         add_in_kvec(con, con->v2.in_buf,
1638                     con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1639                                       CEPH_PREAMBLE_PLAIN_LEN);
1640         con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1641 }
1642
1643 static int prepare_read_control(struct ceph_connection *con)
1644 {
1645         int ctrl_len = con->v2.in_desc.fd_lens[0];
1646         int head_len;
1647         void *buf;
1648
1649         reset_in_kvecs(con);
1650         if (con->state == CEPH_CON_S_V2_HELLO ||
1651             con->state == CEPH_CON_S_V2_AUTH) {
1652                 head_len = head_onwire_len(ctrl_len, false);
1653                 buf = alloc_conn_buf(con, head_len);
1654                 if (!buf)
1655                         return -ENOMEM;
1656
1657                 /* preserve preamble */
1658                 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1659
1660                 add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1661                 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1662                 add_in_sign_kvec(con, buf, head_len);
1663         } else {
1664                 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1665                         buf = alloc_conn_buf(con, ctrl_len);
1666                         if (!buf)
1667                                 return -ENOMEM;
1668
1669                         add_in_kvec(con, buf, ctrl_len);
1670                 } else {
1671                         add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1672                 }
1673                 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1674         }
1675         con->v2.in_state = IN_S_HANDLE_CONTROL;
1676         return 0;
1677 }
1678
1679 static int prepare_read_control_remainder(struct ceph_connection *con)
1680 {
1681         int ctrl_len = con->v2.in_desc.fd_lens[0];
1682         int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1683         void *buf;
1684
1685         buf = alloc_conn_buf(con, ctrl_len);
1686         if (!buf)
1687                 return -ENOMEM;
1688
1689         memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1690
1691         reset_in_kvecs(con);
1692         add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1693         add_in_kvec(con, con->v2.in_buf,
1694                     padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1695         con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1696         return 0;
1697 }
1698
1699 static int prepare_read_data(struct ceph_connection *con)
1700 {
1701         struct bio_vec bv;
1702
1703         con->in_data_crc = -1;
1704         ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1705                                   data_len(con->in_msg));
1706
1707         get_bvec_at(&con->v2.in_cursor, &bv);
1708         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1709                 if (unlikely(!con->bounce_page)) {
1710                         con->bounce_page = alloc_page(GFP_NOIO);
1711                         if (!con->bounce_page) {
1712                                 pr_err("failed to allocate bounce page\n");
1713                                 return -ENOMEM;
1714                         }
1715                 }
1716
1717                 bv.bv_page = con->bounce_page;
1718                 bv.bv_offset = 0;
1719         }
1720         set_in_bvec(con, &bv);
1721         con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1722         return 0;
1723 }
1724
1725 static void prepare_read_data_cont(struct ceph_connection *con)
1726 {
1727         struct bio_vec bv;
1728
1729         if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1730                 con->in_data_crc = crc32c(con->in_data_crc,
1731                                           page_address(con->bounce_page),
1732                                           con->v2.in_bvec.bv_len);
1733
1734                 get_bvec_at(&con->v2.in_cursor, &bv);
1735                 memcpy_to_page(bv.bv_page, bv.bv_offset,
1736                                page_address(con->bounce_page),
1737                                con->v2.in_bvec.bv_len);
1738         } else {
1739                 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1740                                                     con->v2.in_bvec.bv_page,
1741                                                     con->v2.in_bvec.bv_offset,
1742                                                     con->v2.in_bvec.bv_len);
1743         }
1744
1745         ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1746         if (con->v2.in_cursor.total_resid) {
1747                 get_bvec_at(&con->v2.in_cursor, &bv);
1748                 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1749                         bv.bv_page = con->bounce_page;
1750                         bv.bv_offset = 0;
1751                 }
1752                 set_in_bvec(con, &bv);
1753                 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1754                 return;
1755         }
1756
1757         /*
1758          * We've read all data.  Prepare to read epilogue.
1759          */
1760         reset_in_kvecs(con);
1761         add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1762         con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1763 }
1764
1765 static int prepare_read_tail_plain(struct ceph_connection *con)
1766 {
1767         struct ceph_msg *msg = con->in_msg;
1768
1769         if (!front_len(msg) && !middle_len(msg)) {
1770                 WARN_ON(!data_len(msg));
1771                 return prepare_read_data(con);
1772         }
1773
1774         reset_in_kvecs(con);
1775         if (front_len(msg)) {
1776                 add_in_kvec(con, msg->front.iov_base, front_len(msg));
1777                 WARN_ON(msg->front.iov_len != front_len(msg));
1778         }
1779         if (middle_len(msg)) {
1780                 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1781                 WARN_ON(msg->middle->vec.iov_len != middle_len(msg));
1782         }
1783
1784         if (data_len(msg)) {
1785                 con->v2.in_state = IN_S_PREPARE_READ_DATA;
1786         } else {
1787                 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1788                 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1789         }
1790         return 0;
1791 }
1792
1793 static void prepare_read_enc_page(struct ceph_connection *con)
1794 {
1795         struct bio_vec bv;
1796
1797         dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,
1798              con->v2.in_enc_resid);
1799         WARN_ON(!con->v2.in_enc_resid);
1800
1801         bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i],
1802                       min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0);
1803
1804         set_in_bvec(con, &bv);
1805         con->v2.in_enc_i++;
1806         con->v2.in_enc_resid -= bv.bv_len;
1807
1808         if (con->v2.in_enc_resid) {
1809                 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;
1810                 return;
1811         }
1812
1813         /*
1814          * We are set to read the last piece of ciphertext (ending
1815          * with epilogue) + auth tag.
1816          */
1817         WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);
1818         con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1819 }
1820
1821 static int prepare_read_tail_secure(struct ceph_connection *con)
1822 {
1823         struct page **enc_pages;
1824         int enc_page_cnt;
1825         int tail_len;
1826
1827         tail_len = tail_onwire_len(con->in_msg, true);
1828         WARN_ON(!tail_len);
1829
1830         enc_page_cnt = calc_pages_for(0, tail_len);
1831         enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1832         if (IS_ERR(enc_pages))
1833                 return PTR_ERR(enc_pages);
1834
1835         WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);
1836         con->v2.in_enc_pages = enc_pages;
1837         con->v2.in_enc_page_cnt = enc_page_cnt;
1838         con->v2.in_enc_resid = tail_len;
1839         con->v2.in_enc_i = 0;
1840
1841         prepare_read_enc_page(con);
1842         return 0;
1843 }
1844
1845 static void __finish_skip(struct ceph_connection *con)
1846 {
1847         con->in_seq++;
1848         prepare_read_preamble(con);
1849 }
1850
1851 static void prepare_skip_message(struct ceph_connection *con)
1852 {
1853         struct ceph_frame_desc *desc = &con->v2.in_desc;
1854         int tail_len;
1855
1856         dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
1857              desc->fd_lens[2], desc->fd_lens[3]);
1858
1859         tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
1860                                      desc->fd_lens[3], con_secure(con));
1861         if (!tail_len) {
1862                 __finish_skip(con);
1863         } else {
1864                 set_in_skip(con, tail_len);
1865                 con->v2.in_state = IN_S_FINISH_SKIP;
1866         }
1867 }
1868
1869 static int process_banner_prefix(struct ceph_connection *con)
1870 {
1871         int payload_len;
1872         void *p;
1873
1874         WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
1875
1876         p = con->v2.in_kvecs[0].iov_base;
1877         if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
1878                 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
1879                         con->error_msg = "server is speaking msgr1 protocol";
1880                 else
1881                         con->error_msg = "protocol error, bad banner";
1882                 return -EINVAL;
1883         }
1884
1885         p += CEPH_BANNER_V2_LEN;
1886         payload_len = ceph_decode_16(&p);
1887         dout("%s con %p payload_len %d\n", __func__, con, payload_len);
1888
1889         return prepare_read_banner_payload(con, payload_len);
1890 }
1891
1892 static int process_banner_payload(struct ceph_connection *con)
1893 {
1894         void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
1895         u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
1896         u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
1897         u64 server_feat, server_req_feat;
1898         void *p;
1899         int ret;
1900
1901         p = con->v2.in_kvecs[0].iov_base;
1902         ceph_decode_64_safe(&p, end, server_feat, bad);
1903         ceph_decode_64_safe(&p, end, server_req_feat, bad);
1904
1905         dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
1906              __func__, con, server_feat, server_req_feat);
1907
1908         if (req_feat & ~server_feat) {
1909                 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
1910                        server_feat, req_feat & ~server_feat);
1911                 con->error_msg = "missing required protocol features";
1912                 return -EINVAL;
1913         }
1914         if (server_req_feat & ~feat) {
1915                 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
1916                        feat, server_req_feat & ~feat);
1917                 con->error_msg = "missing required protocol features";
1918                 return -EINVAL;
1919         }
1920
1921         /* no reset_out_kvecs() as our banner may still be pending */
1922         ret = prepare_hello(con);
1923         if (ret) {
1924                 pr_err("prepare_hello failed: %d\n", ret);
1925                 return ret;
1926         }
1927
1928         con->state = CEPH_CON_S_V2_HELLO;
1929         prepare_read_preamble(con);
1930         return 0;
1931
1932 bad:
1933         pr_err("failed to decode banner payload\n");
1934         return -EINVAL;
1935 }
1936
1937 static int process_hello(struct ceph_connection *con, void *p, void *end)
1938 {
1939         struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1940         struct ceph_entity_addr addr_for_me;
1941         u8 entity_type;
1942         int ret;
1943
1944         if (con->state != CEPH_CON_S_V2_HELLO) {
1945                 con->error_msg = "protocol error, unexpected hello";
1946                 return -EINVAL;
1947         }
1948
1949         ceph_decode_8_safe(&p, end, entity_type, bad);
1950         ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
1951         if (ret) {
1952                 pr_err("failed to decode addr_for_me: %d\n", ret);
1953                 return ret;
1954         }
1955
1956         dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
1957              entity_type, ceph_pr_addr(&addr_for_me));
1958
1959         if (entity_type != con->peer_name.type) {
1960                 pr_err("bad peer type, want %d, got %d\n",
1961                        con->peer_name.type, entity_type);
1962                 con->error_msg = "wrong peer at address";
1963                 return -EINVAL;
1964         }
1965
1966         /*
1967          * Set our address to the address our first peer (i.e. monitor)
1968          * sees that we are connecting from.  If we are behind some sort
1969          * of NAT and want to be identified by some private (not NATed)
1970          * address, ip option should be used.
1971          */
1972         if (ceph_addr_is_blank(my_addr)) {
1973                 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
1974                        sizeof(my_addr->in_addr));
1975                 ceph_addr_set_port(my_addr, 0);
1976                 dout("%s con %p set my addr %s, as seen by peer %s\n",
1977                      __func__, con, ceph_pr_addr(my_addr),
1978                      ceph_pr_addr(&con->peer_addr));
1979         } else {
1980                 dout("%s con %p my addr already set %s\n",
1981                      __func__, con, ceph_pr_addr(my_addr));
1982         }
1983
1984         WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
1985         WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
1986         WARN_ON(!my_addr->nonce);
1987
1988         /* no reset_out_kvecs() as our hello may still be pending */
1989         ret = prepare_auth_request(con);
1990         if (ret) {
1991                 if (ret != -EAGAIN)
1992                         pr_err("prepare_auth_request failed: %d\n", ret);
1993                 return ret;
1994         }
1995
1996         con->state = CEPH_CON_S_V2_AUTH;
1997         return 0;
1998
1999 bad:
2000         pr_err("failed to decode hello\n");
2001         return -EINVAL;
2002 }
2003
2004 static int process_auth_bad_method(struct ceph_connection *con,
2005                                    void *p, void *end)
2006 {
2007         int allowed_protos[8], allowed_modes[8];
2008         int allowed_proto_cnt, allowed_mode_cnt;
2009         int used_proto, result;
2010         int ret;
2011         int i;
2012
2013         if (con->state != CEPH_CON_S_V2_AUTH) {
2014                 con->error_msg = "protocol error, unexpected auth_bad_method";
2015                 return -EINVAL;
2016         }
2017
2018         ceph_decode_32_safe(&p, end, used_proto, bad);
2019         ceph_decode_32_safe(&p, end, result, bad);
2020         dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
2021              result);
2022
2023         ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
2024         if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
2025                 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
2026                 return -EINVAL;
2027         }
2028         for (i = 0; i < allowed_proto_cnt; i++) {
2029                 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
2030                 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
2031                      i, allowed_protos[i]);
2032         }
2033
2034         ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
2035         if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
2036                 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
2037                 return -EINVAL;
2038         }
2039         for (i = 0; i < allowed_mode_cnt; i++) {
2040                 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
2041                 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
2042                      i, allowed_modes[i]);
2043         }
2044
2045         mutex_unlock(&con->mutex);
2046         ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2047                                                allowed_protos,
2048                                                allowed_proto_cnt,
2049                                                allowed_modes,
2050                                                allowed_mode_cnt);
2051         mutex_lock(&con->mutex);
2052         if (con->state != CEPH_CON_S_V2_AUTH) {
2053                 dout("%s con %p state changed to %d\n", __func__, con,
2054                      con->state);
2055                 return -EAGAIN;
2056         }
2057
2058         dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2059         return ret;
2060
2061 bad:
2062         pr_err("failed to decode auth_bad_method\n");
2063         return -EINVAL;
2064 }
2065
2066 static int process_auth_reply_more(struct ceph_connection *con,
2067                                    void *p, void *end)
2068 {
2069         int payload_len;
2070         int ret;
2071
2072         if (con->state != CEPH_CON_S_V2_AUTH) {
2073                 con->error_msg = "protocol error, unexpected auth_reply_more";
2074                 return -EINVAL;
2075         }
2076
2077         ceph_decode_32_safe(&p, end, payload_len, bad);
2078         ceph_decode_need(&p, end, payload_len, bad);
2079
2080         dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2081
2082         reset_out_kvecs(con);
2083         ret = prepare_auth_request_more(con, p, payload_len);
2084         if (ret) {
2085                 if (ret != -EAGAIN)
2086                         pr_err("prepare_auth_request_more failed: %d\n", ret);
2087                 return ret;
2088         }
2089
2090         return 0;
2091
2092 bad:
2093         pr_err("failed to decode auth_reply_more\n");
2094         return -EINVAL;
2095 }
2096
2097 /*
2098  * Align session_key and con_secret to avoid GFP_ATOMIC allocation
2099  * inside crypto_shash_setkey() and crypto_aead_setkey() called from
2100  * setup_crypto().  __aligned(16) isn't guaranteed to work for stack
2101  * objects, so do it by hand.
2102  */
2103 static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2104 {
2105         u8 session_key_buf[CEPH_KEY_LEN + 16];
2106         u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2107         u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16);
2108         u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
2109         int session_key_len, con_secret_len;
2110         int payload_len;
2111         u64 global_id;
2112         int ret;
2113
2114         if (con->state != CEPH_CON_S_V2_AUTH) {
2115                 con->error_msg = "protocol error, unexpected auth_done";
2116                 return -EINVAL;
2117         }
2118
2119         ceph_decode_64_safe(&p, end, global_id, bad);
2120         ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2121         ceph_decode_32_safe(&p, end, payload_len, bad);
2122
2123         dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2124              __func__, con, global_id, con->v2.con_mode, payload_len);
2125
2126         mutex_unlock(&con->mutex);
2127         session_key_len = 0;
2128         con_secret_len = 0;
2129         ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2130                                          session_key, &session_key_len,
2131                                          con_secret, &con_secret_len);
2132         mutex_lock(&con->mutex);
2133         if (con->state != CEPH_CON_S_V2_AUTH) {
2134                 dout("%s con %p state changed to %d\n", __func__, con,
2135                      con->state);
2136                 ret = -EAGAIN;
2137                 goto out;
2138         }
2139
2140         dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2141         if (ret)
2142                 goto out;
2143
2144         ret = setup_crypto(con, session_key, session_key_len, con_secret,
2145                            con_secret_len);
2146         if (ret)
2147                 goto out;
2148
2149         reset_out_kvecs(con);
2150         ret = prepare_auth_signature(con);
2151         if (ret) {
2152                 pr_err("prepare_auth_signature failed: %d\n", ret);
2153                 goto out;
2154         }
2155
2156         con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2157
2158 out:
2159         memzero_explicit(session_key_buf, sizeof(session_key_buf));
2160         memzero_explicit(con_secret_buf, sizeof(con_secret_buf));
2161         return ret;
2162
2163 bad:
2164         pr_err("failed to decode auth_done\n");
2165         return -EINVAL;
2166 }
2167
2168 static int process_auth_signature(struct ceph_connection *con,
2169                                   void *p, void *end)
2170 {
2171         u8 hmac[SHA256_DIGEST_SIZE];
2172         int ret;
2173
2174         if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2175                 con->error_msg = "protocol error, unexpected auth_signature";
2176                 return -EINVAL;
2177         }
2178
2179         ret = hmac_sha256(con, con->v2.out_sign_kvecs,
2180                           con->v2.out_sign_kvec_cnt, hmac);
2181         if (ret)
2182                 return ret;
2183
2184         ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2185         if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2186                 con->error_msg = "integrity error, bad auth signature";
2187                 return -EBADMSG;
2188         }
2189
2190         dout("%s con %p auth signature ok\n", __func__, con);
2191
2192         /* no reset_out_kvecs() as our auth_signature may still be pending */
2193         if (!con->v2.server_cookie) {
2194                 ret = prepare_client_ident(con);
2195                 if (ret) {
2196                         pr_err("prepare_client_ident failed: %d\n", ret);
2197                         return ret;
2198                 }
2199
2200                 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2201         } else {
2202                 ret = prepare_session_reconnect(con);
2203                 if (ret) {
2204                         pr_err("prepare_session_reconnect failed: %d\n", ret);
2205                         return ret;
2206                 }
2207
2208                 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2209         }
2210
2211         return 0;
2212
2213 bad:
2214         pr_err("failed to decode auth_signature\n");
2215         return -EINVAL;
2216 }
2217
2218 static int process_server_ident(struct ceph_connection *con,
2219                                 void *p, void *end)
2220 {
2221         struct ceph_client *client = from_msgr(con->msgr);
2222         u64 features, required_features;
2223         struct ceph_entity_addr addr;
2224         u64 global_seq;
2225         u64 global_id;
2226         u64 cookie;
2227         u64 flags;
2228         int ret;
2229
2230         if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2231                 con->error_msg = "protocol error, unexpected server_ident";
2232                 return -EINVAL;
2233         }
2234
2235         ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2236         if (ret) {
2237                 pr_err("failed to decode server addrs: %d\n", ret);
2238                 return ret;
2239         }
2240
2241         ceph_decode_64_safe(&p, end, global_id, bad);
2242         ceph_decode_64_safe(&p, end, global_seq, bad);
2243         ceph_decode_64_safe(&p, end, features, bad);
2244         ceph_decode_64_safe(&p, end, required_features, bad);
2245         ceph_decode_64_safe(&p, end, flags, bad);
2246         ceph_decode_64_safe(&p, end, cookie, bad);
2247
2248         dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2249              __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2250              global_id, global_seq, features, required_features, flags, cookie);
2251
2252         /* is this who we intended to talk to? */
2253         if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2254                 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2255                        ceph_pr_addr(&con->peer_addr),
2256                        le32_to_cpu(con->peer_addr.nonce),
2257                        ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2258                 con->error_msg = "wrong peer at address";
2259                 return -EINVAL;
2260         }
2261
2262         if (client->required_features & ~features) {
2263                 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2264                        features, client->required_features & ~features);
2265                 con->error_msg = "missing required protocol features";
2266                 return -EINVAL;
2267         }
2268
2269         /*
2270          * Both name->type and name->num are set in ceph_con_open() but
2271          * name->num may be bogus in the initial monmap.  name->type is
2272          * verified in handle_hello().
2273          */
2274         WARN_ON(!con->peer_name.type);
2275         con->peer_name.num = cpu_to_le64(global_id);
2276         con->v2.peer_global_seq = global_seq;
2277         con->peer_features = features;
2278         WARN_ON(required_features & ~client->supported_features);
2279         con->v2.server_cookie = cookie;
2280
2281         if (flags & CEPH_MSG_CONNECT_LOSSY) {
2282                 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2283                 WARN_ON(con->v2.server_cookie);
2284         } else {
2285                 WARN_ON(!con->v2.server_cookie);
2286         }
2287
2288         clear_in_sign_kvecs(con);
2289         clear_out_sign_kvecs(con);
2290         free_conn_bufs(con);
2291         con->delay = 0;  /* reset backoff memory */
2292
2293         con->state = CEPH_CON_S_OPEN;
2294         con->v2.out_state = OUT_S_GET_NEXT;
2295         return 0;
2296
2297 bad:
2298         pr_err("failed to decode server_ident\n");
2299         return -EINVAL;
2300 }
2301
2302 static int process_ident_missing_features(struct ceph_connection *con,
2303                                           void *p, void *end)
2304 {
2305         struct ceph_client *client = from_msgr(con->msgr);
2306         u64 missing_features;
2307
2308         if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2309                 con->error_msg = "protocol error, unexpected ident_missing_features";
2310                 return -EINVAL;
2311         }
2312
2313         ceph_decode_64_safe(&p, end, missing_features, bad);
2314         pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2315                client->supported_features, missing_features);
2316         con->error_msg = "missing required protocol features";
2317         return -EINVAL;
2318
2319 bad:
2320         pr_err("failed to decode ident_missing_features\n");
2321         return -EINVAL;
2322 }
2323
2324 static int process_session_reconnect_ok(struct ceph_connection *con,
2325                                         void *p, void *end)
2326 {
2327         u64 seq;
2328
2329         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2330                 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2331                 return -EINVAL;
2332         }
2333
2334         ceph_decode_64_safe(&p, end, seq, bad);
2335
2336         dout("%s con %p seq %llu\n", __func__, con, seq);
2337         ceph_con_discard_requeued(con, seq);
2338
2339         clear_in_sign_kvecs(con);
2340         clear_out_sign_kvecs(con);
2341         free_conn_bufs(con);
2342         con->delay = 0;  /* reset backoff memory */
2343
2344         con->state = CEPH_CON_S_OPEN;
2345         con->v2.out_state = OUT_S_GET_NEXT;
2346         return 0;
2347
2348 bad:
2349         pr_err("failed to decode session_reconnect_ok\n");
2350         return -EINVAL;
2351 }
2352
2353 static int process_session_retry(struct ceph_connection *con,
2354                                  void *p, void *end)
2355 {
2356         u64 connect_seq;
2357         int ret;
2358
2359         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2360                 con->error_msg = "protocol error, unexpected session_retry";
2361                 return -EINVAL;
2362         }
2363
2364         ceph_decode_64_safe(&p, end, connect_seq, bad);
2365
2366         dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2367         WARN_ON(connect_seq <= con->v2.connect_seq);
2368         con->v2.connect_seq = connect_seq + 1;
2369
2370         free_conn_bufs(con);
2371
2372         reset_out_kvecs(con);
2373         ret = prepare_session_reconnect(con);
2374         if (ret) {
2375                 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2376                 return ret;
2377         }
2378
2379         return 0;
2380
2381 bad:
2382         pr_err("failed to decode session_retry\n");
2383         return -EINVAL;
2384 }
2385
2386 static int process_session_retry_global(struct ceph_connection *con,
2387                                         void *p, void *end)
2388 {
2389         u64 global_seq;
2390         int ret;
2391
2392         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2393                 con->error_msg = "protocol error, unexpected session_retry_global";
2394                 return -EINVAL;
2395         }
2396
2397         ceph_decode_64_safe(&p, end, global_seq, bad);
2398
2399         dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2400         WARN_ON(global_seq <= con->v2.global_seq);
2401         con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2402
2403         free_conn_bufs(con);
2404
2405         reset_out_kvecs(con);
2406         ret = prepare_session_reconnect(con);
2407         if (ret) {
2408                 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2409                 return ret;
2410         }
2411
2412         return 0;
2413
2414 bad:
2415         pr_err("failed to decode session_retry_global\n");
2416         return -EINVAL;
2417 }
2418
2419 static int process_session_reset(struct ceph_connection *con,
2420                                  void *p, void *end)
2421 {
2422         bool full;
2423         int ret;
2424
2425         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2426                 con->error_msg = "protocol error, unexpected session_reset";
2427                 return -EINVAL;
2428         }
2429
2430         ceph_decode_8_safe(&p, end, full, bad);
2431         if (!full) {
2432                 con->error_msg = "protocol error, bad session_reset";
2433                 return -EINVAL;
2434         }
2435
2436         pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2437                 ceph_pr_addr(&con->peer_addr));
2438         ceph_con_reset_session(con);
2439
2440         mutex_unlock(&con->mutex);
2441         if (con->ops->peer_reset)
2442                 con->ops->peer_reset(con);
2443         mutex_lock(&con->mutex);
2444         if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2445                 dout("%s con %p state changed to %d\n", __func__, con,
2446                      con->state);
2447                 return -EAGAIN;
2448         }
2449
2450         free_conn_bufs(con);
2451
2452         reset_out_kvecs(con);
2453         ret = prepare_client_ident(con);
2454         if (ret) {
2455                 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2456                 return ret;
2457         }
2458
2459         con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2460         return 0;
2461
2462 bad:
2463         pr_err("failed to decode session_reset\n");
2464         return -EINVAL;
2465 }
2466
2467 static int process_keepalive2_ack(struct ceph_connection *con,
2468                                   void *p, void *end)
2469 {
2470         if (con->state != CEPH_CON_S_OPEN) {
2471                 con->error_msg = "protocol error, unexpected keepalive2_ack";
2472                 return -EINVAL;
2473         }
2474
2475         ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2476         ceph_decode_timespec64(&con->last_keepalive_ack, p);
2477
2478         dout("%s con %p timestamp %lld.%09ld\n", __func__, con,
2479              con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);
2480
2481         return 0;
2482
2483 bad:
2484         pr_err("failed to decode keepalive2_ack\n");
2485         return -EINVAL;
2486 }
2487
2488 static int process_ack(struct ceph_connection *con, void *p, void *end)
2489 {
2490         u64 seq;
2491
2492         if (con->state != CEPH_CON_S_OPEN) {
2493                 con->error_msg = "protocol error, unexpected ack";
2494                 return -EINVAL;
2495         }
2496
2497         ceph_decode_64_safe(&p, end, seq, bad);
2498
2499         dout("%s con %p seq %llu\n", __func__, con, seq);
2500         ceph_con_discard_sent(con, seq);
2501         return 0;
2502
2503 bad:
2504         pr_err("failed to decode ack\n");
2505         return -EINVAL;
2506 }
2507
2508 static int process_control(struct ceph_connection *con, void *p, void *end)
2509 {
2510         int tag = con->v2.in_desc.fd_tag;
2511         int ret;
2512
2513         dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2514
2515         switch (tag) {
2516         case FRAME_TAG_HELLO:
2517                 ret = process_hello(con, p, end);
2518                 break;
2519         case FRAME_TAG_AUTH_BAD_METHOD:
2520                 ret = process_auth_bad_method(con, p, end);
2521                 break;
2522         case FRAME_TAG_AUTH_REPLY_MORE:
2523                 ret = process_auth_reply_more(con, p, end);
2524                 break;
2525         case FRAME_TAG_AUTH_DONE:
2526                 ret = process_auth_done(con, p, end);
2527                 break;
2528         case FRAME_TAG_AUTH_SIGNATURE:
2529                 ret = process_auth_signature(con, p, end);
2530                 break;
2531         case FRAME_TAG_SERVER_IDENT:
2532                 ret = process_server_ident(con, p, end);
2533                 break;
2534         case FRAME_TAG_IDENT_MISSING_FEATURES:
2535                 ret = process_ident_missing_features(con, p, end);
2536                 break;
2537         case FRAME_TAG_SESSION_RECONNECT_OK:
2538                 ret = process_session_reconnect_ok(con, p, end);
2539                 break;
2540         case FRAME_TAG_SESSION_RETRY:
2541                 ret = process_session_retry(con, p, end);
2542                 break;
2543         case FRAME_TAG_SESSION_RETRY_GLOBAL:
2544                 ret = process_session_retry_global(con, p, end);
2545                 break;
2546         case FRAME_TAG_SESSION_RESET:
2547                 ret = process_session_reset(con, p, end);
2548                 break;
2549         case FRAME_TAG_KEEPALIVE2_ACK:
2550                 ret = process_keepalive2_ack(con, p, end);
2551                 break;
2552         case FRAME_TAG_ACK:
2553                 ret = process_ack(con, p, end);
2554                 break;
2555         default:
2556                 pr_err("bad tag %d\n", tag);
2557                 con->error_msg = "protocol error, bad tag";
2558                 return -EINVAL;
2559         }
2560         if (ret) {
2561                 dout("%s con %p error %d\n", __func__, con, ret);
2562                 return ret;
2563         }
2564
2565         prepare_read_preamble(con);
2566         return 0;
2567 }
2568
2569 /*
2570  * Return:
2571  *   1 - con->in_msg set, read message
2572  *   0 - skip message
2573  *  <0 - error
2574  */
2575 static int process_message_header(struct ceph_connection *con,
2576                                   void *p, void *end)
2577 {
2578         struct ceph_frame_desc *desc = &con->v2.in_desc;
2579         struct ceph_msg_header2 *hdr2 = p;
2580         struct ceph_msg_header hdr;
2581         int skip;
2582         int ret;
2583         u64 seq;
2584
2585         /* verify seq# */
2586         seq = le64_to_cpu(hdr2->seq);
2587         if ((s64)seq - (s64)con->in_seq < 1) {
2588                 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2589                         ENTITY_NAME(con->peer_name),
2590                         ceph_pr_addr(&con->peer_addr),
2591                         seq, con->in_seq + 1);
2592                 return 0;
2593         }
2594         if ((s64)seq - (s64)con->in_seq > 1) {
2595                 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2596                 con->error_msg = "bad message sequence # for incoming message";
2597                 return -EBADE;
2598         }
2599
2600         ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2601
2602         fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2603                     desc->fd_lens[3], &con->peer_name);
2604         ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2605         if (ret)
2606                 return ret;
2607
2608         WARN_ON(!con->in_msg ^ skip);
2609         if (skip)
2610                 return 0;
2611
2612         WARN_ON(!con->in_msg);
2613         WARN_ON(con->in_msg->con != con);
2614         return 1;
2615 }
2616
2617 static int process_message(struct ceph_connection *con)
2618 {
2619         ceph_con_process_message(con);
2620
2621         /*
2622          * We could have been closed by ceph_con_close() because
2623          * ceph_con_process_message() temporarily drops con->mutex.
2624          */
2625         if (con->state != CEPH_CON_S_OPEN) {
2626                 dout("%s con %p state changed to %d\n", __func__, con,
2627                      con->state);
2628                 return -EAGAIN;
2629         }
2630
2631         prepare_read_preamble(con);
2632         return 0;
2633 }
2634
2635 static int __handle_control(struct ceph_connection *con, void *p)
2636 {
2637         void *end = p + con->v2.in_desc.fd_lens[0];
2638         struct ceph_msg *msg;
2639         int ret;
2640
2641         if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2642                 return process_control(con, p, end);
2643
2644         ret = process_message_header(con, p, end);
2645         if (ret < 0)
2646                 return ret;
2647         if (ret == 0) {
2648                 prepare_skip_message(con);
2649                 return 0;
2650         }
2651
2652         msg = con->in_msg;  /* set in process_message_header() */
2653         if (front_len(msg)) {
2654                 WARN_ON(front_len(msg) > msg->front_alloc_len);
2655                 msg->front.iov_len = front_len(msg);
2656         } else {
2657                 msg->front.iov_len = 0;
2658         }
2659         if (middle_len(msg)) {
2660                 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2661                 msg->middle->vec.iov_len = middle_len(msg);
2662         } else if (msg->middle) {
2663                 msg->middle->vec.iov_len = 0;
2664         }
2665
2666         if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
2667                 return process_message(con);
2668
2669         if (con_secure(con))
2670                 return prepare_read_tail_secure(con);
2671
2672         return prepare_read_tail_plain(con);
2673 }
2674
2675 static int handle_preamble(struct ceph_connection *con)
2676 {
2677         struct ceph_frame_desc *desc = &con->v2.in_desc;
2678         int ret;
2679
2680         if (con_secure(con)) {
2681                 ret = decrypt_preamble(con);
2682                 if (ret) {
2683                         if (ret == -EBADMSG)
2684                                 con->error_msg = "integrity error, bad preamble auth tag";
2685                         return ret;
2686                 }
2687         }
2688
2689         ret = decode_preamble(con->v2.in_buf, desc);
2690         if (ret) {
2691                 if (ret == -EBADMSG)
2692                         con->error_msg = "integrity error, bad crc";
2693                 else
2694                         con->error_msg = "protocol error, bad preamble";
2695                 return ret;
2696         }
2697
2698         dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2699              con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2700              desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2701
2702         if (!con_secure(con))
2703                 return prepare_read_control(con);
2704
2705         if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2706                 return prepare_read_control_remainder(con);
2707
2708         return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2709 }
2710
2711 static int handle_control(struct ceph_connection *con)
2712 {
2713         int ctrl_len = con->v2.in_desc.fd_lens[0];
2714         void *buf;
2715         int ret;
2716
2717         WARN_ON(con_secure(con));
2718
2719         ret = verify_control_crc(con);
2720         if (ret) {
2721                 con->error_msg = "integrity error, bad crc";
2722                 return ret;
2723         }
2724
2725         if (con->state == CEPH_CON_S_V2_AUTH) {
2726                 buf = alloc_conn_buf(con, ctrl_len);
2727                 if (!buf)
2728                         return -ENOMEM;
2729
2730                 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
2731                 return __handle_control(con, buf);
2732         }
2733
2734         return __handle_control(con, con->v2.in_kvecs[0].iov_base);
2735 }
2736
2737 static int handle_control_remainder(struct ceph_connection *con)
2738 {
2739         int ret;
2740
2741         WARN_ON(!con_secure(con));
2742
2743         ret = decrypt_control_remainder(con);
2744         if (ret) {
2745                 if (ret == -EBADMSG)
2746                         con->error_msg = "integrity error, bad control remainder auth tag";
2747                 return ret;
2748         }
2749
2750         return __handle_control(con, con->v2.in_kvecs[0].iov_base -
2751                                      CEPH_PREAMBLE_INLINE_LEN);
2752 }
2753
2754 static int handle_epilogue(struct ceph_connection *con)
2755 {
2756         u32 front_crc, middle_crc, data_crc;
2757         int ret;
2758
2759         if (con_secure(con)) {
2760                 ret = decrypt_tail(con);
2761                 if (ret) {
2762                         if (ret == -EBADMSG)
2763                                 con->error_msg = "integrity error, bad epilogue auth tag";
2764                         return ret;
2765                 }
2766
2767                 /* just late_status */
2768                 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
2769                 if (ret) {
2770                         con->error_msg = "protocol error, bad epilogue";
2771                         return ret;
2772                 }
2773         } else {
2774                 ret = decode_epilogue(con->v2.in_buf, &front_crc,
2775                                       &middle_crc, &data_crc);
2776                 if (ret) {
2777                         con->error_msg = "protocol error, bad epilogue";
2778                         return ret;
2779                 }
2780
2781                 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
2782                                            data_crc);
2783                 if (ret) {
2784                         con->error_msg = "integrity error, bad crc";
2785                         return ret;
2786                 }
2787         }
2788
2789         return process_message(con);
2790 }
2791
2792 static void finish_skip(struct ceph_connection *con)
2793 {
2794         dout("%s con %p\n", __func__, con);
2795
2796         if (con_secure(con))
2797                 gcm_inc_nonce(&con->v2.in_gcm_nonce);
2798
2799         __finish_skip(con);
2800 }
2801
2802 static int populate_in_iter(struct ceph_connection *con)
2803 {
2804         int ret;
2805
2806         dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
2807              con->v2.in_state);
2808         WARN_ON(iov_iter_count(&con->v2.in_iter));
2809
2810         if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
2811                 ret = process_banner_prefix(con);
2812         } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
2813                 ret = process_banner_payload(con);
2814         } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
2815                     con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
2816                    con->state == CEPH_CON_S_OPEN) {
2817                 switch (con->v2.in_state) {
2818                 case IN_S_HANDLE_PREAMBLE:
2819                         ret = handle_preamble(con);
2820                         break;
2821                 case IN_S_HANDLE_CONTROL:
2822                         ret = handle_control(con);
2823                         break;
2824                 case IN_S_HANDLE_CONTROL_REMAINDER:
2825                         ret = handle_control_remainder(con);
2826                         break;
2827                 case IN_S_PREPARE_READ_DATA:
2828                         ret = prepare_read_data(con);
2829                         break;
2830                 case IN_S_PREPARE_READ_DATA_CONT:
2831                         prepare_read_data_cont(con);
2832                         ret = 0;
2833                         break;
2834                 case IN_S_PREPARE_READ_ENC_PAGE:
2835                         prepare_read_enc_page(con);
2836                         ret = 0;
2837                         break;
2838                 case IN_S_HANDLE_EPILOGUE:
2839                         ret = handle_epilogue(con);
2840                         break;
2841                 case IN_S_FINISH_SKIP:
2842                         finish_skip(con);
2843                         ret = 0;
2844                         break;
2845                 default:
2846                         WARN(1, "bad in_state %d", con->v2.in_state);
2847                         return -EINVAL;
2848                 }
2849         } else {
2850                 WARN(1, "bad state %d", con->state);
2851                 return -EINVAL;
2852         }
2853         if (ret) {
2854                 dout("%s con %p error %d\n", __func__, con, ret);
2855                 return ret;
2856         }
2857
2858         if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2859                 return -ENODATA;
2860         dout("%s con %p populated %zu\n", __func__, con,
2861              iov_iter_count(&con->v2.in_iter));
2862         return 1;
2863 }
2864
2865 int ceph_con_v2_try_read(struct ceph_connection *con)
2866 {
2867         int ret;
2868
2869         dout("%s con %p state %d need %zu\n", __func__, con, con->state,
2870              iov_iter_count(&con->v2.in_iter));
2871
2872         if (con->state == CEPH_CON_S_PREOPEN)
2873                 return 0;
2874
2875         /*
2876          * We should always have something pending here.  If not,
2877          * avoid calling populate_in_iter() as if we read something
2878          * (ceph_tcp_recv() would immediately return 1).
2879          */
2880         if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
2881                 return -ENODATA;
2882
2883         for (;;) {
2884                 ret = ceph_tcp_recv(con);
2885                 if (ret <= 0)
2886                         return ret;
2887
2888                 ret = populate_in_iter(con);
2889                 if (ret <= 0) {
2890                         if (ret && ret != -EAGAIN && !con->error_msg)
2891                                 con->error_msg = "read processing error";
2892                         return ret;
2893                 }
2894         }
2895 }
2896
2897 static void queue_data(struct ceph_connection *con)
2898 {
2899         struct bio_vec bv;
2900
2901         con->v2.out_epil.data_crc = -1;
2902         ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,
2903                                   data_len(con->out_msg));
2904
2905         get_bvec_at(&con->v2.out_cursor, &bv);
2906         set_out_bvec(con, &bv, true);
2907         con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
2908 }
2909
2910 static void queue_data_cont(struct ceph_connection *con)
2911 {
2912         struct bio_vec bv;
2913
2914         con->v2.out_epil.data_crc = ceph_crc32c_page(
2915                 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
2916                 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
2917
2918         ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
2919         if (con->v2.out_cursor.total_resid) {
2920                 get_bvec_at(&con->v2.out_cursor, &bv);
2921                 set_out_bvec(con, &bv, true);
2922                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
2923                 return;
2924         }
2925
2926         /*
2927          * We've written all data.  Queue epilogue.  Once it's written,
2928          * we are done.
2929          */
2930         reset_out_kvecs(con);
2931         prepare_epilogue_plain(con, false);
2932         con->v2.out_state = OUT_S_FINISH_MESSAGE;
2933 }
2934
2935 static void queue_enc_page(struct ceph_connection *con)
2936 {
2937         struct bio_vec bv;
2938
2939         dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
2940              con->v2.out_enc_resid);
2941         WARN_ON(!con->v2.out_enc_resid);
2942
2943         bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i],
2944                       min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0);
2945
2946         set_out_bvec(con, &bv, false);
2947         con->v2.out_enc_i++;
2948         con->v2.out_enc_resid -= bv.bv_len;
2949
2950         if (con->v2.out_enc_resid) {
2951                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
2952                 return;
2953         }
2954
2955         /*
2956          * We've queued the last piece of ciphertext (ending with
2957          * epilogue) + auth tag.  Once it's written, we are done.
2958          */
2959         WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
2960         con->v2.out_state = OUT_S_FINISH_MESSAGE;
2961 }
2962
2963 static void queue_zeros(struct ceph_connection *con)
2964 {
2965         dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
2966
2967         if (con->v2.out_zero) {
2968                 set_out_bvec_zero(con);
2969                 con->v2.out_zero -= con->v2.out_bvec.bv_len;
2970                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
2971                 return;
2972         }
2973
2974         /*
2975          * We've zero-filled everything up to epilogue.  Queue epilogue
2976          * with late_status set to ABORTED and crcs adjusted for zeros.
2977          * Once it's written, we are done patching up for the revoke.
2978          */
2979         reset_out_kvecs(con);
2980         prepare_epilogue_plain(con, true);
2981         con->v2.out_state = OUT_S_FINISH_MESSAGE;
2982 }
2983
2984 static void finish_message(struct ceph_connection *con)
2985 {
2986         dout("%s con %p msg %p\n", __func__, con, con->out_msg);
2987
2988         /* we end up here both plain and secure modes */
2989         if (con->v2.out_enc_pages) {
2990                 WARN_ON(!con->v2.out_enc_page_cnt);
2991                 ceph_release_page_vector(con->v2.out_enc_pages,
2992                                          con->v2.out_enc_page_cnt);
2993                 con->v2.out_enc_pages = NULL;
2994                 con->v2.out_enc_page_cnt = 0;
2995         }
2996         /* message may have been revoked */
2997         if (con->out_msg) {
2998                 ceph_msg_put(con->out_msg);
2999                 con->out_msg = NULL;
3000         }
3001
3002         con->v2.out_state = OUT_S_GET_NEXT;
3003 }
3004
3005 static int populate_out_iter(struct ceph_connection *con)
3006 {
3007         int ret;
3008
3009         dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
3010              con->v2.out_state);
3011         WARN_ON(iov_iter_count(&con->v2.out_iter));
3012
3013         if (con->state != CEPH_CON_S_OPEN) {
3014                 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
3015                         con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
3016                 goto nothing_pending;
3017         }
3018
3019         switch (con->v2.out_state) {
3020         case OUT_S_QUEUE_DATA:
3021                 WARN_ON(!con->out_msg);
3022                 queue_data(con);
3023                 goto populated;
3024         case OUT_S_QUEUE_DATA_CONT:
3025                 WARN_ON(!con->out_msg);
3026                 queue_data_cont(con);
3027                 goto populated;
3028         case OUT_S_QUEUE_ENC_PAGE:
3029                 queue_enc_page(con);
3030                 goto populated;
3031         case OUT_S_QUEUE_ZEROS:
3032                 WARN_ON(con->out_msg);  /* revoked */
3033                 queue_zeros(con);
3034                 goto populated;
3035         case OUT_S_FINISH_MESSAGE:
3036                 finish_message(con);
3037                 break;
3038         case OUT_S_GET_NEXT:
3039                 break;
3040         default:
3041                 WARN(1, "bad out_state %d", con->v2.out_state);
3042                 return -EINVAL;
3043         }
3044
3045         WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3046         if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3047                 ret = prepare_keepalive2(con);
3048                 if (ret) {
3049                         pr_err("prepare_keepalive2 failed: %d\n", ret);
3050                         return ret;
3051                 }
3052         } else if (!list_empty(&con->out_queue)) {
3053                 ceph_con_get_out_msg(con);
3054                 ret = prepare_message(con);
3055                 if (ret) {
3056                         pr_err("prepare_message failed: %d\n", ret);
3057                         return ret;
3058                 }
3059         } else if (con->in_seq > con->in_seq_acked) {
3060                 ret = prepare_ack(con);
3061                 if (ret) {
3062                         pr_err("prepare_ack failed: %d\n", ret);
3063                         return ret;
3064                 }
3065         } else {
3066                 goto nothing_pending;
3067         }
3068
3069 populated:
3070         if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3071                 return -ENODATA;
3072         dout("%s con %p populated %zu\n", __func__, con,
3073              iov_iter_count(&con->v2.out_iter));
3074         return 1;
3075
3076 nothing_pending:
3077         WARN_ON(iov_iter_count(&con->v2.out_iter));
3078         dout("%s con %p nothing pending\n", __func__, con);
3079         ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3080         return 0;
3081 }
3082
3083 int ceph_con_v2_try_write(struct ceph_connection *con)
3084 {
3085         int ret;
3086
3087         dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3088              iov_iter_count(&con->v2.out_iter));
3089
3090         /* open the socket first? */
3091         if (con->state == CEPH_CON_S_PREOPEN) {
3092                 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3093
3094                 /*
3095                  * Always bump global_seq.  Bump connect_seq only if
3096                  * there is a session (i.e. we are reconnecting and will
3097                  * send session_reconnect instead of client_ident).
3098                  */
3099                 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3100                 if (con->v2.server_cookie)
3101                         con->v2.connect_seq++;
3102
3103                 ret = prepare_read_banner_prefix(con);
3104                 if (ret) {
3105                         pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3106                         con->error_msg = "connect error";
3107                         return ret;
3108                 }
3109
3110                 reset_out_kvecs(con);
3111                 ret = prepare_banner(con);
3112                 if (ret) {
3113                         pr_err("prepare_banner failed: %d\n", ret);
3114                         con->error_msg = "connect error";
3115                         return ret;
3116                 }
3117
3118                 ret = ceph_tcp_connect(con);
3119                 if (ret) {
3120                         pr_err("ceph_tcp_connect failed: %d\n", ret);
3121                         con->error_msg = "connect error";
3122                         return ret;
3123                 }
3124         }
3125
3126         if (!iov_iter_count(&con->v2.out_iter)) {
3127                 ret = populate_out_iter(con);
3128                 if (ret <= 0) {
3129                         if (ret && ret != -EAGAIN && !con->error_msg)
3130                                 con->error_msg = "write processing error";
3131                         return ret;
3132                 }
3133         }
3134
3135         tcp_sock_set_cork(con->sock->sk, true);
3136         for (;;) {
3137                 ret = ceph_tcp_send(con);
3138                 if (ret <= 0)
3139                         break;
3140
3141                 ret = populate_out_iter(con);
3142                 if (ret <= 0) {
3143                         if (ret && ret != -EAGAIN && !con->error_msg)
3144                                 con->error_msg = "write processing error";
3145                         break;
3146                 }
3147         }
3148
3149         tcp_sock_set_cork(con->sock->sk, false);
3150         return ret;
3151 }
3152
3153 static u32 crc32c_zeros(u32 crc, int zero_len)
3154 {
3155         int len;
3156
3157         while (zero_len) {
3158                 len = min(zero_len, (int)PAGE_SIZE);
3159                 crc = crc32c(crc, page_address(ceph_zero_page), len);
3160                 zero_len -= len;
3161         }
3162
3163         return crc;
3164 }
3165
3166 static void prepare_zero_front(struct ceph_connection *con, int resid)
3167 {
3168         int sent;
3169
3170         WARN_ON(!resid || resid > front_len(con->out_msg));
3171         sent = front_len(con->out_msg) - resid;
3172         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3173
3174         if (sent) {
3175                 con->v2.out_epil.front_crc =
3176                         crc32c(-1, con->out_msg->front.iov_base, sent);
3177                 con->v2.out_epil.front_crc =
3178                         crc32c_zeros(con->v2.out_epil.front_crc, resid);
3179         } else {
3180                 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3181         }
3182
3183         con->v2.out_iter.count -= resid;
3184         out_zero_add(con, resid);
3185 }
3186
3187 static void prepare_zero_middle(struct ceph_connection *con, int resid)
3188 {
3189         int sent;
3190
3191         WARN_ON(!resid || resid > middle_len(con->out_msg));
3192         sent = middle_len(con->out_msg) - resid;
3193         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3194
3195         if (sent) {
3196                 con->v2.out_epil.middle_crc =
3197                         crc32c(-1, con->out_msg->middle->vec.iov_base, sent);
3198                 con->v2.out_epil.middle_crc =
3199                         crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3200         } else {
3201                 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3202         }
3203
3204         con->v2.out_iter.count -= resid;
3205         out_zero_add(con, resid);
3206 }
3207
3208 static void prepare_zero_data(struct ceph_connection *con)
3209 {
3210         dout("%s con %p\n", __func__, con);
3211         con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));
3212         out_zero_add(con, data_len(con->out_msg));
3213 }
3214
3215 static void revoke_at_queue_data(struct ceph_connection *con)
3216 {
3217         int boundary;
3218         int resid;
3219
3220         WARN_ON(!data_len(con->out_msg));
3221         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3222         resid = iov_iter_count(&con->v2.out_iter);
3223
3224         boundary = front_len(con->out_msg) + middle_len(con->out_msg);
3225         if (resid > boundary) {
3226                 resid -= boundary;
3227                 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3228                 dout("%s con %p was sending head\n", __func__, con);
3229                 if (front_len(con->out_msg))
3230                         prepare_zero_front(con, front_len(con->out_msg));
3231                 if (middle_len(con->out_msg))
3232                         prepare_zero_middle(con, middle_len(con->out_msg));
3233                 prepare_zero_data(con);
3234                 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3235                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3236                 return;
3237         }
3238
3239         boundary = middle_len(con->out_msg);
3240         if (resid > boundary) {
3241                 resid -= boundary;
3242                 dout("%s con %p was sending front\n", __func__, con);
3243                 prepare_zero_front(con, resid);
3244                 if (middle_len(con->out_msg))
3245                         prepare_zero_middle(con, middle_len(con->out_msg));
3246                 prepare_zero_data(con);
3247                 queue_zeros(con);
3248                 return;
3249         }
3250
3251         WARN_ON(!resid);
3252         dout("%s con %p was sending middle\n", __func__, con);
3253         prepare_zero_middle(con, resid);
3254         prepare_zero_data(con);
3255         queue_zeros(con);
3256 }
3257
3258 static void revoke_at_queue_data_cont(struct ceph_connection *con)
3259 {
3260         int sent, resid;  /* current piece of data */
3261
3262         WARN_ON(!data_len(con->out_msg));
3263         WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3264         resid = iov_iter_count(&con->v2.out_iter);
3265         WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3266         sent = con->v2.out_bvec.bv_len - resid;
3267         dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3268
3269         if (sent) {
3270                 con->v2.out_epil.data_crc = ceph_crc32c_page(
3271                         con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3272                         con->v2.out_bvec.bv_offset, sent);
3273                 ceph_msg_data_advance(&con->v2.out_cursor, sent);
3274         }
3275         WARN_ON(resid > con->v2.out_cursor.total_resid);
3276         con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3277                                                 con->v2.out_cursor.total_resid);
3278
3279         con->v2.out_iter.count -= resid;
3280         out_zero_add(con, con->v2.out_cursor.total_resid);
3281         queue_zeros(con);
3282 }
3283
3284 static void revoke_at_finish_message(struct ceph_connection *con)
3285 {
3286         int boundary;
3287         int resid;
3288
3289         WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3290         resid = iov_iter_count(&con->v2.out_iter);
3291
3292         if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&
3293             !data_len(con->out_msg)) {
3294                 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3295                 dout("%s con %p was sending head (empty message) - noop\n",
3296                      __func__, con);
3297                 return;
3298         }
3299
3300         boundary = front_len(con->out_msg) + middle_len(con->out_msg) +
3301                    CEPH_EPILOGUE_PLAIN_LEN;
3302         if (resid > boundary) {
3303                 resid -= boundary;
3304                 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3305                 dout("%s con %p was sending head\n", __func__, con);
3306                 if (front_len(con->out_msg))
3307                         prepare_zero_front(con, front_len(con->out_msg));
3308                 if (middle_len(con->out_msg))
3309                         prepare_zero_middle(con, middle_len(con->out_msg));
3310                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3311                 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3312                 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3313                 return;
3314         }
3315
3316         boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3317         if (resid > boundary) {
3318                 resid -= boundary;
3319                 dout("%s con %p was sending front\n", __func__, con);
3320                 prepare_zero_front(con, resid);
3321                 if (middle_len(con->out_msg))
3322                         prepare_zero_middle(con, middle_len(con->out_msg));
3323                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3324                 queue_zeros(con);
3325                 return;
3326         }
3327
3328         boundary = CEPH_EPILOGUE_PLAIN_LEN;
3329         if (resid > boundary) {
3330                 resid -= boundary;
3331                 dout("%s con %p was sending middle\n", __func__, con);
3332                 prepare_zero_middle(con, resid);
3333                 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3334                 queue_zeros(con);
3335                 return;
3336         }
3337
3338         WARN_ON(!resid);
3339         dout("%s con %p was sending epilogue - noop\n", __func__, con);
3340 }
3341
3342 void ceph_con_v2_revoke(struct ceph_connection *con)
3343 {
3344         WARN_ON(con->v2.out_zero);
3345
3346         if (con_secure(con)) {
3347                 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3348                         con->v2.out_state != OUT_S_FINISH_MESSAGE);
3349                 dout("%s con %p secure - noop\n", __func__, con);
3350                 return;
3351         }
3352
3353         switch (con->v2.out_state) {
3354         case OUT_S_QUEUE_DATA:
3355                 revoke_at_queue_data(con);
3356                 break;
3357         case OUT_S_QUEUE_DATA_CONT:
3358                 revoke_at_queue_data_cont(con);
3359                 break;
3360         case OUT_S_FINISH_MESSAGE:
3361                 revoke_at_finish_message(con);
3362                 break;
3363         default:
3364                 WARN(1, "bad out_state %d", con->v2.out_state);
3365                 break;
3366         }
3367 }
3368
3369 static void revoke_at_prepare_read_data(struct ceph_connection *con)
3370 {
3371         int remaining;
3372         int resid;
3373
3374         WARN_ON(con_secure(con));
3375         WARN_ON(!data_len(con->in_msg));
3376         WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3377         resid = iov_iter_count(&con->v2.in_iter);
3378         WARN_ON(!resid);
3379
3380         remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3381         dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3382              remaining);
3383         con->v2.in_iter.count -= resid;
3384         set_in_skip(con, resid + remaining);
3385         con->v2.in_state = IN_S_FINISH_SKIP;
3386 }
3387
3388 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3389 {
3390         int recved, resid;  /* current piece of data */
3391         int remaining;
3392
3393         WARN_ON(con_secure(con));
3394         WARN_ON(!data_len(con->in_msg));
3395         WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3396         resid = iov_iter_count(&con->v2.in_iter);
3397         WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3398         recved = con->v2.in_bvec.bv_len - resid;
3399         dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3400
3401         if (recved)
3402                 ceph_msg_data_advance(&con->v2.in_cursor, recved);
3403         WARN_ON(resid > con->v2.in_cursor.total_resid);
3404
3405         remaining = CEPH_EPILOGUE_PLAIN_LEN;
3406         dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3407              con->v2.in_cursor.total_resid, remaining);
3408         con->v2.in_iter.count -= resid;
3409         set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3410         con->v2.in_state = IN_S_FINISH_SKIP;
3411 }
3412
3413 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)
3414 {
3415         int resid;  /* current enc page (not necessarily data) */
3416
3417         WARN_ON(!con_secure(con));
3418         WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3419         resid = iov_iter_count(&con->v2.in_iter);
3420         WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3421
3422         dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,
3423              con->v2.in_enc_resid);
3424         con->v2.in_iter.count -= resid;
3425         set_in_skip(con, resid + con->v2.in_enc_resid);
3426         con->v2.in_state = IN_S_FINISH_SKIP;
3427 }
3428
3429 static void revoke_at_handle_epilogue(struct ceph_connection *con)
3430 {
3431         int resid;
3432
3433         resid = iov_iter_count(&con->v2.in_iter);
3434         WARN_ON(!resid);
3435
3436         dout("%s con %p resid %d\n", __func__, con, resid);
3437         con->v2.in_iter.count -= resid;
3438         set_in_skip(con, resid);
3439         con->v2.in_state = IN_S_FINISH_SKIP;
3440 }
3441
3442 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3443 {
3444         switch (con->v2.in_state) {
3445         case IN_S_PREPARE_READ_DATA:
3446                 revoke_at_prepare_read_data(con);
3447                 break;
3448         case IN_S_PREPARE_READ_DATA_CONT:
3449                 revoke_at_prepare_read_data_cont(con);
3450                 break;
3451         case IN_S_PREPARE_READ_ENC_PAGE:
3452                 revoke_at_prepare_read_enc_page(con);
3453                 break;
3454         case IN_S_HANDLE_EPILOGUE:
3455                 revoke_at_handle_epilogue(con);
3456                 break;
3457         default:
3458                 WARN(1, "bad in_state %d", con->v2.in_state);
3459                 break;
3460         }
3461 }
3462
3463 bool ceph_con_v2_opened(struct ceph_connection *con)
3464 {
3465         return con->v2.peer_global_seq;
3466 }
3467
3468 void ceph_con_v2_reset_session(struct ceph_connection *con)
3469 {
3470         con->v2.client_cookie = 0;
3471         con->v2.server_cookie = 0;
3472         con->v2.global_seq = 0;
3473         con->v2.connect_seq = 0;
3474         con->v2.peer_global_seq = 0;
3475 }
3476
3477 void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3478 {
3479         iov_iter_truncate(&con->v2.in_iter, 0);
3480         iov_iter_truncate(&con->v2.out_iter, 0);
3481         con->v2.out_zero = 0;
3482
3483         clear_in_sign_kvecs(con);
3484         clear_out_sign_kvecs(con);
3485         free_conn_bufs(con);
3486
3487         if (con->v2.in_enc_pages) {
3488                 WARN_ON(!con->v2.in_enc_page_cnt);
3489                 ceph_release_page_vector(con->v2.in_enc_pages,
3490                                          con->v2.in_enc_page_cnt);
3491                 con->v2.in_enc_pages = NULL;
3492                 con->v2.in_enc_page_cnt = 0;
3493         }
3494         if (con->v2.out_enc_pages) {
3495                 WARN_ON(!con->v2.out_enc_page_cnt);
3496                 ceph_release_page_vector(con->v2.out_enc_pages,
3497                                          con->v2.out_enc_page_cnt);
3498                 con->v2.out_enc_pages = NULL;
3499                 con->v2.out_enc_page_cnt = 0;
3500         }
3501
3502         con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3503         memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3504         memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
3505
3506         if (con->v2.hmac_tfm) {
3507                 crypto_free_shash(con->v2.hmac_tfm);
3508                 con->v2.hmac_tfm = NULL;
3509         }
3510         if (con->v2.gcm_req) {
3511                 aead_request_free(con->v2.gcm_req);
3512                 con->v2.gcm_req = NULL;
3513         }
3514         if (con->v2.gcm_tfm) {
3515                 crypto_free_aead(con->v2.gcm_tfm);
3516                 con->v2.gcm_tfm = NULL;
3517         }
3518 }