1 // SPDX-License-Identifier: GPL-2.0
3 * Ceph msgr2 protocol implementation
5 * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com>
8 #include <linux/ceph/ceph_debug.h>
10 #include <crypto/aead.h>
11 #include <crypto/hash.h>
12 #include <crypto/sha2.h>
13 #include <crypto/utils.h>
14 #include <linux/bvec.h>
15 #include <linux/crc32c.h>
16 #include <linux/net.h>
17 #include <linux/scatterlist.h>
18 #include <linux/socket.h>
19 #include <linux/sched/mm.h>
23 #include <linux/ceph/ceph_features.h>
24 #include <linux/ceph/decode.h>
25 #include <linux/ceph/libceph.h>
26 #include <linux/ceph/messenger.h>
28 #include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */
30 #define FRAME_TAG_HELLO 1
31 #define FRAME_TAG_AUTH_REQUEST 2
32 #define FRAME_TAG_AUTH_BAD_METHOD 3
33 #define FRAME_TAG_AUTH_REPLY_MORE 4
34 #define FRAME_TAG_AUTH_REQUEST_MORE 5
35 #define FRAME_TAG_AUTH_DONE 6
36 #define FRAME_TAG_AUTH_SIGNATURE 7
37 #define FRAME_TAG_CLIENT_IDENT 8
38 #define FRAME_TAG_SERVER_IDENT 9
39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10
40 #define FRAME_TAG_SESSION_RECONNECT 11
41 #define FRAME_TAG_SESSION_RESET 12
42 #define FRAME_TAG_SESSION_RETRY 13
43 #define FRAME_TAG_SESSION_RETRY_GLOBAL 14
44 #define FRAME_TAG_SESSION_RECONNECT_OK 15
45 #define FRAME_TAG_WAIT 16
46 #define FRAME_TAG_MESSAGE 17
47 #define FRAME_TAG_KEEPALIVE2 18
48 #define FRAME_TAG_KEEPALIVE2_ACK 19
49 #define FRAME_TAG_ACK 20
51 #define FRAME_LATE_STATUS_ABORTED 0x1
52 #define FRAME_LATE_STATUS_COMPLETE 0xe
53 #define FRAME_LATE_STATUS_ABORTED_MASK 0xf
55 #define IN_S_HANDLE_PREAMBLE 1
56 #define IN_S_HANDLE_CONTROL 2
57 #define IN_S_HANDLE_CONTROL_REMAINDER 3
58 #define IN_S_PREPARE_READ_DATA 4
59 #define IN_S_PREPARE_READ_DATA_CONT 5
60 #define IN_S_PREPARE_READ_ENC_PAGE 6
61 #define IN_S_PREPARE_SPARSE_DATA 7
62 #define IN_S_PREPARE_SPARSE_DATA_CONT 8
63 #define IN_S_HANDLE_EPILOGUE 9
64 #define IN_S_FINISH_SKIP 10
66 #define OUT_S_QUEUE_DATA 1
67 #define OUT_S_QUEUE_DATA_CONT 2
68 #define OUT_S_QUEUE_ENC_PAGE 3
69 #define OUT_S_QUEUE_ZEROS 4
70 #define OUT_S_FINISH_MESSAGE 5
71 #define OUT_S_GET_NEXT 6
73 #define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN)
74 #define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN)
75 #define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN)
76 #define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN)
78 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL)
80 static int do_recvmsg(struct socket *sock, struct iov_iter *it)
82 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
86 while (iov_iter_count(it)) {
87 ret = sock_recvmsg(sock, &msg, msg.msg_flags);
94 iov_iter_advance(it, ret);
97 WARN_ON(msg_data_left(&msg));
102 * Read as much as possible.
105 * 1 - done, nothing (else) to read
106 * 0 - socket is empty, need to wait
109 static int ceph_tcp_recv(struct ceph_connection *con)
113 dout("%s con %p %s %zu\n", __func__, con,
114 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need",
115 iov_iter_count(&con->v2.in_iter));
116 ret = do_recvmsg(con->sock, &con->v2.in_iter);
117 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
118 iov_iter_count(&con->v2.in_iter));
122 static int do_sendmsg(struct socket *sock, struct iov_iter *it)
124 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
128 while (iov_iter_count(it)) {
129 ret = sock_sendmsg(sock, &msg);
136 iov_iter_advance(it, ret);
139 WARN_ON(msg_data_left(&msg));
143 static int do_try_sendpage(struct socket *sock, struct iov_iter *it)
145 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS };
149 if (WARN_ON(!iov_iter_is_bvec(it)))
152 while (iov_iter_count(it)) {
153 /* iov_iter_iovec() for ITER_BVEC */
154 bvec_set_page(&bv, it->bvec->bv_page,
155 min(iov_iter_count(it),
156 it->bvec->bv_len - it->iov_offset),
157 it->bvec->bv_offset + it->iov_offset);
160 * MSG_SPLICE_PAGES cannot properly handle pages with
161 * page_count == 0, we need to fall back to sendmsg if
164 * Same goes for slab pages: skb_can_coalesce() allows
165 * coalescing neighboring slab objects into a single frag
166 * which triggers one of hardened usercopy checks.
168 if (sendpage_ok(bv.bv_page))
169 msg.msg_flags |= MSG_SPLICE_PAGES;
171 msg.msg_flags &= ~MSG_SPLICE_PAGES;
173 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, bv.bv_len);
174 ret = sock_sendmsg(sock, &msg);
181 iov_iter_advance(it, ret);
188 * Write as much as possible. The socket is expected to be corked,
189 * so we don't bother with MSG_MORE here.
192 * 1 - done, nothing (else) to write
193 * 0 - socket is full, need to wait
196 static int ceph_tcp_send(struct ceph_connection *con)
200 dout("%s con %p have %zu try_sendpage %d\n", __func__, con,
201 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage);
202 if (con->v2.out_iter_sendpage)
203 ret = do_try_sendpage(con->sock, &con->v2.out_iter);
205 ret = do_sendmsg(con->sock, &con->v2.out_iter);
206 dout("%s con %p ret %d left %zu\n", __func__, con, ret,
207 iov_iter_count(&con->v2.out_iter));
211 static void add_in_kvec(struct ceph_connection *con, void *buf, int len)
213 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs));
214 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
216 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf;
217 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len;
218 con->v2.in_kvec_cnt++;
220 con->v2.in_iter.nr_segs++;
221 con->v2.in_iter.count += len;
224 static void reset_in_kvecs(struct ceph_connection *con)
226 WARN_ON(iov_iter_count(&con->v2.in_iter));
228 con->v2.in_kvec_cnt = 0;
229 iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0);
232 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv)
234 WARN_ON(iov_iter_count(&con->v2.in_iter));
236 con->v2.in_bvec = *bv;
237 iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len);
240 static void set_in_skip(struct ceph_connection *con, int len)
242 WARN_ON(iov_iter_count(&con->v2.in_iter));
244 dout("%s con %p len %d\n", __func__, con, len);
245 iov_iter_discard(&con->v2.in_iter, ITER_DEST, len);
248 static void add_out_kvec(struct ceph_connection *con, void *buf, int len)
250 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs));
251 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
252 WARN_ON(con->v2.out_zero);
254 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf;
255 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len;
256 con->v2.out_kvec_cnt++;
258 con->v2.out_iter.nr_segs++;
259 con->v2.out_iter.count += len;
262 static void reset_out_kvecs(struct ceph_connection *con)
264 WARN_ON(iov_iter_count(&con->v2.out_iter));
265 WARN_ON(con->v2.out_zero);
267 con->v2.out_kvec_cnt = 0;
269 iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0);
270 con->v2.out_iter_sendpage = false;
273 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv,
276 WARN_ON(iov_iter_count(&con->v2.out_iter));
277 WARN_ON(con->v2.out_zero);
279 con->v2.out_bvec = *bv;
280 con->v2.out_iter_sendpage = zerocopy;
281 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
282 con->v2.out_bvec.bv_len);
285 static void set_out_bvec_zero(struct ceph_connection *con)
287 WARN_ON(iov_iter_count(&con->v2.out_iter));
288 WARN_ON(!con->v2.out_zero);
290 bvec_set_page(&con->v2.out_bvec, ceph_zero_page,
291 min(con->v2.out_zero, (int)PAGE_SIZE), 0);
292 con->v2.out_iter_sendpage = true;
293 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1,
294 con->v2.out_bvec.bv_len);
297 static void out_zero_add(struct ceph_connection *con, int len)
299 dout("%s con %p len %d\n", __func__, con, len);
300 con->v2.out_zero += len;
303 static void *alloc_conn_buf(struct ceph_connection *con, int len)
307 dout("%s con %p len %d\n", __func__, con, len);
309 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs)))
312 buf = kvmalloc(len, GFP_NOIO);
316 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf;
320 static void free_conn_bufs(struct ceph_connection *con)
322 while (con->v2.conn_buf_cnt)
323 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]);
326 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len)
328 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs));
330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf;
331 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len;
332 con->v2.in_sign_kvec_cnt++;
335 static void clear_in_sign_kvecs(struct ceph_connection *con)
337 con->v2.in_sign_kvec_cnt = 0;
340 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len)
342 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs));
344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf;
345 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len;
346 con->v2.out_sign_kvec_cnt++;
349 static void clear_out_sign_kvecs(struct ceph_connection *con)
351 con->v2.out_sign_kvec_cnt = 0;
354 static bool con_secure(struct ceph_connection *con)
356 return con->v2.con_mode == CEPH_CON_MODE_SECURE;
359 static int front_len(const struct ceph_msg *msg)
361 return le32_to_cpu(msg->hdr.front_len);
364 static int middle_len(const struct ceph_msg *msg)
366 return le32_to_cpu(msg->hdr.middle_len);
369 static int data_len(const struct ceph_msg *msg)
371 return le32_to_cpu(msg->hdr.data_len);
374 static bool need_padding(int len)
376 return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN);
379 static int padded_len(int len)
381 return ALIGN(len, CEPH_GCM_BLOCK_LEN);
384 static int padding_len(int len)
386 return padded_len(len) - len;
389 /* preamble + control segment */
390 static int head_onwire_len(int ctrl_len, bool secure)
395 BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN);
398 head_len = CEPH_PREAMBLE_SECURE_LEN;
399 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
400 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
401 head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN;
404 head_len = CEPH_PREAMBLE_PLAIN_LEN;
406 head_len += ctrl_len + CEPH_CRC_LEN;
411 /* front, middle and data segments + epilogue */
412 static int __tail_onwire_len(int front_len, int middle_len, int data_len,
415 BUG_ON(front_len < 0 || front_len > CEPH_MSG_MAX_FRONT_LEN ||
416 middle_len < 0 || middle_len > CEPH_MSG_MAX_MIDDLE_LEN ||
417 data_len < 0 || data_len > CEPH_MSG_MAX_DATA_LEN);
419 if (!front_len && !middle_len && !data_len)
423 return front_len + middle_len + data_len +
424 CEPH_EPILOGUE_PLAIN_LEN;
426 return padded_len(front_len) + padded_len(middle_len) +
427 padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN;
430 static int tail_onwire_len(const struct ceph_msg *msg, bool secure)
432 return __tail_onwire_len(front_len(msg), middle_len(msg),
433 data_len(msg), secure);
436 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */
437 #define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \
438 sizeof(struct ceph_msg_header2) + \
441 static const int frame_aligns[] = {
449 * Discards trailing empty segments, unless there is just one segment.
450 * A frame always has at least one (possibly empty) segment.
452 static int calc_segment_count(const int *lens, int len_cnt)
456 for (i = len_cnt - 1; i >= 0; i--) {
464 static void init_frame_desc(struct ceph_frame_desc *desc, int tag,
465 const int *lens, int len_cnt)
469 memset(desc, 0, sizeof(*desc));
472 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt);
473 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT);
474 for (i = 0; i < desc->fd_seg_cnt; i++) {
475 desc->fd_lens[i] = lens[i];
476 desc->fd_aligns[i] = frame_aligns[i];
481 * Preamble crc covers everything up to itself (28 bytes) and
482 * is calculated and verified irrespective of the connection mode
483 * (i.e. even if the frame is encrypted).
485 static void encode_preamble(const struct ceph_frame_desc *desc, void *p)
487 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
491 memset(p, 0, CEPH_PREAMBLE_LEN);
493 ceph_encode_8(&p, desc->fd_tag);
494 ceph_encode_8(&p, desc->fd_seg_cnt);
495 for (i = 0; i < desc->fd_seg_cnt; i++) {
496 ceph_encode_32(&p, desc->fd_lens[i]);
497 ceph_encode_16(&p, desc->fd_aligns[i]);
500 put_unaligned_le32(crc32c(0, start, crcp - start), crcp);
503 static int decode_preamble(void *p, struct ceph_frame_desc *desc)
505 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN;
506 u32 crc, expected_crc;
509 crc = crc32c(0, p, crcp - p);
510 expected_crc = get_unaligned_le32(crcp);
511 if (crc != expected_crc) {
512 pr_err("bad preamble crc, calculated %u, expected %u\n",
517 memset(desc, 0, sizeof(*desc));
519 desc->fd_tag = ceph_decode_8(&p);
520 desc->fd_seg_cnt = ceph_decode_8(&p);
521 if (desc->fd_seg_cnt < 1 ||
522 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) {
523 pr_err("bad segment count %d\n", desc->fd_seg_cnt);
526 for (i = 0; i < desc->fd_seg_cnt; i++) {
527 desc->fd_lens[i] = ceph_decode_32(&p);
528 desc->fd_aligns[i] = ceph_decode_16(&p);
531 if (desc->fd_lens[0] < 0 ||
532 desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) {
533 pr_err("bad control segment length %d\n", desc->fd_lens[0]);
536 if (desc->fd_lens[1] < 0 ||
537 desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) {
538 pr_err("bad front segment length %d\n", desc->fd_lens[1]);
541 if (desc->fd_lens[2] < 0 ||
542 desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) {
543 pr_err("bad middle segment length %d\n", desc->fd_lens[2]);
546 if (desc->fd_lens[3] < 0 ||
547 desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) {
548 pr_err("bad data segment length %d\n", desc->fd_lens[3]);
553 * This would fire for FRAME_TAG_WAIT (it has one empty
554 * segment), but we should never get it as client.
556 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) {
557 pr_err("last segment empty, segment count %d\n",
565 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted)
567 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
568 FRAME_LATE_STATUS_COMPLETE;
569 cpu_to_le32s(&con->v2.out_epil.front_crc);
570 cpu_to_le32s(&con->v2.out_epil.middle_crc);
571 cpu_to_le32s(&con->v2.out_epil.data_crc);
574 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted)
576 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil));
577 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED :
578 FRAME_LATE_STATUS_COMPLETE;
581 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc,
586 late_status = ceph_decode_8(&p);
587 if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) !=
588 FRAME_LATE_STATUS_COMPLETE) {
589 /* we should never get an aborted message as client */
590 pr_err("bad late_status 0x%x\n", late_status);
594 if (front_crc && middle_crc && data_crc) {
595 *front_crc = ceph_decode_32(&p);
596 *middle_crc = ceph_decode_32(&p);
597 *data_crc = ceph_decode_32(&p);
603 static void fill_header(struct ceph_msg_header *hdr,
604 const struct ceph_msg_header2 *hdr2,
605 int front_len, int middle_len, int data_len,
606 const struct ceph_entity_name *peer_name)
608 hdr->seq = hdr2->seq;
609 hdr->tid = hdr2->tid;
610 hdr->type = hdr2->type;
611 hdr->priority = hdr2->priority;
612 hdr->version = hdr2->version;
613 hdr->front_len = cpu_to_le32(front_len);
614 hdr->middle_len = cpu_to_le32(middle_len);
615 hdr->data_len = cpu_to_le32(data_len);
616 hdr->data_off = hdr2->data_off;
617 hdr->src = *peer_name;
618 hdr->compat_version = hdr2->compat_version;
623 static void fill_header2(struct ceph_msg_header2 *hdr2,
624 const struct ceph_msg_header *hdr, u64 ack_seq)
626 hdr2->seq = hdr->seq;
627 hdr2->tid = hdr->tid;
628 hdr2->type = hdr->type;
629 hdr2->priority = hdr->priority;
630 hdr2->version = hdr->version;
631 hdr2->data_pre_padding_len = 0;
632 hdr2->data_off = hdr->data_off;
633 hdr2->ack_seq = cpu_to_le64(ack_seq);
635 hdr2->compat_version = hdr->compat_version;
639 static int verify_control_crc(struct ceph_connection *con)
641 int ctrl_len = con->v2.in_desc.fd_lens[0];
642 u32 crc, expected_crc;
644 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len);
645 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN);
647 crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len);
648 expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base);
649 if (crc != expected_crc) {
650 pr_err("bad control crc, calculated %u, expected %u\n",
658 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc,
659 u32 middle_crc, u32 data_crc)
661 if (front_len(con->in_msg)) {
662 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base,
663 front_len(con->in_msg));
665 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg));
666 con->in_front_crc = -1;
669 if (middle_len(con->in_msg))
670 con->in_middle_crc = crc32c(-1,
671 con->in_msg->middle->vec.iov_base,
672 middle_len(con->in_msg));
673 else if (data_len(con->in_msg))
674 con->in_middle_crc = -1;
676 con->in_middle_crc = 0;
678 if (!data_len(con->in_msg))
679 con->in_data_crc = 0;
681 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg,
682 con->in_front_crc, con->in_middle_crc, con->in_data_crc);
684 if (con->in_front_crc != front_crc) {
685 pr_err("bad front crc, calculated %u, expected %u\n",
686 con->in_front_crc, front_crc);
689 if (con->in_middle_crc != middle_crc) {
690 pr_err("bad middle crc, calculated %u, expected %u\n",
691 con->in_middle_crc, middle_crc);
694 if (con->in_data_crc != data_crc) {
695 pr_err("bad data crc, calculated %u, expected %u\n",
696 con->in_data_crc, data_crc);
703 static int setup_crypto(struct ceph_connection *con,
704 const u8 *session_key, int session_key_len,
705 const u8 *con_secret, int con_secret_len)
707 unsigned int noio_flag;
710 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n",
711 __func__, con, con->v2.con_mode, session_key_len, con_secret_len);
712 WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req);
714 if (con->v2.con_mode != CEPH_CON_MODE_CRC &&
715 con->v2.con_mode != CEPH_CON_MODE_SECURE) {
716 pr_err("bad con_mode %d\n", con->v2.con_mode);
720 if (!session_key_len) {
721 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC);
722 WARN_ON(con_secret_len);
723 return 0; /* auth_none */
726 noio_flag = memalloc_noio_save();
727 con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
728 memalloc_noio_restore(noio_flag);
729 if (IS_ERR(con->v2.hmac_tfm)) {
730 ret = PTR_ERR(con->v2.hmac_tfm);
731 con->v2.hmac_tfm = NULL;
732 pr_err("failed to allocate hmac tfm context: %d\n", ret);
736 WARN_ON((unsigned long)session_key &
737 crypto_shash_alignmask(con->v2.hmac_tfm));
738 ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key,
741 pr_err("failed to set hmac key: %d\n", ret);
745 if (con->v2.con_mode == CEPH_CON_MODE_CRC) {
746 WARN_ON(con_secret_len);
747 return 0; /* auth_x, plain mode */
750 if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) {
751 pr_err("con_secret too small %d\n", con_secret_len);
755 noio_flag = memalloc_noio_save();
756 con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0);
757 memalloc_noio_restore(noio_flag);
758 if (IS_ERR(con->v2.gcm_tfm)) {
759 ret = PTR_ERR(con->v2.gcm_tfm);
760 con->v2.gcm_tfm = NULL;
761 pr_err("failed to allocate gcm tfm context: %d\n", ret);
765 WARN_ON((unsigned long)con_secret &
766 crypto_aead_alignmask(con->v2.gcm_tfm));
767 ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN);
769 pr_err("failed to set gcm key: %d\n", ret);
773 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN);
774 ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN);
776 pr_err("failed to set gcm tag size: %d\n", ret);
780 con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO);
781 if (!con->v2.gcm_req) {
782 pr_err("failed to allocate gcm request\n");
786 crypto_init_wait(&con->v2.gcm_wait);
787 aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
788 crypto_req_done, &con->v2.gcm_wait);
790 memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN,
792 memcpy(&con->v2.out_gcm_nonce,
793 con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN,
795 return 0; /* auth_x, secure mode */
798 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs,
799 int kvec_cnt, u8 *hmac)
801 SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */
805 dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con,
806 con->v2.hmac_tfm, kvec_cnt);
808 if (!con->v2.hmac_tfm) {
809 memset(hmac, 0, SHA256_DIGEST_SIZE);
810 return 0; /* auth_none */
813 desc->tfm = con->v2.hmac_tfm;
814 ret = crypto_shash_init(desc);
818 for (i = 0; i < kvec_cnt; i++) {
819 WARN_ON((unsigned long)kvecs[i].iov_base &
820 crypto_shash_alignmask(con->v2.hmac_tfm));
821 ret = crypto_shash_update(desc, kvecs[i].iov_base,
827 ret = crypto_shash_final(desc, hmac);
830 shash_desc_zero(desc);
831 return ret; /* auth_x, both plain and secure modes */
834 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce)
838 counter = le64_to_cpu(nonce->counter);
839 nonce->counter = cpu_to_le64(counter + 1);
842 static int gcm_crypt(struct ceph_connection *con, bool encrypt,
843 struct scatterlist *src, struct scatterlist *dst,
846 struct ceph_gcm_nonce *nonce;
849 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce;
851 aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */
852 aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce);
853 ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) :
854 crypto_aead_decrypt(con->v2.gcm_req),
859 gcm_inc_nonce(nonce);
863 static void get_bvec_at(struct ceph_msg_data_cursor *cursor,
869 WARN_ON(!cursor->total_resid);
871 /* skip zero-length data items */
872 while (!cursor->resid)
873 ceph_msg_data_advance(cursor, 0);
875 /* get a piece of data, cursor isn't advanced */
876 page = ceph_msg_data_next(cursor, &off, &len);
877 bvec_set_page(bv, page, len, off);
880 static int calc_sg_cnt(void *buf, int buf_len)
887 sg_cnt = need_padding(buf_len) ? 1 : 0;
888 if (is_vmalloc_addr(buf)) {
889 WARN_ON(offset_in_page(buf));
890 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT;
898 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor)
900 int data_len = cursor->total_resid;
907 sg_cnt = need_padding(data_len) ? 1 : 0;
909 get_bvec_at(cursor, &bv);
912 ceph_msg_data_advance(cursor, bv.bv_len);
913 } while (cursor->total_resid);
918 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad)
920 void *end = buf + buf_len;
928 if (is_vmalloc_addr(buf)) {
931 page = vmalloc_to_page(p);
932 len = min_t(int, end - p, PAGE_SIZE);
933 WARN_ON(!page || !len || offset_in_page(p));
934 sg_set_page(*sg, page, len, 0);
939 sg_set_buf(*sg, buf, buf_len);
943 if (need_padding(buf_len)) {
944 sg_set_buf(*sg, pad, padding_len(buf_len));
949 static void init_sgs_cursor(struct scatterlist **sg,
950 struct ceph_msg_data_cursor *cursor, u8 *pad)
952 int data_len = cursor->total_resid;
959 get_bvec_at(cursor, &bv);
960 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset);
963 ceph_msg_data_advance(cursor, bv.bv_len);
964 } while (cursor->total_resid);
966 if (need_padding(data_len)) {
967 sg_set_buf(*sg, pad, padding_len(data_len));
973 * init_sgs_pages: set up scatterlist on an array of page pointers
974 * @sg: scatterlist to populate
975 * @pages: pointer to page array
976 * @dpos: position in the array to start (bytes)
977 * @dlen: len to add to sg (bytes)
978 * @pad: pointer to pad destination (if any)
980 * Populate the scatterlist from the page array, starting at an arbitrary
981 * byte in the array and running for a specified length.
983 static void init_sgs_pages(struct scatterlist **sg, struct page **pages,
984 int dpos, int dlen, u8 *pad)
986 int idx = dpos >> PAGE_SHIFT;
987 int off = offset_in_page(dpos);
991 int len = min(resid, (int)PAGE_SIZE - off);
993 sg_set_page(*sg, pages[idx], len, off);
1000 if (need_padding(dlen)) {
1001 sg_set_buf(*sg, pad, padding_len(dlen));
1006 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg,
1007 u8 *front_pad, u8 *middle_pad, u8 *data_pad,
1008 void *epilogue, struct page **pages, int dpos,
1011 struct ceph_msg_data_cursor cursor;
1012 struct scatterlist *cur_sg;
1013 int dlen = data_len(msg);
1017 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
1020 sg_cnt = 1; /* epilogue + [auth tag] */
1022 sg_cnt += calc_sg_cnt(msg->front.iov_base,
1024 if (middle_len(msg))
1025 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base,
1029 sg_cnt += calc_pages_for(dpos, dlen);
1030 if (need_padding(dlen))
1033 ceph_msg_data_cursor_init(&cursor, msg, dlen);
1034 sg_cnt += calc_sg_cnt_cursor(&cursor);
1038 ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO);
1044 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg),
1046 if (middle_len(msg))
1047 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg),
1051 init_sgs_pages(&cur_sg, pages, dpos, dlen, data_pad);
1053 ceph_msg_data_cursor_init(&cursor, msg, dlen);
1054 init_sgs_cursor(&cur_sg, &cursor, data_pad);
1058 WARN_ON(!sg_is_last(cur_sg));
1059 sg_set_buf(cur_sg, epilogue,
1060 CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0));
1064 static int decrypt_preamble(struct ceph_connection *con)
1066 struct scatterlist sg;
1068 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN);
1069 return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN);
1072 static int decrypt_control_remainder(struct ceph_connection *con)
1074 int ctrl_len = con->v2.in_desc.fd_lens[0];
1075 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1076 int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN;
1077 struct scatterlist sgs[2];
1079 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len);
1080 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len);
1082 sg_init_table(sgs, 2);
1083 sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len);
1084 sg_set_buf(&sgs[1], con->v2.in_buf, pt_len);
1086 return gcm_crypt(con, false, sgs, sgs,
1087 padded_len(rem_len) + CEPH_GCM_TAG_LEN);
1090 /* Process sparse read data that lives in a buffer */
1091 static int process_v2_sparse_read(struct ceph_connection *con,
1092 struct page **pages, int spos)
1094 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;
1100 ret = con->ops->sparse_read(con, cursor, &buf);
1104 dout("%s: sparse_read return %x buf %p\n", __func__, ret, buf);
1107 int idx = spos >> PAGE_SHIFT;
1108 int soff = offset_in_page(spos);
1109 struct page *spage = con->v2.in_enc_pages[idx];
1110 int len = min_t(int, ret, PAGE_SIZE - soff);
1113 memcpy_from_page(buf, spage, soff, len);
1118 get_bvec_at(cursor, &bv);
1119 len = min_t(int, len, bv.bv_len);
1120 memcpy_page(bv.bv_page, bv.bv_offset,
1122 ceph_msg_data_advance(cursor, len);
1130 static int decrypt_tail(struct ceph_connection *con)
1132 struct sg_table enc_sgt = {};
1133 struct sg_table sgt = {};
1134 struct page **pages = NULL;
1135 bool sparse = con->in_msg->sparse_read;
1140 tail_len = tail_onwire_len(con->in_msg, true);
1141 ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages,
1142 con->v2.in_enc_page_cnt, 0, tail_len,
1148 dpos = padded_len(front_len(con->in_msg) + padded_len(middle_len(con->in_msg)));
1149 pages = con->v2.in_enc_pages;
1152 ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf),
1153 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf),
1154 con->v2.in_buf, pages, dpos, true);
1158 dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con,
1159 con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents);
1160 ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len);
1164 if (sparse && data_len(con->in_msg)) {
1165 ret = process_v2_sparse_read(con, con->v2.in_enc_pages, dpos);
1170 WARN_ON(!con->v2.in_enc_page_cnt);
1171 ceph_release_page_vector(con->v2.in_enc_pages,
1172 con->v2.in_enc_page_cnt);
1173 con->v2.in_enc_pages = NULL;
1174 con->v2.in_enc_page_cnt = 0;
1177 sg_free_table(&sgt);
1178 sg_free_table(&enc_sgt);
1182 static int prepare_banner(struct ceph_connection *con)
1184 int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8;
1187 buf = alloc_conn_buf(con, buf_len);
1192 ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN);
1193 ceph_encode_16(&p, sizeof(u64) + sizeof(u64));
1194 ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES);
1195 ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES);
1196 WARN_ON(p != buf + buf_len);
1198 add_out_kvec(con, buf, buf_len);
1199 add_out_sign_kvec(con, buf, buf_len);
1200 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1207 * control body (ctrl_len bytes)
1208 * space for control crc
1210 * extdata (optional):
1211 * control body (extdata_len bytes)
1213 * Compute control crc and gather base and extdata into:
1216 * control body (ctrl_len + extdata_len bytes)
1219 * Preamble should already be encoded at the start of base.
1221 static void prepare_head_plain(struct ceph_connection *con, void *base,
1222 int ctrl_len, void *extdata, int extdata_len,
1225 int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN;
1226 void *crcp = base + base_len - CEPH_CRC_LEN;
1229 crc = crc32c(-1, CTRL_BODY(base), ctrl_len);
1231 crc = crc32c(crc, extdata, extdata_len);
1232 put_unaligned_le32(crc, crcp);
1235 add_out_kvec(con, base, base_len);
1237 add_out_sign_kvec(con, base, base_len);
1241 add_out_kvec(con, base, crcp - base);
1242 add_out_kvec(con, extdata, extdata_len);
1243 add_out_kvec(con, crcp, CEPH_CRC_LEN);
1245 add_out_sign_kvec(con, base, crcp - base);
1246 add_out_sign_kvec(con, extdata, extdata_len);
1247 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN);
1251 static int prepare_head_secure_small(struct ceph_connection *con,
1252 void *base, int ctrl_len)
1254 struct scatterlist sg;
1257 /* inline buffer padding? */
1258 if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN)
1259 memset(CTRL_BODY(base) + ctrl_len, 0,
1260 CEPH_PREAMBLE_INLINE_LEN - ctrl_len);
1262 sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN);
1263 ret = gcm_crypt(con, true, &sg, &sg,
1264 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN);
1268 add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN);
1275 * control body (ctrl_len bytes)
1276 * space for padding, if needed
1277 * space for control remainder auth tag
1278 * space for preamble auth tag
1280 * Encrypt preamble and the inline portion, then encrypt the remainder
1284 * control body (48 bytes)
1286 * control body (ctrl_len - 48 bytes)
1287 * zero padding, if needed
1288 * control remainder auth tag
1290 * Preamble should already be encoded at the start of base.
1292 static int prepare_head_secure_big(struct ceph_connection *con,
1293 void *base, int ctrl_len)
1295 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1296 void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN;
1297 void *rem_tag = rem + padded_len(rem_len);
1298 void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN;
1299 struct scatterlist sgs[2];
1302 sg_init_table(sgs, 2);
1303 sg_set_buf(&sgs[0], base, rem - base);
1304 sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN);
1305 ret = gcm_crypt(con, true, sgs, sgs, rem - base);
1309 /* control remainder padding? */
1310 if (need_padding(rem_len))
1311 memset(rem + rem_len, 0, padding_len(rem_len));
1313 sg_init_one(&sgs[0], rem, pmbl_tag - rem);
1314 ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem);
1318 add_out_kvec(con, base, rem - base);
1319 add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN);
1320 add_out_kvec(con, rem, pmbl_tag - rem);
1324 static int __prepare_control(struct ceph_connection *con, int tag,
1325 void *base, int ctrl_len, void *extdata,
1326 int extdata_len, bool to_be_signed)
1328 int total_len = ctrl_len + extdata_len;
1329 struct ceph_frame_desc desc;
1332 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag,
1333 total_len, ctrl_len, extdata_len);
1335 /* extdata may be vmalloc'ed but not base */
1336 if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len))
1339 init_frame_desc(&desc, tag, &total_len, 1);
1340 encode_preamble(&desc, base);
1342 if (con_secure(con)) {
1343 if (WARN_ON(extdata_len || to_be_signed))
1346 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN)
1347 /* fully inlined, inline buffer may need padding */
1348 ret = prepare_head_secure_small(con, base, ctrl_len);
1350 /* partially inlined, inline buffer is full */
1351 ret = prepare_head_secure_big(con, base, ctrl_len);
1355 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len,
1359 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1363 static int prepare_control(struct ceph_connection *con, int tag,
1364 void *base, int ctrl_len)
1366 return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false);
1369 static int prepare_hello(struct ceph_connection *con)
1374 ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr);
1375 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1380 ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT);
1381 ceph_encode_entity_addr(&p, &con->peer_addr);
1382 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1384 return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len,
1388 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */
1389 #define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN)
1391 static int prepare_auth_request(struct ceph_connection *con)
1393 void *authorizer, *authorizer_copy;
1394 int ctrl_len, authorizer_len;
1398 ctrl_len = AUTH_BUF_LEN;
1399 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1403 mutex_unlock(&con->mutex);
1404 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len,
1405 &authorizer, &authorizer_len);
1406 mutex_lock(&con->mutex);
1407 if (con->state != CEPH_CON_S_V2_HELLO) {
1408 dout("%s con %p state changed to %d\n", __func__, con,
1413 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret);
1417 authorizer_copy = alloc_conn_buf(con, authorizer_len);
1418 if (!authorizer_copy)
1421 memcpy(authorizer_copy, authorizer, authorizer_len);
1423 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len,
1424 authorizer_copy, authorizer_len, true);
1427 static int prepare_auth_request_more(struct ceph_connection *con,
1428 void *reply, int reply_len)
1430 int ctrl_len, authorizer_len;
1435 ctrl_len = AUTH_BUF_LEN;
1436 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false));
1440 mutex_unlock(&con->mutex);
1441 ret = con->ops->handle_auth_reply_more(con, reply, reply_len,
1442 CTRL_BODY(buf), &ctrl_len,
1443 &authorizer, &authorizer_len);
1444 mutex_lock(&con->mutex);
1445 if (con->state != CEPH_CON_S_V2_AUTH) {
1446 dout("%s con %p state changed to %d\n", __func__, con,
1451 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret);
1455 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf,
1456 ctrl_len, authorizer, authorizer_len, true);
1459 static int prepare_auth_signature(struct ceph_connection *con)
1464 buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE,
1469 ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt,
1474 return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf,
1475 SHA256_DIGEST_SIZE);
1478 static int prepare_client_ident(struct ceph_connection *con)
1480 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1481 struct ceph_client *client = from_msgr(con->msgr);
1482 u64 global_id = ceph_client_gid(client);
1486 WARN_ON(con->v2.server_cookie);
1487 WARN_ON(con->v2.connect_seq);
1488 WARN_ON(con->v2.peer_global_seq);
1490 if (!con->v2.client_cookie) {
1492 get_random_bytes(&con->v2.client_cookie,
1493 sizeof(con->v2.client_cookie));
1494 } while (!con->v2.client_cookie);
1495 dout("%s con %p generated cookie 0x%llx\n", __func__, con,
1496 con->v2.client_cookie);
1498 dout("%s con %p cookie already set 0x%llx\n", __func__, con,
1499 con->v2.client_cookie);
1502 dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n",
1503 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1504 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce),
1505 global_id, con->v2.global_seq, client->supported_features,
1506 client->required_features, con->v2.client_cookie);
1508 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) +
1509 ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8;
1510 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1515 ceph_encode_8(&p, 2); /* addrvec marker */
1516 ceph_encode_32(&p, 1); /* addr_cnt */
1517 ceph_encode_entity_addr(&p, my_addr);
1518 ceph_encode_entity_addr(&p, &con->peer_addr);
1519 ceph_encode_64(&p, global_id);
1520 ceph_encode_64(&p, con->v2.global_seq);
1521 ceph_encode_64(&p, client->supported_features);
1522 ceph_encode_64(&p, client->required_features);
1523 ceph_encode_64(&p, 0); /* flags */
1524 ceph_encode_64(&p, con->v2.client_cookie);
1525 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1527 return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len);
1530 static int prepare_session_reconnect(struct ceph_connection *con)
1532 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
1536 WARN_ON(!con->v2.client_cookie);
1537 WARN_ON(!con->v2.server_cookie);
1538 WARN_ON(!con->v2.connect_seq);
1539 WARN_ON(!con->v2.peer_global_seq);
1541 dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n",
1542 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce),
1543 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq,
1544 con->v2.connect_seq, con->in_seq);
1546 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8;
1547 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con)));
1552 ceph_encode_8(&p, 2); /* entity_addrvec_t marker */
1553 ceph_encode_32(&p, 1); /* my_addrs len */
1554 ceph_encode_entity_addr(&p, my_addr);
1555 ceph_encode_64(&p, con->v2.client_cookie);
1556 ceph_encode_64(&p, con->v2.server_cookie);
1557 ceph_encode_64(&p, con->v2.global_seq);
1558 ceph_encode_64(&p, con->v2.connect_seq);
1559 ceph_encode_64(&p, con->in_seq);
1560 WARN_ON(p != CTRL_BODY(buf) + ctrl_len);
1562 return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len);
1565 static int prepare_keepalive2(struct ceph_connection *con)
1567 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf);
1568 struct timespec64 now;
1570 ktime_get_real_ts64(&now);
1571 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec,
1574 ceph_encode_timespec64(ts, &now);
1576 reset_out_kvecs(con);
1577 return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf,
1578 sizeof(struct ceph_timespec));
1581 static int prepare_ack(struct ceph_connection *con)
1585 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1586 con->in_seq_acked, con->in_seq);
1587 con->in_seq_acked = con->in_seq;
1589 p = CTRL_BODY(con->v2.out_buf);
1590 ceph_encode_64(&p, con->in_seq_acked);
1592 reset_out_kvecs(con);
1593 return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8);
1596 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted)
1598 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con,
1599 con->out_msg, aborted, con->v2.out_epil.front_crc,
1600 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc);
1602 encode_epilogue_plain(con, aborted);
1603 add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN);
1607 * For "used" empty segments, crc is -1. For unused (trailing)
1608 * segments, crc is 0.
1610 static void prepare_message_plain(struct ceph_connection *con)
1612 struct ceph_msg *msg = con->out_msg;
1614 prepare_head_plain(con, con->v2.out_buf,
1615 sizeof(struct ceph_msg_header2), NULL, 0, false);
1617 if (!front_len(msg) && !middle_len(msg)) {
1618 if (!data_len(msg)) {
1620 * Empty message: once the head is written,
1621 * we are done -- there is no epilogue.
1623 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1627 con->v2.out_epil.front_crc = -1;
1628 con->v2.out_epil.middle_crc = -1;
1629 con->v2.out_state = OUT_S_QUEUE_DATA;
1633 if (front_len(msg)) {
1634 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base,
1636 add_out_kvec(con, msg->front.iov_base, front_len(msg));
1638 /* middle (at least) is there, checked above */
1639 con->v2.out_epil.front_crc = -1;
1642 if (middle_len(msg)) {
1643 con->v2.out_epil.middle_crc =
1644 crc32c(-1, msg->middle->vec.iov_base, middle_len(msg));
1645 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
1647 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0;
1650 if (data_len(msg)) {
1651 con->v2.out_state = OUT_S_QUEUE_DATA;
1653 con->v2.out_epil.data_crc = 0;
1654 prepare_epilogue_plain(con, false);
1655 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1660 * Unfortunately the kernel crypto API doesn't support streaming
1661 * (piecewise) operation for AEAD algorithms, so we can't get away
1662 * with a fixed size buffer and a couple sgs. Instead, we have to
1663 * allocate pages for the entire tail of the message (currently up
1664 * to ~32M) and two sgs arrays (up to ~256K each)...
1666 static int prepare_message_secure(struct ceph_connection *con)
1668 void *zerop = page_address(ceph_zero_page);
1669 struct sg_table enc_sgt = {};
1670 struct sg_table sgt = {};
1671 struct page **enc_pages;
1676 ret = prepare_head_secure_small(con, con->v2.out_buf,
1677 sizeof(struct ceph_msg_header2));
1681 tail_len = tail_onwire_len(con->out_msg, true);
1684 * Empty message: once the head is written,
1685 * we are done -- there is no epilogue.
1687 con->v2.out_state = OUT_S_FINISH_MESSAGE;
1691 encode_epilogue_secure(con, false);
1692 ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop,
1693 &con->v2.out_epil, NULL, 0, false);
1697 enc_page_cnt = calc_pages_for(0, tail_len);
1698 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
1699 if (IS_ERR(enc_pages)) {
1700 ret = PTR_ERR(enc_pages);
1704 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt);
1705 con->v2.out_enc_pages = enc_pages;
1706 con->v2.out_enc_page_cnt = enc_page_cnt;
1707 con->v2.out_enc_resid = tail_len;
1708 con->v2.out_enc_i = 0;
1710 ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt,
1711 0, tail_len, GFP_NOIO);
1715 ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl,
1716 tail_len - CEPH_GCM_TAG_LEN);
1720 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con,
1721 con->out_msg, sgt.orig_nents, enc_page_cnt);
1722 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE;
1725 sg_free_table(&sgt);
1726 sg_free_table(&enc_sgt);
1730 static int prepare_message(struct ceph_connection *con)
1733 sizeof(struct ceph_msg_header2),
1734 front_len(con->out_msg),
1735 middle_len(con->out_msg),
1736 data_len(con->out_msg)
1738 struct ceph_frame_desc desc;
1741 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con,
1742 con->out_msg, lens[0], lens[1], lens[2], lens[3]);
1744 if (con->in_seq > con->in_seq_acked) {
1745 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con,
1746 con->in_seq_acked, con->in_seq);
1747 con->in_seq_acked = con->in_seq;
1750 reset_out_kvecs(con);
1751 init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4);
1752 encode_preamble(&desc, con->v2.out_buf);
1753 fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr,
1756 if (con_secure(con)) {
1757 ret = prepare_message_secure(con);
1761 prepare_message_plain(con);
1764 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING);
1768 static int prepare_read_banner_prefix(struct ceph_connection *con)
1772 buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN);
1776 reset_in_kvecs(con);
1777 add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1778 add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN);
1779 con->state = CEPH_CON_S_V2_BANNER_PREFIX;
1783 static int prepare_read_banner_payload(struct ceph_connection *con,
1788 buf = alloc_conn_buf(con, payload_len);
1792 reset_in_kvecs(con);
1793 add_in_kvec(con, buf, payload_len);
1794 add_in_sign_kvec(con, buf, payload_len);
1795 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD;
1799 static void prepare_read_preamble(struct ceph_connection *con)
1801 reset_in_kvecs(con);
1802 add_in_kvec(con, con->v2.in_buf,
1803 con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN :
1804 CEPH_PREAMBLE_PLAIN_LEN);
1805 con->v2.in_state = IN_S_HANDLE_PREAMBLE;
1808 static int prepare_read_control(struct ceph_connection *con)
1810 int ctrl_len = con->v2.in_desc.fd_lens[0];
1814 reset_in_kvecs(con);
1815 if (con->state == CEPH_CON_S_V2_HELLO ||
1816 con->state == CEPH_CON_S_V2_AUTH) {
1817 head_len = head_onwire_len(ctrl_len, false);
1818 buf = alloc_conn_buf(con, head_len);
1822 /* preserve preamble */
1823 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN);
1825 add_in_kvec(con, CTRL_BODY(buf), ctrl_len);
1826 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN);
1827 add_in_sign_kvec(con, buf, head_len);
1829 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) {
1830 buf = alloc_conn_buf(con, ctrl_len);
1834 add_in_kvec(con, buf, ctrl_len);
1836 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len);
1838 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN);
1840 con->v2.in_state = IN_S_HANDLE_CONTROL;
1844 static int prepare_read_control_remainder(struct ceph_connection *con)
1846 int ctrl_len = con->v2.in_desc.fd_lens[0];
1847 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN;
1850 buf = alloc_conn_buf(con, ctrl_len);
1854 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN);
1856 reset_in_kvecs(con);
1857 add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len);
1858 add_in_kvec(con, con->v2.in_buf,
1859 padding_len(rem_len) + CEPH_GCM_TAG_LEN);
1860 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER;
1864 static int prepare_read_data(struct ceph_connection *con)
1868 con->in_data_crc = -1;
1869 ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg,
1870 data_len(con->in_msg));
1872 get_bvec_at(&con->v2.in_cursor, &bv);
1873 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1874 if (unlikely(!con->bounce_page)) {
1875 con->bounce_page = alloc_page(GFP_NOIO);
1876 if (!con->bounce_page) {
1877 pr_err("failed to allocate bounce page\n");
1882 bv.bv_page = con->bounce_page;
1885 set_in_bvec(con, &bv);
1886 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT;
1890 static void prepare_read_data_cont(struct ceph_connection *con)
1894 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1895 con->in_data_crc = crc32c(con->in_data_crc,
1896 page_address(con->bounce_page),
1897 con->v2.in_bvec.bv_len);
1899 get_bvec_at(&con->v2.in_cursor, &bv);
1900 memcpy_to_page(bv.bv_page, bv.bv_offset,
1901 page_address(con->bounce_page),
1902 con->v2.in_bvec.bv_len);
1904 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1905 con->v2.in_bvec.bv_page,
1906 con->v2.in_bvec.bv_offset,
1907 con->v2.in_bvec.bv_len);
1910 ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len);
1911 if (con->v2.in_cursor.total_resid) {
1912 get_bvec_at(&con->v2.in_cursor, &bv);
1913 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1914 bv.bv_page = con->bounce_page;
1917 set_in_bvec(con, &bv);
1918 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT);
1923 * We've read all data. Prepare to read epilogue.
1925 reset_in_kvecs(con);
1926 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1927 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1930 static int prepare_sparse_read_cont(struct ceph_connection *con)
1935 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor;
1937 WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT);
1939 if (iov_iter_is_bvec(&con->v2.in_iter)) {
1940 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1941 con->in_data_crc = crc32c(con->in_data_crc,
1942 page_address(con->bounce_page),
1943 con->v2.in_bvec.bv_len);
1944 get_bvec_at(cursor, &bv);
1945 memcpy_to_page(bv.bv_page, bv.bv_offset,
1946 page_address(con->bounce_page),
1947 con->v2.in_bvec.bv_len);
1949 con->in_data_crc = ceph_crc32c_page(con->in_data_crc,
1950 con->v2.in_bvec.bv_page,
1951 con->v2.in_bvec.bv_offset,
1952 con->v2.in_bvec.bv_len);
1955 ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len);
1956 cursor->sr_resid -= con->v2.in_bvec.bv_len;
1957 dout("%s: advance by 0x%x sr_resid 0x%x\n", __func__,
1958 con->v2.in_bvec.bv_len, cursor->sr_resid);
1959 WARN_ON_ONCE(cursor->sr_resid > cursor->total_resid);
1960 if (cursor->sr_resid) {
1961 get_bvec_at(cursor, &bv);
1962 if (bv.bv_len > cursor->sr_resid)
1963 bv.bv_len = cursor->sr_resid;
1964 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
1965 bv.bv_page = con->bounce_page;
1968 set_in_bvec(con, &bv);
1969 con->v2.data_len_remain -= bv.bv_len;
1972 } else if (iov_iter_is_kvec(&con->v2.in_iter)) {
1973 /* On first call, we have no kvec so don't compute crc */
1974 if (con->v2.in_kvec_cnt) {
1975 WARN_ON_ONCE(con->v2.in_kvec_cnt > 1);
1976 con->in_data_crc = crc32c(con->in_data_crc,
1977 con->v2.in_kvecs[0].iov_base,
1978 con->v2.in_kvecs[0].iov_len);
1984 /* get next extent */
1985 ret = con->ops->sparse_read(con, cursor, &buf);
1990 reset_in_kvecs(con);
1991 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
1992 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
1997 /* receive into buffer */
1998 reset_in_kvecs(con);
1999 add_in_kvec(con, buf, ret);
2000 con->v2.data_len_remain -= ret;
2004 if (ret > cursor->total_resid) {
2005 pr_warn("%s: ret 0x%x total_resid 0x%zx resid 0x%zx\n",
2006 __func__, ret, cursor->total_resid, cursor->resid);
2009 get_bvec_at(cursor, &bv);
2010 if (bv.bv_len > cursor->sr_resid)
2011 bv.bv_len = cursor->sr_resid;
2012 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) {
2013 if (unlikely(!con->bounce_page)) {
2014 con->bounce_page = alloc_page(GFP_NOIO);
2015 if (!con->bounce_page) {
2016 pr_err("failed to allocate bounce page\n");
2021 bv.bv_page = con->bounce_page;
2024 set_in_bvec(con, &bv);
2025 con->v2.data_len_remain -= ret;
2029 static int prepare_sparse_read_data(struct ceph_connection *con)
2031 struct ceph_msg *msg = con->in_msg;
2033 dout("%s: starting sparse read\n", __func__);
2035 if (WARN_ON_ONCE(!con->ops->sparse_read))
2038 if (!con_secure(con))
2039 con->in_data_crc = -1;
2041 reset_in_kvecs(con);
2042 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT;
2043 con->v2.data_len_remain = data_len(msg);
2044 return prepare_sparse_read_cont(con);
2047 static int prepare_read_tail_plain(struct ceph_connection *con)
2049 struct ceph_msg *msg = con->in_msg;
2051 if (!front_len(msg) && !middle_len(msg)) {
2052 WARN_ON(!data_len(msg));
2053 return prepare_read_data(con);
2056 reset_in_kvecs(con);
2057 if (front_len(msg)) {
2058 add_in_kvec(con, msg->front.iov_base, front_len(msg));
2059 WARN_ON(msg->front.iov_len != front_len(msg));
2061 if (middle_len(msg)) {
2062 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg));
2063 WARN_ON(msg->middle->vec.iov_len != middle_len(msg));
2066 if (data_len(msg)) {
2067 if (msg->sparse_read)
2068 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA;
2070 con->v2.in_state = IN_S_PREPARE_READ_DATA;
2072 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN);
2073 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2078 static void prepare_read_enc_page(struct ceph_connection *con)
2082 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i,
2083 con->v2.in_enc_resid);
2084 WARN_ON(!con->v2.in_enc_resid);
2086 bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i],
2087 min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0);
2089 set_in_bvec(con, &bv);
2091 con->v2.in_enc_resid -= bv.bv_len;
2093 if (con->v2.in_enc_resid) {
2094 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE;
2099 * We are set to read the last piece of ciphertext (ending
2100 * with epilogue) + auth tag.
2102 WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt);
2103 con->v2.in_state = IN_S_HANDLE_EPILOGUE;
2106 static int prepare_read_tail_secure(struct ceph_connection *con)
2108 struct page **enc_pages;
2112 tail_len = tail_onwire_len(con->in_msg, true);
2115 enc_page_cnt = calc_pages_for(0, tail_len);
2116 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO);
2117 if (IS_ERR(enc_pages))
2118 return PTR_ERR(enc_pages);
2120 WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt);
2121 con->v2.in_enc_pages = enc_pages;
2122 con->v2.in_enc_page_cnt = enc_page_cnt;
2123 con->v2.in_enc_resid = tail_len;
2124 con->v2.in_enc_i = 0;
2126 prepare_read_enc_page(con);
2130 static void __finish_skip(struct ceph_connection *con)
2133 prepare_read_preamble(con);
2136 static void prepare_skip_message(struct ceph_connection *con)
2138 struct ceph_frame_desc *desc = &con->v2.in_desc;
2141 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1],
2142 desc->fd_lens[2], desc->fd_lens[3]);
2144 tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2],
2145 desc->fd_lens[3], con_secure(con));
2149 set_in_skip(con, tail_len);
2150 con->v2.in_state = IN_S_FINISH_SKIP;
2154 static int process_banner_prefix(struct ceph_connection *con)
2159 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN);
2161 p = con->v2.in_kvecs[0].iov_base;
2162 if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) {
2163 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN))
2164 con->error_msg = "server is speaking msgr1 protocol";
2166 con->error_msg = "protocol error, bad banner";
2170 p += CEPH_BANNER_V2_LEN;
2171 payload_len = ceph_decode_16(&p);
2172 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2174 return prepare_read_banner_payload(con, payload_len);
2177 static int process_banner_payload(struct ceph_connection *con)
2179 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len;
2180 u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES;
2181 u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES;
2182 u64 server_feat, server_req_feat;
2186 p = con->v2.in_kvecs[0].iov_base;
2187 ceph_decode_64_safe(&p, end, server_feat, bad);
2188 ceph_decode_64_safe(&p, end, server_req_feat, bad);
2190 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n",
2191 __func__, con, server_feat, server_req_feat);
2193 if (req_feat & ~server_feat) {
2194 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2195 server_feat, req_feat & ~server_feat);
2196 con->error_msg = "missing required protocol features";
2199 if (server_req_feat & ~feat) {
2200 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2201 feat, server_req_feat & ~feat);
2202 con->error_msg = "missing required protocol features";
2206 /* no reset_out_kvecs() as our banner may still be pending */
2207 ret = prepare_hello(con);
2209 pr_err("prepare_hello failed: %d\n", ret);
2213 con->state = CEPH_CON_S_V2_HELLO;
2214 prepare_read_preamble(con);
2218 pr_err("failed to decode banner payload\n");
2222 static int process_hello(struct ceph_connection *con, void *p, void *end)
2224 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr;
2225 struct ceph_entity_addr addr_for_me;
2229 if (con->state != CEPH_CON_S_V2_HELLO) {
2230 con->error_msg = "protocol error, unexpected hello";
2234 ceph_decode_8_safe(&p, end, entity_type, bad);
2235 ret = ceph_decode_entity_addr(&p, end, &addr_for_me);
2237 pr_err("failed to decode addr_for_me: %d\n", ret);
2241 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con,
2242 entity_type, ceph_pr_addr(&addr_for_me));
2244 if (entity_type != con->peer_name.type) {
2245 pr_err("bad peer type, want %d, got %d\n",
2246 con->peer_name.type, entity_type);
2247 con->error_msg = "wrong peer at address";
2252 * Set our address to the address our first peer (i.e. monitor)
2253 * sees that we are connecting from. If we are behind some sort
2254 * of NAT and want to be identified by some private (not NATed)
2255 * address, ip option should be used.
2257 if (ceph_addr_is_blank(my_addr)) {
2258 memcpy(&my_addr->in_addr, &addr_for_me.in_addr,
2259 sizeof(my_addr->in_addr));
2260 ceph_addr_set_port(my_addr, 0);
2261 dout("%s con %p set my addr %s, as seen by peer %s\n",
2262 __func__, con, ceph_pr_addr(my_addr),
2263 ceph_pr_addr(&con->peer_addr));
2265 dout("%s con %p my addr already set %s\n",
2266 __func__, con, ceph_pr_addr(my_addr));
2269 WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr));
2270 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY);
2271 WARN_ON(!my_addr->nonce);
2273 /* no reset_out_kvecs() as our hello may still be pending */
2274 ret = prepare_auth_request(con);
2277 pr_err("prepare_auth_request failed: %d\n", ret);
2281 con->state = CEPH_CON_S_V2_AUTH;
2285 pr_err("failed to decode hello\n");
2289 static int process_auth_bad_method(struct ceph_connection *con,
2292 int allowed_protos[8], allowed_modes[8];
2293 int allowed_proto_cnt, allowed_mode_cnt;
2294 int used_proto, result;
2298 if (con->state != CEPH_CON_S_V2_AUTH) {
2299 con->error_msg = "protocol error, unexpected auth_bad_method";
2303 ceph_decode_32_safe(&p, end, used_proto, bad);
2304 ceph_decode_32_safe(&p, end, result, bad);
2305 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto,
2308 ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad);
2309 if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) {
2310 pr_err("allowed_protos too big %d\n", allowed_proto_cnt);
2313 for (i = 0; i < allowed_proto_cnt; i++) {
2314 ceph_decode_32_safe(&p, end, allowed_protos[i], bad);
2315 dout("%s con %p allowed_protos[%d] %d\n", __func__, con,
2316 i, allowed_protos[i]);
2319 ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad);
2320 if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) {
2321 pr_err("allowed_modes too big %d\n", allowed_mode_cnt);
2324 for (i = 0; i < allowed_mode_cnt; i++) {
2325 ceph_decode_32_safe(&p, end, allowed_modes[i], bad);
2326 dout("%s con %p allowed_modes[%d] %d\n", __func__, con,
2327 i, allowed_modes[i]);
2330 mutex_unlock(&con->mutex);
2331 ret = con->ops->handle_auth_bad_method(con, used_proto, result,
2336 mutex_lock(&con->mutex);
2337 if (con->state != CEPH_CON_S_V2_AUTH) {
2338 dout("%s con %p state changed to %d\n", __func__, con,
2343 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret);
2347 pr_err("failed to decode auth_bad_method\n");
2351 static int process_auth_reply_more(struct ceph_connection *con,
2357 if (con->state != CEPH_CON_S_V2_AUTH) {
2358 con->error_msg = "protocol error, unexpected auth_reply_more";
2362 ceph_decode_32_safe(&p, end, payload_len, bad);
2363 ceph_decode_need(&p, end, payload_len, bad);
2365 dout("%s con %p payload_len %d\n", __func__, con, payload_len);
2367 reset_out_kvecs(con);
2368 ret = prepare_auth_request_more(con, p, payload_len);
2371 pr_err("prepare_auth_request_more failed: %d\n", ret);
2378 pr_err("failed to decode auth_reply_more\n");
2383 * Align session_key and con_secret to avoid GFP_ATOMIC allocation
2384 * inside crypto_shash_setkey() and crypto_aead_setkey() called from
2385 * setup_crypto(). __aligned(16) isn't guaranteed to work for stack
2386 * objects, so do it by hand.
2388 static int process_auth_done(struct ceph_connection *con, void *p, void *end)
2390 u8 session_key_buf[CEPH_KEY_LEN + 16];
2391 u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16];
2392 u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16);
2393 u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16);
2394 int session_key_len, con_secret_len;
2399 if (con->state != CEPH_CON_S_V2_AUTH) {
2400 con->error_msg = "protocol error, unexpected auth_done";
2404 ceph_decode_64_safe(&p, end, global_id, bad);
2405 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad);
2406 ceph_decode_32_safe(&p, end, payload_len, bad);
2408 dout("%s con %p global_id %llu con_mode %d payload_len %d\n",
2409 __func__, con, global_id, con->v2.con_mode, payload_len);
2411 mutex_unlock(&con->mutex);
2412 session_key_len = 0;
2414 ret = con->ops->handle_auth_done(con, global_id, p, payload_len,
2415 session_key, &session_key_len,
2416 con_secret, &con_secret_len);
2417 mutex_lock(&con->mutex);
2418 if (con->state != CEPH_CON_S_V2_AUTH) {
2419 dout("%s con %p state changed to %d\n", __func__, con,
2425 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret);
2429 ret = setup_crypto(con, session_key, session_key_len, con_secret,
2434 reset_out_kvecs(con);
2435 ret = prepare_auth_signature(con);
2437 pr_err("prepare_auth_signature failed: %d\n", ret);
2441 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE;
2444 memzero_explicit(session_key_buf, sizeof(session_key_buf));
2445 memzero_explicit(con_secret_buf, sizeof(con_secret_buf));
2449 pr_err("failed to decode auth_done\n");
2453 static int process_auth_signature(struct ceph_connection *con,
2456 u8 hmac[SHA256_DIGEST_SIZE];
2459 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) {
2460 con->error_msg = "protocol error, unexpected auth_signature";
2464 ret = hmac_sha256(con, con->v2.out_sign_kvecs,
2465 con->v2.out_sign_kvec_cnt, hmac);
2469 ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad);
2470 if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) {
2471 con->error_msg = "integrity error, bad auth signature";
2475 dout("%s con %p auth signature ok\n", __func__, con);
2477 /* no reset_out_kvecs() as our auth_signature may still be pending */
2478 if (!con->v2.server_cookie) {
2479 ret = prepare_client_ident(con);
2481 pr_err("prepare_client_ident failed: %d\n", ret);
2485 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2487 ret = prepare_session_reconnect(con);
2489 pr_err("prepare_session_reconnect failed: %d\n", ret);
2493 con->state = CEPH_CON_S_V2_SESSION_RECONNECT;
2499 pr_err("failed to decode auth_signature\n");
2503 static int process_server_ident(struct ceph_connection *con,
2506 struct ceph_client *client = from_msgr(con->msgr);
2507 u64 features, required_features;
2508 struct ceph_entity_addr addr;
2515 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2516 con->error_msg = "protocol error, unexpected server_ident";
2520 ret = ceph_decode_entity_addrvec(&p, end, true, &addr);
2522 pr_err("failed to decode server addrs: %d\n", ret);
2526 ceph_decode_64_safe(&p, end, global_id, bad);
2527 ceph_decode_64_safe(&p, end, global_seq, bad);
2528 ceph_decode_64_safe(&p, end, features, bad);
2529 ceph_decode_64_safe(&p, end, required_features, bad);
2530 ceph_decode_64_safe(&p, end, flags, bad);
2531 ceph_decode_64_safe(&p, end, cookie, bad);
2533 dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n",
2534 __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce),
2535 global_id, global_seq, features, required_features, flags, cookie);
2537 /* is this who we intended to talk to? */
2538 if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) {
2539 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n",
2540 ceph_pr_addr(&con->peer_addr),
2541 le32_to_cpu(con->peer_addr.nonce),
2542 ceph_pr_addr(&addr), le32_to_cpu(addr.nonce));
2543 con->error_msg = "wrong peer at address";
2547 if (client->required_features & ~features) {
2548 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n",
2549 features, client->required_features & ~features);
2550 con->error_msg = "missing required protocol features";
2555 * Both name->type and name->num are set in ceph_con_open() but
2556 * name->num may be bogus in the initial monmap. name->type is
2557 * verified in handle_hello().
2559 WARN_ON(!con->peer_name.type);
2560 con->peer_name.num = cpu_to_le64(global_id);
2561 con->v2.peer_global_seq = global_seq;
2562 con->peer_features = features;
2563 WARN_ON(required_features & ~client->supported_features);
2564 con->v2.server_cookie = cookie;
2566 if (flags & CEPH_MSG_CONNECT_LOSSY) {
2567 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX);
2568 WARN_ON(con->v2.server_cookie);
2570 WARN_ON(!con->v2.server_cookie);
2573 clear_in_sign_kvecs(con);
2574 clear_out_sign_kvecs(con);
2575 free_conn_bufs(con);
2576 con->delay = 0; /* reset backoff memory */
2578 con->state = CEPH_CON_S_OPEN;
2579 con->v2.out_state = OUT_S_GET_NEXT;
2583 pr_err("failed to decode server_ident\n");
2587 static int process_ident_missing_features(struct ceph_connection *con,
2590 struct ceph_client *client = from_msgr(con->msgr);
2591 u64 missing_features;
2593 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) {
2594 con->error_msg = "protocol error, unexpected ident_missing_features";
2598 ceph_decode_64_safe(&p, end, missing_features, bad);
2599 pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n",
2600 client->supported_features, missing_features);
2601 con->error_msg = "missing required protocol features";
2605 pr_err("failed to decode ident_missing_features\n");
2609 static int process_session_reconnect_ok(struct ceph_connection *con,
2614 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2615 con->error_msg = "protocol error, unexpected session_reconnect_ok";
2619 ceph_decode_64_safe(&p, end, seq, bad);
2621 dout("%s con %p seq %llu\n", __func__, con, seq);
2622 ceph_con_discard_requeued(con, seq);
2624 clear_in_sign_kvecs(con);
2625 clear_out_sign_kvecs(con);
2626 free_conn_bufs(con);
2627 con->delay = 0; /* reset backoff memory */
2629 con->state = CEPH_CON_S_OPEN;
2630 con->v2.out_state = OUT_S_GET_NEXT;
2634 pr_err("failed to decode session_reconnect_ok\n");
2638 static int process_session_retry(struct ceph_connection *con,
2644 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2645 con->error_msg = "protocol error, unexpected session_retry";
2649 ceph_decode_64_safe(&p, end, connect_seq, bad);
2651 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq);
2652 WARN_ON(connect_seq <= con->v2.connect_seq);
2653 con->v2.connect_seq = connect_seq + 1;
2655 free_conn_bufs(con);
2657 reset_out_kvecs(con);
2658 ret = prepare_session_reconnect(con);
2660 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret);
2667 pr_err("failed to decode session_retry\n");
2671 static int process_session_retry_global(struct ceph_connection *con,
2677 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2678 con->error_msg = "protocol error, unexpected session_retry_global";
2682 ceph_decode_64_safe(&p, end, global_seq, bad);
2684 dout("%s con %p global_seq %llu\n", __func__, con, global_seq);
2685 WARN_ON(global_seq <= con->v2.global_seq);
2686 con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq);
2688 free_conn_bufs(con);
2690 reset_out_kvecs(con);
2691 ret = prepare_session_reconnect(con);
2693 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret);
2700 pr_err("failed to decode session_retry_global\n");
2704 static int process_session_reset(struct ceph_connection *con,
2710 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2711 con->error_msg = "protocol error, unexpected session_reset";
2715 ceph_decode_8_safe(&p, end, full, bad);
2717 con->error_msg = "protocol error, bad session_reset";
2721 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name),
2722 ceph_pr_addr(&con->peer_addr));
2723 ceph_con_reset_session(con);
2725 mutex_unlock(&con->mutex);
2726 if (con->ops->peer_reset)
2727 con->ops->peer_reset(con);
2728 mutex_lock(&con->mutex);
2729 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) {
2730 dout("%s con %p state changed to %d\n", __func__, con,
2735 free_conn_bufs(con);
2737 reset_out_kvecs(con);
2738 ret = prepare_client_ident(con);
2740 pr_err("prepare_client_ident (rst) failed: %d\n", ret);
2744 con->state = CEPH_CON_S_V2_SESSION_CONNECT;
2748 pr_err("failed to decode session_reset\n");
2752 static int process_keepalive2_ack(struct ceph_connection *con,
2755 if (con->state != CEPH_CON_S_OPEN) {
2756 con->error_msg = "protocol error, unexpected keepalive2_ack";
2760 ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad);
2761 ceph_decode_timespec64(&con->last_keepalive_ack, p);
2763 dout("%s con %p timestamp %lld.%09ld\n", __func__, con,
2764 con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec);
2769 pr_err("failed to decode keepalive2_ack\n");
2773 static int process_ack(struct ceph_connection *con, void *p, void *end)
2777 if (con->state != CEPH_CON_S_OPEN) {
2778 con->error_msg = "protocol error, unexpected ack";
2782 ceph_decode_64_safe(&p, end, seq, bad);
2784 dout("%s con %p seq %llu\n", __func__, con, seq);
2785 ceph_con_discard_sent(con, seq);
2789 pr_err("failed to decode ack\n");
2793 static int process_control(struct ceph_connection *con, void *p, void *end)
2795 int tag = con->v2.in_desc.fd_tag;
2798 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p));
2801 case FRAME_TAG_HELLO:
2802 ret = process_hello(con, p, end);
2804 case FRAME_TAG_AUTH_BAD_METHOD:
2805 ret = process_auth_bad_method(con, p, end);
2807 case FRAME_TAG_AUTH_REPLY_MORE:
2808 ret = process_auth_reply_more(con, p, end);
2810 case FRAME_TAG_AUTH_DONE:
2811 ret = process_auth_done(con, p, end);
2813 case FRAME_TAG_AUTH_SIGNATURE:
2814 ret = process_auth_signature(con, p, end);
2816 case FRAME_TAG_SERVER_IDENT:
2817 ret = process_server_ident(con, p, end);
2819 case FRAME_TAG_IDENT_MISSING_FEATURES:
2820 ret = process_ident_missing_features(con, p, end);
2822 case FRAME_TAG_SESSION_RECONNECT_OK:
2823 ret = process_session_reconnect_ok(con, p, end);
2825 case FRAME_TAG_SESSION_RETRY:
2826 ret = process_session_retry(con, p, end);
2828 case FRAME_TAG_SESSION_RETRY_GLOBAL:
2829 ret = process_session_retry_global(con, p, end);
2831 case FRAME_TAG_SESSION_RESET:
2832 ret = process_session_reset(con, p, end);
2834 case FRAME_TAG_KEEPALIVE2_ACK:
2835 ret = process_keepalive2_ack(con, p, end);
2838 ret = process_ack(con, p, end);
2841 pr_err("bad tag %d\n", tag);
2842 con->error_msg = "protocol error, bad tag";
2846 dout("%s con %p error %d\n", __func__, con, ret);
2850 prepare_read_preamble(con);
2856 * 1 - con->in_msg set, read message
2860 static int process_message_header(struct ceph_connection *con,
2863 struct ceph_frame_desc *desc = &con->v2.in_desc;
2864 struct ceph_msg_header2 *hdr2 = p;
2865 struct ceph_msg_header hdr;
2871 seq = le64_to_cpu(hdr2->seq);
2872 if ((s64)seq - (s64)con->in_seq < 1) {
2873 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n",
2874 ENTITY_NAME(con->peer_name),
2875 ceph_pr_addr(&con->peer_addr),
2876 seq, con->in_seq + 1);
2879 if ((s64)seq - (s64)con->in_seq > 1) {
2880 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1);
2881 con->error_msg = "bad message sequence # for incoming message";
2885 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq));
2887 fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2],
2888 desc->fd_lens[3], &con->peer_name);
2889 ret = ceph_con_in_msg_alloc(con, &hdr, &skip);
2893 WARN_ON(!con->in_msg ^ skip);
2897 WARN_ON(!con->in_msg);
2898 WARN_ON(con->in_msg->con != con);
2902 static int process_message(struct ceph_connection *con)
2904 ceph_con_process_message(con);
2907 * We could have been closed by ceph_con_close() because
2908 * ceph_con_process_message() temporarily drops con->mutex.
2910 if (con->state != CEPH_CON_S_OPEN) {
2911 dout("%s con %p state changed to %d\n", __func__, con,
2916 prepare_read_preamble(con);
2920 static int __handle_control(struct ceph_connection *con, void *p)
2922 void *end = p + con->v2.in_desc.fd_lens[0];
2923 struct ceph_msg *msg;
2926 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE)
2927 return process_control(con, p, end);
2929 ret = process_message_header(con, p, end);
2933 prepare_skip_message(con);
2937 msg = con->in_msg; /* set in process_message_header() */
2938 if (front_len(msg)) {
2939 WARN_ON(front_len(msg) > msg->front_alloc_len);
2940 msg->front.iov_len = front_len(msg);
2942 msg->front.iov_len = 0;
2944 if (middle_len(msg)) {
2945 WARN_ON(middle_len(msg) > msg->middle->alloc_len);
2946 msg->middle->vec.iov_len = middle_len(msg);
2947 } else if (msg->middle) {
2948 msg->middle->vec.iov_len = 0;
2951 if (!front_len(msg) && !middle_len(msg) && !data_len(msg))
2952 return process_message(con);
2954 if (con_secure(con))
2955 return prepare_read_tail_secure(con);
2957 return prepare_read_tail_plain(con);
2960 static int handle_preamble(struct ceph_connection *con)
2962 struct ceph_frame_desc *desc = &con->v2.in_desc;
2965 if (con_secure(con)) {
2966 ret = decrypt_preamble(con);
2968 if (ret == -EBADMSG)
2969 con->error_msg = "integrity error, bad preamble auth tag";
2974 ret = decode_preamble(con->v2.in_buf, desc);
2976 if (ret == -EBADMSG)
2977 con->error_msg = "integrity error, bad crc";
2979 con->error_msg = "protocol error, bad preamble";
2983 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__,
2984 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0],
2985 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]);
2987 if (!con_secure(con))
2988 return prepare_read_control(con);
2990 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN)
2991 return prepare_read_control_remainder(con);
2993 return __handle_control(con, CTRL_BODY(con->v2.in_buf));
2996 static int handle_control(struct ceph_connection *con)
2998 int ctrl_len = con->v2.in_desc.fd_lens[0];
3002 WARN_ON(con_secure(con));
3004 ret = verify_control_crc(con);
3006 con->error_msg = "integrity error, bad crc";
3010 if (con->state == CEPH_CON_S_V2_AUTH) {
3011 buf = alloc_conn_buf(con, ctrl_len);
3015 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len);
3016 return __handle_control(con, buf);
3019 return __handle_control(con, con->v2.in_kvecs[0].iov_base);
3022 static int handle_control_remainder(struct ceph_connection *con)
3026 WARN_ON(!con_secure(con));
3028 ret = decrypt_control_remainder(con);
3030 if (ret == -EBADMSG)
3031 con->error_msg = "integrity error, bad control remainder auth tag";
3035 return __handle_control(con, con->v2.in_kvecs[0].iov_base -
3036 CEPH_PREAMBLE_INLINE_LEN);
3039 static int handle_epilogue(struct ceph_connection *con)
3041 u32 front_crc, middle_crc, data_crc;
3044 if (con_secure(con)) {
3045 ret = decrypt_tail(con);
3047 if (ret == -EBADMSG)
3048 con->error_msg = "integrity error, bad epilogue auth tag";
3052 /* just late_status */
3053 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL);
3055 con->error_msg = "protocol error, bad epilogue";
3059 ret = decode_epilogue(con->v2.in_buf, &front_crc,
3060 &middle_crc, &data_crc);
3062 con->error_msg = "protocol error, bad epilogue";
3066 ret = verify_epilogue_crcs(con, front_crc, middle_crc,
3069 con->error_msg = "integrity error, bad crc";
3074 return process_message(con);
3077 static void finish_skip(struct ceph_connection *con)
3079 dout("%s con %p\n", __func__, con);
3081 if (con_secure(con))
3082 gcm_inc_nonce(&con->v2.in_gcm_nonce);
3087 static int populate_in_iter(struct ceph_connection *con)
3091 dout("%s con %p state %d in_state %d\n", __func__, con, con->state,
3093 WARN_ON(iov_iter_count(&con->v2.in_iter));
3095 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) {
3096 ret = process_banner_prefix(con);
3097 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) {
3098 ret = process_banner_payload(con);
3099 } else if ((con->state >= CEPH_CON_S_V2_HELLO &&
3100 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) ||
3101 con->state == CEPH_CON_S_OPEN) {
3102 switch (con->v2.in_state) {
3103 case IN_S_HANDLE_PREAMBLE:
3104 ret = handle_preamble(con);
3106 case IN_S_HANDLE_CONTROL:
3107 ret = handle_control(con);
3109 case IN_S_HANDLE_CONTROL_REMAINDER:
3110 ret = handle_control_remainder(con);
3112 case IN_S_PREPARE_READ_DATA:
3113 ret = prepare_read_data(con);
3115 case IN_S_PREPARE_READ_DATA_CONT:
3116 prepare_read_data_cont(con);
3119 case IN_S_PREPARE_READ_ENC_PAGE:
3120 prepare_read_enc_page(con);
3123 case IN_S_PREPARE_SPARSE_DATA:
3124 ret = prepare_sparse_read_data(con);
3126 case IN_S_PREPARE_SPARSE_DATA_CONT:
3127 ret = prepare_sparse_read_cont(con);
3129 case IN_S_HANDLE_EPILOGUE:
3130 ret = handle_epilogue(con);
3132 case IN_S_FINISH_SKIP:
3137 WARN(1, "bad in_state %d", con->v2.in_state);
3141 WARN(1, "bad state %d", con->state);
3145 dout("%s con %p error %d\n", __func__, con, ret);
3149 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3151 dout("%s con %p populated %zu\n", __func__, con,
3152 iov_iter_count(&con->v2.in_iter));
3156 int ceph_con_v2_try_read(struct ceph_connection *con)
3160 dout("%s con %p state %d need %zu\n", __func__, con, con->state,
3161 iov_iter_count(&con->v2.in_iter));
3163 if (con->state == CEPH_CON_S_PREOPEN)
3167 * We should always have something pending here. If not,
3168 * avoid calling populate_in_iter() as if we read something
3169 * (ceph_tcp_recv() would immediately return 1).
3171 if (WARN_ON(!iov_iter_count(&con->v2.in_iter)))
3175 ret = ceph_tcp_recv(con);
3179 ret = populate_in_iter(con);
3181 if (ret && ret != -EAGAIN && !con->error_msg)
3182 con->error_msg = "read processing error";
3188 static void queue_data(struct ceph_connection *con)
3192 con->v2.out_epil.data_crc = -1;
3193 ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg,
3194 data_len(con->out_msg));
3196 get_bvec_at(&con->v2.out_cursor, &bv);
3197 set_out_bvec(con, &bv, true);
3198 con->v2.out_state = OUT_S_QUEUE_DATA_CONT;
3201 static void queue_data_cont(struct ceph_connection *con)
3205 con->v2.out_epil.data_crc = ceph_crc32c_page(
3206 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3207 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len);
3209 ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len);
3210 if (con->v2.out_cursor.total_resid) {
3211 get_bvec_at(&con->v2.out_cursor, &bv);
3212 set_out_bvec(con, &bv, true);
3213 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT);
3218 * We've written all data. Queue epilogue. Once it's written,
3221 reset_out_kvecs(con);
3222 prepare_epilogue_plain(con, false);
3223 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3226 static void queue_enc_page(struct ceph_connection *con)
3230 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i,
3231 con->v2.out_enc_resid);
3232 WARN_ON(!con->v2.out_enc_resid);
3234 bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i],
3235 min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0);
3237 set_out_bvec(con, &bv, false);
3238 con->v2.out_enc_i++;
3239 con->v2.out_enc_resid -= bv.bv_len;
3241 if (con->v2.out_enc_resid) {
3242 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE);
3247 * We've queued the last piece of ciphertext (ending with
3248 * epilogue) + auth tag. Once it's written, we are done.
3250 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt);
3251 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3254 static void queue_zeros(struct ceph_connection *con)
3256 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero);
3258 if (con->v2.out_zero) {
3259 set_out_bvec_zero(con);
3260 con->v2.out_zero -= con->v2.out_bvec.bv_len;
3261 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3266 * We've zero-filled everything up to epilogue. Queue epilogue
3267 * with late_status set to ABORTED and crcs adjusted for zeros.
3268 * Once it's written, we are done patching up for the revoke.
3270 reset_out_kvecs(con);
3271 prepare_epilogue_plain(con, true);
3272 con->v2.out_state = OUT_S_FINISH_MESSAGE;
3275 static void finish_message(struct ceph_connection *con)
3277 dout("%s con %p msg %p\n", __func__, con, con->out_msg);
3279 /* we end up here both plain and secure modes */
3280 if (con->v2.out_enc_pages) {
3281 WARN_ON(!con->v2.out_enc_page_cnt);
3282 ceph_release_page_vector(con->v2.out_enc_pages,
3283 con->v2.out_enc_page_cnt);
3284 con->v2.out_enc_pages = NULL;
3285 con->v2.out_enc_page_cnt = 0;
3287 /* message may have been revoked */
3289 ceph_msg_put(con->out_msg);
3290 con->out_msg = NULL;
3293 con->v2.out_state = OUT_S_GET_NEXT;
3296 static int populate_out_iter(struct ceph_connection *con)
3300 dout("%s con %p state %d out_state %d\n", __func__, con, con->state,
3302 WARN_ON(iov_iter_count(&con->v2.out_iter));
3304 if (con->state != CEPH_CON_S_OPEN) {
3305 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX ||
3306 con->state > CEPH_CON_S_V2_SESSION_RECONNECT);
3307 goto nothing_pending;
3310 switch (con->v2.out_state) {
3311 case OUT_S_QUEUE_DATA:
3312 WARN_ON(!con->out_msg);
3315 case OUT_S_QUEUE_DATA_CONT:
3316 WARN_ON(!con->out_msg);
3317 queue_data_cont(con);
3319 case OUT_S_QUEUE_ENC_PAGE:
3320 queue_enc_page(con);
3322 case OUT_S_QUEUE_ZEROS:
3323 WARN_ON(con->out_msg); /* revoked */
3326 case OUT_S_FINISH_MESSAGE:
3327 finish_message(con);
3329 case OUT_S_GET_NEXT:
3332 WARN(1, "bad out_state %d", con->v2.out_state);
3336 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT);
3337 if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) {
3338 ret = prepare_keepalive2(con);
3340 pr_err("prepare_keepalive2 failed: %d\n", ret);
3343 } else if (!list_empty(&con->out_queue)) {
3344 ceph_con_get_out_msg(con);
3345 ret = prepare_message(con);
3347 pr_err("prepare_message failed: %d\n", ret);
3350 } else if (con->in_seq > con->in_seq_acked) {
3351 ret = prepare_ack(con);
3353 pr_err("prepare_ack failed: %d\n", ret);
3357 goto nothing_pending;
3361 if (WARN_ON(!iov_iter_count(&con->v2.out_iter)))
3363 dout("%s con %p populated %zu\n", __func__, con,
3364 iov_iter_count(&con->v2.out_iter));
3368 WARN_ON(iov_iter_count(&con->v2.out_iter));
3369 dout("%s con %p nothing pending\n", __func__, con);
3370 ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING);
3374 int ceph_con_v2_try_write(struct ceph_connection *con)
3378 dout("%s con %p state %d have %zu\n", __func__, con, con->state,
3379 iov_iter_count(&con->v2.out_iter));
3381 /* open the socket first? */
3382 if (con->state == CEPH_CON_S_PREOPEN) {
3383 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2);
3386 * Always bump global_seq. Bump connect_seq only if
3387 * there is a session (i.e. we are reconnecting and will
3388 * send session_reconnect instead of client_ident).
3390 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0);
3391 if (con->v2.server_cookie)
3392 con->v2.connect_seq++;
3394 ret = prepare_read_banner_prefix(con);
3396 pr_err("prepare_read_banner_prefix failed: %d\n", ret);
3397 con->error_msg = "connect error";
3401 reset_out_kvecs(con);
3402 ret = prepare_banner(con);
3404 pr_err("prepare_banner failed: %d\n", ret);
3405 con->error_msg = "connect error";
3409 ret = ceph_tcp_connect(con);
3411 pr_err("ceph_tcp_connect failed: %d\n", ret);
3412 con->error_msg = "connect error";
3417 if (!iov_iter_count(&con->v2.out_iter)) {
3418 ret = populate_out_iter(con);
3420 if (ret && ret != -EAGAIN && !con->error_msg)
3421 con->error_msg = "write processing error";
3426 tcp_sock_set_cork(con->sock->sk, true);
3428 ret = ceph_tcp_send(con);
3432 ret = populate_out_iter(con);
3434 if (ret && ret != -EAGAIN && !con->error_msg)
3435 con->error_msg = "write processing error";
3440 tcp_sock_set_cork(con->sock->sk, false);
3444 static u32 crc32c_zeros(u32 crc, int zero_len)
3449 len = min(zero_len, (int)PAGE_SIZE);
3450 crc = crc32c(crc, page_address(ceph_zero_page), len);
3457 static void prepare_zero_front(struct ceph_connection *con, int resid)
3461 WARN_ON(!resid || resid > front_len(con->out_msg));
3462 sent = front_len(con->out_msg) - resid;
3463 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3466 con->v2.out_epil.front_crc =
3467 crc32c(-1, con->out_msg->front.iov_base, sent);
3468 con->v2.out_epil.front_crc =
3469 crc32c_zeros(con->v2.out_epil.front_crc, resid);
3471 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid);
3474 con->v2.out_iter.count -= resid;
3475 out_zero_add(con, resid);
3478 static void prepare_zero_middle(struct ceph_connection *con, int resid)
3482 WARN_ON(!resid || resid > middle_len(con->out_msg));
3483 sent = middle_len(con->out_msg) - resid;
3484 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3487 con->v2.out_epil.middle_crc =
3488 crc32c(-1, con->out_msg->middle->vec.iov_base, sent);
3489 con->v2.out_epil.middle_crc =
3490 crc32c_zeros(con->v2.out_epil.middle_crc, resid);
3492 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid);
3495 con->v2.out_iter.count -= resid;
3496 out_zero_add(con, resid);
3499 static void prepare_zero_data(struct ceph_connection *con)
3501 dout("%s con %p\n", __func__, con);
3502 con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg));
3503 out_zero_add(con, data_len(con->out_msg));
3506 static void revoke_at_queue_data(struct ceph_connection *con)
3511 WARN_ON(!data_len(con->out_msg));
3512 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3513 resid = iov_iter_count(&con->v2.out_iter);
3515 boundary = front_len(con->out_msg) + middle_len(con->out_msg);
3516 if (resid > boundary) {
3518 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3519 dout("%s con %p was sending head\n", __func__, con);
3520 if (front_len(con->out_msg))
3521 prepare_zero_front(con, front_len(con->out_msg));
3522 if (middle_len(con->out_msg))
3523 prepare_zero_middle(con, middle_len(con->out_msg));
3524 prepare_zero_data(con);
3525 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3526 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3530 boundary = middle_len(con->out_msg);
3531 if (resid > boundary) {
3533 dout("%s con %p was sending front\n", __func__, con);
3534 prepare_zero_front(con, resid);
3535 if (middle_len(con->out_msg))
3536 prepare_zero_middle(con, middle_len(con->out_msg));
3537 prepare_zero_data(con);
3543 dout("%s con %p was sending middle\n", __func__, con);
3544 prepare_zero_middle(con, resid);
3545 prepare_zero_data(con);
3549 static void revoke_at_queue_data_cont(struct ceph_connection *con)
3551 int sent, resid; /* current piece of data */
3553 WARN_ON(!data_len(con->out_msg));
3554 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter));
3555 resid = iov_iter_count(&con->v2.out_iter);
3556 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len);
3557 sent = con->v2.out_bvec.bv_len - resid;
3558 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid);
3561 con->v2.out_epil.data_crc = ceph_crc32c_page(
3562 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page,
3563 con->v2.out_bvec.bv_offset, sent);
3564 ceph_msg_data_advance(&con->v2.out_cursor, sent);
3566 WARN_ON(resid > con->v2.out_cursor.total_resid);
3567 con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc,
3568 con->v2.out_cursor.total_resid);
3570 con->v2.out_iter.count -= resid;
3571 out_zero_add(con, con->v2.out_cursor.total_resid);
3575 static void revoke_at_finish_message(struct ceph_connection *con)
3580 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter));
3581 resid = iov_iter_count(&con->v2.out_iter);
3583 if (!front_len(con->out_msg) && !middle_len(con->out_msg) &&
3584 !data_len(con->out_msg)) {
3585 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN);
3586 dout("%s con %p was sending head (empty message) - noop\n",
3591 boundary = front_len(con->out_msg) + middle_len(con->out_msg) +
3592 CEPH_EPILOGUE_PLAIN_LEN;
3593 if (resid > boundary) {
3595 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN);
3596 dout("%s con %p was sending head\n", __func__, con);
3597 if (front_len(con->out_msg))
3598 prepare_zero_front(con, front_len(con->out_msg));
3599 if (middle_len(con->out_msg))
3600 prepare_zero_middle(con, middle_len(con->out_msg));
3601 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3602 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid);
3603 con->v2.out_state = OUT_S_QUEUE_ZEROS;
3607 boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3608 if (resid > boundary) {
3610 dout("%s con %p was sending front\n", __func__, con);
3611 prepare_zero_front(con, resid);
3612 if (middle_len(con->out_msg))
3613 prepare_zero_middle(con, middle_len(con->out_msg));
3614 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3619 boundary = CEPH_EPILOGUE_PLAIN_LEN;
3620 if (resid > boundary) {
3622 dout("%s con %p was sending middle\n", __func__, con);
3623 prepare_zero_middle(con, resid);
3624 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN;
3630 dout("%s con %p was sending epilogue - noop\n", __func__, con);
3633 void ceph_con_v2_revoke(struct ceph_connection *con)
3635 WARN_ON(con->v2.out_zero);
3637 if (con_secure(con)) {
3638 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE &&
3639 con->v2.out_state != OUT_S_FINISH_MESSAGE);
3640 dout("%s con %p secure - noop\n", __func__, con);
3644 switch (con->v2.out_state) {
3645 case OUT_S_QUEUE_DATA:
3646 revoke_at_queue_data(con);
3648 case OUT_S_QUEUE_DATA_CONT:
3649 revoke_at_queue_data_cont(con);
3651 case OUT_S_FINISH_MESSAGE:
3652 revoke_at_finish_message(con);
3655 WARN(1, "bad out_state %d", con->v2.out_state);
3660 static void revoke_at_prepare_read_data(struct ceph_connection *con)
3665 WARN_ON(con_secure(con));
3666 WARN_ON(!data_len(con->in_msg));
3667 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter));
3668 resid = iov_iter_count(&con->v2.in_iter);
3671 remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN;
3672 dout("%s con %p resid %d remaining %d\n", __func__, con, resid,
3674 con->v2.in_iter.count -= resid;
3675 set_in_skip(con, resid + remaining);
3676 con->v2.in_state = IN_S_FINISH_SKIP;
3679 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con)
3681 int recved, resid; /* current piece of data */
3684 WARN_ON(con_secure(con));
3685 WARN_ON(!data_len(con->in_msg));
3686 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3687 resid = iov_iter_count(&con->v2.in_iter);
3688 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3689 recved = con->v2.in_bvec.bv_len - resid;
3690 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid);
3693 ceph_msg_data_advance(&con->v2.in_cursor, recved);
3694 WARN_ON(resid > con->v2.in_cursor.total_resid);
3696 remaining = CEPH_EPILOGUE_PLAIN_LEN;
3697 dout("%s con %p total_resid %zu remaining %d\n", __func__, con,
3698 con->v2.in_cursor.total_resid, remaining);
3699 con->v2.in_iter.count -= resid;
3700 set_in_skip(con, con->v2.in_cursor.total_resid + remaining);
3701 con->v2.in_state = IN_S_FINISH_SKIP;
3704 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con)
3706 int resid; /* current enc page (not necessarily data) */
3708 WARN_ON(!con_secure(con));
3709 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3710 resid = iov_iter_count(&con->v2.in_iter);
3711 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len);
3713 dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid,
3714 con->v2.in_enc_resid);
3715 con->v2.in_iter.count -= resid;
3716 set_in_skip(con, resid + con->v2.in_enc_resid);
3717 con->v2.in_state = IN_S_FINISH_SKIP;
3720 static void revoke_at_prepare_sparse_data(struct ceph_connection *con)
3722 int resid; /* current piece of data */
3725 WARN_ON(con_secure(con));
3726 WARN_ON(!data_len(con->in_msg));
3727 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter));
3728 resid = iov_iter_count(&con->v2.in_iter);
3729 dout("%s con %p resid %d\n", __func__, con, resid);
3731 remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain;
3732 con->v2.in_iter.count -= resid;
3733 set_in_skip(con, resid + remaining);
3734 con->v2.in_state = IN_S_FINISH_SKIP;
3737 static void revoke_at_handle_epilogue(struct ceph_connection *con)
3741 resid = iov_iter_count(&con->v2.in_iter);
3744 dout("%s con %p resid %d\n", __func__, con, resid);
3745 con->v2.in_iter.count -= resid;
3746 set_in_skip(con, resid);
3747 con->v2.in_state = IN_S_FINISH_SKIP;
3750 void ceph_con_v2_revoke_incoming(struct ceph_connection *con)
3752 switch (con->v2.in_state) {
3753 case IN_S_PREPARE_SPARSE_DATA:
3754 case IN_S_PREPARE_READ_DATA:
3755 revoke_at_prepare_read_data(con);
3757 case IN_S_PREPARE_READ_DATA_CONT:
3758 revoke_at_prepare_read_data_cont(con);
3760 case IN_S_PREPARE_READ_ENC_PAGE:
3761 revoke_at_prepare_read_enc_page(con);
3763 case IN_S_PREPARE_SPARSE_DATA_CONT:
3764 revoke_at_prepare_sparse_data(con);
3766 case IN_S_HANDLE_EPILOGUE:
3767 revoke_at_handle_epilogue(con);
3770 WARN(1, "bad in_state %d", con->v2.in_state);
3775 bool ceph_con_v2_opened(struct ceph_connection *con)
3777 return con->v2.peer_global_seq;
3780 void ceph_con_v2_reset_session(struct ceph_connection *con)
3782 con->v2.client_cookie = 0;
3783 con->v2.server_cookie = 0;
3784 con->v2.global_seq = 0;
3785 con->v2.connect_seq = 0;
3786 con->v2.peer_global_seq = 0;
3789 void ceph_con_v2_reset_protocol(struct ceph_connection *con)
3791 iov_iter_truncate(&con->v2.in_iter, 0);
3792 iov_iter_truncate(&con->v2.out_iter, 0);
3793 con->v2.out_zero = 0;
3795 clear_in_sign_kvecs(con);
3796 clear_out_sign_kvecs(con);
3797 free_conn_bufs(con);
3799 if (con->v2.in_enc_pages) {
3800 WARN_ON(!con->v2.in_enc_page_cnt);
3801 ceph_release_page_vector(con->v2.in_enc_pages,
3802 con->v2.in_enc_page_cnt);
3803 con->v2.in_enc_pages = NULL;
3804 con->v2.in_enc_page_cnt = 0;
3806 if (con->v2.out_enc_pages) {
3807 WARN_ON(!con->v2.out_enc_page_cnt);
3808 ceph_release_page_vector(con->v2.out_enc_pages,
3809 con->v2.out_enc_page_cnt);
3810 con->v2.out_enc_pages = NULL;
3811 con->v2.out_enc_page_cnt = 0;
3814 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN;
3815 memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN);
3816 memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN);
3818 if (con->v2.hmac_tfm) {
3819 crypto_free_shash(con->v2.hmac_tfm);
3820 con->v2.hmac_tfm = NULL;
3822 if (con->v2.gcm_req) {
3823 aead_request_free(con->v2.gcm_req);
3824 con->v2.gcm_req = NULL;
3826 if (con->v2.gcm_tfm) {
3827 crypto_free_aead(con->v2.gcm_tfm);
3828 con->v2.gcm_tfm = NULL;