2 * Copyright (C) 2013-2015 Kay Sievers
3 * Copyright (C) 2013-2015 Greg Kroah-Hartman <gregkh@linuxfoundation.org>
4 * Copyright (C) 2013-2015 Daniel Mack <daniel@zonque.org>
5 * Copyright (C) 2013-2015 David Herrmann <dh.herrmann@gmail.com>
6 * Copyright (C) 2013-2015 Linux Foundation
7 * Copyright (C) 2014-2015 Djalal Harouni <tixxdz@opendz.org>
9 * kdbus is free software; you can redistribute it and/or modify it under
10 * the terms of the GNU Lesser General Public License as published by the
11 * Free Software Foundation; either version 2.1 of the License, or (at
12 * your option) any later version.
15 #include <linux/capability.h>
16 #include <linux/cgroup.h>
17 #include <linux/cred.h>
18 #include <linux/file.h>
19 #include <linux/init.h>
20 #include <linux/module.h>
21 #include <linux/mutex.h>
22 #include <linux/sched.h>
23 #include <linux/shmem_fs.h>
24 #include <linux/sizes.h>
25 #include <linux/slab.h>
26 #include <linux/uaccess.h>
30 #include "connection.h"
40 static const char * const zeros = "\0\0\0\0\0\0\0";
42 static struct kdbus_gaps *kdbus_gaps_new(size_t n_memfds, size_t n_fds)
44 size_t size_offsets, size_memfds, size_fds, size;
45 struct kdbus_gaps *gaps;
47 size_offsets = n_memfds * sizeof(*gaps->memfd_offsets);
48 size_memfds = n_memfds * sizeof(*gaps->memfd_files);
49 size_fds = n_fds * sizeof(*gaps->fd_files);
50 size = sizeof(*gaps) + size_offsets + size_memfds + size_fds;
52 gaps = kzalloc(size, GFP_KERNEL);
54 return ERR_PTR(-ENOMEM);
56 kref_init(&gaps->kref);
57 gaps->n_memfds = 0; /* we reserve n_memfds, but don't enforce them */
58 gaps->memfd_offsets = (void *)(gaps + 1);
59 gaps->memfd_files = (void *)((u8 *)gaps->memfd_offsets + size_offsets);
60 gaps->n_fds = 0; /* we reserve n_fds, but don't enforce them */
61 gaps->fd_files = (void *)((u8 *)gaps->memfd_files + size_memfds);
66 static void kdbus_gaps_free(struct kref *kref)
68 struct kdbus_gaps *gaps = container_of(kref, struct kdbus_gaps, kref);
71 for (i = 0; i < gaps->n_fds; ++i)
72 if (gaps->fd_files[i])
73 fput(gaps->fd_files[i]);
74 for (i = 0; i < gaps->n_memfds; ++i)
75 if (gaps->memfd_files[i])
76 fput(gaps->memfd_files[i]);
82 * kdbus_gaps_ref() - gain reference
85 * Return: @gaps is returned
87 struct kdbus_gaps *kdbus_gaps_ref(struct kdbus_gaps *gaps)
90 kref_get(&gaps->kref);
95 * kdbus_gaps_unref() - drop reference
100 struct kdbus_gaps *kdbus_gaps_unref(struct kdbus_gaps *gaps)
103 kref_put(&gaps->kref, kdbus_gaps_free);
108 * kdbus_gaps_install() - install file-descriptors
109 * @gaps: gaps object, or NULL
110 * @slice: pool slice that contains the message
111 * @out_incomplete output variable to note incomplete fds
113 * This function installs all file-descriptors of @gaps into the current
114 * process and copies the file-descriptor numbers into the target pool slice.
116 * If the file-descriptors were only partially installed, then @out_incomplete
117 * will be set to true. Otherwise, it's set to false.
119 * Return: 0 on success, negative error code on failure
121 int kdbus_gaps_install(struct kdbus_gaps *gaps, struct kdbus_pool_slice *slice,
122 bool *out_incomplete)
124 bool incomplete_fds = false;
131 *out_incomplete = incomplete_fds;
135 n_fds = gaps->n_fds + gaps->n_memfds;
138 *out_incomplete = incomplete_fds;
142 fds = kmalloc_array(n_fds, sizeof(*fds), GFP_KERNEL);
147 /* 1) allocate fds and copy them over */
149 if (gaps->n_fds > 0) {
150 for (i = 0; i < gaps->n_fds; ++i) {
153 fd = get_unused_fd_flags(O_CLOEXEC);
155 incomplete_fds = true;
157 WARN_ON(!gaps->fd_files[i]);
159 fds[n_fds++] = fd < 0 ? -1 : fd;
163 * The file-descriptor array can only be present once per
164 * message. Hence, prepare all fds and then copy them over with
168 WARN_ON(!gaps->fd_offset);
171 kvec.iov_len = gaps->n_fds * sizeof(*fds);
172 ret = kdbus_pool_slice_copy_kvec(slice, gaps->fd_offset,
173 &kvec, 1, kvec.iov_len);
178 for (i = 0; i < gaps->n_memfds; ++i) {
181 memfd = get_unused_fd_flags(O_CLOEXEC);
183 incomplete_fds = true;
184 /* memfds are initialized to -1, skip copying it */
188 fds[n_fds++] = memfd;
191 * memfds have to be copied individually as they each are put
192 * into a separate item. This should not be an issue, though,
193 * as usually there is no need to send more than one memfd per
197 WARN_ON(!gaps->memfd_offsets[i]);
198 WARN_ON(!gaps->memfd_files[i]);
200 kvec.iov_base = &memfd;
201 kvec.iov_len = sizeof(memfd);
202 ret = kdbus_pool_slice_copy_kvec(slice, gaps->memfd_offsets[i],
203 &kvec, 1, kvec.iov_len);
208 /* 2) install fds now that everything was successful */
210 for (i = 0; i < gaps->n_fds; ++i)
212 fd_install(fds[i], get_file(gaps->fd_files[i]));
213 for (i = 0; i < gaps->n_memfds; ++i)
214 if (fds[gaps->n_fds + i] >= 0)
215 fd_install(fds[gaps->n_fds + i],
216 get_file(gaps->memfd_files[i]));
222 for (i = 0; i < n_fds; ++i)
223 put_unused_fd(fds[i]);
225 *out_incomplete = incomplete_fds;
229 static struct file *kdbus_get_fd(int fd)
231 struct file *f, *ret;
234 return ERR_PTR(-EBADF);
238 return ERR_PTR(-EBADF);
240 if (f->f_mode & FMODE_PATH)
241 ret = f; /* O_PATH is always allowed */
242 else if (f->f_op == &kdbus_handle_ops)
243 ret = ERR_PTR(-EOPNOTSUPP); /* disallow kdbus-fd over kdbus */
245 ret = f; /* all other are allowed */
253 static struct file *kdbus_get_memfd(const struct kdbus_memfd *memfd)
255 const int m = F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_WRITE | F_SEAL_SEAL;
256 struct file *f, *ret;
260 return ERR_PTR(-EBADF);
264 return ERR_PTR(-EBADF);
266 s = shmem_get_seals(f);
268 ret = ERR_PTR(-EMEDIUMTYPE);
269 else if ((s & m) != m)
270 ret = ERR_PTR(-ETXTBSY);
271 else if (memfd->start + memfd->size > (u64)i_size_read(file_inode(f)))
272 ret = ERR_PTR(-EFAULT);
282 static int kdbus_msg_examine(struct kdbus_msg *msg, struct kdbus_bus *bus,
283 struct kdbus_cmd_send *cmd, size_t *out_n_memfds,
284 size_t *out_n_fds, size_t *out_n_parts)
286 struct kdbus_item *item, *fds = NULL, *bloom = NULL, *dstname = NULL;
287 u64 n_parts, n_memfds, n_fds, vec_size;
291 * Validate the message and command parameters.
294 /* KDBUS_PAYLOAD_KERNEL is reserved to kernel messages */
295 if (msg->payload_type == KDBUS_PAYLOAD_KERNEL)
298 if (msg->dst_id == KDBUS_DST_ID_BROADCAST) {
299 /* broadcasts must be marked as signals */
300 if (!(msg->flags & KDBUS_MSG_SIGNAL))
302 /* broadcasts cannot have timeouts */
303 if (msg->timeout_ns > 0)
307 if (msg->flags & KDBUS_MSG_EXPECT_REPLY) {
308 /* if you expect a reply, you must specify a timeout */
309 if (msg->timeout_ns == 0 || !msg->cookie)
311 /* signals cannot have replies */
312 if (msg->flags & KDBUS_MSG_SIGNAL)
315 /* must expect reply if sent as synchronous call */
316 if (cmd->flags & KDBUS_SEND_SYNC_REPLY)
318 /* cannot mark replies as signal */
319 if (msg->cookie_reply && (msg->flags & KDBUS_MSG_SIGNAL))
325 * Validate all passed items. While at it, select some statistics that
326 * are required to allocate state objects later on.
328 * Generic item validation has already been done via
329 * kdbus_item_validate(). Furthermore, the number of items is naturally
330 * limited by the maximum message size. Hence, only non-generic item
331 * checks are performed here (mainly integer overflow tests).
339 KDBUS_ITEMS_FOREACH(item, msg->items, KDBUS_ITEMS_SIZE(msg, items)) {
340 switch (item->type) {
341 case KDBUS_ITEM_PAYLOAD_VEC: {
342 void __force __user *ptr = KDBUS_PTR(item->vec.address);
343 u64 size = item->vec.size;
345 if (vec_size + size < vec_size)
347 if (vec_size + size > KDBUS_MSG_MAX_PAYLOAD_VEC_SIZE)
349 if (ptr && unlikely(!access_ok(VERIFY_READ, ptr, size)))
352 if (ptr || size % 8) /* data or padding */
356 case KDBUS_ITEM_PAYLOAD_MEMFD: {
357 u64 start = item->memfd.start;
358 u64 size = item->memfd.size;
360 if (start + size < start)
362 if (n_memfds >= KDBUS_MSG_MAX_MEMFD_ITEMS)
366 if (size % 8) /* vec-padding required */
370 case KDBUS_ITEM_FDS: {
375 n_fds = KDBUS_ITEM_PAYLOAD_SIZE(item) / sizeof(int);
376 if (n_fds > KDBUS_CONN_MAX_FDS_PER_USER)
381 case KDBUS_ITEM_BLOOM_FILTER: {
388 bloom_size = KDBUS_ITEM_PAYLOAD_SIZE(item) -
389 offsetof(struct kdbus_bloom_filter, data);
390 if (!KDBUS_IS_ALIGNED8(bloom_size))
392 if (bloom_size != bus->bloom.size)
397 case KDBUS_ITEM_DST_NAME: {
402 if (!kdbus_name_is_valid(item->str, false))
404 if (msg->dst_id == KDBUS_DST_ID_BROADCAST)
416 * Validate that required items were actually passed, and that no item
417 * contradicts the message flags.
420 /* bloom filters must be attached _iff_ it's a signal */
421 if (!(msg->flags & KDBUS_MSG_SIGNAL) != !bloom)
423 /* destination name is required if no ID is given */
424 if (msg->dst_id == KDBUS_DST_ID_NAME && !dstname)
425 return -EDESTADDRREQ;
426 /* cannot send file-descriptors attached to broadcasts */
427 if (msg->dst_id == KDBUS_DST_ID_BROADCAST && fds)
430 *out_n_memfds = n_memfds;
432 *out_n_parts = n_parts;
437 static bool kdbus_staging_merge_vecs(struct kdbus_staging *staging,
438 struct kdbus_item **prev_item,
439 struct iovec **prev_vec,
440 const struct kdbus_item *merge)
442 void __user *ptr = (void __user *)KDBUS_PTR(merge->vec.address);
443 u64 padding = merge->vec.size % 8;
444 struct kdbus_item *prev = *prev_item;
445 struct iovec *vec = *prev_vec;
447 /* XXX: merging is disabled so far */
448 if (0 && prev && prev->type == KDBUS_ITEM_PAYLOAD_OFF &&
449 !merge->vec.address == !prev->vec.address) {
451 * If we merge two VECs, we can always drop the second
452 * PAYLOAD_VEC item. Hence, include its size in the previous
455 prev->vec.size += merge->vec.size;
459 * If we merge two data VECs, we need two iovecs to copy
460 * the data. But the items can be easily merged by
461 * summing their lengths.
463 vec = &staging->parts[staging->n_parts++];
464 vec->iov_len = merge->vec.size;
466 staging->n_payload += vec->iov_len;
467 } else if (padding) {
469 * If we merge two 0-vecs with the second 0-vec
470 * requiring padding, we need to insert an iovec to copy
471 * the 0-padding. We try merging it with the previous
472 * 0-padding iovec. This might end up with an
473 * iov_len==0, in which case we simply drop the iovec.
476 staging->n_payload -= vec->iov_len;
477 vec->iov_len = prev->vec.size % 8;
482 staging->n_payload += vec->iov_len;
485 vec = &staging->parts[staging->n_parts++];
486 vec->iov_len = padding;
487 vec->iov_base = (char __user *)zeros;
488 staging->n_payload += vec->iov_len;
492 * If we merge two 0-vecs with the second 0-vec having
493 * no padding, we know the padding of the first stays
494 * the same. Hence, @vec needs no adjustment.
498 /* successfully merged with previous item */
502 * If we cannot merge the payload item with the previous one,
503 * we simply insert a new iovec for the data/padding.
506 vec = &staging->parts[staging->n_parts++];
507 vec->iov_len = merge->vec.size;
509 staging->n_payload += vec->iov_len;
510 } else if (padding) {
511 vec = &staging->parts[staging->n_parts++];
512 vec->iov_len = padding;
513 vec->iov_base = (char __user *)zeros;
514 staging->n_payload += vec->iov_len;
520 *prev_item = (struct kdbus_item *)merge;
523 return merge == prev;
526 static int kdbus_staging_import(struct kdbus_staging *staging)
528 struct kdbus_item *it, *item, *last, *prev_payload;
529 struct kdbus_gaps *gaps = staging->gaps;
530 struct kdbus_msg *msg = staging->msg;
531 struct iovec *part, *prev_part;
540 * We modify msg->items along the way; make sure to use @item as offset
541 * to the next item (instead of the iterator @it).
543 for (it = item = msg->items;
545 (u8 *)it < (u8 *)msg + msg->size &&
546 (u8 *)it + it->size <= (u8 *)msg + msg->size; ) {
548 * If we dropped items along the way, move current item to
549 * front. We must not access @it afterwards, but use @item
553 memmove(item, it, it->size);
554 it = (void *)((u8 *)it + KDBUS_ALIGN8(item->size));
556 switch (item->type) {
557 case KDBUS_ITEM_PAYLOAD_VEC: {
558 size_t offset = staging->n_payload;
560 if (kdbus_staging_merge_vecs(staging, &prev_payload,
563 } else if (item->vec.address) {
564 /* real offset is patched later on */
565 item->type = KDBUS_ITEM_PAYLOAD_OFF;
566 item->vec.offset = offset;
568 item->type = KDBUS_ITEM_PAYLOAD_OFF;
569 item->vec.offset = ~0ULL;
574 case KDBUS_ITEM_PAYLOAD_MEMFD: {
577 f = kdbus_get_memfd(&item->memfd);
581 gaps->memfd_files[gaps->n_memfds] = f;
582 gaps->memfd_offsets[gaps->n_memfds] =
583 (u8 *)&item->memfd.fd - (u8 *)msg;
586 /* memfds cannot be merged */
590 /* insert padding to make following VECs aligned */
591 if (item->memfd.size % 8) {
592 part = &staging->parts[staging->n_parts++];
593 part->iov_len = item->memfd.size % 8;
594 part->iov_base = (char __user *)zeros;
595 staging->n_payload += part->iov_len;
600 case KDBUS_ITEM_FDS: {
603 n_fds = KDBUS_ITEM_PAYLOAD_SIZE(item) / sizeof(int);
604 for (i = 0; i < n_fds; ++i) {
607 f = kdbus_get_fd(item->fds[i]);
611 gaps->fd_files[gaps->n_fds++] = f;
614 gaps->fd_offset = (u8 *)item->fds - (u8 *)msg;
618 case KDBUS_ITEM_BLOOM_FILTER:
619 staging->bloom_filter = &item->bloom_filter;
621 case KDBUS_ITEM_DST_NAME:
622 staging->dst_name = item->str;
626 /* drop item if we merged it with a previous one */
631 item = KDBUS_ITEM_NEXT(item);
635 /* adjust message size regarding dropped items */
636 msg->size = offsetof(struct kdbus_msg, items);
638 msg->size += ((u8 *)last - (u8 *)msg->items) + last->size;
643 static void kdbus_staging_reserve(struct kdbus_staging *staging)
647 part = &staging->parts[staging->n_parts++];
648 part->iov_base = (void __user *)zeros;
652 static struct kdbus_staging *kdbus_staging_new(struct kdbus_bus *bus,
654 size_t msg_extra_size)
656 const size_t reserved_parts = 5; /* see below for explanation */
657 struct kdbus_staging *staging;
660 n_parts += reserved_parts;
662 staging = kzalloc(sizeof(*staging) + n_parts * sizeof(*staging->parts) +
663 msg_extra_size, GFP_KERNEL);
665 return ERR_PTR(-ENOMEM);
667 staging->msg_seqnum = atomic64_inc_return(&bus->last_message_id);
668 staging->n_parts = 0; /* we reserve n_parts, but don't enforce them */
669 staging->parts = (void *)(staging + 1);
671 if (msg_extra_size) /* if requested, allocate message, too */
672 staging->msg = (void *)((u8 *)staging->parts +
673 n_parts * sizeof(*staging->parts));
675 staging->meta_proc = kdbus_meta_proc_new();
676 if (IS_ERR(staging->meta_proc)) {
677 ret = PTR_ERR(staging->meta_proc);
678 staging->meta_proc = NULL;
682 staging->meta_conn = kdbus_meta_conn_new();
683 if (IS_ERR(staging->meta_conn)) {
684 ret = PTR_ERR(staging->meta_conn);
685 staging->meta_conn = NULL;
690 * Prepare iovecs to copy the message into the target pool. We use the
692 * * iovec to copy "kdbus_msg.size"
693 * * iovec to copy "struct kdbus_msg" (minus size) plus items
694 * * iovec for possible padding after the items
695 * * iovec for metadata items
696 * * iovec for possible padding after the items
698 * Make sure to update @reserved_parts if you add more parts here.
701 kdbus_staging_reserve(staging); /* msg.size */
702 kdbus_staging_reserve(staging); /* msg (minus msg.size) plus items */
703 kdbus_staging_reserve(staging); /* msg padding */
704 kdbus_staging_reserve(staging); /* meta */
705 kdbus_staging_reserve(staging); /* meta padding */
710 kdbus_staging_free(staging);
714 struct kdbus_staging *kdbus_staging_new_kernel(struct kdbus_bus *bus,
715 u64 dst, u64 cookie_timeout,
716 size_t it_size, size_t it_type)
718 struct kdbus_staging *staging;
721 size = offsetof(struct kdbus_msg, items) +
722 KDBUS_ITEM_HEADER_SIZE + it_size;
724 staging = kdbus_staging_new(bus, 0, KDBUS_ALIGN8(size));
726 return ERR_CAST(staging);
728 staging->msg->size = size;
729 staging->msg->flags = (dst == KDBUS_DST_ID_BROADCAST) ?
730 KDBUS_MSG_SIGNAL : 0;
731 staging->msg->dst_id = dst;
732 staging->msg->src_id = KDBUS_SRC_ID_KERNEL;
733 staging->msg->payload_type = KDBUS_PAYLOAD_KERNEL;
734 staging->msg->cookie_reply = cookie_timeout;
735 staging->notify = staging->msg->items;
736 staging->notify->size = KDBUS_ITEM_HEADER_SIZE + it_size;
737 staging->notify->type = it_type;
742 struct kdbus_staging *kdbus_staging_new_user(struct kdbus_bus *bus,
743 struct kdbus_cmd_send *cmd,
744 struct kdbus_msg *msg)
746 const size_t reserved_parts = 1; /* see below for explanation */
747 size_t n_memfds, n_fds, n_parts;
748 struct kdbus_staging *staging;
752 * Examine user-supplied message and figure out how many resources we
753 * need to allocate in our staging area. This requires us to iterate
754 * the message twice, but saves us from re-allocating our resources
758 ret = kdbus_msg_examine(msg, bus, cmd, &n_memfds, &n_fds, &n_parts);
762 n_parts += reserved_parts;
765 * Allocate staging area with the number of required resources. Make
766 * sure that we have enough iovecs for all required parts pre-allocated
767 * so this will hopefully be the only memory allocation for this
768 * message transaction.
771 staging = kdbus_staging_new(bus, n_parts, 0);
773 return ERR_CAST(staging);
778 * If the message contains memfds or fd items, we need to remember some
779 * state so we can fill in the requested information at RECV time.
780 * File-descriptors cannot be passed at SEND time. Hence, allocate a
781 * gaps-object to remember that state. That gaps object is linked to
782 * from the staging area, but will also be linked to from the message
783 * queue of each peer. Hence, each receiver owns a reference to it, and
784 * it will later be used to fill the 'gaps' in message that couldn't be
785 * filled at SEND time.
786 * Note that the 'gaps' object is read-only once the staging-allocator
787 * returns. There might be connections receiving a queued message while
788 * the sender still broadcasts the message to other receivers.
791 if (n_memfds > 0 || n_fds > 0) {
792 staging->gaps = kdbus_gaps_new(n_memfds, n_fds);
793 if (IS_ERR(staging->gaps)) {
794 ret = PTR_ERR(staging->gaps);
795 staging->gaps = NULL;
796 kdbus_staging_free(staging);
802 * kdbus_staging_new() already reserves parts for message setup. For
803 * user-supplied messages, we add the following iovecs:
804 * ... variable number of iovecs for payload ...
805 * * final iovec for possible padding of payload
807 * Make sure to update @reserved_parts if you add more parts here.
810 ret = kdbus_staging_import(staging); /* payload */
811 kdbus_staging_reserve(staging); /* payload padding */
819 kdbus_staging_free(staging);
823 struct kdbus_staging *kdbus_staging_free(struct kdbus_staging *staging)
828 kdbus_meta_conn_unref(staging->meta_conn);
829 kdbus_meta_proc_unref(staging->meta_proc);
830 kdbus_gaps_unref(staging->gaps);
836 static int kdbus_staging_collect_metadata(struct kdbus_staging *staging,
837 struct kdbus_conn *src,
838 struct kdbus_conn *dst,
845 attach = kdbus_meta_msg_mask(src, dst);
847 attach = KDBUS_ATTACH_TIMESTAMP; /* metadata for kernel msgs */
849 if (src && !src->meta_fake) {
850 ret = kdbus_meta_proc_collect(staging->meta_proc, attach);
855 ret = kdbus_meta_conn_collect(staging->meta_conn, src,
856 staging->msg_seqnum, attach);
860 *out_attach = attach;
865 * kdbus_staging_emit() - emit linearized message in target pool
866 * @staging: staging object to create message from
867 * @src: sender of the message (or NULL)
868 * @dst: target connection to allocate message for
870 * This allocates a pool-slice for @dst and copies the message provided by
871 * @staging into it. The new slice is then returned to the caller for further
872 * processing. It's not linked into any queue, yet.
874 * Return: Newly allocated slice or ERR_PTR on failure.
876 struct kdbus_pool_slice *kdbus_staging_emit(struct kdbus_staging *staging,
877 struct kdbus_conn *src,
878 struct kdbus_conn *dst)
880 struct kdbus_item *item, *meta_items = NULL;
881 struct kdbus_pool_slice *slice = NULL;
882 size_t off, size, meta_size;
884 u64 attach, msg_size;
886 size_t kvec_parts_count, size_before_payload = 0;
890 * Collect metadata from @src depending on the attach-flags allowed for
891 * @dst. Translate it into the namespaces pinned by @dst.
894 ret = kdbus_staging_collect_metadata(staging, src, dst, &attach);
898 ret = kdbus_meta_emit(staging->meta_proc, NULL, staging->meta_conn,
899 dst, attach, &meta_items, &meta_size);
905 * Setup iovecs for the message. See kdbus_staging_new() for allocation
906 * of those iovecs. All reserved iovecs have been initialized with
907 * iov_len=0 + iov_base=zeros. Furthermore, the iovecs to copy the
908 * actual message payload have already been initialized and need not be
913 msg_size = staging->msg->size;
916 v->iov_len = sizeof(msg_size);
917 v->iov_base = (void __user *)&msg_size;
920 /* msg (after msg.size) plus items */
921 v->iov_len = staging->msg->size - sizeof(staging->msg->size);
922 v->iov_base = (void __user *)((u8 *)staging->msg +
923 sizeof(staging->msg->size));
926 /* padding after msg */
927 v->iov_len = KDBUS_ALIGN8(staging->msg->size) - staging->msg->size;
928 v->iov_base = (void __user *)zeros;
933 v->iov_len = meta_size;
934 v->iov_base = (void __user *)meta_items;
937 /* padding after metadata */
938 v->iov_len = KDBUS_ALIGN8(meta_size) - meta_size;
939 v->iov_base = (void __user *)zeros;
942 msg_size = KDBUS_ALIGN8(msg_size) + meta_size;
946 v->iov_base = (void __user *)zeros;
949 /* padding after metadata */
951 v->iov_base = (void __user *)zeros;
955 kvec_parts_count = v - staging->parts;
957 /* ... payload iovecs are already filled in ... */
959 /* compute overall size and fill in padding after payload */
960 size = KDBUS_ALIGN8(msg_size);
962 if (staging->n_payload > 0) {
963 size += staging->n_payload;
965 v = &staging->parts[staging->n_parts - 1];
966 v->iov_len = KDBUS_ALIGN8(size) - size;
967 v->iov_base = (void __user *)zeros;
969 size = KDBUS_ALIGN8(size);
974 * The PAYLOAD_OFF items in the message contain a relative 'offset'
975 * field that tells the receiver where to find the actual payload. This
976 * offset is relative to the start of the message, and as such depends
977 * on the size of the metadata items we inserted. This size is variable
978 * and changes for each peer we send the message to. Hence, we remember
979 * the last relative offset that was used to calculate the 'offset'
980 * fields. For each message, we re-calculate it and patch all items, in
984 off = KDBUS_ALIGN8(msg_size);
986 if (off != staging->i_payload) {
987 KDBUS_ITEMS_FOREACH(item, staging->msg->items,
988 KDBUS_ITEMS_SIZE(staging->msg, items)) {
989 if (item->type != KDBUS_ITEM_PAYLOAD_OFF)
992 item->vec.offset -= staging->i_payload;
993 item->vec.offset += off;
996 staging->i_payload = off;
1001 * Allocate pool slice and copy over all data. Make sure to properly
1002 * account on user quota.
1005 ret = kdbus_conn_quota_inc(dst, src ? src->user : NULL, size,
1006 staging->gaps ? staging->gaps->n_fds : 0);
1010 slice = kdbus_pool_slice_alloc(dst->pool, size, true);
1011 if (IS_ERR(slice)) {
1012 ret = PTR_ERR(slice);
1017 WARN_ON(kdbus_pool_slice_size(slice) != size);
1021 for (count_parts = 0; count_parts < kvec_parts_count; count_parts++)
1022 size_before_payload += staging->parts[count_parts].iov_len;
1025 ret = kdbus_pool_slice_copy_kvec(slice, 0, (struct kvec *)staging->parts,
1026 kvec_parts_count, size_before_payload);
1030 for (; kvec_parts_count < staging->n_parts; kvec_parts_count++) {
1031 struct iovec *part = &staging->parts[kvec_parts_count];
1032 if (part->iov_base != zeros)
1033 ret = kdbus_pool_slice_copy_iovec(slice, size_before_payload,
1034 part, 1, part->iov_len);
1036 ret = kdbus_pool_slice_copy_kvec(slice, size_before_payload,
1037 (struct kvec *)&part, 1, part->iov_len);
1040 size_before_payload += part->iov_len;
1043 /* all done, return slice to caller */
1048 kdbus_conn_quota_dec(dst, src ? src->user : NULL, size,
1049 staging->gaps ? staging->gaps->n_fds : 0);
1050 kdbus_pool_slice_release(slice);
1051 slice = ERR_PTR(ret);