drivers/dma-buf/udmabuf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/cred.h>
   3 #include <linux/device.h>
   4 #include <linux/dma-buf.h>
   5 #include <linux/highmem.h>
   6 #include <linux/init.h>
   7 #include <linux/kernel.h>
   8 #include <linux/memfd.h>
   9 #include <linux/miscdevice.h>
  10 #include <linux/module.h>
  11 #include <linux/shmem_fs.h>
  12 #include <linux/slab.h>
  13 #include <linux/udmabuf.h>
  14 #include <linux/hugetlb.h>
  15
  16 static int list_limit = 1024;
  17 module_param(list_limit, int, 0644);
  18 MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024.");
  19
  20 static int size_limit_mb = 64;
  21 module_param(size_limit_mb, int, 0644);
  22 MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64.");
  23
  24 struct udmabuf {
  25         pgoff_t pagecount;
  26         struct page **pages;
  27         struct sg_table *sg;
  28         struct miscdevice *device;
  29 };
  30
  31 static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
  32 {
  33         struct vm_area_struct *vma = vmf->vma;
  34         struct udmabuf *ubuf = vma->vm_private_data;
  35         pgoff_t pgoff = vmf->pgoff;
  36
  37         if (pgoff >= ubuf->pagecount)
  38                 return VM_FAULT_SIGBUS;
  39         vmf->page = ubuf->pages[pgoff];
  40         get_page(vmf->page);
  41         return 0;
  42 }
  43
  44 static const struct vm_operations_struct udmabuf_vm_ops = {
  45         .fault = udmabuf_vm_fault,
  46 };
  47
  48 static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma)
  49 {
  50         struct udmabuf *ubuf = buf->priv;
  51
  52         if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
  53                 return -EINVAL;
  54
  55         vma->vm_ops = &udmabuf_vm_ops;
  56         vma->vm_private_data = ubuf;
  57         return 0;
  58 }
  59
  60 static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,
  61                                      enum dma_data_direction direction)
  62 {
  63         struct udmabuf *ubuf = buf->priv;
  64         struct sg_table *sg;
  65         int ret;
  66
  67         sg = kzalloc(sizeof(*sg), GFP_KERNEL);
  68         if (!sg)
  69                 return ERR_PTR(-ENOMEM);
  70         ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount,
  71                                         0, ubuf->pagecount << PAGE_SHIFT,
  72                                         GFP_KERNEL);
  73         if (ret < 0)
  74                 goto err;
  75         ret = dma_map_sgtable(dev, sg, direction, 0);
  76         if (ret < 0)
  77                 goto err;
  78         return sg;
  79
  80 err:
  81         sg_free_table(sg);
  82         kfree(sg);
  83         return ERR_PTR(ret);
  84 }
  85
  86 static void put_sg_table(struct device *dev, struct sg_table *sg,
  87                          enum dma_data_direction direction)
  88 {
  89         dma_unmap_sgtable(dev, sg, direction, 0);
  90         sg_free_table(sg);
  91         kfree(sg);
  92 }
  93
  94 static struct sg_table *map_udmabuf(struct dma_buf_attachment *at,
  95                                     enum dma_data_direction direction)
  96 {
  97         return get_sg_table(at->dev, at->dmabuf, direction);
  98 }
  99
 100 static void unmap_udmabuf(struct dma_buf_attachment *at,
 101                           struct sg_table *sg,
 102                           enum dma_data_direction direction)
 103 {
 104         return put_sg_table(at->dev, sg, direction);
 105 }
 106
 107 static void release_udmabuf(struct dma_buf *buf)
 108 {
 109         struct udmabuf *ubuf = buf->priv;
 110         struct device *dev = ubuf->device->this_device;
 111         pgoff_t pg;
 112
 113         if (ubuf->sg)
 114                 put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL);
 115
 116         for (pg = 0; pg < ubuf->pagecount; pg++)
 117                 put_page(ubuf->pages[pg]);
 118         kfree(ubuf->pages);
 119         kfree(ubuf);
 120 }
 121
 122 static int begin_cpu_udmabuf(struct dma_buf *buf,
 123                              enum dma_data_direction direction)
 124 {
 125         struct udmabuf *ubuf = buf->priv;
 126         struct device *dev = ubuf->device->this_device;
 127
 128         if (!ubuf->sg) {
 129                 ubuf->sg = get_sg_table(dev, buf, direction);
 130                 if (IS_ERR(ubuf->sg))
 131                         return PTR_ERR(ubuf->sg);
 132         } else {
 133                 dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents,
 134                                     direction);
 135         }
 136
 137         return 0;
 138 }
 139
 140 static int end_cpu_udmabuf(struct dma_buf *buf,
 141                            enum dma_data_direction direction)
 142 {
 143         struct udmabuf *ubuf = buf->priv;
 144         struct device *dev = ubuf->device->this_device;
 145
 146         if (!ubuf->sg)
 147                 return -EINVAL;
 148
 149         dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction);
 150         return 0;
 151 }
 152
 153 static const struct dma_buf_ops udmabuf_ops = {
 154         .cache_sgt_mapping = true,
 155         .map_dma_buf       = map_udmabuf,
 156         .unmap_dma_buf     = unmap_udmabuf,
 157         .release           = release_udmabuf,
 158         .mmap              = mmap_udmabuf,
 159         .begin_cpu_access  = begin_cpu_udmabuf,
 160         .end_cpu_access    = end_cpu_udmabuf,
 161 };
 162
 163 #define SEALS_WANTED (F_SEAL_SHRINK)
 164 #define SEALS_DENIED (F_SEAL_WRITE)
 165
 166 static long udmabuf_create(struct miscdevice *device,
 167                            struct udmabuf_create_list *head,
 168                            struct udmabuf_create_item *list)
 169 {
 170         DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 171         struct file *memfd = NULL;
 172         struct address_space *mapping = NULL;
 173         struct udmabuf *ubuf;
 174         struct dma_buf *buf;
 175         pgoff_t pgoff, pgcnt, pgidx, pgbuf = 0, pglimit;
 176         struct page *page, *hpage = NULL;
 177         pgoff_t subpgoff, maxsubpgs;
 178         struct hstate *hpstate;
 179         int seals, ret = -EINVAL;
 180         u32 i, flags;
 181
 182         ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL);
 183         if (!ubuf)
 184                 return -ENOMEM;
 185
 186         pglimit = (size_limit_mb * 1024 * 1024) >> PAGE_SHIFT;
 187         for (i = 0; i < head->count; i++) {
 188                 if (!IS_ALIGNED(list[i].offset, PAGE_SIZE))
 189                         goto err;
 190                 if (!IS_ALIGNED(list[i].size, PAGE_SIZE))
 191                         goto err;
 192                 ubuf->pagecount += list[i].size >> PAGE_SHIFT;
 193                 if (ubuf->pagecount > pglimit)
 194                         goto err;
 195         }
 196
 197         if (!ubuf->pagecount)
 198                 goto err;
 199
 200         ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->pages),
 201                                     GFP_KERNEL);
 202         if (!ubuf->pages) {
 203                 ret = -ENOMEM;
 204                 goto err;
 205         }
 206
 207         pgbuf = 0;
 208         for (i = 0; i < head->count; i++) {
 209                 ret = -EBADFD;
 210                 memfd = fget(list[i].memfd);
 211                 if (!memfd)
 212                         goto err;
 213                 mapping = file_inode(memfd)->i_mapping;
 214                 if (!shmem_mapping(mapping) && !is_file_hugepages(memfd))
 215                         goto err;
 216                 seals = memfd_fcntl(memfd, F_GET_SEALS, 0);
 217                 if (seals == -EINVAL)
 218                         goto err;
 219                 ret = -EINVAL;
 220                 if ((seals & SEALS_WANTED) != SEALS_WANTED ||
 221                     (seals & SEALS_DENIED) != 0)
 222                         goto err;
 223                 pgoff = list[i].offset >> PAGE_SHIFT;
 224                 pgcnt = list[i].size   >> PAGE_SHIFT;
 225                 if (is_file_hugepages(memfd)) {
 226                         hpstate = hstate_file(memfd);
 227                         pgoff = list[i].offset >> huge_page_shift(hpstate);
 228                         subpgoff = (list[i].offset &
 229                                     ~huge_page_mask(hpstate)) >> PAGE_SHIFT;
 230                         maxsubpgs = huge_page_size(hpstate) >> PAGE_SHIFT;
 231                 }
 232                 for (pgidx = 0; pgidx < pgcnt; pgidx++) {
 233                         if (is_file_hugepages(memfd)) {
 234                                 if (!hpage) {
 235                                         hpage = find_get_page_flags(mapping, pgoff,
 236                                                                     FGP_ACCESSED);
 237                                         if (!hpage) {
 238                                                 ret = -EINVAL;
 239                                                 goto err;
 240                                         }
 241                                 }
 242                                 page = hpage + subpgoff;
 243                                 get_page(page);
 244                                 subpgoff++;
 245                                 if (subpgoff == maxsubpgs) {
 246                                         put_page(hpage);
 247                                         hpage = NULL;
 248                                         subpgoff = 0;
 249                                         pgoff++;
 250                                 }
 251                         } else {
 252                                 page = shmem_read_mapping_page(mapping,
 253                                                                pgoff + pgidx);
 254                                 if (IS_ERR(page)) {
 255                                         ret = PTR_ERR(page);
 256                                         goto err;
 257                                 }
 258                         }
 259                         ubuf->pages[pgbuf++] = page;
 260                 }
 261                 fput(memfd);
 262                 memfd = NULL;
 263                 if (hpage) {
 264                         put_page(hpage);
 265                         hpage = NULL;
 266                 }
 267         }
 268
 269         exp_info.ops  = &udmabuf_ops;
 270         exp_info.size = ubuf->pagecount << PAGE_SHIFT;
 271         exp_info.priv = ubuf;
 272         exp_info.flags = O_RDWR;
 273
 274         ubuf->device = device;
 275         buf = dma_buf_export(&exp_info);
 276         if (IS_ERR(buf)) {
 277                 ret = PTR_ERR(buf);
 278                 goto err;
 279         }
 280
 281         flags = 0;
 282         if (head->flags & UDMABUF_FLAGS_CLOEXEC)
 283                 flags |= O_CLOEXEC;
 284         return dma_buf_fd(buf, flags);
 285
 286 err:
 287         while (pgbuf > 0)
 288                 put_page(ubuf->pages[--pgbuf]);
 289         if (memfd)
 290                 fput(memfd);
 291         kfree(ubuf->pages);
 292         kfree(ubuf);
 293         return ret;
 294 }
 295
 296 static long udmabuf_ioctl_create(struct file *filp, unsigned long arg)
 297 {
 298         struct udmabuf_create create;
 299         struct udmabuf_create_list head;
 300         struct udmabuf_create_item list;
 301
 302         if (copy_from_user(&create, (void __user *)arg,
 303                            sizeof(create)))
 304                 return -EFAULT;
 305
 306         head.flags  = create.flags;
 307         head.count  = 1;
 308         list.memfd  = create.memfd;
 309         list.offset = create.offset;
 310         list.size   = create.size;
 311
 312         return udmabuf_create(filp->private_data, &head, &list);
 313 }
 314
 315 static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg)
 316 {
 317         struct udmabuf_create_list head;
 318         struct udmabuf_create_item *list;
 319         int ret = -EINVAL;
 320         u32 lsize;
 321
 322         if (copy_from_user(&head, (void __user *)arg, sizeof(head)))
 323                 return -EFAULT;
 324         if (head.count > list_limit)
 325                 return -EINVAL;
 326         lsize = sizeof(struct udmabuf_create_item) * head.count;
 327         list = memdup_user((void __user *)(arg + sizeof(head)), lsize);
 328         if (IS_ERR(list))
 329                 return PTR_ERR(list);
 330
 331         ret = udmabuf_create(filp->private_data, &head, list);
 332         kfree(list);
 333         return ret;
 334 }
 335
 336 static long udmabuf_ioctl(struct file *filp, unsigned int ioctl,
 337                           unsigned long arg)
 338 {
 339         long ret;
 340
 341         switch (ioctl) {
 342         case UDMABUF_CREATE:
 343                 ret = udmabuf_ioctl_create(filp, arg);
 344                 break;
 345         case UDMABUF_CREATE_LIST:
 346                 ret = udmabuf_ioctl_create_list(filp, arg);
 347                 break;
 348         default:
 349                 ret = -ENOTTY;
 350                 break;
 351         }
 352         return ret;
 353 }
 354
 355 static const struct file_operations udmabuf_fops = {
 356         .owner          = THIS_MODULE,
 357         .unlocked_ioctl = udmabuf_ioctl,
 358 #ifdef CONFIG_COMPAT
 359         .compat_ioctl   = udmabuf_ioctl,
 360 #endif
 361 };
 362
 363 static struct miscdevice udmabuf_misc = {
 364         .minor          = MISC_DYNAMIC_MINOR,
 365         .name           = "udmabuf",
 366         .fops           = &udmabuf_fops,
 367 };
 368
 369 static int __init udmabuf_dev_init(void)
 370 {
 371         int ret;
 372
 373         ret = misc_register(&udmabuf_misc);
 374         if (ret < 0) {
 375                 pr_err("Could not initialize udmabuf device\n");
 376                 return ret;
 377         }
 378
 379         ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device,
 380                                            DMA_BIT_MASK(64));
 381         if (ret < 0) {
 382                 pr_err("Could not setup DMA mask for udmabuf device\n");
 383                 misc_deregister(&udmabuf_misc);
 384                 return ret;
 385         }
 386
 387         return 0;
 388 }
 389
 390 static void __exit udmabuf_dev_exit(void)
 391 {
 392         misc_deregister(&udmabuf_misc);
 393 }
 394
 395 module_init(udmabuf_dev_init)
 396 module_exit(udmabuf_dev_exit)
 397
 398 MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>");
 399 MODULE_LICENSE("GPL v2");