1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */
3 #include <linux/init.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
7 #include <linux/device.h>
8 #include <linux/sched/task.h>
9 #include <linux/io-64-nonatomic-lo-hi.h>
10 #include <linux/cdev.h>
12 #include <linux/poll.h>
13 #include <linux/iommu.h>
14 #include <linux/highmem.h>
15 #include <uapi/linux/idxd.h>
16 #include <linux/xarray.h>
17 #include "registers.h"
20 struct idxd_cdev_context {
27 * Since user file names are global in DSA devices, define their ida's as
28 * global to avoid conflict file names.
30 static DEFINE_IDA(file_ida);
31 static DEFINE_MUTEX(ida_lock);
34 * ictx is an array based off of accelerator types. enum idxd_type
37 static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = {
42 struct idxd_user_context {
44 struct task_struct *task;
48 struct iommu_sva *sva;
49 struct idxd_dev idxd_dev;
50 u64 counters[COUNTER_MAX];
55 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid);
56 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);
58 static inline struct idxd_user_context *dev_to_uctx(struct device *dev)
60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev);
62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev);
65 static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf)
67 struct idxd_user_context *ctx = dev_to_uctx(dev);
69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]);
71 static DEVICE_ATTR_RO(cr_faults);
73 static ssize_t cr_fault_failures_show(struct device *dev,
74 struct device_attribute *attr, char *buf)
76 struct idxd_user_context *ctx = dev_to_uctx(dev);
78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]);
80 static DEVICE_ATTR_RO(cr_fault_failures);
82 static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf)
84 struct idxd_user_context *ctx = dev_to_uctx(dev);
86 return sysfs_emit(buf, "%u\n", ctx->pid);
88 static DEVICE_ATTR_RO(pid);
90 static struct attribute *cdev_file_attributes[] = {
91 &dev_attr_cr_faults.attr,
92 &dev_attr_cr_fault_failures.attr,
97 static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n)
99 struct device *dev = container_of(kobj, typeof(*dev), kobj);
100 struct idxd_user_context *ctx = dev_to_uctx(dev);
101 struct idxd_wq *wq = ctx->wq;
103 if (!wq_pasid_enabled(wq))
109 static const struct attribute_group cdev_file_attribute_group = {
110 .attrs = cdev_file_attributes,
111 .is_visible = cdev_file_attr_visible,
114 static const struct attribute_group *cdev_file_attribute_groups[] = {
115 &cdev_file_attribute_group,
119 static void idxd_file_dev_release(struct device *dev)
121 struct idxd_user_context *ctx = dev_to_uctx(dev);
122 struct idxd_wq *wq = ctx->wq;
123 struct idxd_device *idxd = wq->idxd;
126 mutex_lock(&ida_lock);
127 ida_free(&file_ida, ctx->id);
128 mutex_unlock(&ida_lock);
130 /* Wait for in-flight operations to complete. */
132 idxd_device_drain_pasid(idxd, ctx->pasid);
134 if (device_user_pasid_enabled(idxd)) {
135 /* The wq disable in the disable pasid function will drain the wq */
136 rc = idxd_wq_disable_pasid(wq);
138 dev_err(dev, "wq disable pasid failed.\n");
145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid);
146 iommu_sva_unbind_device(ctx->sva);
147 idxd_xa_pasid_remove(ctx);
150 mutex_lock(&wq->wq_lock);
152 mutex_unlock(&wq->wq_lock);
155 static struct device_type idxd_cdev_file_type = {
157 .release = idxd_file_dev_release,
158 .groups = cdev_file_attribute_groups,
161 static void idxd_cdev_dev_release(struct device *dev)
163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev);
164 struct idxd_cdev_context *cdev_ctx;
165 struct idxd_wq *wq = idxd_cdev->wq;
167 cdev_ctx = &ictx[wq->idxd->data->type];
168 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor);
172 static struct device_type idxd_cdev_device_type = {
174 .release = idxd_cdev_dev_release,
177 static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode)
179 struct cdev *cdev = inode->i_cdev;
181 return container_of(cdev, struct idxd_cdev, cdev);
184 static inline struct idxd_wq *inode_wq(struct inode *inode)
186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode);
188 return idxd_cdev->wq;
191 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx)
193 struct idxd_wq *wq = ctx->wq;
196 mutex_lock(&wq->uc_lock);
197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);
198 if (ptr != (void *)ctx)
199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",
201 mutex_unlock(&wq->uc_lock);
204 void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)
206 struct idxd_user_context *ctx;
208 if (index >= COUNTER_MAX)
211 mutex_lock(&wq->uc_lock);
212 ctx = xa_load(&wq->upasid_xa, pasid);
214 mutex_unlock(&wq->uc_lock);
217 ctx->counters[index]++;
218 mutex_unlock(&wq->uc_lock);
221 static int idxd_cdev_open(struct inode *inode, struct file *filp)
223 struct idxd_user_context *ctx;
224 struct idxd_device *idxd;
226 struct device *dev, *fdev;
228 struct iommu_sva *sva;
230 struct idxd_cdev *idxd_cdev;
232 wq = inode_wq(inode);
234 dev = &idxd->pdev->dev;
236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq));
238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
242 mutex_lock(&wq->wq_lock);
244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) {
250 filp->private_data = ctx;
251 ctx->pid = current->pid;
253 if (device_user_pasid_enabled(idxd)) {
254 sva = iommu_sva_bind_device(dev, current->mm);
257 dev_err(dev, "pasid allocation failed: %d\n", rc);
261 pasid = iommu_sva_get_pasid(sva);
262 if (pasid == IOMMU_PASID_INVALID) {
264 goto failed_get_pasid;
269 ctx->mm = current->mm;
271 mutex_lock(&wq->uc_lock);
272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);
273 mutex_unlock(&wq->uc_lock);
275 dev_warn(dev, "PASID entry already exist in xarray.\n");
277 if (wq_dedicated(wq)) {
278 rc = idxd_wq_set_pasid(wq, pasid);
280 iommu_sva_unbind_device(sva);
281 dev_err(dev, "wq set pasid failed: %d\n", rc);
282 goto failed_set_pasid;
287 idxd_cdev = wq->idxd_cdev;
288 mutex_lock(&ida_lock);
289 ctx->id = ida_alloc(&file_ida, GFP_KERNEL);
290 mutex_unlock(&ida_lock);
292 dev_warn(dev, "ida alloc failure\n");
295 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE;
296 fdev = user_ctx_dev(ctx);
297 device_initialize(fdev);
298 fdev->parent = cdev_dev(idxd_cdev);
299 fdev->bus = &dsa_bus_type;
300 fdev->type = &idxd_cdev_file_type;
302 rc = dev_set_name(fdev, "file%d", ctx->id);
304 dev_warn(dev, "set name failure\n");
305 goto failed_dev_name;
308 rc = device_add(fdev);
310 dev_warn(dev, "file device add failure\n");
315 mutex_unlock(&wq->wq_lock);
323 if (device_user_pasid_enabled(idxd))
324 idxd_xa_pasid_remove(ctx);
326 if (device_user_pasid_enabled(idxd))
327 iommu_sva_unbind_device(sva);
329 mutex_unlock(&wq->wq_lock);
334 static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid)
336 struct idxd_device *idxd = wq->idxd;
337 struct idxd_evl *evl = idxd->evl;
338 union evl_status_reg status;
340 int ent_size = evl_ent_size(idxd);
341 struct __evl_entry *entry_head;
346 spin_lock(&evl->lock);
347 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET);
353 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size));
354 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id)
355 set_bit(h, evl->bmap);
358 spin_unlock(&evl->lock);
360 drain_workqueue(wq->wq);
363 static int idxd_cdev_release(struct inode *node, struct file *filep)
365 struct idxd_user_context *ctx = filep->private_data;
366 struct idxd_wq *wq = ctx->wq;
367 struct idxd_device *idxd = wq->idxd;
368 struct device *dev = &idxd->pdev->dev;
370 dev_dbg(dev, "%s called\n", __func__);
371 filep->private_data = NULL;
373 device_unregister(user_ctx_dev(ctx));
378 static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma,
381 struct device *dev = &wq->idxd->pdev->dev;
383 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
384 dev_info_ratelimited(dev,
385 "%s: %s: mapping too large: %lu\n",
387 vma->vm_end - vma->vm_start);
394 static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma)
396 struct idxd_user_context *ctx = filp->private_data;
397 struct idxd_wq *wq = ctx->wq;
398 struct idxd_device *idxd = wq->idxd;
399 struct pci_dev *pdev = idxd->pdev;
400 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR);
404 dev_dbg(&pdev->dev, "%s called\n", __func__);
405 rc = check_vma(wq, vma, __func__);
409 vm_flags_set(vma, VM_DONTCOPY);
410 pfn = (base + idxd_get_wq_portal_full_offset(wq->id,
411 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT;
412 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
413 vma->vm_private_data = ctx;
415 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE,
419 static __poll_t idxd_cdev_poll(struct file *filp,
420 struct poll_table_struct *wait)
422 struct idxd_user_context *ctx = filp->private_data;
423 struct idxd_wq *wq = ctx->wq;
424 struct idxd_device *idxd = wq->idxd;
427 poll_wait(filp, &wq->err_queue, wait);
428 spin_lock(&idxd->dev_lock);
429 if (idxd->sw_err.valid)
430 out = EPOLLIN | EPOLLRDNORM;
431 spin_unlock(&idxd->dev_lock);
436 static const struct file_operations idxd_cdev_fops = {
437 .owner = THIS_MODULE,
438 .open = idxd_cdev_open,
439 .release = idxd_cdev_release,
440 .mmap = idxd_cdev_mmap,
441 .poll = idxd_cdev_poll,
444 int idxd_cdev_get_major(struct idxd_device *idxd)
446 return MAJOR(ictx[idxd->data->type].devt);
449 int idxd_wq_add_cdev(struct idxd_wq *wq)
451 struct idxd_device *idxd = wq->idxd;
452 struct idxd_cdev *idxd_cdev;
455 struct idxd_cdev_context *cdev_ctx;
458 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL);
462 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV;
464 cdev = &idxd_cdev->cdev;
465 dev = cdev_dev(idxd_cdev);
466 cdev_ctx = &ictx[wq->idxd->data->type];
467 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL);
472 idxd_cdev->minor = minor;
474 device_initialize(dev);
475 dev->parent = wq_confdev(wq);
476 dev->bus = &dsa_bus_type;
477 dev->type = &idxd_cdev_device_type;
478 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor);
480 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id);
484 wq->idxd_cdev = idxd_cdev;
485 cdev_init(cdev, &idxd_cdev_fops);
486 rc = cdev_device_add(cdev, dev);
488 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc);
496 wq->idxd_cdev = NULL;
500 void idxd_wq_del_cdev(struct idxd_wq *wq)
502 struct idxd_cdev *idxd_cdev;
504 idxd_cdev = wq->idxd_cdev;
505 ida_destroy(&file_ida);
506 wq->idxd_cdev = NULL;
507 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev));
508 put_device(cdev_dev(idxd_cdev));
511 static int idxd_user_drv_probe(struct idxd_dev *idxd_dev)
513 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
514 struct idxd_device *idxd = wq->idxd;
517 if (idxd->state != IDXD_DEV_ENABLED)
521 * User type WQ is enabled only when SVA is enabled for two reasons:
522 * - If no IOMMU or IOMMU Passthrough without SVA, userspace
523 * can directly access physical address through the WQ.
524 * - The IDXD cdev driver does not provide any ways to pin
525 * user pages and translate the address from user VA to IOVA or
526 * PA without IOMMU SVA. Therefore the application has no way
527 * to instruct the device to perform DMA function. This makes
528 * the cdev not usable for normal application usage.
530 if (!device_user_pasid_enabled(idxd)) {
531 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU;
532 dev_dbg(&idxd->pdev->dev,
533 "User type WQ cannot be enabled without SVA.\n");
538 mutex_lock(&wq->wq_lock);
540 wq->wq = create_workqueue(dev_name(wq_confdev(wq)));
546 wq->type = IDXD_WQT_USER;
547 rc = drv_enable_wq(wq);
551 rc = idxd_wq_add_cdev(wq);
553 idxd->cmd_status = IDXD_SCMD_CDEV_ERR;
557 idxd->cmd_status = 0;
558 mutex_unlock(&wq->wq_lock);
564 destroy_workqueue(wq->wq);
565 wq->type = IDXD_WQT_NONE;
567 mutex_unlock(&wq->wq_lock);
571 static void idxd_user_drv_remove(struct idxd_dev *idxd_dev)
573 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
575 mutex_lock(&wq->wq_lock);
576 idxd_wq_del_cdev(wq);
578 wq->type = IDXD_WQT_NONE;
579 destroy_workqueue(wq->wq);
581 mutex_unlock(&wq->wq_lock);
584 static enum idxd_dev_type dev_types[] = {
589 struct idxd_device_driver idxd_user_drv = {
590 .probe = idxd_user_drv_probe,
591 .remove = idxd_user_drv_remove,
595 EXPORT_SYMBOL_GPL(idxd_user_drv);
597 int idxd_cdev_register(void)
601 for (i = 0; i < IDXD_TYPE_MAX; i++) {
602 ida_init(&ictx[i].minor_ida);
603 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK,
606 goto err_free_chrdev_region;
611 err_free_chrdev_region:
612 for (i--; i >= 0; i--)
613 unregister_chrdev_region(ictx[i].devt, MINORMASK);
618 void idxd_cdev_remove(void)
622 for (i = 0; i < IDXD_TYPE_MAX; i++) {
623 unregister_chrdev_region(ictx[i].devt, MINORMASK);
624 ida_destroy(&ictx[i].minor_ida);
629 * idxd_copy_cr - copy completion record to user address space found by wq and
633 * @addr: user fault address to write
634 * @cr: completion record
635 * @len: number of bytes to copy
637 * This is called by a work that handles completion record fault.
639 * Return: number of bytes copied.
641 int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr,
644 struct device *dev = &wq->idxd->pdev->dev;
645 int left = len, status_size = 1;
646 struct idxd_user_context *ctx;
647 struct mm_struct *mm;
649 mutex_lock(&wq->uc_lock);
651 ctx = xa_load(&wq->upasid_xa, pasid);
653 dev_warn(dev, "No user context\n");
659 * The completion record fault handling work is running in kernel
660 * thread context. It temporarily switches to the mm to copy cr
664 left = copy_to_user((void __user *)addr + status_size, cr + status_size,
667 * Copy status only after the rest of completion record is copied
668 * successfully so that the user gets the complete completion record
669 * when a non-zero status is polled.
675 * Ensure that the completion record's status field is written
676 * after the rest of the completion record has been written.
677 * This ensures that the user receives the correct completion
678 * record information once polling for a non-zero status.
682 if (put_user(status, (u8 __user *)addr))
687 kthread_unuse_mm(mm);
690 mutex_unlock(&wq->uc_lock);