1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES
5 #include <linux/iommu.h>
6 #include <uapi/linux/iommufd.h>
8 #include "iommufd_private.h"
10 void iommufd_hw_pagetable_destroy(struct iommufd_object *obj)
12 struct iommufd_hw_pagetable *hwpt =
13 container_of(obj, struct iommufd_hw_pagetable, obj);
15 if (!list_empty(&hwpt->hwpt_item)) {
16 mutex_lock(&hwpt->ioas->mutex);
17 list_del(&hwpt->hwpt_item);
18 mutex_unlock(&hwpt->ioas->mutex);
20 iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
24 iommu_domain_free(hwpt->domain);
26 refcount_dec(&hwpt->ioas->obj.users);
29 void iommufd_hw_pagetable_abort(struct iommufd_object *obj)
31 struct iommufd_hw_pagetable *hwpt =
32 container_of(obj, struct iommufd_hw_pagetable, obj);
34 /* The ioas->mutex must be held until finalize is called. */
35 lockdep_assert_held(&hwpt->ioas->mutex);
37 if (!list_empty(&hwpt->hwpt_item)) {
38 list_del_init(&hwpt->hwpt_item);
39 iopt_table_remove_domain(&hwpt->ioas->iopt, hwpt->domain);
41 iommufd_hw_pagetable_destroy(obj);
44 int iommufd_hw_pagetable_enforce_cc(struct iommufd_hw_pagetable *hwpt)
46 if (hwpt->enforce_cache_coherency)
49 if (hwpt->domain->ops->enforce_cache_coherency)
50 hwpt->enforce_cache_coherency =
51 hwpt->domain->ops->enforce_cache_coherency(
53 if (!hwpt->enforce_cache_coherency)
59 * iommufd_hw_pagetable_alloc() - Get an iommu_domain for a device
60 * @ictx: iommufd context
61 * @ioas: IOAS to associate the domain with
62 * @idev: Device to get an iommu_domain for
63 * @immediate_attach: True if idev should be attached to the hwpt
65 * Allocate a new iommu_domain and return it as a hw_pagetable. The HWPT
66 * will be linked to the given ioas and upon return the underlying iommu_domain
67 * is fully popoulated.
69 * The caller must hold the ioas->mutex until after
70 * iommufd_object_abort_and_destroy() or iommufd_object_finalize() is called on
73 struct iommufd_hw_pagetable *
74 iommufd_hw_pagetable_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas,
75 struct iommufd_device *idev, bool immediate_attach)
77 struct iommufd_hw_pagetable *hwpt;
80 lockdep_assert_held(&ioas->mutex);
82 hwpt = iommufd_object_alloc(ictx, hwpt, IOMMUFD_OBJ_HW_PAGETABLE);
86 INIT_LIST_HEAD(&hwpt->hwpt_item);
87 /* Pairs with iommufd_hw_pagetable_destroy() */
88 refcount_inc(&ioas->obj.users);
91 hwpt->domain = iommu_domain_alloc(idev->dev->bus);
98 * Set the coherency mode before we do iopt_table_add_domain() as some
99 * iommus have a per-PTE bit that controls it and need to decide before
100 * doing any maps. It is an iommu driver bug to report
101 * IOMMU_CAP_ENFORCE_CACHE_COHERENCY but fail enforce_cache_coherency on
104 if (idev->enforce_cache_coherency) {
105 rc = iommufd_hw_pagetable_enforce_cc(hwpt);
111 * immediate_attach exists only to accommodate iommu drivers that cannot
112 * directly allocate a domain. These drivers do not finish creating the
113 * domain until attach is completed. Thus we must have this call
114 * sequence. Once those drivers are fixed this should be removed.
116 if (immediate_attach) {
117 rc = iommufd_hw_pagetable_attach(hwpt, idev);
122 rc = iopt_table_add_domain(&hwpt->ioas->iopt, hwpt->domain);
125 list_add_tail(&hwpt->hwpt_item, &hwpt->ioas->hwpt_list);
129 if (immediate_attach)
130 iommufd_hw_pagetable_detach(idev);
132 iommufd_object_abort_and_destroy(ictx, &hwpt->obj);
136 int iommufd_hwpt_alloc(struct iommufd_ucmd *ucmd)
138 struct iommu_hwpt_alloc *cmd = ucmd->cmd;
139 struct iommufd_hw_pagetable *hwpt;
140 struct iommufd_device *idev;
141 struct iommufd_ioas *ioas;
144 if (cmd->flags || cmd->__reserved)
147 idev = iommufd_get_device(ucmd, cmd->dev_id);
149 return PTR_ERR(idev);
151 ioas = iommufd_get_ioas(ucmd->ictx, cmd->pt_id);
157 mutex_lock(&ioas->mutex);
158 hwpt = iommufd_hw_pagetable_alloc(ucmd->ictx, ioas, idev, false);
164 cmd->out_hwpt_id = hwpt->obj.id;
165 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
168 iommufd_object_finalize(ucmd->ictx, &hwpt->obj);
172 iommufd_object_abort_and_destroy(ucmd->ictx, &hwpt->obj);
174 mutex_unlock(&ioas->mutex);
175 iommufd_put_object(&ioas->obj);
177 iommufd_put_object(&idev->obj);