iommu/vt-d: Remove DEFER_DEVICE_DOMAIN_INFO

Allocate and set the per-device iommu private data during iommu device
probe. This makes the per-device iommu private data always available
during iommu_probe_device() and iommu_release_device(). With this changed,
the dummy DEFER_DEVICE_DOMAIN_INFO pointer could be removed. The wrappers
for getting the private data and domain are also cleaned.

Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/20220214025704.3184654-1-baolu.lu@linux.intel.com
Link: https://lore.kernel.org/r/20220301020159.633356-6-baolu.lu@linux.intel.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/intel/debugfs.c b/drivers/iommu/intel/debugfs.c
index db7a0ca736267c7304d6f6f14d4e25812e20b45b..ed796eea4581305ff1b7684eb0b9ef8dd2295eda 100644 (file)
--- a/drivers/iommu/intel/debugfs.c
+++ b/drivers/iommu/intel/debugfs.c
@@ -344,7 +344,8 @@ static void pgtable_walk_level(struct seq_file *m, struct dma_pte *pde,
 
 static int show_device_domain_translation(struct device *dev, void *data)
 {
-       struct dmar_domain *domain = find_domain(dev);
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
+       struct dmar_domain *domain = info->domain;
        struct seq_file *m = data;
        u64 path[6] = { 0 };
 

diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 51db26d8606c6712992703b02c07a0be8b67d12e..d9965e72d9a833b9886ec2cf9557fcb47459646a 100644 (file)
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -342,21 +342,6 @@ static int iommu_skip_te_disable;
 int intel_iommu_gfx_mapped;
 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
 
-#define DEFER_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-2))
-struct device_domain_info *get_domain_info(struct device *dev)
-{
-       struct device_domain_info *info;
-
-       if (!dev)
-               return NULL;
-
-       info = dev_iommu_priv_get(dev);
-       if (unlikely(info == DEFER_DEVICE_DOMAIN_INFO))
-               return NULL;
-
-       return info;
-}
-
 DEFINE_SPINLOCK(device_domain_lock);
 static LIST_HEAD(device_domain_list);
 
@@ -741,11 +726,6 @@ struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
        return &context[devfn];
 }
 
-static bool attach_deferred(struct device *dev)
-{
-       return dev_iommu_priv_get(dev) == DEFER_DEVICE_DOMAIN_INFO;
-}
-
 /**
  * is_downstream_to_pci_bridge - test if a device belongs to the PCI
  *                              sub-hierarchy of a candidate PCI-PCI bridge
@@ -2432,15 +2412,6 @@ static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8
        __iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);
 }
 
-static inline void unlink_domain_info(struct device_domain_info *info)
-{
-       assert_spin_locked(&device_domain_lock);
-       list_del(&info->link);
-       list_del(&info->global);
-       if (info->dev)
-               dev_iommu_priv_set(info->dev, NULL);
-}
-
 static void domain_remove_dev_info(struct dmar_domain *domain)
 {
        struct device_domain_info *info, *tmp;
@@ -2452,24 +2423,6 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
        spin_unlock_irqrestore(&device_domain_lock, flags);
 }
 
-struct dmar_domain *find_domain(struct device *dev)
-{
-       struct device_domain_info *info;
-
-       if (unlikely(!dev || !dev->iommu))
-               return NULL;
-
-       if (unlikely(attach_deferred(dev)))
-               return NULL;
-
-       /* No lock here, assumes no domain exit in normal case */
-       info = get_domain_info(dev);
-       if (likely(info))
-               return info->domain;
-
-       return NULL;
-}
-
 static inline struct device_domain_info *
 dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
 {
@@ -2530,66 +2483,20 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,
                                                    struct device *dev,
                                                    struct dmar_domain *domain)
 {
-       struct device_domain_info *info;
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
        unsigned long flags;
        int ret;
 
-       info = kzalloc(sizeof(*info), GFP_KERNEL);
-       if (!info)
-               return NULL;
-
-       if (!dev_is_real_dma_subdevice(dev)) {
-               info->bus = bus;
-               info->devfn = devfn;
-               info->segment = iommu->segment;
-       } else {
-               struct pci_dev *pdev = to_pci_dev(dev);
-
-               info->bus = pdev->bus->number;
-               info->devfn = pdev->devfn;
-               info->segment = pci_domain_nr(pdev->bus);
-       }
-
-       info->dev = dev;
-       info->domain = domain;
-       info->iommu = iommu;
-
-       if (dev && dev_is_pci(dev)) {
-               struct pci_dev *pdev = to_pci_dev(info->dev);
-
-               if (ecap_dev_iotlb_support(iommu->ecap) &&
-                   pci_ats_supported(pdev) &&
-                   dmar_find_matched_atsr_unit(pdev))
-                       info->ats_supported = 1;
-
-               if (sm_supported(iommu)) {
-                       if (pasid_supported(iommu)) {
-                               int features = pci_pasid_features(pdev);
-                               if (features >= 0)
-                                       info->pasid_supported = features | 1;
-                       }
-
-                       if (info->ats_supported && ecap_prs(iommu->ecap) &&
-                           pci_pri_supported(pdev))
-                               info->pri_supported = 1;
-               }
-       }
-
        spin_lock_irqsave(&device_domain_lock, flags);
+       info->domain = domain;
        spin_lock(&iommu->lock);
        ret = domain_attach_iommu(domain, iommu);
        spin_unlock(&iommu->lock);
-
        if (ret) {
                spin_unlock_irqrestore(&device_domain_lock, flags);
-               kfree(info);
                return NULL;
        }
-
        list_add(&info->link, &domain->devices);
-       list_add(&info->global, &device_domain_list);
-       if (dev)
-               dev_iommu_priv_set(dev, info);
        spin_unlock_irqrestore(&device_domain_lock, flags);
 
        /* PASID table is mandatory for a PCI device in scalable mode. */
@@ -4336,13 +4243,11 @@ static void __dmar_remove_one_dev_info(struct device_domain_info *info)
                intel_pasid_free_table(info->dev);
        }
 
-       unlink_domain_info(info);
+       list_del(&info->link);
 
        spin_lock_irqsave(&iommu->lock, flags);
        domain_detach_iommu(domain, iommu);
        spin_unlock_irqrestore(&iommu->lock, flags);
-
-       kfree(info);
 }
 
 static void dmar_remove_one_dev_info(struct device *dev)
@@ -4351,7 +4256,7 @@ static void dmar_remove_one_dev_info(struct device *dev)
        unsigned long flags;
 
        spin_lock_irqsave(&device_domain_lock, flags);
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        if (info)
                __dmar_remove_one_dev_info(info);
        spin_unlock_irqrestore(&device_domain_lock, flags);
@@ -4475,10 +4380,9 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
 
        /* normally dev is not mapped */
        if (unlikely(domain_context_mapped(dev))) {
-               struct dmar_domain *old_domain;
+               struct device_domain_info *info = dev_iommu_priv_get(dev);
 
-               old_domain = find_domain(dev);
-               if (old_domain)
+               if (info->domain)
                        dmar_remove_one_dev_info(dev);
        }
 
@@ -4642,28 +4546,73 @@ static bool intel_iommu_capable(enum iommu_cap cap)
 
 static struct iommu_device *intel_iommu_probe_device(struct device *dev)
 {
+       struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
+       struct device_domain_info *info;
        struct intel_iommu *iommu;
+       unsigned long flags;
+       u8 bus, devfn;
 
-       iommu = device_to_iommu(dev, NULL, NULL);
+       iommu = device_to_iommu(dev, &bus, &devfn);
        if (!iommu)
                return ERR_PTR(-ENODEV);
 
-       if (translation_pre_enabled(iommu))
-               dev_iommu_priv_set(dev, DEFER_DEVICE_DOMAIN_INFO);
+       info = kzalloc(sizeof(*info), GFP_KERNEL);
+       if (!info)
+               return ERR_PTR(-ENOMEM);
+
+       if (dev_is_real_dma_subdevice(dev)) {
+               info->bus = pdev->bus->number;
+               info->devfn = pdev->devfn;
+               info->segment = pci_domain_nr(pdev->bus);
+       } else {
+               info->bus = bus;
+               info->devfn = devfn;
+               info->segment = iommu->segment;
+       }
+
+       info->dev = dev;
+       info->iommu = iommu;
+       if (dev_is_pci(dev)) {
+               if (ecap_dev_iotlb_support(iommu->ecap) &&
+                   pci_ats_supported(pdev) &&
+                   dmar_find_matched_atsr_unit(pdev))
+                       info->ats_supported = 1;
+
+               if (sm_supported(iommu)) {
+                       if (pasid_supported(iommu)) {
+                               int features = pci_pasid_features(pdev);
+
+                               if (features >= 0)
+                                       info->pasid_supported = features | 1;
+                       }
+
+                       if (info->ats_supported && ecap_prs(iommu->ecap) &&
+                           pci_pri_supported(pdev))
+                               info->pri_supported = 1;
+               }
+       }
+
+       spin_lock_irqsave(&device_domain_lock, flags);
+       list_add(&info->global, &device_domain_list);
+       dev_iommu_priv_set(dev, info);
+       spin_unlock_irqrestore(&device_domain_lock, flags);
 
        return &iommu->iommu;
 }
 
 static void intel_iommu_release_device(struct device *dev)
 {
-       struct intel_iommu *iommu;
-
-       iommu = device_to_iommu(dev, NULL, NULL);
-       if (!iommu)
-               return;
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
+       unsigned long flags;
 
        dmar_remove_one_dev_info(dev);
 
+       spin_lock_irqsave(&device_domain_lock, flags);
+       dev_iommu_priv_set(dev, NULL);
+       list_del(&info->global);
+       spin_unlock_irqrestore(&device_domain_lock, flags);
+
+       kfree(info);
        set_dma_ops(dev, NULL);
 }
 
@@ -4732,14 +4681,14 @@ static void intel_iommu_get_resv_regions(struct device *device,
 
 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
 {
-       struct device_domain_info *info;
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
        struct context_entry *context;
        struct dmar_domain *domain;
        unsigned long flags;
        u64 ctx_lo;
        int ret;
 
-       domain = find_domain(dev);
+       domain = info->domain;
        if (!domain)
                return -EINVAL;
 
@@ -4747,8 +4696,7 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
        spin_lock(&iommu->lock);
 
        ret = -EINVAL;
-       info = get_domain_info(dev);
-       if (!info || !info->pasid_supported)
+       if (!info->pasid_supported)
                goto out;
 
        context = iommu_context_addr(iommu, info->bus, info->devfn, 0);
@@ -4790,7 +4738,7 @@ static struct iommu_group *intel_iommu_device_group(struct device *dev)
 
 static int intel_iommu_enable_sva(struct device *dev)
 {
-       struct device_domain_info *info = get_domain_info(dev);
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
        struct intel_iommu *iommu;
        int ret;
 
@@ -4819,7 +4767,7 @@ static int intel_iommu_enable_sva(struct device *dev)
 
 static int intel_iommu_disable_sva(struct device *dev)
 {
-       struct device_domain_info *info = get_domain_info(dev);
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
        struct intel_iommu *iommu = info->iommu;
        int ret;
 
@@ -4832,7 +4780,7 @@ static int intel_iommu_disable_sva(struct device *dev)
 
 static int intel_iommu_enable_iopf(struct device *dev)
 {
-       struct device_domain_info *info = get_domain_info(dev);
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
 
        if (info && info->pri_supported)
                return 0;
@@ -4872,7 +4820,9 @@ intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
 
 static bool intel_iommu_is_attach_deferred(struct device *dev)
 {
-       return attach_deferred(dev);
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
+
+       return translation_pre_enabled(info->iommu) && !info->domain;
 }
 
 /*

diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c
index 10fb82ea467d8f58145ee26301465cbe61a156e5..f8d215d85695b286a0ce469e1bf32ec90e97fa02 100644 (file)
--- a/drivers/iommu/intel/pasid.c
+++ b/drivers/iommu/intel/pasid.c
@@ -150,7 +150,7 @@ int intel_pasid_alloc_table(struct device *dev)
        int size;
 
        might_sleep();
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
                return -EINVAL;
 
@@ -197,7 +197,7 @@ void intel_pasid_free_table(struct device *dev)
        struct pasid_entry *table;
        int i, max_pde;
 
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        if (!info || !dev_is_pci(dev) || !info->pasid_table)
                return;
 
@@ -223,7 +223,7 @@ struct pasid_table *intel_pasid_get_table(struct device *dev)
 {
        struct device_domain_info *info;
 
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        if (!info)
                return NULL;
 
@@ -234,7 +234,7 @@ static int intel_pasid_get_dev_max_id(struct device *dev)
 {
        struct device_domain_info *info;
 
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        if (!info || !info->pasid_table)
                return 0;
 
@@ -254,7 +254,7 @@ static struct pasid_entry *intel_pasid_get_entry(struct device *dev, u32 pasid)
                return NULL;
 
        dir = pasid_table->table;
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        dir_index = pasid >> PASID_PDE_SHIFT;
        index = pasid & PASID_PTE_MASK;
 
@@ -487,7 +487,7 @@ devtlb_invalidation_with_pasid(struct intel_iommu *iommu,
        struct device_domain_info *info;
        u16 sid, qdep, pfsid;
 
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        if (!info || !info->ats_enabled)
                return;
 

diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c
index d04c83dd3a58d3ed4c49bf3346d9c9b3d993334f..944e2408b6d2dd92f9fe0ed537dbb4c9e5c0e26d 100644 (file)
--- a/drivers/iommu/intel/svm.c
+++ b/drivers/iommu/intel/svm.c
@@ -200,7 +200,7 @@ static void __flush_svm_range_dev(struct intel_svm *svm,
                                  unsigned long address,
                                  unsigned long pages, int ih)
 {
-       struct device_domain_info *info = get_domain_info(sdev->dev);
+       struct device_domain_info *info = dev_iommu_priv_get(sdev->dev);
 
        if (WARN_ON(!pages))
                return;
@@ -337,7 +337,7 @@ static struct iommu_sva *intel_svm_bind_mm(struct intel_iommu *iommu,
                                           struct mm_struct *mm,
                                           unsigned int flags)
 {
-       struct device_domain_info *info = get_domain_info(dev);
+       struct device_domain_info *info = dev_iommu_priv_get(dev);
        unsigned long iflags, sflags;
        struct intel_svm_dev *sdev;
        struct intel_svm *svm;
@@ -545,7 +545,7 @@ static void intel_svm_drain_prq(struct device *dev, u32 pasid)
        u16 sid, did;
        int qdep;
 
-       info = get_domain_info(dev);
+       info = dev_iommu_priv_get(dev);
        if (WARN_ON(!info || !dev_is_pci(dev)))
                return;
 

diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 8c7591b5f3e2445db2415c089aa593451ceaf297..03f1134fc2fedf0f41e4d5cb41d60eda1a0841eb 100644 (file)
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -733,8 +733,6 @@ int for_each_device_domain(int (*fn)(struct device_domain_info *info,
                                     void *data), void *data);
 void iommu_flush_write_buffer(struct intel_iommu *iommu);
 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);
-struct dmar_domain *find_domain(struct device *dev);
-struct device_domain_info *get_domain_info(struct device *dev);
 struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);
 
 #ifdef CONFIG_INTEL_IOMMU_SVM