iommu: Pass struct iommu_iotlb_gather to ->unmap() and ->iotlb_sync()

To allow IOMMU drivers to batch up TLB flushing operations and postpone
them until ->iotlb_sync() is called, extend the prototypes for the
->unmap() and ->iotlb_sync() IOMMU ops callbacks to take a pointer to
the current iommu_iotlb_gather structure.

All affected IOMMU drivers are updated, but there should be no
functional change since the extra parameter is ignored for now.

Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index f93b148cf55e78320392d40656b02159d3597e0a..29eeea91466080a1a328accbd435dd673ab0866a 100644 (file)
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -3055,7 +3055,8 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
 }
 
 static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
-                          size_t page_size)
+                             size_t page_size,
+                             struct iommu_iotlb_gather *gather)
 {
        struct protection_domain *domain = to_pdomain(dom);
        size_t unmap_size;
@@ -3196,6 +3197,12 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
        domain_flush_complete(dom);
 }
 
+static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
+                                struct iommu_iotlb_gather *gather)
+{
+       amd_iommu_flush_iotlb_all(domain);
+}
+
 const struct iommu_ops amd_iommu_ops = {
        .capable = amd_iommu_capable,
        .domain_alloc = amd_iommu_domain_alloc,
@@ -3214,7 +3221,7 @@ const struct iommu_ops amd_iommu_ops = {
        .is_attach_deferred = amd_iommu_is_attach_deferred,
        .pgsize_bitmap  = AMD_IOMMU_PGSIZES,
        .flush_iotlb_all = amd_iommu_flush_iotlb_all,
-       .iotlb_sync = amd_iommu_flush_iotlb_all,
+       .iotlb_sync = amd_iommu_iotlb_sync,
 };
 
 /*****************************************************************************

diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 7e137e1e28f18ea776f0eb983764c92da2523837..80753b8ca0548a8ef8ea87fbf35c35da21f946c3 100644 (file)
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -1985,8 +1985,8 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
        return ops->map(ops, iova, paddr, size, prot);
 }
 
-static size_t
-arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size)
+static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
+                            size_t size, struct iommu_iotlb_gather *gather)
 {
        int ret;
        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
@@ -2010,7 +2010,8 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
                arm_smmu_tlb_inv_context(smmu_domain);
 }
 
-static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
+static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
+                               struct iommu_iotlb_gather *gather)
 {
        struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
 

diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index dc08db347ef3add4676877c4b8ee32ac6c74c9c8..e535ae2a9e656da7ec9acd51e0d0e678e1356260 100644 (file)
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1301,7 +1301,7 @@ static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
 }
 
 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
-                            size_t size)
+                            size_t size, struct iommu_iotlb_gather *gather)
 {
        struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
        struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
@@ -1329,7 +1329,8 @@ static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
        }
 }
 
-static void arm_smmu_iotlb_sync(struct iommu_domain *domain)
+static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
+                               struct iommu_iotlb_gather *gather)
 {
        struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
        struct arm_smmu_device *smmu = smmu_domain->smmu;

diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index b0c1e5f9daae5acc353ef91c5d0ab44f773fa318..cf5af34cb6819bf25501bb235b1cb58b6a21614e 100644 (file)
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1130,7 +1130,8 @@ static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain
 }
 
 static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
-                                unsigned long l_iova, size_t size)
+                                unsigned long l_iova, size_t size,
+                                struct iommu_iotlb_gather *gather)
 {
        struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
        sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;

diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index ac4172c0224487b69cfbee8c6e292d7755eb55e9..b9fb8d6ddc6e7393ec77ed600558b251362c3264 100644 (file)
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -5147,7 +5147,8 @@ static int intel_iommu_map(struct iommu_domain *domain,
 }
 
 static size_t intel_iommu_unmap(struct iommu_domain *domain,
-                               unsigned long iova, size_t size)
+                               unsigned long iova, size_t size,
+                               struct iommu_iotlb_gather *gather)
 {
        struct dmar_domain *dmar_domain = to_dmar_domain(domain);
        struct page *freelist = NULL;

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index d67222fdfe4423e2ef8733c1337e9be7c9754964..70bfbcc09248257ac1fc955458097e672756ab75 100644 (file)
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -1899,7 +1899,7 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
        while (unmapped < size) {
                size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
 
-               unmapped_page = ops->unmap(domain, iova, pgsize);
+               unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather);
                if (!unmapped_page)
                        break;
 

diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 2c14a2c65b22257aa27867eb21460afa28c3731c..a9332b893ce208db171ff7e57bc9d50f3516e33b 100644 (file)
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -733,14 +733,14 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
 }
 
 static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
-                         size_t size)
+                         size_t size, struct iommu_iotlb_gather *gather)
 {
        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
 
        return domain->iop->unmap(domain->iop, iova, size);
 }
 
-static void ipmmu_iotlb_sync(struct iommu_domain *io_domain)
+static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain)
 {
        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
 
@@ -748,6 +748,12 @@ static void ipmmu_iotlb_sync(struct iommu_domain *io_domain)
                ipmmu_tlb_flush_all(domain);
 }
 
+static void ipmmu_iotlb_sync(struct iommu_domain *io_domain,
+                            struct iommu_iotlb_gather *gather)
+{
+       ipmmu_flush_iotlb_all(io_domain);
+}
+
 static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
                                      dma_addr_t iova)
 {
@@ -957,7 +963,7 @@ static const struct iommu_ops ipmmu_ops = {
        .detach_dev = ipmmu_detach_device,
        .map = ipmmu_map,
        .unmap = ipmmu_unmap,
-       .flush_iotlb_all = ipmmu_iotlb_sync,
+       .flush_iotlb_all = ipmmu_flush_iotlb_all,
        .iotlb_sync = ipmmu_iotlb_sync,
        .iova_to_phys = ipmmu_iova_to_phys,
        .add_device = ipmmu_add_device,

diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index 8b602384a385b5f0613352a37b24cb562a2b9d39..681ab3d3376db5536f929afd60725836fab7bcae 100644 (file)
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -509,7 +509,7 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 
 static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
-                             size_t len)
+                             size_t len, struct iommu_iotlb_gather *gather)
 {
        struct msm_priv *priv = to_msm_priv(domain);
        unsigned long flags;

diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index fed77658d67e076c4c873684b22e986ebe11cad4..c870f16749030e03ae312522461c8460803cf61a 100644 (file)
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -371,7 +371,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 
 static size_t mtk_iommu_unmap(struct iommu_domain *domain,
-                             unsigned long iova, size_t size)
+                             unsigned long iova, size_t size,
+                             struct iommu_iotlb_gather *gather)
 {
        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
        unsigned long flags;
@@ -384,7 +385,13 @@ static size_t mtk_iommu_unmap(struct iommu_domain *domain,
        return unmapsz;
 }
 
-static void mtk_iommu_iotlb_sync(struct iommu_domain *domain)
+static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain)
+{
+       mtk_iommu_tlb_sync(mtk_iommu_get_m4u_data());
+}
+
+static void mtk_iommu_iotlb_sync(struct iommu_domain *domain,
+                                struct iommu_iotlb_gather *gather)
 {
        mtk_iommu_tlb_sync(mtk_iommu_get_m4u_data());
 }
@@ -490,7 +497,7 @@ static const struct iommu_ops mtk_iommu_ops = {
        .detach_dev     = mtk_iommu_detach_device,
        .map            = mtk_iommu_map,
        .unmap          = mtk_iommu_unmap,
-       .flush_iotlb_all = mtk_iommu_iotlb_sync,
+       .flush_iotlb_all = mtk_iommu_flush_iotlb_all,
        .iotlb_sync     = mtk_iommu_iotlb_sync,
        .iova_to_phys   = mtk_iommu_iova_to_phys,
        .add_device     = mtk_iommu_add_device,

diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index abeeac488372c8e5735f8838deb0ed7f5eef0eca..7b92ddd5d9fd9ea4b5c354e8b30f1c1fd308ba51 100644 (file)
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -324,7 +324,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 
 static size_t mtk_iommu_unmap(struct iommu_domain *domain,
-                             unsigned long iova, size_t size)
+                             unsigned long iova, size_t size,
+                             struct iommu_iotlb_gather *gather)
 {
        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
        unsigned long flags;

diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index dfb961d8c21b7fd95feaa102a3235950950c0e5f..8039bc5ee42575f02acc2f6872041c03ccf29ca3 100644 (file)
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -1149,7 +1149,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
 }
 
 static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
-                              size_t size)
+                              size_t size, struct iommu_iotlb_gather *gather)
 {
        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        struct device *dev = omap_domain->dev;

diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
index fd9d9f4da735f0f2f219cf96f034a7de4c00876e..a7432991fa044cd1314a3b9c6cfdd21e0b311506 100644 (file)
--- a/drivers/iommu/qcom_iommu.c
+++ b/drivers/iommu/qcom_iommu.c
@@ -417,7 +417,7 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 
 static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
-                              size_t size)
+                              size_t size, struct iommu_iotlb_gather *gather)
 {
        size_t ret;
        unsigned long flags;
@@ -441,7 +441,7 @@ static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
        return ret;
 }
 
-static void qcom_iommu_iotlb_sync(struct iommu_domain *domain)
+static void qcom_iommu_flush_iotlb_all(struct iommu_domain *domain)
 {
        struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
        struct io_pgtable *pgtable = container_of(qcom_domain->pgtbl_ops,
@@ -454,6 +454,12 @@ static void qcom_iommu_iotlb_sync(struct iommu_domain *domain)
        pm_runtime_put_sync(qcom_domain->iommu->dev);
 }
 
+static void qcom_iommu_iotlb_sync(struct iommu_domain *domain,
+                                 struct iommu_iotlb_gather *gather)
+{
+       qcom_iommu_flush_iotlb_all(domain);
+}
+
 static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
                                           dma_addr_t iova)
 {
@@ -581,7 +587,7 @@ static const struct iommu_ops qcom_iommu_ops = {
        .detach_dev     = qcom_iommu_detach_dev,
        .map            = qcom_iommu_map,
        .unmap          = qcom_iommu_unmap,
-       .flush_iotlb_all = qcom_iommu_iotlb_sync,
+       .flush_iotlb_all = qcom_iommu_flush_iotlb_all,
        .iotlb_sync     = qcom_iommu_iotlb_sync,
        .iova_to_phys   = qcom_iommu_iova_to_phys,
        .add_device     = qcom_iommu_add_device,

diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index dc26d74d79c2c7c337b717aae3145810d13ff552..26290f310f90bd8c6d05da9b255ec02d144cd56f 100644 (file)
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -794,7 +794,7 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
 }
 
 static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
-                            size_t size)
+                            size_t size, struct iommu_iotlb_gather *gather)
 {
        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        unsigned long flags;

diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 22d4db302c1c8c2cc6b19a11a5ee8baf8c132acc..3b0b18e23187c19d7c787389fb33d73ca957a429 100644 (file)
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -314,7 +314,8 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
 }
 
 static size_t s390_iommu_unmap(struct iommu_domain *domain,
-                              unsigned long iova, size_t size)
+                              unsigned long iova, size_t size,
+                              struct iommu_iotlb_gather *gather)
 {
        struct s390_domain *s390_domain = to_s390_domain(domain);
        int flags = ZPCI_PTE_INVALID;

diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index 6d40bc1b38bf0ccc2f90a441d04729d492c4d3a5..3924f7c055440edd3ee64be0b02710102bd7efcd 100644 (file)
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -207,7 +207,7 @@ static inline int __gart_iommu_unmap(struct gart_device *gart,
 }
 
 static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
-                              size_t bytes)
+                              size_t bytes, struct iommu_iotlb_gather *gather)
 {
        struct gart_device *gart = gart_handle;
        int err;
@@ -273,11 +273,17 @@ static int gart_iommu_of_xlate(struct device *dev,
        return 0;
 }
 
-static void gart_iommu_sync(struct iommu_domain *domain)
+static void gart_iommu_sync_map(struct iommu_domain *domain)
 {
        FLUSH_GART_REGS(gart_handle);
 }
 
+static void gart_iommu_sync(struct iommu_domain *domain,
+                           struct iommu_iotlb_gather *gather)
+{
+       gart_iommu_sync_map(domain);
+}
+
 static const struct iommu_ops gart_iommu_ops = {
        .capable        = gart_iommu_capable,
        .domain_alloc   = gart_iommu_domain_alloc,
@@ -292,7 +298,7 @@ static const struct iommu_ops gart_iommu_ops = {
        .iova_to_phys   = gart_iommu_iova_to_phys,
        .pgsize_bitmap  = GART_IOMMU_PGSIZES,
        .of_xlate       = gart_iommu_of_xlate,
-       .iotlb_sync_map = gart_iommu_sync,
+       .iotlb_sync_map = gart_iommu_sync_map,
        .iotlb_sync     = gart_iommu_sync,
 };
 

diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index c4a652b227f8d6a794fa54a85ac9edf867f3f05f..7293fc3f796d69f61f8cbe8a4ef3176d3b9ff5b0 100644 (file)
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -680,7 +680,7 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
 }
 
 static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
-                              size_t size)
+                              size_t size, struct iommu_iotlb_gather *gather)
 {
        struct tegra_smmu_as *as = to_smmu_as(domain);
        dma_addr_t pte_dma;

diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
index 433f4d2ee9565be632c1e780e7cdedcdf4254e1d..5f9f91a4d7f3eb75a1f06fd15b995000f3ba8d2f 100644 (file)
--- a/drivers/iommu/virtio-iommu.c
+++ b/drivers/iommu/virtio-iommu.c
@@ -742,7 +742,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
 }
 
 static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova,
-                          size_t size)
+                          size_t size, struct iommu_iotlb_gather *gather)
 {
        int ret = 0;
        size_t unmapped;
@@ -788,7 +788,8 @@ static phys_addr_t viommu_iova_to_phys(struct iommu_domain *domain,
        return paddr;
 }
 
-static void viommu_iotlb_sync(struct iommu_domain *domain)
+static void viommu_iotlb_sync(struct iommu_domain *domain,
+                             struct iommu_iotlb_gather *gather)
 {
        struct viommu_domain *vdomain = to_viommu_domain(domain);
 

diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index ad41aee55bc63ff6d03fb65d8fb1cb6466b1d41e..64ebaff33455fa38aaeab596e6bf49c96b9276d1 100644 (file)
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -258,10 +258,11 @@ struct iommu_ops {
        int (*map)(struct iommu_domain *domain, unsigned long iova,
                   phys_addr_t paddr, size_t size, int prot);
        size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
-                    size_t size);
+                    size_t size, struct iommu_iotlb_gather *iotlb_gather);
        void (*flush_iotlb_all)(struct iommu_domain *domain);
        void (*iotlb_sync_map)(struct iommu_domain *domain);
-       void (*iotlb_sync)(struct iommu_domain *domain);
+       void (*iotlb_sync)(struct iommu_domain *domain,
+                          struct iommu_iotlb_gather *iotlb_gather);
        phys_addr_t (*iova_to_phys)(struct iommu_domain *domain, dma_addr_t iova);
        int (*add_device)(struct device *dev);
        void (*remove_device)(struct device *dev);
@@ -502,7 +503,7 @@ static inline void iommu_tlb_sync(struct iommu_domain *domain,
                                  struct iommu_iotlb_gather *iotlb_gather)
 {
        if (domain->ops->iotlb_sync)
-               domain->ops->iotlb_sync(domain);
+               domain->ops->iotlb_sync(domain, iotlb_gather);
 
        iommu_iotlb_gather_init(iotlb_gather);
 }