iommu/s390: Push the gfp parameter to the kmem_cache_alloc()'s

dma_alloc_cpu_table() and dma_alloc_page_table() are eventually called by
iommufd through s390_iommu_map_pages() and it should not be forced to
atomic. Thread the gfp parameter through the call chain starting from
s390_iommu_map_pages().

Reviewed-by: Niklas Schnelle <schnelle@linux.ibm.com>
Reviewed-by: Matthew Rosato <mjrosato@linux.ibm.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Link: https://lore.kernel.org/r/9-v3-76b587fe28df+6e3-iommu_map_gfp_jgg@nvidia.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/arch/s390/include/asm/pci_dma.h b/arch/s390/include/asm/pci_dma.h
index 91e6342..7119c04 100644 (file)
--- a/arch/s390/include/asm/pci_dma.h
+++ b/arch/s390/include/asm/pci_dma.h
@@ -186,9 +186,10 @@ static inline unsigned long *get_st_pto(unsigned long entry)
 
 /* Prototypes */
 void dma_free_seg_table(unsigned long);
-unsigned long *dma_alloc_cpu_table(void);
+unsigned long *dma_alloc_cpu_table(gfp_t gfp);
 void dma_cleanup_tables(unsigned long *);
-unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr);
+unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr,
+                                 gfp_t gfp);
 void dma_update_cpu_trans(unsigned long *entry, phys_addr_t page_addr, int flags);
 
 extern const struct dma_map_ops s390_pci_dma_ops;

diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c
index ea478d1..2f6d05d 100644 (file)
--- a/arch/s390/pci/pci_dma.c
+++ b/arch/s390/pci/pci_dma.c
@@ -27,11 +27,11 @@ static int zpci_refresh_global(struct zpci_dev *zdev)
                                  zdev->iommu_pages * PAGE_SIZE);
 }
 
-unsigned long *dma_alloc_cpu_table(void)
+unsigned long *dma_alloc_cpu_table(gfp_t gfp)
 {
        unsigned long *table, *entry;
 
-       table = kmem_cache_alloc(dma_region_table_cache, GFP_ATOMIC);
+       table = kmem_cache_alloc(dma_region_table_cache, gfp);
        if (!table)
                return NULL;
 
@@ -45,11 +45,11 @@ static void dma_free_cpu_table(void *table)
        kmem_cache_free(dma_region_table_cache, table);
 }
 
-static unsigned long *dma_alloc_page_table(void)
+static unsigned long *dma_alloc_page_table(gfp_t gfp)
 {
        unsigned long *table, *entry;
 
-       table = kmem_cache_alloc(dma_page_table_cache, GFP_ATOMIC);
+       table = kmem_cache_alloc(dma_page_table_cache, gfp);
        if (!table)
                return NULL;
 
@@ -63,7 +63,7 @@ static void dma_free_page_table(void *table)
        kmem_cache_free(dma_page_table_cache, table);
 }
 
-static unsigned long *dma_get_seg_table_origin(unsigned long *rtep)
+static unsigned long *dma_get_seg_table_origin(unsigned long *rtep, gfp_t gfp)
 {
        unsigned long old_rte, rte;
        unsigned long *sto;
@@ -72,7 +72,7 @@ static unsigned long *dma_get_seg_table_origin(unsigned long *rtep)
        if (reg_entry_isvalid(rte)) {
                sto = get_rt_sto(rte);
        } else {
-               sto = dma_alloc_cpu_table();
+               sto = dma_alloc_cpu_table(gfp);
                if (!sto)
                        return NULL;
 
@@ -90,7 +90,7 @@ static unsigned long *dma_get_seg_table_origin(unsigned long *rtep)
        return sto;
 }
 
-static unsigned long *dma_get_page_table_origin(unsigned long *step)
+static unsigned long *dma_get_page_table_origin(unsigned long *step, gfp_t gfp)
 {
        unsigned long old_ste, ste;
        unsigned long *pto;
@@ -99,7 +99,7 @@ static unsigned long *dma_get_page_table_origin(unsigned long *step)
        if (reg_entry_isvalid(ste)) {
                pto = get_st_pto(ste);
        } else {
-               pto = dma_alloc_page_table();
+               pto = dma_alloc_page_table(gfp);
                if (!pto)
                        return NULL;
                set_st_pto(&ste, virt_to_phys(pto));
@@ -116,18 +116,19 @@ static unsigned long *dma_get_page_table_origin(unsigned long *step)
        return pto;
 }
 
-unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr)
+unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr,
+                                 gfp_t gfp)
 {
        unsigned long *sto, *pto;
        unsigned int rtx, sx, px;
 
        rtx = calc_rtx(dma_addr);
-       sto = dma_get_seg_table_origin(&rto[rtx]);
+       sto = dma_get_seg_table_origin(&rto[rtx], gfp);
        if (!sto)
                return NULL;
 
        sx = calc_sx(dma_addr);
-       pto = dma_get_page_table_origin(&sto[sx]);
+       pto = dma_get_page_table_origin(&sto[sx], gfp);
        if (!pto)
                return NULL;
 
@@ -170,7 +171,8 @@ static int __dma_update_trans(struct zpci_dev *zdev, phys_addr_t pa,
                return -EINVAL;
 
        for (i = 0; i < nr_pages; i++) {
-               entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);
+               entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr,
+                                          GFP_ATOMIC);
                if (!entry) {
                        rc = -ENOMEM;
                        goto undo_cpu_trans;
@@ -186,7 +188,8 @@ undo_cpu_trans:
                while (i-- > 0) {
                        page_addr -= PAGE_SIZE;
                        dma_addr -= PAGE_SIZE;
-                       entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr);
+                       entry = dma_walk_cpu_trans(zdev->dma_table, dma_addr,
+                                                  GFP_ATOMIC);
                        if (!entry)
                                break;
                        dma_update_cpu_trans(entry, page_addr, flags);
@@ -576,7 +579,7 @@ int zpci_dma_init_device(struct zpci_dev *zdev)
 
        spin_lock_init(&zdev->iommu_bitmap_lock);
 
-       zdev->dma_table = dma_alloc_cpu_table();
+       zdev->dma_table = dma_alloc_cpu_table(GFP_ATOMIC);
        if (!zdev->dma_table) {
                rc = -ENOMEM;
                goto out;

diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index ed33c6c..654ec44 100644 (file)
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -52,7 +52,7 @@ static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
        if (!s390_domain)
                return NULL;
 
-       s390_domain->dma_table = dma_alloc_cpu_table();
+       s390_domain->dma_table = dma_alloc_cpu_table(GFP_ATOMIC);
        if (!s390_domain->dma_table) {
                kfree(s390_domain);
                return NULL;
@@ -260,7 +260,8 @@ static void s390_iommu_iotlb_sync_map(struct iommu_domain *domain,
 
 static int s390_iommu_validate_trans(struct s390_domain *s390_domain,
                                     phys_addr_t pa, dma_addr_t dma_addr,
-                                    unsigned long nr_pages, int flags)
+                                    unsigned long nr_pages, int flags,
+                                    gfp_t gfp)
 {
        phys_addr_t page_addr = pa & PAGE_MASK;
        unsigned long *entry;
@@ -268,7 +269,8 @@ static int s390_iommu_validate_trans(struct s390_domain *s390_domain,
        int rc;
 
        for (i = 0; i < nr_pages; i++) {
-               entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
+               entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr,
+                                          gfp);
                if (unlikely(!entry)) {
                        rc = -ENOMEM;
                        goto undo_cpu_trans;
@@ -284,7 +286,7 @@ undo_cpu_trans:
        while (i-- > 0) {
                dma_addr -= PAGE_SIZE;
                entry = dma_walk_cpu_trans(s390_domain->dma_table,
-                                          dma_addr);
+                                          dma_addr, gfp);
                if (!entry)
                        break;
                dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID);
@@ -301,7 +303,8 @@ static int s390_iommu_invalidate_trans(struct s390_domain *s390_domain,
        int rc = 0;
 
        for (i = 0; i < nr_pages; i++) {
-               entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr);
+               entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr,
+                                          GFP_ATOMIC);
                if (unlikely(!entry)) {
                        rc = -EINVAL;
                        break;
@@ -339,7 +342,7 @@ static int s390_iommu_map_pages(struct iommu_domain *domain,
                flags |= ZPCI_TABLE_PROTECTED;
 
        rc = s390_iommu_validate_trans(s390_domain, paddr, iova,
-                                      pgcount, flags);
+                                      pgcount, flags, gfp);
        if (!rc)
                *mapped = size;