Merge tag 'dma-mapping-5.20-2022-08-06' of git://git.infradead.org/users/hch/dma...

[platform/kernel/linux-starfive.git] / kernel / dma / direct.c

diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 8d0b68a..63859a1 100644 (file)
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -453,29 +453,60 @@ void dma_direct_sync_sg_for_cpu(struct device *dev,
                arch_sync_dma_for_cpu_all();
 }
 
+/*
+ * Unmaps segments, except for ones marked as pci_p2pdma which do not
+ * require any further action as they contain a bus address.
+ */
 void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl,
                int nents, enum dma_data_direction dir, unsigned long attrs)
 {
        struct scatterlist *sg;
        int i;
 
-       for_each_sg(sgl, sg, nents, i)
-               dma_direct_unmap_page(dev, sg->dma_address, sg_dma_len(sg), dir,
-                            attrs);
+       for_each_sg(sgl,  sg, nents, i) {
+               if (sg_is_dma_bus_address(sg))
+                       sg_dma_unmark_bus_address(sg);
+               else
+                       dma_direct_unmap_page(dev, sg->dma_address,
+                                             sg_dma_len(sg), dir, attrs);
+       }
 }
 #endif
 
 int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
                enum dma_data_direction dir, unsigned long attrs)
 {
-       int i;
+       struct pci_p2pdma_map_state p2pdma_state = {};
+       enum pci_p2pdma_map_type map;
        struct scatterlist *sg;
+       int i, ret;
 
        for_each_sg(sgl, sg, nents, i) {
+               if (is_pci_p2pdma_page(sg_page(sg))) {
+                       map = pci_p2pdma_map_segment(&p2pdma_state, dev, sg);
+                       switch (map) {
+                       case PCI_P2PDMA_MAP_BUS_ADDR:
+                               continue;
+                       case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
+                               /*
+                                * Any P2P mapping that traverses the PCI
+                                * host bridge must be mapped with CPU physical
+                                * address and not PCI bus addresses. This is
+                                * done with dma_direct_map_page() below.
+                                */
+                               break;
+                       default:
+                               ret = -EREMOTEIO;
+                               goto out_unmap;
+                       }
+               }
+
                sg->dma_address = dma_direct_map_page(dev, sg_page(sg),
                                sg->offset, sg->length, dir, attrs);
-               if (sg->dma_address == DMA_MAPPING_ERROR)
+               if (sg->dma_address == DMA_MAPPING_ERROR) {
+                       ret = -EIO;
                        goto out_unmap;
+               }
                sg_dma_len(sg) = sg->length;
        }
 
@@ -483,7 +514,7 @@ int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents,
 
 out_unmap:
        dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
-       return -EIO;
+       return ret;
 }
 
 dma_addr_t dma_direct_map_resource(struct device *dev, phys_addr_t paddr,