Merge tag 'dma-mapping-6.4-2023-04-28' of git://git.infradead.org/users/hch/dma-mapping

[platform/kernel/linux-starfive.git] / kernel / dma / swiotlb.c

diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 0fb16bd..af2e304 100644 (file)
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -566,6 +566,40 @@ static unsigned int wrap_area_index(struct io_tlb_mem *mem, unsigned int index)
 }
 
 /*
+ * Track the total used slots with a global atomic value in order to have
+ * correct information to determine the high water mark. The mem_used()
+ * function gives imprecise results because there's no locking across
+ * multiple areas.
+ */
+#ifdef CONFIG_DEBUG_FS
+static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
+{
+       unsigned long old_hiwater, new_used;
+
+       new_used = atomic_long_add_return(nslots, &mem->total_used);
+       old_hiwater = atomic_long_read(&mem->used_hiwater);
+       do {
+               if (new_used <= old_hiwater)
+                       break;
+       } while (!atomic_long_try_cmpxchg(&mem->used_hiwater,
+                                         &old_hiwater, new_used));
+}
+
+static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
+{
+       atomic_long_sub(nslots, &mem->total_used);
+}
+
+#else /* !CONFIG_DEBUG_FS */
+static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
+{
+}
+static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
+
+/*
  * Find a suitable number of IO TLB entries size that will fit this request and
  * allocate a buffer from that IO TLB pool.
  */
@@ -659,6 +693,8 @@ found:
        area->index = wrap_area_index(mem, index + nslots);
        area->used += nslots;
        spin_unlock_irqrestore(&area->lock, flags);
+
+       inc_used_and_hiwater(mem, nslots);
        return slot_index;
 }
 
@@ -791,6 +827,8 @@ static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
                mem->slots[i].list = ++count;
        area->used -= nslots;
        spin_unlock_irqrestore(&area->lock, flags);
+
+       dec_used(mem, nslots);
 }
 
 /*
@@ -885,34 +923,73 @@ bool is_swiotlb_active(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(is_swiotlb_active);
 
+#ifdef CONFIG_DEBUG_FS
+
 static int io_tlb_used_get(void *data, u64 *val)
 {
-       *val = mem_used(&io_tlb_default_mem);
+       struct io_tlb_mem *mem = data;
+
+       *val = mem_used(mem);
        return 0;
 }
+
+static int io_tlb_hiwater_get(void *data, u64 *val)
+{
+       struct io_tlb_mem *mem = data;
+
+       *val = atomic_long_read(&mem->used_hiwater);
+       return 0;
+}
+
+static int io_tlb_hiwater_set(void *data, u64 val)
+{
+       struct io_tlb_mem *mem = data;
+
+       /* Only allow setting to zero */
+       if (val != 0)
+               return -EINVAL;
+
+       atomic_long_set(&mem->used_hiwater, val);
+       return 0;
+}
+
 DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
+DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
+                               io_tlb_hiwater_set, "%llu\n");
 
 static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
                                         const char *dirname)
 {
+       atomic_long_set(&mem->total_used, 0);
+       atomic_long_set(&mem->used_hiwater, 0);
+
        mem->debugfs = debugfs_create_dir(dirname, io_tlb_default_mem.debugfs);
        if (!mem->nslabs)
                return;
 
        debugfs_create_ulong("io_tlb_nslabs", 0400, mem->debugfs, &mem->nslabs);
-       debugfs_create_file("io_tlb_used", 0400, mem->debugfs, NULL,
+       debugfs_create_file("io_tlb_used", 0400, mem->debugfs, mem,
                        &fops_io_tlb_used);
+       debugfs_create_file("io_tlb_used_hiwater", 0600, mem->debugfs, mem,
+                       &fops_io_tlb_hiwater);
 }
 
-static int __init __maybe_unused swiotlb_create_default_debugfs(void)
+static int __init swiotlb_create_default_debugfs(void)
 {
        swiotlb_create_debugfs_files(&io_tlb_default_mem, "swiotlb");
        return 0;
 }
 
-#ifdef CONFIG_DEBUG_FS
 late_initcall(swiotlb_create_default_debugfs);
-#endif
+
+#else  /* !CONFIG_DEBUG_FS */
+
+static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
+                                               const char *dirname)
+{
+}
+
+#endif /* CONFIG_DEBUG_FS */
 
 #ifdef CONFIG_DMA_RESTRICTED_POOL
 
@@ -955,6 +1032,11 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
        /* Set Per-device io tlb area to one */
        unsigned int nareas = 1;
 
+       if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
+               dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
+               return -EINVAL;
+       }
+
        /*
         * Since multiple devices can share the same pool, the private data,
         * io_tlb_mem struct, will be initialized by the first device attached
@@ -1016,11 +1098,6 @@ static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
            of_get_flat_dt_prop(node, "no-map", NULL))
                return -EINVAL;
 
-       if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
-               pr_err("Restricted DMA pool must be accessible within the linear mapping.");
-               return -EINVAL;
-       }
-
        rmem->ops = &rmem_swiotlb_ops;
        pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
                &rmem->base, (unsigned long)rmem->size / SZ_1M);