habanalabs: MMU invalidation h/w is per device

The code used the mmu mutex to protect access to the context's page
tables and invalidation of the MMU cache. Because pgt are per
context, the mmu mutex was a member of the context object.

The problem is that the device has a single MMU invalidation h/w
(per MMU). Therefore, the mmu mutex should not be a property of the
context but a property of the device.

Signed-off-by: Oded Gabbay <ogabbay@kernel.org>

diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c
index d16de18..2b33299 100644 (file)
--- a/drivers/misc/habanalabs/common/command_buffer.c
+++ b/drivers/misc/habanalabs/common/command_buffer.c
@@ -44,20 +44,20 @@ static int cb_map_mem(struct hl_ctx *ctx, struct hl_cb *cb)
                return -ENOMEM;
        }
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        rc = hl_mmu_map_contiguous(ctx, cb->virtual_addr, cb->bus_address, cb->roundup_size);
        if (rc) {
                dev_err(hdev->dev, "Failed to map VA %#llx to CB\n", cb->virtual_addr);
                goto err_va_umap;
        }
        rc = hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        cb->is_mmu_mapped = true;
        return rc;
 
 err_va_umap:
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
        gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size);
        return rc;
 }
@@ -66,10 +66,10 @@ static void cb_unmap_mem(struct hl_ctx *ctx, struct hl_cb *cb)
 {
        struct hl_device *hdev = ctx->hdev;
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        hl_mmu_unmap_contiguous(ctx, cb->virtual_addr, cb->roundup_size);
        hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        gen_pool_free(ctx->cb_va_pool, cb->virtual_addr, cb->roundup_size);
 }

diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 959e361..9c2123d 100644 (file)
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -1793,8 +1793,6 @@ struct hl_cs_outcome_store {
  *                 command submissions for a long time after CS id wraparound.
  * @va_range: holds available virtual addresses for host and dram mappings.
  * @mem_hash_lock: protects the mem_hash.
- * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the
- *            MMU hash or walking the PGT requires talking this lock.
  * @hw_block_list_lock: protects the HW block memory list.
  * @debugfs_list: node in debugfs list of contexts.
  * @hw_block_mem_list: list of HW block virtual mapped addresses.
@@ -1831,7 +1829,6 @@ struct hl_ctx {
        struct hl_cs_outcome_store      outcome_store;
        struct hl_va_range              *va_range[HL_VA_RANGE_TYPE_MAX];
        struct mutex                    mem_hash_lock;
-       struct mutex                    mmu_lock;
        struct mutex                    hw_block_list_lock;
        struct list_head                debugfs_list;
        struct list_head                hw_block_mem_list;
@@ -3079,6 +3076,12 @@ struct hl_reset_info {
  * @asid_mutex: protects asid_bitmap.
  * @send_cpu_message_lock: enforces only one message in Host <-> CPU-CP queue.
  * @debug_lock: protects critical section of setting debug mode for device
+ * @mmu_lock: protects the MMU page tables and invalidation h/w. Although the
+ *            page tables are per context, the invalidation h/w is per MMU.
+ *            Therefore, we can't allow multiple contexts (we only have two,
+ *            user and kernel) to access the invalidation h/w at the same time.
+ *            In addition, any change to the PGT, modifying the MMU hash or
+ *            walking the PGT requires talking this lock.
  * @asic_prop: ASIC specific immutable properties.
  * @asic_funcs: ASIC specific functions.
  * @asic_specific: ASIC specific information to use only from ASIC files.
@@ -3244,6 +3247,7 @@ struct hl_device {
        struct mutex                    asid_mutex;
        struct mutex                    send_cpu_message_lock;
        struct mutex                    debug_lock;
+       struct mutex                    mmu_lock;
        struct asic_fixed_properties    asic_prop;
        const struct hl_asic_funcs      *asic_funcs;
        void                            *asic_specific;

diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c
index 0a653ff..096fa3c 100644 (file)
--- a/drivers/misc/habanalabs/common/memory.c
+++ b/drivers/misc/habanalabs/common/memory.c
@@ -1210,18 +1210,18 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, u64 *device
                goto va_block_err;
        }
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
 
        rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
        if (rc) {
                dev_err(hdev->dev, "mapping page pack failed for handle %u\n", handle);
-               mutex_unlock(&ctx->mmu_lock);
+               mutex_unlock(&hdev->mmu_lock);
                goto map_err;
        }
 
        rc = hl_mmu_invalidate_cache_range(hdev, false, *vm_type | MMU_OP_SKIP_LOW_CACHE_INV,
                                ctx->asid, ret_vaddr, phys_pg_pack->total_size);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
        if (rc)
                goto map_err;
 
@@ -1362,7 +1362,7 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
        else
                vaddr &= ~(((u64) phys_pg_pack->page_size) - 1);
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
 
        unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack);
 
@@ -1375,7 +1375,7 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
                rc = hl_mmu_invalidate_cache_range(hdev, true, *vm_type, ctx->asid, vaddr,
                                                        phys_pg_pack->total_size);
 
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        /*
         * If the context is closing we don't need to check for the MMU cache
@@ -2771,13 +2771,13 @@ void hl_vm_ctx_fini(struct hl_ctx *ctx)
                unmap_device_va(ctx, &args, true);
        }
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
 
        /* invalidate the cache once after the unmapping loop */
        hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
        hl_mmu_invalidate_cache(hdev, true, MMU_OP_PHYS_PACK);
 
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        INIT_LIST_HEAD(&free_list);
 

diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c
index 264f3b9..cf89462 100644 (file)
--- a/drivers/misc/habanalabs/common/mmu/mmu.c
+++ b/drivers/misc/habanalabs/common/mmu/mmu.c
@@ -47,6 +47,8 @@ int hl_mmu_init(struct hl_device *hdev)
        if (!hdev->mmu_enable)
                return 0;
 
+       mutex_init(&hdev->mmu_lock);
+
        if (hdev->mmu_func[MMU_DR_PGT].init != NULL) {
                rc = hdev->mmu_func[MMU_DR_PGT].init(hdev);
                if (rc)
@@ -88,6 +90,8 @@ void hl_mmu_fini(struct hl_device *hdev)
 
        if (hdev->mmu_func[MMU_HR_PGT].fini != NULL)
                hdev->mmu_func[MMU_HR_PGT].fini(hdev);
+
+       mutex_destroy(&hdev->mmu_lock);
 }
 
 /**
@@ -106,8 +110,6 @@ int hl_mmu_ctx_init(struct hl_ctx *ctx)
        if (!hdev->mmu_enable)
                return 0;
 
-       mutex_init(&ctx->mmu_lock);
-
        if (hdev->mmu_func[MMU_DR_PGT].ctx_init != NULL) {
                rc = hdev->mmu_func[MMU_DR_PGT].ctx_init(ctx);
                if (rc)
@@ -151,8 +153,6 @@ void hl_mmu_ctx_fini(struct hl_ctx *ctx)
 
        if (hdev->mmu_func[MMU_HR_PGT].ctx_fini != NULL)
                hdev->mmu_func[MMU_HR_PGT].ctx_fini(ctx);
-
-       mutex_destroy(&ctx->mmu_lock);
 }
 
 /*
@@ -609,9 +609,9 @@ int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr,
        pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT;
        mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr);
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        rc = mmu_funcs->get_tlb_info(ctx, virt_addr, hops);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        if (rc)
                return rc;
@@ -701,16 +701,16 @@ static void hl_mmu_prefetch_work_function(struct work_struct *work)
 {
        struct hl_prefetch_work *pfw = container_of(work, struct hl_prefetch_work, pf_work);
        struct hl_ctx *ctx = pfw->ctx;
+       struct hl_device *hdev = ctx->hdev;
 
-       if (!hl_device_operational(ctx->hdev, NULL))
+       if (!hl_device_operational(hdev, NULL))
                goto put_ctx;
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
 
-       ctx->hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid,
-                                                               pfw->va, pfw->size);
+       hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, pfw->va, pfw->size);
 
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
 put_ctx:
        /*

diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
index 2b328cb..48ff3b1 100644 (file)
--- a/drivers/misc/habanalabs/gaudi/gaudi.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -8405,13 +8405,13 @@ static int gaudi_internal_cb_pool_init(struct hl_device *hdev,
                goto destroy_internal_cb_pool;
        }
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base,
                        hdev->internal_cb_pool_dma_addr,
                        HOST_SPACE_INTERNAL_CB_SZ);
 
        hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        if (rc)
                goto unreserve_internal_cb_pool;
@@ -8438,13 +8438,13 @@ static void gaudi_internal_cb_pool_fini(struct hl_device *hdev,
        if (!(gaudi->hw_cap_initialized & HW_CAP_MMU))
                return;
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base,
                        HOST_SPACE_INTERNAL_CB_SZ);
        hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base,
                        HOST_SPACE_INTERNAL_CB_SZ);
        hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        gen_pool_destroy(hdev->internal_cb_pool);
 

diff --git a/drivers/misc/habanalabs/gaudi2/gaudi2.c b/drivers/misc/habanalabs/gaudi2/gaudi2.c
index f749f73..5761ca5 100644 (file)
--- a/drivers/misc/habanalabs/gaudi2/gaudi2.c
+++ b/drivers/misc/habanalabs/gaudi2/gaudi2.c
@@ -9302,12 +9302,12 @@ static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, v
        }
 
        /* Create mapping on asic side */
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        rc = hl_mmu_map_contiguous(ctx, reserved_va_base, host_mem_dma_addr, SZ_2M);
        hl_mmu_invalidate_cache_range(hdev, false,
                                      MMU_OP_USERPTR | MMU_OP_SKIP_LOW_CACHE_INV,
                                      ctx->asid, reserved_va_base, SZ_2M);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
        if (rc) {
                dev_err(hdev->dev, "Failed to create mapping on asic mmu\n");
                goto unreserve_va;
@@ -9340,11 +9340,11 @@ static int gaudi2_debugfs_read_dma(struct hl_device *hdev, u64 addr, u32 size, v
 
        gaudi2_kdma_set_mmbp_asid(hdev, true, HL_KERNEL_ASID_ID);
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        hl_mmu_unmap_contiguous(ctx, reserved_va_base, SZ_2M);
        hl_mmu_invalidate_cache_range(hdev, false, MMU_OP_USERPTR,
                                      ctx->asid, reserved_va_base, SZ_2M);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 unreserve_va:
        hl_unreserve_va_block(hdev, ctx, reserved_va_base, SZ_2M);
 free_data_buffer:
@@ -9397,11 +9397,11 @@ static int gaudi2_internal_cb_pool_init(struct hl_device *hdev, struct hl_ctx *c
                goto destroy_internal_cb_pool;
        }
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr,
                                        HOST_SPACE_INTERNAL_CB_SZ);
        hl_mmu_invalidate_cache(hdev, false, MMU_OP_USERPTR);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        if (rc)
                goto unreserve_internal_cb_pool;
@@ -9426,11 +9426,11 @@ static void gaudi2_internal_cb_pool_fini(struct hl_device *hdev, struct hl_ctx *
        if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU))
                return;
 
-       mutex_lock(&ctx->mmu_lock);
+       mutex_lock(&hdev->mmu_lock);
        hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
        hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ);
        hl_mmu_invalidate_cache(hdev, true, MMU_OP_USERPTR);
-       mutex_unlock(&ctx->mmu_lock);
+       mutex_unlock(&hdev->mmu_lock);
 
        gen_pool_destroy(hdev->internal_cb_pool);