drivers/gpu/drm/i915/gt/intel_tlb.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2023 Intel Corporation
   4  */
   5
   6 #include "i915_drv.h"
   7 #include "i915_perf_oa_regs.h"
   8 #include "intel_engine_pm.h"
   9 #include "intel_gt.h"
  10 #include "intel_gt_mcr.h"
  11 #include "intel_gt_pm.h"
  12 #include "intel_gt_print.h"
  13 #include "intel_gt_regs.h"
  14 #include "intel_tlb.h"
  15
  16 /*
  17  * HW architecture suggest typical invalidation time at 40us,
  18  * with pessimistic cases up to 100us and a recommendation to
  19  * cap at 1ms. We go a bit higher just in case.
  20  */
  21 #define TLB_INVAL_TIMEOUT_US 100
  22 #define TLB_INVAL_TIMEOUT_MS 4
  23
  24 /*
  25  * On Xe_HP the TLB invalidation registers are located at the same MMIO offsets
  26  * but are now considered MCR registers.  Since they exist within a GAM range,
  27  * the primary instance of the register rolls up the status from each unit.
  28  */
  29 static int wait_for_invalidate(struct intel_engine_cs *engine)
  30 {
  31         if (engine->tlb_inv.mcr)
  32                 return intel_gt_mcr_wait_for_reg(engine->gt,
  33                                                  engine->tlb_inv.reg.mcr_reg,
  34                                                  engine->tlb_inv.done,
  35                                                  0,
  36                                                  TLB_INVAL_TIMEOUT_US,
  37                                                  TLB_INVAL_TIMEOUT_MS);
  38         else
  39                 return __intel_wait_for_register_fw(engine->gt->uncore,
  40                                                     engine->tlb_inv.reg.reg,
  41                                                     engine->tlb_inv.done,
  42                                                     0,
  43                                                     TLB_INVAL_TIMEOUT_US,
  44                                                     TLB_INVAL_TIMEOUT_MS,
  45                                                     NULL);
  46 }
  47
  48 static void mmio_invalidate_full(struct intel_gt *gt)
  49 {
  50         struct drm_i915_private *i915 = gt->i915;
  51         struct intel_uncore *uncore = gt->uncore;
  52         struct intel_engine_cs *engine;
  53         intel_engine_mask_t awake, tmp;
  54         enum intel_engine_id id;
  55         unsigned long flags;
  56
  57         if (GRAPHICS_VER(i915) < 8)
  58                 return;
  59
  60         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
  61
  62         intel_gt_mcr_lock(gt, &flags);
  63         spin_lock(&uncore->lock); /* serialise invalidate with GT reset */
  64
  65         awake = 0;
  66         for_each_engine(engine, gt, id) {
  67                 if (!intel_engine_pm_is_awake(engine))
  68                         continue;
  69
  70                 if (engine->tlb_inv.mcr)
  71                         intel_gt_mcr_multicast_write_fw(gt,
  72                                                         engine->tlb_inv.reg.mcr_reg,
  73                                                         engine->tlb_inv.request);
  74                 else
  75                         intel_uncore_write_fw(uncore,
  76                                               engine->tlb_inv.reg.reg,
  77                                               engine->tlb_inv.request);
  78
  79                 awake |= engine->mask;
  80         }
  81
  82         GT_TRACE(gt, "invalidated engines %08x\n", awake);
  83
  84         /* Wa_2207587034:tgl,dg1,rkl,adl-s,adl-p */
  85         if (awake &&
  86             (IS_TIGERLAKE(i915) ||
  87              IS_DG1(i915) ||
  88              IS_ROCKETLAKE(i915) ||
  89              IS_ALDERLAKE_S(i915) ||
  90              IS_ALDERLAKE_P(i915)))
  91                 intel_uncore_write_fw(uncore, GEN12_OA_TLB_INV_CR, 1);
  92
  93         spin_unlock(&uncore->lock);
  94         intel_gt_mcr_unlock(gt, flags);
  95
  96         for_each_engine_masked(engine, gt, awake, tmp) {
  97                 if (wait_for_invalidate(engine))
  98                         gt_err_ratelimited(gt,
  99                                            "%s TLB invalidation did not complete in %ums!\n",
 100                                            engine->name, TLB_INVAL_TIMEOUT_MS);
 101         }
 102
 103         /*
 104          * Use delayed put since a) we mostly expect a flurry of TLB
 105          * invalidations so it is good to avoid paying the forcewake cost and
 106          * b) it works around a bug in Icelake which cannot cope with too rapid
 107          * transitions.
 108          */
 109         intel_uncore_forcewake_put_delayed(uncore, FORCEWAKE_ALL);
 110 }
 111
 112 static bool tlb_seqno_passed(const struct intel_gt *gt, u32 seqno)
 113 {
 114         u32 cur = intel_gt_tlb_seqno(gt);
 115
 116         /* Only skip if a *full* TLB invalidate barrier has passed */
 117         return (s32)(cur - ALIGN(seqno, 2)) > 0;
 118 }
 119
 120 void intel_gt_invalidate_tlb_full(struct intel_gt *gt, u32 seqno)
 121 {
 122         intel_wakeref_t wakeref;
 123
 124         if (I915_SELFTEST_ONLY(gt->awake == -ENODEV))
 125                 return;
 126
 127         if (intel_gt_is_wedged(gt))
 128                 return;
 129
 130         if (tlb_seqno_passed(gt, seqno))
 131                 return;
 132
 133         with_intel_gt_pm_if_awake(gt, wakeref) {
 134                 mutex_lock(&gt->tlb.invalidate_lock);
 135                 if (tlb_seqno_passed(gt, seqno))
 136                         goto unlock;
 137
 138                 mmio_invalidate_full(gt);
 139
 140                 write_seqcount_invalidate(&gt->tlb.seqno);
 141 unlock:
 142                 mutex_unlock(&gt->tlb.invalidate_lock);
 143         }
 144 }
 145
 146 void intel_gt_init_tlb(struct intel_gt *gt)
 147 {
 148         mutex_init(&gt->tlb.invalidate_lock);
 149         seqcount_mutex_init(&gt->tlb.seqno, &gt->tlb.invalidate_lock);
 150 }
 151
 152 void intel_gt_fini_tlb(struct intel_gt *gt)
 153 {
 154         mutex_destroy(&gt->tlb.invalidate_lock);
 155 }
 156
 157 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 158 #include "selftest_tlb.c"
 159 #endif