On XE_LPM+ platforms the media engines are carved out into a separate
GT but have a common GGTMMADR address range which essentially makes
the GGTT address space to be shared between media and render GT. As a
result any updates in GGTT shall invalidate TLB of GTs sharing it and
similarly any operation on GGTT requiring an action on a GT will have to
involve all GTs sharing it. setup_private_pat was being done on a per
GGTT based as that doesn't touch any GGTT structures moved it to per GT
based.
BSPEC: 63834
v2:
1. Add details to commit msg
2. includes fix for failure to add item to ggtt->gt_list, as suggested
by Lucas
3. as ggtt_flush() is used only for ggtt drop i915_is_ggtt check within
it.
4. setup_private_pat moved out of intel_gt_tiles_init
v3:
1. Move out for_each_gt from i915_driver.c (Jani Nikula)
v4: drop using RCU primitives on ggtt->gt_list as it is not an RCU list
(Matt Roper)
Cc: Matt Roper <matthew.d.roper@intel.com>
Signed-off-by: Aravind Iddamsetty <aravind.iddamsetty@intel.com>
Reviewed-by: Matt Roper <matthew.d.roper@intel.com>
Signed-off-by: Lucas De Marchi <lucas.demarchi@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20221122070126.4813-1-aravind.iddamsetty@intel.com
#include <linux/types.h>
#include <linux/stop_machine.h>
+#include <drm/drm_managed.h>
#include <drm/i915_drm.h>
#include <drm/intel-gtt.h>
void i915_ggtt_suspend(struct i915_ggtt *ggtt)
{
+ struct intel_gt *gt;
+
i915_ggtt_suspend_vm(&ggtt->vm);
ggtt->invalidate(ggtt);
- intel_gt_check_and_clear_faults(ggtt->vm.gt);
+ list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
+ intel_gt_check_and_clear_faults(gt);
}
void gen6_ggtt_invalidate(struct i915_ggtt *ggtt)
static void guc_ggtt_invalidate(struct i915_ggtt *ggtt)
{
- struct intel_uncore *uncore = ggtt->vm.gt->uncore;
struct drm_i915_private *i915 = ggtt->vm.i915;
gen8_ggtt_invalidate(ggtt);
- if (GRAPHICS_VER(i915) >= 12)
- intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR,
- GEN12_GUC_TLB_INV_CR_INVALIDATE);
- else
- intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+ if (GRAPHICS_VER(i915) >= 12) {
+ struct intel_gt *gt;
+
+ list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
+ intel_uncore_write_fw(gt->uncore,
+ GEN12_GUC_TLB_INV_CR,
+ GEN12_GUC_TLB_INV_CR_INVALIDATE);
+ } else {
+ intel_uncore_write_fw(ggtt->vm.gt->uncore,
+ GEN8_GTCR, GEN8_GTCR_INVALIDATE);
+ }
}
u64 gen8_ggtt_pte_encode(dma_addr_t addr,
ggtt->vm.pte_encode = gen8_ggtt_pte_encode;
- setup_private_pat(ggtt->vm.gt);
-
return ggtt_probe_common(ggtt, size);
}
*/
int i915_ggtt_probe_hw(struct drm_i915_private *i915)
{
- int ret;
+ struct intel_gt *gt;
+ int ret, i;
+
+ for_each_gt(gt, i915, i) {
+ ret = intel_gt_assign_ggtt(gt);
+ if (ret)
+ return ret;
+ }
ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915));
if (ret)
return 0;
}
+struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915)
+{
+ struct i915_ggtt *ggtt;
+
+ ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL);
+ if (!ggtt)
+ return ERR_PTR(-ENOMEM);
+
+ INIT_LIST_HEAD(&ggtt->gt_list);
+
+ return ggtt;
+}
+
int i915_ggtt_enable_hw(struct drm_i915_private *i915)
{
if (GRAPHICS_VER(i915) < 6)
void i915_ggtt_resume(struct i915_ggtt *ggtt)
{
+ struct intel_gt *gt;
bool flush;
- intel_gt_check_and_clear_faults(ggtt->vm.gt);
+ list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
+ intel_gt_check_and_clear_faults(gt);
flush = i915_ggtt_resume_vm(&ggtt->vm);
if (flush)
wbinvd_on_all_cpus();
- if (GRAPHICS_VER(ggtt->vm.i915) >= 8)
- setup_private_pat(ggtt->vm.gt);
-
intel_ggtt_restore_fences(ggtt);
}
int intel_gt_assign_ggtt(struct intel_gt *gt)
{
- gt->ggtt = drmm_kzalloc(>->i915->drm, sizeof(*gt->ggtt), GFP_KERNEL);
+ /* Media GT shares primary GT's GGTT */
+ if (gt->type == GT_MEDIA) {
+ gt->ggtt = to_gt(gt->i915)->ggtt;
+ } else {
+ gt->ggtt = i915_ggtt_create(gt->i915);
+ if (IS_ERR(gt->ggtt))
+ return PTR_ERR(gt->ggtt);
+ }
- return gt->ggtt ? 0 : -ENOMEM;
+ list_add_tail(>->ggtt_link, >->ggtt->gt_list);
+
+ return 0;
}
int intel_gt_init_mmio(struct intel_gt *gt)
struct kobject *sysfs_defaults;
struct i915_perf_gt perf;
+
+ /** link: &ggtt.gt_list */
+ struct list_head ggtt_link;
};
struct intel_gt_definition {
struct mutex error_mutex;
struct drm_mm_node error_capture;
struct drm_mm_node uc_fw;
+
+ /** List of GTs mapping this GGTT */
+ struct list_head gt_list;
};
struct i915_ppgtt {
int i915_init_ggtt(struct drm_i915_private *i915);
void i915_ggtt_driver_release(struct drm_i915_private *i915);
void i915_ggtt_driver_late_release(struct drm_i915_private *i915);
+struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915);
static inline bool i915_ggtt_has_aperture(const struct i915_ggtt *ggtt)
{
i915_perf_init(dev_priv);
- ret = intel_gt_assign_ggtt(to_gt(dev_priv));
- if (ret)
- goto err_perf;
-
ret = i915_ggtt_probe_hw(dev_priv);
if (ret)
goto err_perf;
static int i915_drm_resume(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = to_i915(dev);
- int ret;
+ struct intel_gt *gt;
+ int ret, i;
disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
drm_err(&dev_priv->drm, "failed to re-enable GGTT\n");
i915_ggtt_resume(to_gt(dev_priv)->ggtt);
+
+ for_each_gt(gt, dev_priv, i)
+ if (GRAPHICS_VER(gt->i915) >= 8)
+ setup_private_pat(gt);
+
/* Must be called after GGTT is resumed. */
intel_dpt_resume(dev_priv);
for_each_gt(gt, dev_priv, i) {
intel_uc_fetch_firmwares(>->uc);
intel_wopcm_init(>->wopcm);
+ if (GRAPHICS_VER(dev_priv) >= 8)
+ setup_private_pat(gt);
}
ret = i915_init_ggtt(dev_priv);
return !kref_read(&vma->obj->base.refcount);
}
-static int ggtt_flush(struct intel_gt *gt)
+static int ggtt_flush(struct i915_address_space *vm)
{
- /*
- * Not everything in the GGTT is tracked via vma (otherwise we
- * could evict as required with minimal stalling) so we are forced
- * to idle the GPU and explicitly retire outstanding requests in
- * the hopes that we can then remove contexts and the like only
- * bound by their active reference.
- */
- return intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
+ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
+ struct intel_gt *gt;
+ int ret = 0;
+
+ list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) {
+ /*
+ * Not everything in the GGTT is tracked via vma (otherwise we
+ * could evict as required with minimal stalling) so we are forced
+ * to idle the GPU and explicitly retire outstanding requests in
+ * the hopes that we can then remove contexts and the like only
+ * bound by their active reference.
+ */
+ ret = intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
+ if (ret)
+ return ret;
+ }
+ return ret;
}
static bool grab_vma(struct i915_vma *vma, struct i915_gem_ww_ctx *ww)
struct drm_mm_node *node;
enum drm_mm_insert_mode mode;
struct i915_vma *active;
+ struct intel_gt *gt;
int ret;
lockdep_assert_held(&vm->mutex);
min_size, alignment, color,
start, end, mode);
- intel_gt_retire_requests(vm->gt);
+ if (i915_is_ggtt(vm)) {
+ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
+
+ list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
+ intel_gt_retire_requests(gt);
+ } else {
+ intel_gt_retire_requests(vm->gt);
+ }
search_again:
active = NULL;
if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy))
return -EBUSY;
- ret = ggtt_flush(vm->gt);
+ ret = ggtt_flush(vm);
if (ret)
return ret;
* a stray pin (preventing eviction) that can only be resolved by
* retiring.
*/
- intel_gt_retire_requests(vm->gt);
+ if (i915_is_ggtt(vm)) {
+ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
+ struct intel_gt *gt;
+
+ list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
+ intel_gt_retire_requests(gt);
+ } else {
+ intel_gt_retire_requests(vm->gt);
+ }
if (i915_vm_has_cache_coloring(vm)) {
/* Expand search to cover neighbouring guard pages (or lack!) */
* switch otherwise is ineffective.
*/
if (i915_is_ggtt(vm)) {
- ret = ggtt_flush(vm->gt);
+ ret = ggtt_flush(vm);
if (ret)
return ret;
}
u32 align, unsigned int flags)
{
struct i915_address_space *vm = vma->vm;
+ struct intel_gt *gt;
+ struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
int err;
do {
}
/* Unlike i915_vma_pin, we don't take no for an answer! */
- flush_idle_contexts(vm->gt);
+ list_for_each_entry(gt, &ggtt->gt_list, ggtt_link)
+ flush_idle_contexts(gt);
if (mutex_lock_interruptible(&vm->mutex) == 0) {
/*
* We pass NULL ww here, as we don't want to unbind
*/
with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
i915_ggtt_resume(to_gt(i915)->ggtt);
+ if (GRAPHICS_VER(i915) >= 8)
+ setup_private_pat(to_gt(i915));
i915_gem_resume(i915);
}
}