drivers/gpu/drm/i915/gt/intel_context.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_context.h"
   7 #include "gem/i915_gem_pm.h"
   8
   9 #include "i915_drv.h"
  10 #include "i915_trace.h"
  11
  12 #include "intel_context.h"
  13 #include "intel_engine.h"
  14 #include "intel_engine_pm.h"
  15 #include "intel_ring.h"
  16
  17 static struct kmem_cache *slab_ce;
  18
  19 static struct intel_context *intel_context_alloc(void)
  20 {
  21         return kmem_cache_zalloc(slab_ce, GFP_KERNEL);
  22 }
  23
  24 static void rcu_context_free(struct rcu_head *rcu)
  25 {
  26         struct intel_context *ce = container_of(rcu, typeof(*ce), rcu);
  27
  28         trace_intel_context_free(ce);
  29         kmem_cache_free(slab_ce, ce);
  30 }
  31
  32 void intel_context_free(struct intel_context *ce)
  33 {
  34         call_rcu(&ce->rcu, rcu_context_free);
  35 }
  36
  37 struct intel_context *
  38 intel_context_create(struct intel_engine_cs *engine)
  39 {
  40         struct intel_context *ce;
  41
  42         ce = intel_context_alloc();
  43         if (!ce)
  44                 return ERR_PTR(-ENOMEM);
  45
  46         intel_context_init(ce, engine);
  47         trace_intel_context_create(ce);
  48         return ce;
  49 }
  50
  51 int intel_context_alloc_state(struct intel_context *ce)
  52 {
  53         int err = 0;
  54
  55         if (mutex_lock_interruptible(&ce->pin_mutex))
  56                 return -EINTR;
  57
  58         if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
  59                 if (intel_context_is_banned(ce)) {
  60                         err = -EIO;
  61                         goto unlock;
  62                 }
  63
  64                 err = ce->ops->alloc(ce);
  65                 if (unlikely(err))
  66                         goto unlock;
  67
  68                 set_bit(CONTEXT_ALLOC_BIT, &ce->flags);
  69         }
  70
  71 unlock:
  72         mutex_unlock(&ce->pin_mutex);
  73         return err;
  74 }
  75
  76 static int intel_context_active_acquire(struct intel_context *ce)
  77 {
  78         int err;
  79
  80         __i915_active_acquire(&ce->active);
  81
  82         if (intel_context_is_barrier(ce) || intel_engine_uses_guc(ce->engine))
  83                 return 0;
  84
  85         /* Preallocate tracking nodes */
  86         err = i915_active_acquire_preallocate_barrier(&ce->active,
  87                                                       ce->engine);
  88         if (err)
  89                 i915_active_release(&ce->active);
  90
  91         return err;
  92 }
  93
  94 static void intel_context_active_release(struct intel_context *ce)
  95 {
  96         /* Nodes preallocated in intel_context_active() */
  97         i915_active_acquire_barrier(&ce->active);
  98         i915_active_release(&ce->active);
  99 }
 100
 101 static int __context_pin_state(struct i915_vma *vma, struct i915_gem_ww_ctx *ww)
 102 {
 103         unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
 104         int err;
 105
 106         err = i915_ggtt_pin(vma, ww, 0, bias | PIN_HIGH);
 107         if (err)
 108                 return err;
 109
 110         err = i915_active_acquire(&vma->active);
 111         if (err)
 112                 goto err_unpin;
 113
 114         /*
 115          * And mark it as a globally pinned object to let the shrinker know
 116          * it cannot reclaim the object until we release it.
 117          */
 118         i915_vma_make_unshrinkable(vma);
 119         vma->obj->mm.dirty = true;
 120
 121         return 0;
 122
 123 err_unpin:
 124         i915_vma_unpin(vma);
 125         return err;
 126 }
 127
 128 static void __context_unpin_state(struct i915_vma *vma)
 129 {
 130         i915_vma_make_shrinkable(vma);
 131         i915_active_release(&vma->active);
 132         __i915_vma_unpin(vma);
 133 }
 134
 135 static int __ring_active(struct intel_ring *ring,
 136                          struct i915_gem_ww_ctx *ww)
 137 {
 138         int err;
 139
 140         err = intel_ring_pin(ring, ww);
 141         if (err)
 142                 return err;
 143
 144         err = i915_active_acquire(&ring->vma->active);
 145         if (err)
 146                 goto err_pin;
 147
 148         return 0;
 149
 150 err_pin:
 151         intel_ring_unpin(ring);
 152         return err;
 153 }
 154
 155 static void __ring_retire(struct intel_ring *ring)
 156 {
 157         i915_active_release(&ring->vma->active);
 158         intel_ring_unpin(ring);
 159 }
 160
 161 static int intel_context_pre_pin(struct intel_context *ce,
 162                                  struct i915_gem_ww_ctx *ww)
 163 {
 164         int err;
 165
 166         CE_TRACE(ce, "active\n");
 167
 168         err = __ring_active(ce->ring, ww);
 169         if (err)
 170                 return err;
 171
 172         err = intel_timeline_pin(ce->timeline, ww);
 173         if (err)
 174                 goto err_ring;
 175
 176         if (!ce->state)
 177                 return 0;
 178
 179         err = __context_pin_state(ce->state, ww);
 180         if (err)
 181                 goto err_timeline;
 182
 183
 184         return 0;
 185
 186 err_timeline:
 187         intel_timeline_unpin(ce->timeline);
 188 err_ring:
 189         __ring_retire(ce->ring);
 190         return err;
 191 }
 192
 193 static void intel_context_post_unpin(struct intel_context *ce)
 194 {
 195         if (ce->state)
 196                 __context_unpin_state(ce->state);
 197
 198         intel_timeline_unpin(ce->timeline);
 199         __ring_retire(ce->ring);
 200 }
 201
 202 int __intel_context_do_pin_ww(struct intel_context *ce,
 203                               struct i915_gem_ww_ctx *ww)
 204 {
 205         bool handoff = false;
 206         void *vaddr;
 207         int err = 0;
 208
 209         if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
 210                 err = intel_context_alloc_state(ce);
 211                 if (err)
 212                         return err;
 213         }
 214
 215         /*
 216          * We always pin the context/ring/timeline here, to ensure a pin
 217          * refcount for __intel_context_active(), which prevent a lock
 218          * inversion of ce->pin_mutex vs dma_resv_lock().
 219          */
 220
 221         err = i915_gem_object_lock(ce->timeline->hwsp_ggtt->obj, ww);
 222         if (!err && ce->ring->vma->obj)
 223                 err = i915_gem_object_lock(ce->ring->vma->obj, ww);
 224         if (!err && ce->state)
 225                 err = i915_gem_object_lock(ce->state->obj, ww);
 226         if (!err)
 227                 err = intel_context_pre_pin(ce, ww);
 228         if (err)
 229                 return err;
 230
 231         err = i915_active_acquire(&ce->active);
 232         if (err)
 233                 goto err_ctx_unpin;
 234
 235         err = ce->ops->pre_pin(ce, ww, &vaddr);
 236         if (err)
 237                 goto err_release;
 238
 239         err = mutex_lock_interruptible(&ce->pin_mutex);
 240         if (err)
 241                 goto err_post_unpin;
 242
 243         if (unlikely(intel_context_is_closed(ce))) {
 244                 err = -ENOENT;
 245                 goto err_unlock;
 246         }
 247
 248         if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
 249                 err = intel_context_active_acquire(ce);
 250                 if (unlikely(err))
 251                         goto err_unlock;
 252
 253                 err = ce->ops->pin(ce, vaddr);
 254                 if (err) {
 255                         intel_context_active_release(ce);
 256                         goto err_unlock;
 257                 }
 258
 259                 CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
 260                          i915_ggtt_offset(ce->ring->vma),
 261                          ce->ring->head, ce->ring->tail);
 262
 263                 handoff = true;
 264                 smp_mb__before_atomic(); /* flush pin before it is visible */
 265                 atomic_inc(&ce->pin_count);
 266         }
 267
 268         GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
 269
 270         trace_intel_context_do_pin(ce);
 271
 272 err_unlock:
 273         mutex_unlock(&ce->pin_mutex);
 274 err_post_unpin:
 275         if (!handoff)
 276                 ce->ops->post_unpin(ce);
 277 err_release:
 278         i915_active_release(&ce->active);
 279 err_ctx_unpin:
 280         intel_context_post_unpin(ce);
 281
 282         /*
 283          * Unlock the hwsp_ggtt object since it's shared.
 284          * In principle we can unlock all the global state locked above
 285          * since it's pinned and doesn't need fencing, and will
 286          * thus remain resident until it is explicitly unpinned.
 287          */
 288         i915_gem_ww_unlock_single(ce->timeline->hwsp_ggtt->obj);
 289
 290         return err;
 291 }
 292
 293 int __intel_context_do_pin(struct intel_context *ce)
 294 {
 295         struct i915_gem_ww_ctx ww;
 296         int err;
 297
 298         i915_gem_ww_ctx_init(&ww, true);
 299 retry:
 300         err = __intel_context_do_pin_ww(ce, &ww);
 301         if (err == -EDEADLK) {
 302                 err = i915_gem_ww_ctx_backoff(&ww);
 303                 if (!err)
 304                         goto retry;
 305         }
 306         i915_gem_ww_ctx_fini(&ww);
 307         return err;
 308 }
 309
 310 void __intel_context_do_unpin(struct intel_context *ce, int sub)
 311 {
 312         if (!atomic_sub_and_test(sub, &ce->pin_count))
 313                 return;
 314
 315         CE_TRACE(ce, "unpin\n");
 316         ce->ops->unpin(ce);
 317         ce->ops->post_unpin(ce);
 318
 319         /*
 320          * Once released, we may asynchronously drop the active reference.
 321          * As that may be the only reference keeping the context alive,
 322          * take an extra now so that it is not freed before we finish
 323          * dereferencing it.
 324          */
 325         intel_context_get(ce);
 326         intel_context_active_release(ce);
 327         trace_intel_context_do_unpin(ce);
 328         intel_context_put(ce);
 329 }
 330
 331 static void __intel_context_retire(struct i915_active *active)
 332 {
 333         struct intel_context *ce = container_of(active, typeof(*ce), active);
 334
 335         CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n",
 336                  intel_context_get_total_runtime_ns(ce),
 337                  intel_context_get_avg_runtime_ns(ce));
 338
 339         set_bit(CONTEXT_VALID_BIT, &ce->flags);
 340         intel_context_post_unpin(ce);
 341         intel_context_put(ce);
 342 }
 343
 344 static int __intel_context_active(struct i915_active *active)
 345 {
 346         struct intel_context *ce = container_of(active, typeof(*ce), active);
 347
 348         intel_context_get(ce);
 349
 350         /* everything should already be activated by intel_context_pre_pin() */
 351         GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->ring->vma->active));
 352         __intel_ring_pin(ce->ring);
 353
 354         __intel_timeline_pin(ce->timeline);
 355
 356         if (ce->state) {
 357                 GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->state->active));
 358                 __i915_vma_pin(ce->state);
 359                 i915_vma_make_unshrinkable(ce->state);
 360         }
 361
 362         return 0;
 363 }
 364
 365 static int __i915_sw_fence_call
 366 sw_fence_dummy_notify(struct i915_sw_fence *sf,
 367                       enum i915_sw_fence_notify state)
 368 {
 369         return NOTIFY_DONE;
 370 }
 371
 372 void
 373 intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine)
 374 {
 375         GEM_BUG_ON(!engine->cops);
 376         GEM_BUG_ON(!engine->gt->vm);
 377
 378         kref_init(&ce->ref);
 379
 380         ce->engine = engine;
 381         ce->ops = engine->cops;
 382         ce->sseu = engine->sseu;
 383         ce->ring = NULL;
 384         ce->ring_size = SZ_4K;
 385
 386         ewma_runtime_init(&ce->runtime.avg);
 387
 388         ce->vm = i915_vm_get(engine->gt->vm);
 389
 390         /* NB ce->signal_link/lock is used under RCU */
 391         spin_lock_init(&ce->signal_lock);
 392         INIT_LIST_HEAD(&ce->signals);
 393
 394         mutex_init(&ce->pin_mutex);
 395
 396         spin_lock_init(&ce->guc_state.lock);
 397         INIT_LIST_HEAD(&ce->guc_state.fences);
 398
 399         spin_lock_init(&ce->guc_active.lock);
 400         INIT_LIST_HEAD(&ce->guc_active.requests);
 401
 402         ce->guc_id = GUC_INVALID_LRC_ID;
 403         INIT_LIST_HEAD(&ce->guc_id_link);
 404
 405         /*
 406          * Initialize fence to be complete as this is expected to be complete
 407          * unless there is a pending schedule disable outstanding.
 408          */
 409         i915_sw_fence_init(&ce->guc_blocked, sw_fence_dummy_notify);
 410         i915_sw_fence_commit(&ce->guc_blocked);
 411
 412         i915_active_init(&ce->active,
 413                          __intel_context_active, __intel_context_retire, 0);
 414 }
 415
 416 void intel_context_fini(struct intel_context *ce)
 417 {
 418         if (ce->timeline)
 419                 intel_timeline_put(ce->timeline);
 420         i915_vm_put(ce->vm);
 421
 422         mutex_destroy(&ce->pin_mutex);
 423         i915_active_fini(&ce->active);
 424 }
 425
 426 void i915_context_module_exit(void)
 427 {
 428         kmem_cache_destroy(slab_ce);
 429 }
 430
 431 int __init i915_context_module_init(void)
 432 {
 433         slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
 434         if (!slab_ce)
 435                 return -ENOMEM;
 436
 437         return 0;
 438 }
 439
 440 void intel_context_enter_engine(struct intel_context *ce)
 441 {
 442         intel_engine_pm_get(ce->engine);
 443         intel_timeline_enter(ce->timeline);
 444 }
 445
 446 void intel_context_exit_engine(struct intel_context *ce)
 447 {
 448         intel_timeline_exit(ce->timeline);
 449         intel_engine_pm_put(ce->engine);
 450 }
 451
 452 int intel_context_prepare_remote_request(struct intel_context *ce,
 453                                          struct i915_request *rq)
 454 {
 455         struct intel_timeline *tl = ce->timeline;
 456         int err;
 457
 458         /* Only suitable for use in remotely modifying this context */
 459         GEM_BUG_ON(rq->context == ce);
 460
 461         if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
 462                 /* Queue this switch after current activity by this context. */
 463                 err = i915_active_fence_set(&tl->last_request, rq);
 464                 if (err)
 465                         return err;
 466         }
 467
 468         /*
 469          * Guarantee context image and the timeline remains pinned until the
 470          * modifying request is retired by setting the ce activity tracker.
 471          *
 472          * But we only need to take one pin on the account of it. Or in other
 473          * words transfer the pinned ce object to tracked active request.
 474          */
 475         GEM_BUG_ON(i915_active_is_idle(&ce->active));
 476         return i915_active_add_request(&ce->active, rq);
 477 }
 478
 479 struct i915_request *intel_context_create_request(struct intel_context *ce)
 480 {
 481         struct i915_gem_ww_ctx ww;
 482         struct i915_request *rq;
 483         int err;
 484
 485         i915_gem_ww_ctx_init(&ww, true);
 486 retry:
 487         err = intel_context_pin_ww(ce, &ww);
 488         if (!err) {
 489                 rq = i915_request_create(ce);
 490                 intel_context_unpin(ce);
 491         } else if (err == -EDEADLK) {
 492                 err = i915_gem_ww_ctx_backoff(&ww);
 493                 if (!err)
 494                         goto retry;
 495                 rq = ERR_PTR(err);
 496         } else {
 497                 rq = ERR_PTR(err);
 498         }
 499
 500         i915_gem_ww_ctx_fini(&ww);
 501
 502         if (IS_ERR(rq))
 503                 return rq;
 504
 505         /*
 506          * timeline->mutex should be the inner lock, but is used as outer lock.
 507          * Hack around this to shut up lockdep in selftests..
 508          */
 509         lockdep_unpin_lock(&ce->timeline->mutex, rq->cookie);
 510         mutex_release(&ce->timeline->mutex.dep_map, _RET_IP_);
 511         mutex_acquire(&ce->timeline->mutex.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_);
 512         rq->cookie = lockdep_pin_lock(&ce->timeline->mutex);
 513
 514         return rq;
 515 }
 516
 517 struct i915_request *intel_context_find_active_request(struct intel_context *ce)
 518 {
 519         struct i915_request *rq, *active = NULL;
 520         unsigned long flags;
 521
 522         GEM_BUG_ON(!intel_engine_uses_guc(ce->engine));
 523
 524         spin_lock_irqsave(&ce->guc_active.lock, flags);
 525         list_for_each_entry_reverse(rq, &ce->guc_active.requests,
 526                                     sched.link) {
 527                 if (i915_request_completed(rq))
 528                         break;
 529
 530                 active = rq;
 531         }
 532         spin_unlock_irqrestore(&ce->guc_active.lock, flags);
 533
 534         return active;
 535 }
 536
 537 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 538 #include "selftest_context.c"
 539 #endif