drivers/gpu/drm/i915/gt/intel_context.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include "gem/i915_gem_context.h"
   7 #include "gem/i915_gem_pm.h"
   8
   9 #include "i915_drv.h"
  10 #include "i915_trace.h"
  11
  12 #include "intel_context.h"
  13 #include "intel_engine.h"
  14 #include "intel_engine_pm.h"
  15 #include "intel_ring.h"
  16
  17 static struct kmem_cache *slab_ce;
  18
  19 static struct intel_context *intel_context_alloc(void)
  20 {
  21         return kmem_cache_zalloc(slab_ce, GFP_KERNEL);
  22 }
  23
  24 static void rcu_context_free(struct rcu_head *rcu)
  25 {
  26         struct intel_context *ce = container_of(rcu, typeof(*ce), rcu);
  27
  28         trace_intel_context_free(ce);
  29         kmem_cache_free(slab_ce, ce);
  30 }
  31
  32 void intel_context_free(struct intel_context *ce)
  33 {
  34         call_rcu(&ce->rcu, rcu_context_free);
  35 }
  36
  37 struct intel_context *
  38 intel_context_create(struct intel_engine_cs *engine)
  39 {
  40         struct intel_context *ce;
  41
  42         ce = intel_context_alloc();
  43         if (!ce)
  44                 return ERR_PTR(-ENOMEM);
  45
  46         intel_context_init(ce, engine);
  47         trace_intel_context_create(ce);
  48         return ce;
  49 }
  50
  51 int intel_context_alloc_state(struct intel_context *ce)
  52 {
  53         int err = 0;
  54
  55         if (mutex_lock_interruptible(&ce->pin_mutex))
  56                 return -EINTR;
  57
  58         if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) {
  59                 if (intel_context_is_banned(ce)) {
  60                         err = -EIO;
  61                         goto unlock;
  62                 }
  63
  64                 err = ce->ops->alloc(ce);
  65                 if (unlikely(err))
  66                         goto unlock;
  67
  68                 set_bit(CONTEXT_ALLOC_BIT, &ce->flags);
  69         }
  70
  71 unlock:
  72         mutex_unlock(&ce->pin_mutex);
  73         return err;
  74 }
  75
  76 static int intel_context_active_acquire(struct intel_context *ce)
  77 {
  78         int err;
  79
  80         __i915_active_acquire(&ce->active);
  81
  82         if (intel_context_is_barrier(ce) || intel_engine_uses_guc(ce->engine))
  83                 return 0;
  84
  85         /* Preallocate tracking nodes */
  86         err = i915_active_acquire_preallocate_barrier(&ce->active,
  87                                                       ce->engine);
  88         if (err)
  89                 i915_active_release(&ce->active);
  90
  91         return err;
  92 }
  93
  94 static void intel_context_active_release(struct intel_context *ce)
  95 {
  96         /* Nodes preallocated in intel_context_active() */
  97         i915_active_acquire_barrier(&ce->active);
  98         i915_active_release(&ce->active);
  99 }
 100
 101 static int __context_pin_state(struct i915_vma *vma, struct i915_gem_ww_ctx *ww)
 102 {
 103         unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS;
 104         int err;
 105
 106         err = i915_ggtt_pin(vma, ww, 0, bias | PIN_HIGH);
 107         if (err)
 108                 return err;
 109
 110         err = i915_active_acquire(&vma->active);
 111         if (err)
 112                 goto err_unpin;
 113
 114         /*
 115          * And mark it as a globally pinned object to let the shrinker know
 116          * it cannot reclaim the object until we release it.
 117          */
 118         i915_vma_make_unshrinkable(vma);
 119         vma->obj->mm.dirty = true;
 120
 121         return 0;
 122
 123 err_unpin:
 124         i915_vma_unpin(vma);
 125         return err;
 126 }
 127
 128 static void __context_unpin_state(struct i915_vma *vma)
 129 {
 130         i915_vma_make_shrinkable(vma);
 131         i915_active_release(&vma->active);
 132         __i915_vma_unpin(vma);
 133 }
 134
 135 static int __ring_active(struct intel_ring *ring,
 136                          struct i915_gem_ww_ctx *ww)
 137 {
 138         int err;
 139
 140         err = intel_ring_pin(ring, ww);
 141         if (err)
 142                 return err;
 143
 144         err = i915_active_acquire(&ring->vma->active);
 145         if (err)
 146                 goto err_pin;
 147
 148         return 0;
 149
 150 err_pin:
 151         intel_ring_unpin(ring);
 152         return err;
 153 }
 154
 155 static void __ring_retire(struct intel_ring *ring)
 156 {
 157         i915_active_release(&ring->vma->active);
 158         intel_ring_unpin(ring);
 159 }
 160
 161 static int intel_context_pre_pin(struct intel_context *ce,
 162                                  struct i915_gem_ww_ctx *ww)
 163 {
 164         int err;
 165
 166         CE_TRACE(ce, "active\n");
 167
 168         err = __ring_active(ce->ring, ww);
 169         if (err)
 170                 return err;
 171
 172         err = intel_timeline_pin(ce->timeline, ww);
 173         if (err)
 174                 goto err_ring;
 175
 176         if (!ce->state)
 177                 return 0;
 178
 179         err = __context_pin_state(ce->state, ww);
 180         if (err)
 181                 goto err_timeline;
 182
 183
 184         return 0;
 185
 186 err_timeline:
 187         intel_timeline_unpin(ce->timeline);
 188 err_ring:
 189         __ring_retire(ce->ring);
 190         return err;
 191 }
 192
 193 static void intel_context_post_unpin(struct intel_context *ce)
 194 {
 195         if (ce->state)
 196                 __context_unpin_state(ce->state);
 197
 198         intel_timeline_unpin(ce->timeline);
 199         __ring_retire(ce->ring);
 200 }
 201
 202 int __intel_context_do_pin_ww(struct intel_context *ce,
 203                               struct i915_gem_ww_ctx *ww)
 204 {
 205         bool handoff = false;
 206         void *vaddr;
 207         int err = 0;
 208
 209         if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) {
 210                 err = intel_context_alloc_state(ce);
 211                 if (err)
 212                         return err;
 213         }
 214
 215         /*
 216          * We always pin the context/ring/timeline here, to ensure a pin
 217          * refcount for __intel_context_active(), which prevent a lock
 218          * inversion of ce->pin_mutex vs dma_resv_lock().
 219          */
 220
 221         err = i915_gem_object_lock(ce->timeline->hwsp_ggtt->obj, ww);
 222         if (!err && ce->ring->vma->obj)
 223                 err = i915_gem_object_lock(ce->ring->vma->obj, ww);
 224         if (!err && ce->state)
 225                 err = i915_gem_object_lock(ce->state->obj, ww);
 226         if (!err)
 227                 err = intel_context_pre_pin(ce, ww);
 228         if (err)
 229                 return err;
 230
 231         err = i915_active_acquire(&ce->active);
 232         if (err)
 233                 goto err_ctx_unpin;
 234
 235         err = ce->ops->pre_pin(ce, ww, &vaddr);
 236         if (err)
 237                 goto err_release;
 238
 239         err = mutex_lock_interruptible(&ce->pin_mutex);
 240         if (err)
 241                 goto err_post_unpin;
 242
 243         if (unlikely(intel_context_is_closed(ce))) {
 244                 err = -ENOENT;
 245                 goto err_unlock;
 246         }
 247
 248         if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) {
 249                 err = intel_context_active_acquire(ce);
 250                 if (unlikely(err))
 251                         goto err_unlock;
 252
 253                 err = ce->ops->pin(ce, vaddr);
 254                 if (err) {
 255                         intel_context_active_release(ce);
 256                         goto err_unlock;
 257                 }
 258
 259                 CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n",
 260                          i915_ggtt_offset(ce->ring->vma),
 261                          ce->ring->head, ce->ring->tail);
 262
 263                 handoff = true;
 264                 smp_mb__before_atomic(); /* flush pin before it is visible */
 265                 atomic_inc(&ce->pin_count);
 266         }
 267
 268         GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */
 269
 270         trace_intel_context_do_pin(ce);
 271
 272 err_unlock:
 273         mutex_unlock(&ce->pin_mutex);
 274 err_post_unpin:
 275         if (!handoff)
 276                 ce->ops->post_unpin(ce);
 277 err_release:
 278         i915_active_release(&ce->active);
 279 err_ctx_unpin:
 280         intel_context_post_unpin(ce);
 281
 282         /*
 283          * Unlock the hwsp_ggtt object since it's shared.
 284          * In principle we can unlock all the global state locked above
 285          * since it's pinned and doesn't need fencing, and will
 286          * thus remain resident until it is explicitly unpinned.
 287          */
 288         i915_gem_ww_unlock_single(ce->timeline->hwsp_ggtt->obj);
 289
 290         return err;
 291 }
 292
 293 int __intel_context_do_pin(struct intel_context *ce)
 294 {
 295         struct i915_gem_ww_ctx ww;
 296         int err;
 297
 298         i915_gem_ww_ctx_init(&ww, true);
 299 retry:
 300         err = __intel_context_do_pin_ww(ce, &ww);
 301         if (err == -EDEADLK) {
 302                 err = i915_gem_ww_ctx_backoff(&ww);
 303                 if (!err)
 304                         goto retry;
 305         }
 306         i915_gem_ww_ctx_fini(&ww);
 307         return err;
 308 }
 309
 310 void __intel_context_do_unpin(struct intel_context *ce, int sub)
 311 {
 312         if (!atomic_sub_and_test(sub, &ce->pin_count))
 313                 return;
 314
 315         CE_TRACE(ce, "unpin\n");
 316         ce->ops->unpin(ce);
 317         ce->ops->post_unpin(ce);
 318
 319         /*
 320          * Once released, we may asynchronously drop the active reference.
 321          * As that may be the only reference keeping the context alive,
 322          * take an extra now so that it is not freed before we finish
 323          * dereferencing it.
 324          */
 325         intel_context_get(ce);
 326         intel_context_active_release(ce);
 327         trace_intel_context_do_unpin(ce);
 328         intel_context_put(ce);
 329 }
 330
 331 static void __intel_context_retire(struct i915_active *active)
 332 {
 333         struct intel_context *ce = container_of(active, typeof(*ce), active);
 334
 335         CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n",
 336                  intel_context_get_total_runtime_ns(ce),
 337                  intel_context_get_avg_runtime_ns(ce));
 338
 339         set_bit(CONTEXT_VALID_BIT, &ce->flags);
 340         intel_context_post_unpin(ce);
 341         intel_context_put(ce);
 342 }
 343
 344 static int __intel_context_active(struct i915_active *active)
 345 {
 346         struct intel_context *ce = container_of(active, typeof(*ce), active);
 347
 348         intel_context_get(ce);
 349
 350         /* everything should already be activated by intel_context_pre_pin() */
 351         GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->ring->vma->active));
 352         __intel_ring_pin(ce->ring);
 353
 354         __intel_timeline_pin(ce->timeline);
 355
 356         if (ce->state) {
 357                 GEM_WARN_ON(!i915_active_acquire_if_busy(&ce->state->active));
 358                 __i915_vma_pin(ce->state);
 359                 i915_vma_make_unshrinkable(ce->state);
 360         }
 361
 362         return 0;
 363 }
 364
 365 static int sw_fence_dummy_notify(struct i915_sw_fence *sf,
 366                                  enum i915_sw_fence_notify state)
 367 {
 368         return NOTIFY_DONE;
 369 }
 370
 371 void
 372 intel_context_init(struct intel_context *ce, struct intel_engine_cs *engine)
 373 {
 374         GEM_BUG_ON(!engine->cops);
 375         GEM_BUG_ON(!engine->gt->vm);
 376
 377         kref_init(&ce->ref);
 378
 379         ce->engine = engine;
 380         ce->ops = engine->cops;
 381         ce->sseu = engine->sseu;
 382         ce->ring = NULL;
 383         ce->ring_size = SZ_4K;
 384
 385         ewma_runtime_init(&ce->runtime.avg);
 386
 387         ce->vm = i915_vm_get(engine->gt->vm);
 388
 389         /* NB ce->signal_link/lock is used under RCU */
 390         spin_lock_init(&ce->signal_lock);
 391         INIT_LIST_HEAD(&ce->signals);
 392
 393         mutex_init(&ce->pin_mutex);
 394
 395         spin_lock_init(&ce->guc_state.lock);
 396         INIT_LIST_HEAD(&ce->guc_state.fences);
 397
 398         spin_lock_init(&ce->guc_active.lock);
 399         INIT_LIST_HEAD(&ce->guc_active.requests);
 400
 401         ce->guc_id = GUC_INVALID_LRC_ID;
 402         INIT_LIST_HEAD(&ce->guc_id_link);
 403
 404         /*
 405          * Initialize fence to be complete as this is expected to be complete
 406          * unless there is a pending schedule disable outstanding.
 407          */
 408         i915_sw_fence_init(&ce->guc_blocked, sw_fence_dummy_notify);
 409         i915_sw_fence_commit(&ce->guc_blocked);
 410
 411         i915_active_init(&ce->active,
 412                          __intel_context_active, __intel_context_retire, 0);
 413 }
 414
 415 void intel_context_fini(struct intel_context *ce)
 416 {
 417         if (ce->timeline)
 418                 intel_timeline_put(ce->timeline);
 419         i915_vm_put(ce->vm);
 420
 421         mutex_destroy(&ce->pin_mutex);
 422         i915_active_fini(&ce->active);
 423 }
 424
 425 void i915_context_module_exit(void)
 426 {
 427         kmem_cache_destroy(slab_ce);
 428 }
 429
 430 int __init i915_context_module_init(void)
 431 {
 432         slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN);
 433         if (!slab_ce)
 434                 return -ENOMEM;
 435
 436         return 0;
 437 }
 438
 439 void intel_context_enter_engine(struct intel_context *ce)
 440 {
 441         intel_engine_pm_get(ce->engine);
 442         intel_timeline_enter(ce->timeline);
 443 }
 444
 445 void intel_context_exit_engine(struct intel_context *ce)
 446 {
 447         intel_timeline_exit(ce->timeline);
 448         intel_engine_pm_put(ce->engine);
 449 }
 450
 451 int intel_context_prepare_remote_request(struct intel_context *ce,
 452                                          struct i915_request *rq)
 453 {
 454         struct intel_timeline *tl = ce->timeline;
 455         int err;
 456
 457         /* Only suitable for use in remotely modifying this context */
 458         GEM_BUG_ON(rq->context == ce);
 459
 460         if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */
 461                 /* Queue this switch after current activity by this context. */
 462                 err = i915_active_fence_set(&tl->last_request, rq);
 463                 if (err)
 464                         return err;
 465         }
 466
 467         /*
 468          * Guarantee context image and the timeline remains pinned until the
 469          * modifying request is retired by setting the ce activity tracker.
 470          *
 471          * But we only need to take one pin on the account of it. Or in other
 472          * words transfer the pinned ce object to tracked active request.
 473          */
 474         GEM_BUG_ON(i915_active_is_idle(&ce->active));
 475         return i915_active_add_request(&ce->active, rq);
 476 }
 477
 478 struct i915_request *intel_context_create_request(struct intel_context *ce)
 479 {
 480         struct i915_gem_ww_ctx ww;
 481         struct i915_request *rq;
 482         int err;
 483
 484         i915_gem_ww_ctx_init(&ww, true);
 485 retry:
 486         err = intel_context_pin_ww(ce, &ww);
 487         if (!err) {
 488                 rq = i915_request_create(ce);
 489                 intel_context_unpin(ce);
 490         } else if (err == -EDEADLK) {
 491                 err = i915_gem_ww_ctx_backoff(&ww);
 492                 if (!err)
 493                         goto retry;
 494                 rq = ERR_PTR(err);
 495         } else {
 496                 rq = ERR_PTR(err);
 497         }
 498
 499         i915_gem_ww_ctx_fini(&ww);
 500
 501         if (IS_ERR(rq))
 502                 return rq;
 503
 504         /*
 505          * timeline->mutex should be the inner lock, but is used as outer lock.
 506          * Hack around this to shut up lockdep in selftests..
 507          */
 508         lockdep_unpin_lock(&ce->timeline->mutex, rq->cookie);
 509         mutex_release(&ce->timeline->mutex.dep_map, _RET_IP_);
 510         mutex_acquire(&ce->timeline->mutex.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_);
 511         rq->cookie = lockdep_pin_lock(&ce->timeline->mutex);
 512
 513         return rq;
 514 }
 515
 516 struct i915_request *intel_context_find_active_request(struct intel_context *ce)
 517 {
 518         struct i915_request *rq, *active = NULL;
 519         unsigned long flags;
 520
 521         GEM_BUG_ON(!intel_engine_uses_guc(ce->engine));
 522
 523         spin_lock_irqsave(&ce->guc_active.lock, flags);
 524         list_for_each_entry_reverse(rq, &ce->guc_active.requests,
 525                                     sched.link) {
 526                 if (i915_request_completed(rq))
 527                         break;
 528
 529                 active = rq;
 530         }
 531         spin_unlock_irqrestore(&ce->guc_active.lock, flags);
 532
 533         return active;
 534 }
 535
 536 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 537 #include "selftest_context.c"
 538 #endif