drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include <linux/ktime.h>
  28 #include <linux/hrtimer.h>
  29 #include <linux/sched/signal.h>
  30 #include <trace/events/dma_fence.h>
  31
  32 #include <nvif/if0020.h>
  33
  34 #include "nouveau_drv.h"
  35 #include "nouveau_dma.h"
  36 #include "nouveau_fence.h"
  37
  38 static const struct dma_fence_ops nouveau_fence_ops_uevent;
  39 static const struct dma_fence_ops nouveau_fence_ops_legacy;
  40
  41 static inline struct nouveau_fence *
  42 from_fence(struct dma_fence *fence)
  43 {
  44         return container_of(fence, struct nouveau_fence, base);
  45 }
  46
  47 static inline struct nouveau_fence_chan *
  48 nouveau_fctx(struct nouveau_fence *fence)
  49 {
  50         return container_of(fence->base.lock, struct nouveau_fence_chan, lock);
  51 }
  52
  53 static int
  54 nouveau_fence_signal(struct nouveau_fence *fence)
  55 {
  56         int drop = 0;
  57
  58         dma_fence_signal_locked(&fence->base);
  59         list_del(&fence->head);
  60         rcu_assign_pointer(fence->channel, NULL);
  61
  62         if (test_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags)) {
  63                 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
  64
  65                 if (!--fctx->notify_ref)
  66                         drop = 1;
  67         }
  68
  69         dma_fence_put(&fence->base);
  70         return drop;
  71 }
  72
  73 static struct nouveau_fence *
  74 nouveau_local_fence(struct dma_fence *fence, struct nouveau_drm *drm)
  75 {
  76         if (fence->ops != &nouveau_fence_ops_legacy &&
  77             fence->ops != &nouveau_fence_ops_uevent)
  78                 return NULL;
  79
  80         return from_fence(fence);
  81 }
  82
  83 void
  84 nouveau_fence_context_kill(struct nouveau_fence_chan *fctx, int error)
  85 {
  86         struct nouveau_fence *fence;
  87         unsigned long flags;
  88
  89         spin_lock_irqsave(&fctx->lock, flags);
  90         while (!list_empty(&fctx->pending)) {
  91                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
  92
  93                 if (error)
  94                         dma_fence_set_error(&fence->base, error);
  95
  96                 if (nouveau_fence_signal(fence))
  97                         nvif_event_block(&fctx->event);
  98         }
  99         fctx->killed = 1;
 100         spin_unlock_irqrestore(&fctx->lock, flags);
 101 }
 102
 103 void
 104 nouveau_fence_context_del(struct nouveau_fence_chan *fctx)
 105 {
 106         nouveau_fence_context_kill(fctx, 0);
 107         nvif_event_dtor(&fctx->event);
 108         fctx->dead = 1;
 109
 110         /*
 111          * Ensure that all accesses to fence->channel complete before freeing
 112          * the channel.
 113          */
 114         synchronize_rcu();
 115 }
 116
 117 static void
 118 nouveau_fence_context_put(struct kref *fence_ref)
 119 {
 120         kfree(container_of(fence_ref, struct nouveau_fence_chan, fence_ref));
 121 }
 122
 123 void
 124 nouveau_fence_context_free(struct nouveau_fence_chan *fctx)
 125 {
 126         kref_put(&fctx->fence_ref, nouveau_fence_context_put);
 127 }
 128
 129 static int
 130 nouveau_fence_update(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
 131 {
 132         struct nouveau_fence *fence;
 133         int drop = 0;
 134         u32 seq = fctx->read(chan);
 135
 136         while (!list_empty(&fctx->pending)) {
 137                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
 138
 139                 if ((int)(seq - fence->base.seqno) < 0)
 140                         break;
 141
 142                 drop |= nouveau_fence_signal(fence);
 143         }
 144
 145         return drop;
 146 }
 147
 148 static int
 149 nouveau_fence_wait_uevent_handler(struct nvif_event *event, void *repv, u32 repc)
 150 {
 151         struct nouveau_fence_chan *fctx = container_of(event, typeof(*fctx), event);
 152         unsigned long flags;
 153         int ret = NVIF_EVENT_KEEP;
 154
 155         spin_lock_irqsave(&fctx->lock, flags);
 156         if (!list_empty(&fctx->pending)) {
 157                 struct nouveau_fence *fence;
 158                 struct nouveau_channel *chan;
 159
 160                 fence = list_entry(fctx->pending.next, typeof(*fence), head);
 161                 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
 162                 if (nouveau_fence_update(chan, fctx))
 163                         ret = NVIF_EVENT_DROP;
 164         }
 165         spin_unlock_irqrestore(&fctx->lock, flags);
 166
 167         return ret;
 168 }
 169
 170 void
 171 nouveau_fence_context_new(struct nouveau_channel *chan, struct nouveau_fence_chan *fctx)
 172 {
 173         struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
 174         struct nouveau_cli *cli = (void *)chan->user.client;
 175         struct {
 176                 struct nvif_event_v0 base;
 177                 struct nvif_chan_event_v0 host;
 178         } args;
 179         int ret;
 180
 181         INIT_LIST_HEAD(&fctx->flip);
 182         INIT_LIST_HEAD(&fctx->pending);
 183         spin_lock_init(&fctx->lock);
 184         fctx->context = chan->drm->runl[chan->runlist].context_base + chan->chid;
 185
 186         if (chan == chan->drm->cechan)
 187                 strcpy(fctx->name, "copy engine channel");
 188         else if (chan == chan->drm->channel)
 189                 strcpy(fctx->name, "generic kernel channel");
 190         else
 191                 strcpy(fctx->name, nvxx_client(&cli->base)->name);
 192
 193         kref_init(&fctx->fence_ref);
 194         if (!priv->uevent)
 195                 return;
 196
 197         args.host.version = 0;
 198         args.host.type = NVIF_CHAN_EVENT_V0_NON_STALL_INTR;
 199
 200         ret = nvif_event_ctor(&chan->user, "fenceNonStallIntr", (chan->runlist << 16) | chan->chid,
 201                               nouveau_fence_wait_uevent_handler, false,
 202                               &args.base, sizeof(args), &fctx->event);
 203
 204         WARN_ON(ret);
 205 }
 206
 207 int
 208 nouveau_fence_emit(struct nouveau_fence *fence)
 209 {
 210         struct nouveau_channel *chan = unrcu_pointer(fence->channel);
 211         struct nouveau_fence_chan *fctx = chan->fence;
 212         struct nouveau_fence_priv *priv = (void*)chan->drm->fence;
 213         int ret;
 214
 215         fence->timeout  = jiffies + (15 * HZ);
 216
 217         if (priv->uevent)
 218                 dma_fence_init(&fence->base, &nouveau_fence_ops_uevent,
 219                                &fctx->lock, fctx->context, ++fctx->sequence);
 220         else
 221                 dma_fence_init(&fence->base, &nouveau_fence_ops_legacy,
 222                                &fctx->lock, fctx->context, ++fctx->sequence);
 223         kref_get(&fctx->fence_ref);
 224
 225         ret = fctx->emit(fence);
 226         if (!ret) {
 227                 dma_fence_get(&fence->base);
 228                 spin_lock_irq(&fctx->lock);
 229
 230                 if (unlikely(fctx->killed)) {
 231                         spin_unlock_irq(&fctx->lock);
 232                         dma_fence_put(&fence->base);
 233                         return -ENODEV;
 234                 }
 235
 236                 if (nouveau_fence_update(chan, fctx))
 237                         nvif_event_block(&fctx->event);
 238
 239                 list_add_tail(&fence->head, &fctx->pending);
 240                 spin_unlock_irq(&fctx->lock);
 241         }
 242
 243         return ret;
 244 }
 245
 246 bool
 247 nouveau_fence_done(struct nouveau_fence *fence)
 248 {
 249         if (fence->base.ops == &nouveau_fence_ops_legacy ||
 250             fence->base.ops == &nouveau_fence_ops_uevent) {
 251                 struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 252                 struct nouveau_channel *chan;
 253                 unsigned long flags;
 254
 255                 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->base.flags))
 256                         return true;
 257
 258                 spin_lock_irqsave(&fctx->lock, flags);
 259                 chan = rcu_dereference_protected(fence->channel, lockdep_is_held(&fctx->lock));
 260                 if (chan && nouveau_fence_update(chan, fctx))
 261                         nvif_event_block(&fctx->event);
 262                 spin_unlock_irqrestore(&fctx->lock, flags);
 263         }
 264         return dma_fence_is_signaled(&fence->base);
 265 }
 266
 267 static long
 268 nouveau_fence_wait_legacy(struct dma_fence *f, bool intr, long wait)
 269 {
 270         struct nouveau_fence *fence = from_fence(f);
 271         unsigned long sleep_time = NSEC_PER_MSEC / 1000;
 272         unsigned long t = jiffies, timeout = t + wait;
 273
 274         while (!nouveau_fence_done(fence)) {
 275                 ktime_t kt;
 276
 277                 t = jiffies;
 278
 279                 if (wait != MAX_SCHEDULE_TIMEOUT && time_after_eq(t, timeout)) {
 280                         __set_current_state(TASK_RUNNING);
 281                         return 0;
 282                 }
 283
 284                 __set_current_state(intr ? TASK_INTERRUPTIBLE :
 285                                            TASK_UNINTERRUPTIBLE);
 286
 287                 kt = sleep_time;
 288                 schedule_hrtimeout(&kt, HRTIMER_MODE_REL);
 289                 sleep_time *= 2;
 290                 if (sleep_time > NSEC_PER_MSEC)
 291                         sleep_time = NSEC_PER_MSEC;
 292
 293                 if (intr && signal_pending(current))
 294                         return -ERESTARTSYS;
 295         }
 296
 297         __set_current_state(TASK_RUNNING);
 298
 299         return timeout - t;
 300 }
 301
 302 static int
 303 nouveau_fence_wait_busy(struct nouveau_fence *fence, bool intr)
 304 {
 305         int ret = 0;
 306
 307         while (!nouveau_fence_done(fence)) {
 308                 if (time_after_eq(jiffies, fence->timeout)) {
 309                         ret = -EBUSY;
 310                         break;
 311                 }
 312
 313                 __set_current_state(intr ?
 314                                     TASK_INTERRUPTIBLE :
 315                                     TASK_UNINTERRUPTIBLE);
 316
 317                 if (intr && signal_pending(current)) {
 318                         ret = -ERESTARTSYS;
 319                         break;
 320                 }
 321         }
 322
 323         __set_current_state(TASK_RUNNING);
 324         return ret;
 325 }
 326
 327 int
 328 nouveau_fence_wait(struct nouveau_fence *fence, bool lazy, bool intr)
 329 {
 330         long ret;
 331
 332         if (!lazy)
 333                 return nouveau_fence_wait_busy(fence, intr);
 334
 335         ret = dma_fence_wait_timeout(&fence->base, intr, 15 * HZ);
 336         if (ret < 0)
 337                 return ret;
 338         else if (!ret)
 339                 return -EBUSY;
 340         else
 341                 return 0;
 342 }
 343
 344 int
 345 nouveau_fence_sync(struct nouveau_bo *nvbo, struct nouveau_channel *chan,
 346                    bool exclusive, bool intr)
 347 {
 348         struct nouveau_fence_chan *fctx = chan->fence;
 349         struct dma_resv *resv = nvbo->bo.base.resv;
 350         int i, ret;
 351
 352         ret = dma_resv_reserve_fences(resv, 1);
 353         if (ret)
 354                 return ret;
 355
 356         /* Waiting for the writes first causes performance regressions
 357          * under some circumstances. So manually wait for the reads first.
 358          */
 359         for (i = 0; i < 2; ++i) {
 360                 struct dma_resv_iter cursor;
 361                 struct dma_fence *fence;
 362
 363                 dma_resv_for_each_fence(&cursor, resv,
 364                                         dma_resv_usage_rw(exclusive),
 365                                         fence) {
 366                         enum dma_resv_usage usage;
 367                         struct nouveau_fence *f;
 368
 369                         usage = dma_resv_iter_usage(&cursor);
 370                         if (i == 0 && usage == DMA_RESV_USAGE_WRITE)
 371                                 continue;
 372
 373                         f = nouveau_local_fence(fence, chan->drm);
 374                         if (f) {
 375                                 struct nouveau_channel *prev;
 376                                 bool must_wait = true;
 377
 378                                 rcu_read_lock();
 379                                 prev = rcu_dereference(f->channel);
 380                                 if (prev && (prev == chan ||
 381                                              fctx->sync(f, prev, chan) == 0))
 382                                         must_wait = false;
 383                                 rcu_read_unlock();
 384                                 if (!must_wait)
 385                                         continue;
 386                         }
 387
 388                         ret = dma_fence_wait(fence, intr);
 389                         if (ret)
 390                                 return ret;
 391                 }
 392         }
 393
 394         return 0;
 395 }
 396
 397 void
 398 nouveau_fence_unref(struct nouveau_fence **pfence)
 399 {
 400         if (*pfence)
 401                 dma_fence_put(&(*pfence)->base);
 402         *pfence = NULL;
 403 }
 404
 405 int
 406 nouveau_fence_create(struct nouveau_fence **pfence,
 407                      struct nouveau_channel *chan)
 408 {
 409         struct nouveau_fence *fence;
 410
 411         if (unlikely(!chan->fence))
 412                 return -ENODEV;
 413
 414         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 415         if (!fence)
 416                 return -ENOMEM;
 417
 418         fence->channel = chan;
 419
 420         *pfence = fence;
 421         return 0;
 422 }
 423
 424 int
 425 nouveau_fence_new(struct nouveau_fence **pfence,
 426                   struct nouveau_channel *chan)
 427 {
 428         int ret = 0;
 429
 430         ret = nouveau_fence_create(pfence, chan);
 431         if (ret)
 432                 return ret;
 433
 434         ret = nouveau_fence_emit(*pfence);
 435         if (ret)
 436                 nouveau_fence_unref(pfence);
 437
 438         return ret;
 439 }
 440
 441 static const char *nouveau_fence_get_get_driver_name(struct dma_fence *fence)
 442 {
 443         return "nouveau";
 444 }
 445
 446 static const char *nouveau_fence_get_timeline_name(struct dma_fence *f)
 447 {
 448         struct nouveau_fence *fence = from_fence(f);
 449         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 450
 451         return !fctx->dead ? fctx->name : "dead channel";
 452 }
 453
 454 /*
 455  * In an ideal world, read would not assume the channel context is still alive.
 456  * This function may be called from another device, running into free memory as a
 457  * result. The drm node should still be there, so we can derive the index from
 458  * the fence context.
 459  */
 460 static bool nouveau_fence_is_signaled(struct dma_fence *f)
 461 {
 462         struct nouveau_fence *fence = from_fence(f);
 463         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 464         struct nouveau_channel *chan;
 465         bool ret = false;
 466
 467         rcu_read_lock();
 468         chan = rcu_dereference(fence->channel);
 469         if (chan)
 470                 ret = (int)(fctx->read(chan) - fence->base.seqno) >= 0;
 471         rcu_read_unlock();
 472
 473         return ret;
 474 }
 475
 476 static bool nouveau_fence_no_signaling(struct dma_fence *f)
 477 {
 478         struct nouveau_fence *fence = from_fence(f);
 479
 480         /*
 481          * caller should have a reference on the fence,
 482          * else fence could get freed here
 483          */
 484         WARN_ON(kref_read(&fence->base.refcount) <= 1);
 485
 486         /*
 487          * This needs uevents to work correctly, but dma_fence_add_callback relies on
 488          * being able to enable signaling. It will still get signaled eventually,
 489          * just not right away.
 490          */
 491         if (nouveau_fence_is_signaled(f)) {
 492                 list_del(&fence->head);
 493
 494                 dma_fence_put(&fence->base);
 495                 return false;
 496         }
 497
 498         return true;
 499 }
 500
 501 static void nouveau_fence_release(struct dma_fence *f)
 502 {
 503         struct nouveau_fence *fence = from_fence(f);
 504         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 505
 506         kref_put(&fctx->fence_ref, nouveau_fence_context_put);
 507         dma_fence_free(&fence->base);
 508 }
 509
 510 static const struct dma_fence_ops nouveau_fence_ops_legacy = {
 511         .get_driver_name = nouveau_fence_get_get_driver_name,
 512         .get_timeline_name = nouveau_fence_get_timeline_name,
 513         .enable_signaling = nouveau_fence_no_signaling,
 514         .signaled = nouveau_fence_is_signaled,
 515         .wait = nouveau_fence_wait_legacy,
 516         .release = nouveau_fence_release
 517 };
 518
 519 static bool nouveau_fence_enable_signaling(struct dma_fence *f)
 520 {
 521         struct nouveau_fence *fence = from_fence(f);
 522         struct nouveau_fence_chan *fctx = nouveau_fctx(fence);
 523         bool ret;
 524
 525         if (!fctx->notify_ref++)
 526                 nvif_event_allow(&fctx->event);
 527
 528         ret = nouveau_fence_no_signaling(f);
 529         if (ret)
 530                 set_bit(DMA_FENCE_FLAG_USER_BITS, &fence->base.flags);
 531         else if (!--fctx->notify_ref)
 532                 nvif_event_block(&fctx->event);
 533
 534         return ret;
 535 }
 536
 537 static const struct dma_fence_ops nouveau_fence_ops_uevent = {
 538         .get_driver_name = nouveau_fence_get_get_driver_name,
 539         .get_timeline_name = nouveau_fence_get_timeline_name,
 540         .enable_signaling = nouveau_fence_enable_signaling,
 541         .signaled = nouveau_fence_is_signaled,
 542         .release = nouveau_fence_release
 543 };