drivers/gpu/drm/img/img-rogue/services/server/env/linux/pvr_fence.c

   1 /*
   2  * @File
   3  * @Title       PowerVR Linux fence interface
   4  * @Codingstyle LinuxKernel
   5  * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   6  * @License     Dual MIT/GPLv2
   7  *
   8  * The contents of this file are subject to the MIT license as set out below.
   9  *
  10  * Permission is hereby granted, free of charge, to any person obtaining a copy
  11  * of this software and associated documentation files (the "Software"), to deal
  12  * in the Software without restriction, including without limitation the rights
  13  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14  * copies of the Software, and to permit persons to whom the Software is
  15  * furnished to do so, subject to the following conditions:
  16  *
  17  * The above copyright notice and this permission notice shall be included in
  18  * all copies or substantial portions of the Software.
  19  *
  20  * Alternatively, the contents of this file may be used under the terms of
  21  * the GNU General Public License Version 2 ("GPL") in which case the provisions
  22  * of GPL are applicable instead of those above.
  23  *
  24  * If you wish to allow use of your version of this file only under the terms of
  25  * GPL, and not to allow others to use your version of this file under the terms
  26  * of the MIT license, indicate your decision by deleting the provisions above
  27  * and replace them with the notice and other provisions required by GPL as set
  28  * out in the file called "GPL-COPYING" included in this distribution. If you do
  29  * not delete the provisions above, a recipient may use your version of this file
  30  * under the terms of either the MIT license or GPL.
  31  *
  32  * This License is also included in this distribution in the file called
  33  * "MIT-COPYING".
  34  *
  35  * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36  * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37  * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38  * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40  * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42  */
  43
  44 #include <linux/kernel.h>
  45 #include <linux/jiffies.h>
  46 #include <linux/module.h>
  47 #include <linux/slab.h>
  48
  49 #include "pvr_fence.h"
  50 #include "services_kernel_client.h"
  51 #include "sync_checkpoint_external.h"
  52
  53 #define CREATE_TRACE_POINTS
  54 #include "pvr_fence_trace.h"
  55
  56 /* This header must always be included last */
  57 #include "kernel_compatibility.h"
  58
  59 /* Global kmem_cache for pvr_fence object allocations */
  60 static struct kmem_cache *pvr_fence_cache;
  61 static DEFINE_MUTEX(pvr_fence_cache_mutex);
  62 static u32 pvr_fence_cache_refcount;
  63
  64 #define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
  65         do {                                                             \
  66                 if (pfnDumpDebugPrintf)                                  \
  67                         pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
  68                                            ## __VA_ARGS__);              \
  69                 else                                                     \
  70                         pr_err(fmt "\n", ## __VA_ARGS__);                \
  71         } while (0)
  72
  73 static inline void
  74 pvr_fence_sync_signal(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
  75 {
  76         SyncCheckpointSignal(pvr_fence->sync_checkpoint, fence_sync_flags);
  77 }
  78
  79 static inline bool
  80 pvr_fence_sync_is_signaled(struct pvr_fence *pvr_fence, u32 fence_sync_flags)
  81 {
  82         return SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
  83                                          fence_sync_flags);
  84 }
  85
  86 static inline u32
  87 pvr_fence_sync_value(struct pvr_fence *pvr_fence)
  88 {
  89         if (SyncCheckpointIsErrored(pvr_fence->sync_checkpoint,
  90                                     PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
  91                 return PVRSRV_SYNC_CHECKPOINT_ERRORED;
  92         else if (SyncCheckpointIsSignalled(pvr_fence->sync_checkpoint,
  93                                            PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
  94                 return PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
  95         else
  96                 return PVRSRV_SYNC_CHECKPOINT_ACTIVE;
  97 }
  98
  99 static void
 100 pvr_fence_context_check_status(struct work_struct *data)
 101 {
 102         PVRSRVCheckStatus(NULL);
 103 }
 104
 105 void
 106 pvr_context_value_str(struct pvr_fence_context *fctx, char *str, int size)
 107 {
 108         snprintf(str, size,
 109                  "%u ctx=%llu refs=%u",
 110                  atomic_read(&fctx->fence_seqno),
 111                  fctx->fence_context,
 112                  refcount_read(&fctx->kref.refcount));
 113 }
 114
 115 static void
 116 pvr_fence_context_fences_dump(struct pvr_fence_context *fctx,
 117                               DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
 118                               void *pvDumpDebugFile)
 119 {
 120         struct pvr_fence *pvr_fence;
 121         unsigned long flags;
 122         char value[128];
 123
 124         spin_lock_irqsave(&fctx->list_lock, flags);
 125         pvr_context_value_str(fctx, value, sizeof(value));
 126         PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
 127                          "%s: @%s", fctx->name, value);
 128         list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
 129                 struct dma_fence *fence = pvr_fence->fence;
 130                 const char *timeline_value_str = "unknown timeline value";
 131                 const char *fence_value_str = "unknown fence value";
 132
 133                 pvr_fence->base.ops->fence_value_str(&pvr_fence->base, value,
 134                                                      sizeof(value));
 135                 PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
 136                                   " @%s", value);
 137
 138                 if (is_pvr_fence(fence))
 139                         continue;
 140
 141                 if (fence->ops->timeline_value_str) {
 142                         fence->ops->timeline_value_str(fence, value,
 143                                                        sizeof(value));
 144                         timeline_value_str = value;
 145                 }
 146
 147                 PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
 148                                   " | %s: %s (driver: %s)",
 149                                   fence->ops->get_timeline_name(fence),
 150                                   timeline_value_str,
 151                                   fence->ops->get_driver_name(fence));
 152
 153                 if (fence->ops->fence_value_str) {
 154                         fence->ops->fence_value_str(fence, value,
 155                                                     sizeof(value));
 156                         fence_value_str = value;
 157                 }
 158
 159                 PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
 160                                   " |  @%s (foreign)", value);
 161         }
 162         spin_unlock_irqrestore(&fctx->list_lock, flags);
 163 }
 164
 165 static inline unsigned int
 166 pvr_fence_context_seqno_next(struct pvr_fence_context *fctx)
 167 {
 168         return atomic_inc_return(&fctx->fence_seqno) - 1;
 169 }
 170
 171 /* This function prepends seqno to fence name */
 172 static inline void
 173 pvr_fence_prepare_name(char *fence_name, size_t fence_name_size,
 174                 const char *name, unsigned int seqno)
 175 {
 176         unsigned int len;
 177
 178         len = OSStringUINT32ToStr(fence_name, fence_name_size, seqno);
 179         if (likely((len > 0) && (fence_name_size >= (len + 1)))) {
 180                 fence_name[len] = '-';
 181                 fence_name[len + 1] = '\0';
 182         }
 183         strlcat(fence_name, name, fence_name_size);
 184 }
 185
 186 static void
 187 pvr_fence_sched_free(struct rcu_head *rcu)
 188 {
 189         struct pvr_fence *pvr_fence = container_of(rcu, struct pvr_fence, rcu);
 190
 191         kmem_cache_free(pvr_fence_cache, pvr_fence);
 192 }
 193
 194 static inline void
 195 pvr_fence_context_free_deferred(struct pvr_fence_context *fctx)
 196 {
 197         struct pvr_fence *pvr_fence, *tmp;
 198         LIST_HEAD(deferred_free_list);
 199         unsigned long flags;
 200
 201         spin_lock_irqsave(&fctx->list_lock, flags);
 202         list_for_each_entry_safe(pvr_fence, tmp,
 203                                  &fctx->deferred_free_list,
 204                                  fence_head)
 205                 list_move(&pvr_fence->fence_head, &deferred_free_list);
 206         spin_unlock_irqrestore(&fctx->list_lock, flags);
 207
 208         list_for_each_entry_safe(pvr_fence, tmp,
 209                                  &deferred_free_list,
 210                                  fence_head) {
 211                 list_del(&pvr_fence->fence_head);
 212                 SyncCheckpointFree(pvr_fence->sync_checkpoint);
 213                 call_rcu(&pvr_fence->rcu, pvr_fence_sched_free);
 214                 module_put(THIS_MODULE);
 215         }
 216 }
 217
 218 void
 219 pvr_fence_context_free_deferred_callback(void *data)
 220 {
 221         struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
 222
 223         /*
 224          * Free up any fence objects we have deferred freeing.
 225          */
 226         pvr_fence_context_free_deferred(fctx);
 227 }
 228
 229 static void
 230 pvr_fence_context_signal_fences(void *data)
 231 {
 232         struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
 233         struct pvr_fence *pvr_fence, *tmp;
 234         unsigned long flags1;
 235
 236         LIST_HEAD(signal_list);
 237
 238         /*
 239          * We can't call fence_signal while holding the lock as we can end up
 240          * in a situation whereby pvr_fence_foreign_signal_sync, which also
 241          * takes the list lock, ends up being called as a result of the
 242          * fence_signal below, i.e. fence_signal(fence) -> fence->callback()
 243          *  -> fence_signal(foreign_fence) -> foreign_fence->callback() where
 244          * the foreign_fence callback is pvr_fence_foreign_signal_sync.
 245          *
 246          * So extract the items we intend to signal and add them to their own
 247          * queue.
 248          */
 249         spin_lock_irqsave(&fctx->list_lock, flags1);
 250         list_for_each_entry_safe(pvr_fence, tmp, &fctx->signal_list, signal_head) {
 251                 if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
 252                         list_move_tail(&pvr_fence->signal_head, &signal_list);
 253         }
 254         spin_unlock_irqrestore(&fctx->list_lock, flags1);
 255
 256         list_for_each_entry_safe(pvr_fence, tmp, &signal_list, signal_head) {
 257
 258                 PVR_FENCE_TRACE(&pvr_fence->base, "signalled fence (%s)\n",
 259                                 pvr_fence->name);
 260                 trace_pvr_fence_signal_fence(pvr_fence);
 261                 spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags1);
 262                 list_del(&pvr_fence->signal_head);
 263                 spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags1);
 264                 dma_fence_signal(pvr_fence->fence);
 265                 dma_fence_put(pvr_fence->fence);
 266         }
 267
 268         /*
 269          * Take this opportunity to free up any fence objects we
 270          * have deferred freeing.
 271          */
 272         pvr_fence_context_free_deferred(fctx);
 273 }
 274
 275 void
 276 pvr_fence_context_signal_fences_nohw(void *data)
 277 {
 278         pvr_fence_context_signal_fences(data);
 279 }
 280
 281 static void
 282 pvr_fence_context_destroy_internal(struct pvr_fence_context *fctx)
 283 {
 284         pvr_fence_context_free_deferred(fctx);
 285
 286         if (WARN_ON(!list_empty_careful(&fctx->fence_list)))
 287                 pvr_fence_context_fences_dump(fctx, NULL, NULL);
 288
 289         PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
 290
 291         // wait for all fences to be freed before kmem_cache_destroy() is called
 292         rcu_barrier();
 293
 294         /* Destroy pvr_fence object cache, if no one is using it */
 295         WARN_ON(pvr_fence_cache == NULL);
 296         mutex_lock(&pvr_fence_cache_mutex);
 297         if (--pvr_fence_cache_refcount == 0)
 298                 kmem_cache_destroy(pvr_fence_cache);
 299         mutex_unlock(&pvr_fence_cache_mutex);
 300
 301         kfree(fctx);
 302 }
 303
 304 static void
 305 pvr_fence_context_unregister_dbg(void *dbg_request_handle)
 306 {
 307         PVRSRVUnregisterDeviceDbgRequestNotify(dbg_request_handle);
 308 }
 309
 310 static void
 311 pvr_fence_foreign_context_destroy_work(struct work_struct *data)
 312 {
 313         struct pvr_fence_context *fctx =
 314                 container_of(data, struct pvr_fence_context, destroy_work);
 315
 316         pvr_fence_context_destroy_internal(fctx);
 317 }
 318
 319 static void
 320 pvr_fence_context_destroy_work(struct work_struct *data)
 321 {
 322         struct pvr_fence_context *fctx =
 323                 container_of(data, struct pvr_fence_context, destroy_work);
 324
 325         pvr_fence_context_unregister_dbg(fctx->dbg_request_handle);
 326         pvr_fence_context_destroy_internal(fctx);
 327 }
 328
 329 static void
 330 pvr_fence_context_debug_request(void *data, u32 verbosity,
 331                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
 332                                 void *pvDumpDebugFile)
 333 {
 334         struct pvr_fence_context *fctx = (struct pvr_fence_context *)data;
 335
 336         if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
 337                 pvr_fence_context_fences_dump(fctx, pfnDumpDebugPrintf,
 338                                               pvDumpDebugFile);
 339 }
 340
 341 static struct pvr_fence_context *
 342 pvr_fence_context_create_internal(struct workqueue_struct *fence_status_wq,
 343                         const char *name,
 344                         work_func_t destroy_callback)
 345 {
 346         struct pvr_fence_context *fctx;
 347         PVRSRV_ERROR srv_err;
 348
 349         fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
 350         if (!fctx)
 351                 return NULL;
 352
 353         spin_lock_init(&fctx->lock);
 354         atomic_set(&fctx->fence_seqno, 0);
 355         INIT_WORK(&fctx->check_status_work, pvr_fence_context_check_status);
 356         INIT_WORK(&fctx->destroy_work, destroy_callback);
 357         spin_lock_init(&fctx->list_lock);
 358         INIT_LIST_HEAD(&fctx->signal_list);
 359         INIT_LIST_HEAD(&fctx->fence_list);
 360         INIT_LIST_HEAD(&fctx->deferred_free_list);
 361
 362         fctx->fence_wq = fence_status_wq;
 363
 364         fctx->fence_context = dma_fence_context_alloc(1);
 365         strlcpy(fctx->name, name, sizeof(fctx->name));
 366
 367         srv_err = PVRSRVRegisterCmdCompleteNotify(&fctx->cmd_complete_handle,
 368                                 pvr_fence_context_signal_fences,
 369                                 fctx);
 370         if (srv_err != PVRSRV_OK) {
 371                 pr_err("%s: failed to register command complete callback (%s)\n",
 372                        __func__, PVRSRVGetErrorString(srv_err));
 373                 goto err_free_fctx;
 374         }
 375
 376         /* Create pvr_fence object cache, if not already created */
 377         mutex_lock(&pvr_fence_cache_mutex);
 378         if (pvr_fence_cache_refcount == 0) {
 379                 pvr_fence_cache = KMEM_CACHE(pvr_fence, 0);
 380                 if (!pvr_fence_cache) {
 381                         pr_err("%s: failed to allocate pvr_fence cache\n",
 382                                         __func__);
 383                         mutex_unlock(&pvr_fence_cache_mutex);
 384                         goto err_unregister_cmd_complete_notify;
 385                 }
 386         }
 387         pvr_fence_cache_refcount++;
 388         mutex_unlock(&pvr_fence_cache_mutex);
 389
 390         kref_init(&fctx->kref);
 391
 392         PVR_FENCE_CTX_TRACE(fctx, "created fence context (%s)\n", name);
 393         trace_pvr_fence_context_create(fctx);
 394
 395         return fctx;
 396
 397 err_unregister_cmd_complete_notify:
 398         PVRSRVUnregisterCmdCompleteNotify(fctx->cmd_complete_handle);
 399 err_free_fctx:
 400         kfree(fctx);
 401         return NULL;
 402 }
 403
 404 /**
 405  * pvr_fence_context_register_dbg - registers the debug handler for a
 406  * fence context
 407  *
 408  * @dbg_request_handle: handle used to keep a reference for deregister
 409  * @dev: device to attach the debug notifier.
 410  * @pvr_fence_context: context used as data to the callback for debug
 411  *
 412  * Registers a debug notifier for a given context for a given device.
 413  *
 414  * Returns PVRSRV_OK if successful.
 415  */
 416 PVRSRV_ERROR pvr_fence_context_register_dbg(void *dbg_request_handle,
 417                                 void *dev,
 418                                 struct pvr_fence_context *fctx)
 419 {
 420         PVRSRV_ERROR srv_err;
 421
 422         srv_err = PVRSRVRegisterDeviceDbgRequestNotify(dbg_request_handle,
 423                                 dev,
 424                                 pvr_fence_context_debug_request,
 425                                 DEBUG_REQUEST_LINUXFENCE,
 426                                 fctx);
 427         if (srv_err != PVRSRV_OK) {
 428                 pr_err("%s: failed to register debug request callback (%s)\n",
 429                        __func__, PVRSRVGetErrorString(srv_err));
 430         }
 431
 432         return srv_err;
 433 }
 434
 435 /**
 436  * pvr_fence_foreign_context_create - creates a PVR fence context
 437  * @fence_status_wq: linux workqueue used to signal foreign fences
 438  * @name: context name (used for debugging)
 439  *
 440  * Creates a PVR foreign fence context that can be used to create PVR fences
 441  * or to create PVR fences from an existing fence.
 442  *
 443  * pvr_fence_context_destroy should be called to clean up the fence context.
 444  *
 445  * Returns NULL if a context cannot be created.
 446  */
 447 struct pvr_fence_context *
 448 pvr_fence_foreign_context_create(struct workqueue_struct *fence_status_wq,
 449                 const char *name)
 450 {
 451         return pvr_fence_context_create_internal(fence_status_wq, name,
 452                                                         pvr_fence_foreign_context_destroy_work);
 453 }
 454
 455 /**
 456  * pvr_fence_context_create - creates a PVR fence context
 457  * @dev_cookie: services device cookie
 458  * @fence_status_wq: Status workqueue to queue fence update CBs.
 459  * @name: context name (used for debugging)
 460  *
 461  * Creates a PVR fence context that can be used to create PVR fences or to
 462  * create PVR fences from an existing fence.
 463  *
 464  * pvr_fence_context_destroy should be called to clean up the fence context.
 465  *
 466  * Returns NULL if a context cannot be created.
 467  */
 468 struct pvr_fence_context *
 469 pvr_fence_context_create(void *dev_cookie,
 470                          struct workqueue_struct *fence_status_wq,
 471                          const char *name)
 472 {
 473         struct pvr_fence_context *fctx;
 474         PVRSRV_ERROR eError;
 475
 476         fctx = pvr_fence_context_create_internal(fence_status_wq, name,
 477                                         pvr_fence_context_destroy_work);
 478         if (fctx == NULL) {
 479                 pr_err("%s: failed to create fence context", __func__);
 480                 goto err_out;
 481         }
 482
 483         eError = pvr_fence_context_register_dbg(&fctx->dbg_request_handle,
 484                                         dev_cookie,
 485                                         fctx);
 486         if (eError != PVRSRV_OK) {
 487                 pr_err("%s: failed to register fence context debug (%s)\n",
 488                        __func__, PVRSRVGetErrorString(eError));
 489                 goto err_destroy_ctx;
 490         }
 491
 492         return fctx;
 493
 494 err_destroy_ctx:
 495         pvr_fence_context_destroy(fctx);
 496 err_out:
 497         return NULL;
 498 }
 499
 500 static void pvr_fence_context_destroy_kref(struct kref *kref)
 501 {
 502         struct pvr_fence_context *fctx =
 503                 container_of(kref, struct pvr_fence_context, kref);
 504
 505         PVR_FENCE_CTX_TRACE(fctx, "destroyed fence context (%s)\n", fctx->name);
 506
 507         trace_pvr_fence_context_destroy_kref(fctx);
 508
 509         schedule_work(&fctx->destroy_work);
 510 }
 511
 512 /**
 513  * pvr_fence_context_destroy - destroys a context
 514  * @fctx: PVR fence context to destroy
 515  *
 516  * Destroys a PVR fence context with the expectation that all fences have been
 517  * destroyed.
 518  */
 519 void
 520 pvr_fence_context_destroy(struct pvr_fence_context *fctx)
 521 {
 522         trace_pvr_fence_context_destroy(fctx);
 523
 524         kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
 525 }
 526
 527 static const char *
 528 pvr_fence_get_driver_name(struct dma_fence *fence)
 529 {
 530         return PVR_LDM_DRIVER_REGISTRATION_NAME;
 531 }
 532
 533 static const char *
 534 pvr_fence_get_timeline_name(struct dma_fence *fence)
 535 {
 536         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 537
 538         if (pvr_fence)
 539                 return pvr_fence->fctx->name;
 540         return NULL;
 541 }
 542
 543 static
 544 void pvr_fence_fence_value_str(struct dma_fence *fence, char *str, int size)
 545 {
 546         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 547
 548         if (!pvr_fence)
 549                 return;
 550
 551         snprintf(str, size,
 552                  "%llu: (%s%s) refs=%u fwaddr=%#08x enqueue=%u status=%-9s %s%s",
 553                  (u64) pvr_fence->fence->seqno,
 554                  test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 555                           &pvr_fence->fence->flags) ? "+" : "-",
 556                  test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 557                           &pvr_fence->fence->flags) ? "+" : "-",
 558                  refcount_read(&pvr_fence->fence->refcount.refcount),
 559                  SyncCheckpointGetFirmwareAddr(
 560                          pvr_fence->sync_checkpoint),
 561                  SyncCheckpointGetEnqueuedCount(pvr_fence->sync_checkpoint),
 562                  SyncCheckpointGetStateString(pvr_fence->sync_checkpoint),
 563                  pvr_fence->name,
 564                  (&pvr_fence->base != pvr_fence->fence) ?
 565                  "(foreign)" : "");
 566 }
 567
 568 static
 569 void pvr_fence_timeline_value_str(struct dma_fence *fence, char *str, int size)
 570 {
 571         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 572
 573         if (pvr_fence)
 574                 pvr_context_value_str(pvr_fence->fctx, str, size);
 575 }
 576
 577 static bool
 578 pvr_fence_enable_signaling(struct dma_fence *fence)
 579 {
 580         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 581         unsigned long flags;
 582
 583         if (!pvr_fence)
 584                 return false;
 585
 586         WARN_ON_SMP(!spin_is_locked(&pvr_fence->fctx->lock));
 587
 588         if (pvr_fence_sync_is_signaled(pvr_fence, PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT))
 589                 return false;
 590
 591         dma_fence_get(&pvr_fence->base);
 592
 593         spin_lock_irqsave(&pvr_fence->fctx->list_lock, flags);
 594         list_add_tail(&pvr_fence->signal_head, &pvr_fence->fctx->signal_list);
 595         spin_unlock_irqrestore(&pvr_fence->fctx->list_lock, flags);
 596
 597         PVR_FENCE_TRACE(&pvr_fence->base, "signalling enabled (%s)\n",
 598                         pvr_fence->name);
 599         trace_pvr_fence_enable_signaling(pvr_fence);
 600
 601         return true;
 602 }
 603
 604 static bool
 605 pvr_fence_is_signaled(struct dma_fence *fence)
 606 {
 607         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 608
 609         if (pvr_fence)
 610                 return pvr_fence_sync_is_signaled(pvr_fence,
 611                                                   PVRSRV_FENCE_FLAG_SUPPRESS_HWP_PKT);
 612         return false;
 613 }
 614
 615 static void
 616 pvr_fence_release(struct dma_fence *fence)
 617 {
 618         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 619         unsigned long flags;
 620
 621         if (pvr_fence) {
 622                 struct pvr_fence_context *fctx = pvr_fence->fctx;
 623
 624                 PVR_FENCE_TRACE(&pvr_fence->base, "released fence (%s)\n",
 625                                 pvr_fence->name);
 626                 trace_pvr_fence_release(pvr_fence);
 627
 628                 spin_lock_irqsave(&fctx->list_lock, flags);
 629                 list_move(&pvr_fence->fence_head,
 630                           &fctx->deferred_free_list);
 631                 spin_unlock_irqrestore(&fctx->list_lock, flags);
 632
 633                 kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
 634         }
 635 }
 636
 637 const struct dma_fence_ops pvr_fence_ops = {
 638         .get_driver_name = pvr_fence_get_driver_name,
 639         .get_timeline_name = pvr_fence_get_timeline_name,
 640         .fence_value_str = pvr_fence_fence_value_str,
 641         .timeline_value_str = pvr_fence_timeline_value_str,
 642         .enable_signaling = pvr_fence_enable_signaling,
 643         .signaled = pvr_fence_is_signaled,
 644         .wait = dma_fence_default_wait,
 645         .release = pvr_fence_release,
 646 };
 647
 648 /**
 649  * pvr_fence_create - creates a PVR fence
 650  * @fctx: PVR fence context on which the PVR fence should be created
 651  * @sync_checkpoint_ctx: context in which to create sync checkpoints
 652  * @timeline_fd: timeline on which the PVR fence should be created
 653  * @name: PVR fence name (used for debugging)
 654  *
 655  * Creates a PVR fence.
 656  *
 657  * Once the fence is finished with, pvr_fence_destroy should be called.
 658  *
 659  * Returns NULL if a PVR fence cannot be created.
 660  */
 661 struct pvr_fence *
 662 pvr_fence_create(struct pvr_fence_context *fctx,
 663                 struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
 664                 int timeline_fd, const char *name)
 665 {
 666         struct pvr_fence *pvr_fence;
 667         unsigned int seqno;
 668         unsigned long flags;
 669         PVRSRV_ERROR srv_err;
 670
 671         if (!try_module_get(THIS_MODULE))
 672                 goto err_exit;
 673
 674         /* Note: As kmem_cache is used to allocate pvr_fence objects,
 675          * make sure that all members of pvr_fence struct are initialized
 676          * here
 677          */
 678         pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
 679         if (unlikely(!pvr_fence))
 680                 goto err_module_put;
 681
 682         srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
 683                                       (PVRSRV_TIMELINE) timeline_fd, PVRSRV_NO_FENCE,
 684                                       name, &pvr_fence->sync_checkpoint);
 685         if (unlikely(srv_err != PVRSRV_OK))
 686                 goto err_free_fence;
 687
 688         INIT_LIST_HEAD(&pvr_fence->fence_head);
 689         INIT_LIST_HEAD(&pvr_fence->signal_head);
 690         pvr_fence->fctx = fctx;
 691         seqno = pvr_fence_context_seqno_next(fctx);
 692         /* Add the seqno to the fence name for easier debugging */
 693         pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
 694                         name, seqno);
 695
 696         /* Reset cb to zero */
 697         memset(&pvr_fence->cb, 0, sizeof(pvr_fence->cb));
 698         pvr_fence->fence = &pvr_fence->base;
 699
 700         dma_fence_init(&pvr_fence->base, &pvr_fence_ops, &fctx->lock,
 701                        fctx->fence_context, seqno);
 702
 703         spin_lock_irqsave(&fctx->list_lock, flags);
 704         list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
 705         spin_unlock_irqrestore(&fctx->list_lock, flags);
 706
 707         kref_get(&fctx->kref);
 708
 709         PVR_FENCE_TRACE(&pvr_fence->base, "created fence (%s)\n", name);
 710         trace_pvr_fence_create(pvr_fence);
 711
 712         return pvr_fence;
 713
 714 err_free_fence:
 715         kmem_cache_free(pvr_fence_cache, pvr_fence);
 716 err_module_put:
 717         module_put(THIS_MODULE);
 718 err_exit:
 719         return NULL;
 720 }
 721
 722 static const char *
 723 pvr_fence_foreign_get_driver_name(struct dma_fence *fence)
 724 {
 725         return PVR_LDM_DRIVER_REGISTRATION_NAME;
 726 }
 727
 728 static const char *
 729 pvr_fence_foreign_get_timeline_name(struct dma_fence *fence)
 730 {
 731         return "foreign";
 732 }
 733
 734 static
 735 void pvr_fence_foreign_fence_value_str(struct dma_fence *fence, char *str,
 736                                        int size)
 737 {
 738         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 739         u32 sync_addr = 0;
 740         u32 sync_value_next;
 741
 742         if (WARN_ON(!pvr_fence))
 743                 return;
 744
 745         sync_addr = SyncCheckpointGetFirmwareAddr(pvr_fence->sync_checkpoint);
 746         sync_value_next = PVRSRV_SYNC_CHECKPOINT_SIGNALLED;
 747
 748         /*
 749          * Include the fence flag bits from the foreign fence instead of our
 750          * shadow copy. This is done as the shadow fence flag bits aren't used.
 751          */
 752         snprintf(str, size,
 753                  "%llu: (%s%s) refs=%u fwaddr=%#08x cur=%#08x nxt=%#08x %s",
 754                  (u64) fence->seqno,
 755                  test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
 756                           &pvr_fence->fence->flags) ? "+" : "-",
 757                  test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 758                           &pvr_fence->fence->flags) ? "+" : "-",
 759                  refcount_read(&fence->refcount.refcount),
 760                  sync_addr,
 761                  pvr_fence_sync_value(pvr_fence),
 762                  sync_value_next,
 763                  pvr_fence->name);
 764 }
 765
 766 static
 767 void pvr_fence_foreign_timeline_value_str(struct dma_fence *fence, char *str,
 768                                           int size)
 769 {
 770         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 771
 772         if (pvr_fence)
 773                 pvr_context_value_str(pvr_fence->fctx, str, size);
 774 }
 775
 776 static bool
 777 pvr_fence_foreign_enable_signaling(struct dma_fence *fence)
 778 {
 779         WARN_ON("cannot enable signalling on foreign fence");
 780         return false;
 781 }
 782
 783 static signed long
 784 pvr_fence_foreign_wait(struct dma_fence *fence, bool intr, signed long timeout)
 785 {
 786         WARN_ON("cannot wait on foreign fence");
 787         return 0;
 788 }
 789
 790 static void
 791 pvr_fence_foreign_release(struct dma_fence *fence)
 792 {
 793         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 794         unsigned long flags;
 795
 796         if (pvr_fence) {
 797                 struct pvr_fence_context *fctx = pvr_fence->fctx;
 798                 struct dma_fence *foreign_fence = pvr_fence->fence;
 799
 800                 PVR_FENCE_TRACE(&pvr_fence->base,
 801                                 "released fence for foreign fence %llu#%d (%s)\n",
 802                                 (u64) pvr_fence->fence->context,
 803                                 pvr_fence->fence->seqno, pvr_fence->name);
 804                 trace_pvr_fence_foreign_release(pvr_fence);
 805
 806                 spin_lock_irqsave(&fctx->list_lock, flags);
 807                 list_move(&pvr_fence->fence_head,
 808                           &fctx->deferred_free_list);
 809                 spin_unlock_irqrestore(&fctx->list_lock, flags);
 810
 811                 dma_fence_put(foreign_fence);
 812
 813                 kref_put(&fctx->kref,
 814                          pvr_fence_context_destroy_kref);
 815         }
 816 }
 817
 818 const struct dma_fence_ops pvr_fence_foreign_ops = {
 819         .get_driver_name = pvr_fence_foreign_get_driver_name,
 820         .get_timeline_name = pvr_fence_foreign_get_timeline_name,
 821         .fence_value_str = pvr_fence_foreign_fence_value_str,
 822         .timeline_value_str = pvr_fence_foreign_timeline_value_str,
 823         .enable_signaling = pvr_fence_foreign_enable_signaling,
 824         .wait = pvr_fence_foreign_wait,
 825         .release = pvr_fence_foreign_release,
 826 };
 827
 828 static void
 829 pvr_fence_foreign_signal_sync(struct dma_fence *fence, struct dma_fence_cb *cb)
 830 {
 831         struct pvr_fence *pvr_fence = container_of(cb, struct pvr_fence, cb);
 832         struct pvr_fence_context *fctx = pvr_fence->fctx;
 833
 834         WARN_ON_ONCE(is_pvr_fence(fence));
 835
 836         /* Callback registered by dma_fence_add_callback can be called from an atomic ctx */
 837         pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_CTX_ATOMIC);
 838
 839         trace_pvr_fence_foreign_signal(pvr_fence);
 840
 841         queue_work(fctx->fence_wq, &fctx->check_status_work);
 842
 843         PVR_FENCE_TRACE(&pvr_fence->base,
 844                         "foreign fence %llu#%d signalled (%s)\n",
 845                         (u64) pvr_fence->fence->context,
 846                         pvr_fence->fence->seqno, pvr_fence->name);
 847
 848         /* Drop the reference on the base fence */
 849         dma_fence_put(&pvr_fence->base);
 850 }
 851
 852 /**
 853  * pvr_fence_create_from_fence - creates a PVR fence from a fence
 854  * @fctx: PVR fence context on which the PVR fence should be created
 855  * @sync_checkpoint_ctx: context in which to create sync checkpoints
 856  * @fence: fence from which the PVR fence should be created
 857  * @fence_fd: fd for the sync file to which the fence belongs. If it doesn't
 858  *            belong to a sync file then PVRSRV_NO_FENCE should be given
 859  *            instead.
 860  * @name: PVR fence name (used for debugging)
 861  *
 862  * Creates a PVR fence from an existing fence. If the fence is a foreign fence,
 863  * i.e. one that doesn't originate from a PVR fence context, then a new PVR
 864  * fence will be created using the specified sync_checkpoint_context.
 865  * Otherwise, a reference will be taken on the underlying fence and the PVR
 866  * fence will be returned.
 867  *
 868  * Once the fence is finished with, pvr_fence_destroy should be called.
 869  *
 870  * Returns NULL if a PVR fence cannot be created.
 871  */
 872
 873 struct pvr_fence *
 874 pvr_fence_create_from_fence(struct pvr_fence_context *fctx,
 875                             struct SYNC_CHECKPOINT_CONTEXT_TAG *sync_checkpoint_ctx,
 876                             struct dma_fence *fence,
 877                             PVRSRV_FENCE fence_fd,
 878                             const char *name)
 879 {
 880         struct pvr_fence *pvr_fence = to_pvr_fence(fence);
 881         unsigned int seqno;
 882         unsigned long flags;
 883         PVRSRV_ERROR srv_err;
 884         int err;
 885
 886         if (pvr_fence) {
 887                 if (WARN_ON(fence->ops == &pvr_fence_foreign_ops))
 888                         return NULL;
 889                 dma_fence_get(fence);
 890
 891                 PVR_FENCE_TRACE(fence, "created fence from PVR fence (%s)\n",
 892                                 name);
 893                 return pvr_fence;
 894         }
 895
 896         if (!try_module_get(THIS_MODULE))
 897                 goto err_exit;
 898
 899         /* Note: As kmem_cache is used to allocate pvr_fence objects,
 900          * make sure that all members of pvr_fence struct are initialized
 901          * here
 902          */
 903         pvr_fence = kmem_cache_alloc(pvr_fence_cache, GFP_KERNEL);
 904         if (!pvr_fence)
 905                 goto err_module_put;
 906
 907         srv_err = SyncCheckpointAlloc(sync_checkpoint_ctx,
 908                                           SYNC_CHECKPOINT_FOREIGN_CHECKPOINT,
 909                                           fence_fd,
 910                                           name, &pvr_fence->sync_checkpoint);
 911         if (srv_err != PVRSRV_OK)
 912                 goto err_free_pvr_fence;
 913
 914         INIT_LIST_HEAD(&pvr_fence->fence_head);
 915         INIT_LIST_HEAD(&pvr_fence->signal_head);
 916         pvr_fence->fctx = fctx;
 917         pvr_fence->fence = dma_fence_get(fence);
 918         seqno = pvr_fence_context_seqno_next(fctx);
 919         /* Add the seqno to the fence name for easier debugging */
 920         pvr_fence_prepare_name(pvr_fence->name, sizeof(pvr_fence->name),
 921                         name, seqno);
 922
 923         /*
 924          * We use the base fence to refcount the PVR fence and to do the
 925          * necessary clean up once the refcount drops to 0.
 926          */
 927         dma_fence_init(&pvr_fence->base, &pvr_fence_foreign_ops, &fctx->lock,
 928                        fctx->fence_context, seqno);
 929
 930         /*
 931          * Take an extra reference on the base fence that gets dropped when the
 932          * foreign fence is signalled.
 933          */
 934         dma_fence_get(&pvr_fence->base);
 935
 936         spin_lock_irqsave(&fctx->list_lock, flags);
 937         list_add_tail(&pvr_fence->fence_head, &fctx->fence_list);
 938         spin_unlock_irqrestore(&fctx->list_lock, flags);
 939         kref_get(&fctx->kref);
 940
 941         PVR_FENCE_TRACE(&pvr_fence->base,
 942                         "created fence from foreign fence %llu#%d (%s)\n",
 943                         (u64) pvr_fence->fence->context,
 944                         pvr_fence->fence->seqno, name);
 945
 946         err = dma_fence_add_callback(fence, &pvr_fence->cb,
 947                                      pvr_fence_foreign_signal_sync);
 948         if (err) {
 949                 if (err != -ENOENT) {
 950                         pr_err("%s: failed to add fence callback (err=%d)",
 951                                __func__, err);
 952                         goto err_put_ref;
 953                 }
 954
 955                 /*
 956                  * The fence has already signalled so set the sync as signalled.
 957                  * The "signalled" hwperf packet should be emitted because the
 958                  * callback won't be called for already signalled fence hence,
 959                  * PVRSRV_FENCE_FLAG_NONE flag.
 960                  */
 961                 pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
 962                 PVR_FENCE_TRACE(&pvr_fence->base,
 963                                 "foreign fence %llu#%d already signaled (%s)\n",
 964                                 (u64) pvr_fence->fence->context,
 965                                 pvr_fence->fence->seqno,
 966                                 name);
 967                 dma_fence_put(&pvr_fence->base);
 968         }
 969
 970         trace_pvr_fence_foreign_create(pvr_fence);
 971
 972         return pvr_fence;
 973
 974 err_put_ref:
 975         kref_put(&fctx->kref, pvr_fence_context_destroy_kref);
 976         spin_lock_irqsave(&fctx->list_lock, flags);
 977         list_del(&pvr_fence->fence_head);
 978         spin_unlock_irqrestore(&fctx->list_lock, flags);
 979         SyncCheckpointFree(pvr_fence->sync_checkpoint);
 980 err_free_pvr_fence:
 981         kmem_cache_free(pvr_fence_cache, pvr_fence);
 982 err_module_put:
 983         module_put(THIS_MODULE);
 984 err_exit:
 985         return NULL;
 986 }
 987
 988 /**
 989  * pvr_fence_destroy - destroys a PVR fence
 990  * @pvr_fence: PVR fence to destroy
 991  *
 992  * Destroys a PVR fence. Upon return, the PVR fence may still exist if something
 993  * else still references the underlying fence, e.g. a reservation object, or if
 994  * software signalling has been enabled and the fence hasn't yet been signalled.
 995  */
 996 void
 997 pvr_fence_destroy(struct pvr_fence *pvr_fence)
 998 {
 999         PVR_FENCE_TRACE(&pvr_fence->base, "destroyed fence (%s)\n",
1000                         pvr_fence->name);
1001
1002         dma_fence_put(&pvr_fence->base);
1003 }
1004
1005 /**
1006  * pvr_fence_sw_signal - signals a PVR fence sync
1007  * @pvr_fence: PVR fence to signal
1008  *
1009  * Sets the PVR fence sync value to signalled.
1010  *
1011  * Returns -EINVAL if the PVR fence represents a foreign fence.
1012  */
1013 int
1014 pvr_fence_sw_signal(struct pvr_fence *pvr_fence)
1015 {
1016         if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
1017                 return -EINVAL;
1018
1019         pvr_fence_sync_signal(pvr_fence, PVRSRV_FENCE_FLAG_NONE);
1020
1021         queue_work(pvr_fence->fctx->fence_wq,
1022                    &pvr_fence->fctx->check_status_work);
1023
1024         PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync signalled (%s)\n",
1025                         pvr_fence->name);
1026
1027         return 0;
1028 }
1029
1030 /**
1031  * pvr_fence_sw_error - errors the sync checkpoint backing a PVR fence
1032  * @pvr_fence: PVR fence to error
1033  *
1034  * Sets the PVR fence sync checkpoint value to errored.
1035  *
1036  * Returns -EINVAL if the PVR fence represents a foreign fence.
1037  */
1038 int
1039 pvr_fence_sw_error(struct pvr_fence *pvr_fence)
1040 {
1041         if (!is_our_fence(pvr_fence->fctx, &pvr_fence->base))
1042                 return -EINVAL;
1043
1044         SyncCheckpointError(pvr_fence->sync_checkpoint, PVRSRV_FENCE_FLAG_NONE);
1045         PVR_FENCE_TRACE(&pvr_fence->base, "sw set fence sync errored (%s)\n",
1046                         pvr_fence->name);
1047
1048         return 0;
1049 }
1050
1051 int
1052 pvr_fence_get_checkpoints(struct pvr_fence **pvr_fences, u32 nr_fences,
1053                           struct SYNC_CHECKPOINT_TAG **fence_checkpoints)
1054 {
1055         struct SYNC_CHECKPOINT_TAG **next_fence_checkpoint = fence_checkpoints;
1056         struct pvr_fence **next_pvr_fence = pvr_fences;
1057         int fence_checkpoint_idx;
1058
1059         if (nr_fences > 0) {
1060
1061                 for (fence_checkpoint_idx = 0; fence_checkpoint_idx < nr_fences;
1062                      fence_checkpoint_idx++) {
1063                         struct pvr_fence *next_fence = *next_pvr_fence++;
1064                         *next_fence_checkpoint++ = next_fence->sync_checkpoint;
1065                         /* Take reference on sync checkpoint (will be dropped
1066                          * later by kick code)
1067                          */
1068                         SyncCheckpointTakeRef(next_fence->sync_checkpoint);
1069                 }
1070         }
1071
1072         return 0;
1073 }
1074
1075 struct SYNC_CHECKPOINT_TAG *
1076 pvr_fence_get_checkpoint(struct pvr_fence *update_fence)
1077 {
1078         return update_fence->sync_checkpoint;
1079 }
1080
1081 /**
1082  * pvr_fence_dump_info_on_stalled_ufos - displays debug
1083  * information on a native fence associated with any of
1084  * the ufos provided. This function will be called from
1085  * pvr_sync_file.c if the driver determines any GPU work
1086  * is stuck waiting for a sync checkpoint representing a
1087  * foreign sync to be signalled.
1088  * @nr_ufos: number of ufos in vaddrs
1089  * @vaddrs:  array of FW addresses of UFOs which the
1090  *           driver is waiting on.
1091  *
1092  * Output debug information to kernel log on linux fences
1093  * which would be responsible for signalling the sync
1094  * checkpoints indicated by the ufo vaddresses.
1095  *
1096  * Returns the number of ufos in the array which were found
1097  * to be associated with foreign syncs.
1098  */
1099 u32 pvr_fence_dump_info_on_stalled_ufos(struct pvr_fence_context *fctx,
1100                                         u32 nr_ufos, u32 *vaddrs)
1101 {
1102         int our_ufo_ct = 0;
1103         struct pvr_fence *pvr_fence;
1104         unsigned long flags;
1105
1106         spin_lock_irqsave(&fctx->list_lock, flags);
1107         /* dump info on any ufos in our active list */
1108         list_for_each_entry(pvr_fence, &fctx->fence_list, fence_head) {
1109                 u32 *this_ufo_vaddr = vaddrs;
1110                 int ufo_num;
1111                 DUMPDEBUG_PRINTF_FUNC *pfnDummy = NULL;
1112
1113                 for (ufo_num = 0; ufo_num < nr_ufos; ufo_num++) {
1114                         struct SYNC_CHECKPOINT_TAG *checkpoint =
1115                                 pvr_fence->sync_checkpoint;
1116                         const u32 fence_ufo_addr =
1117                                 SyncCheckpointGetFirmwareAddr(checkpoint);
1118
1119                         if (fence_ufo_addr != this_ufo_vaddr[ufo_num])
1120                                 continue;
1121
1122                         /* Dump sync info */
1123                         PVR_DUMPDEBUG_LOG(pfnDummy, NULL,
1124                                           "\tSyncID = %d, FWAddr = 0x%08x: TLID = %d (Foreign Fence - [%p] %s)",
1125                                           SyncCheckpointGetId(checkpoint),
1126                                           fence_ufo_addr,
1127                                           SyncCheckpointGetTimeline(checkpoint),
1128                                           pvr_fence->fence,
1129                                           pvr_fence->name);
1130                         our_ufo_ct++;
1131                 }
1132         }
1133         spin_unlock_irqrestore(&fctx->list_lock, flags);
1134         return our_ufo_ct;
1135 }