drivers/gpu/drm/img/img-rogue/services/server/env/linux/pvr_sync_file.c

   1 /*
   2  * @File        pvr_sync_file.c
   3  * @Title       Kernel driver for Android's sync mechanism
   4  * @Codingstyle LinuxKernel
   5  * @Copyright   Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   6  * @License     Dual MIT/GPLv2
   7  *
   8  * The contents of this file are subject to the MIT license as set out below.
   9  *
  10  * Permission is hereby granted, free of charge, to any person obtaining a copy
  11  * of this software and associated documentation files (the "Software"), to deal
  12  * in the Software without restriction, including without limitation the rights
  13  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14  * copies of the Software, and to permit persons to whom the Software is
  15  * furnished to do so, subject to the following conditions:
  16  *
  17  * The above copyright notice and this permission notice shall be included in
  18  * all copies or substantial portions of the Software.
  19  *
  20  * Alternatively, the contents of this file may be used under the terms of
  21  * the GNU General Public License Version 2 ("GPL") in which case the provisions
  22  * of GPL are applicable instead of those above.
  23  *
  24  * If you wish to allow use of your version of this file only under the terms of
  25  * GPL, and not to allow others to use your version of this file under the terms
  26  * of the MIT license, indicate your decision by deleting the provisions above
  27  * and replace them with the notice and other provisions required by GPL as set
  28  * out in the file called "GPL-COPYING" included in this distribution. If you do
  29  * not delete the provisions above, a recipient may use your version of this file
  30  * under the terms of either the MIT license or GPL.
  31  *
  32  * This License is also included in this distribution in the file called
  33  * "MIT-COPYING".
  34  *
  35  * EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36  * PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37  * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38  * PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39  * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40  * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42  */
  43
  44 #include "services_kernel_client.h"
  45 #include "pvr_drv.h"
  46 #include "pvr_sync.h"
  47 #include "pvr_fence.h"
  48 #include "pvr_counting_timeline.h"
  49
  50 #include "linux_sw_sync.h"
  51
  52 #include <linux/version.h>
  53 #include <linux/slab.h>
  54 #include <linux/fs.h>
  55 #include <linux/sync_file.h>
  56 #include <linux/file.h>
  57 #include <linux/miscdevice.h>
  58 #include <linux/uaccess.h>
  59
  60 #include "pvr_sync_api.h"
  61
  62 /* This header must always be included last */
  63 #include "kernel_compatibility.h"
  64
  65 #if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0)) && !defined(CHROMIUMOS_KERNEL)
  66 #define sync_file_user_name(s)  ((s)->name)
  67 #else
  68 #define sync_file_user_name(s)  ((s)->user_name)
  69 #endif
  70
  71 #define PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile, fmt, ...) \
  72         do {                                                             \
  73                 if (pfnDumpDebugPrintf)                                  \
  74                         pfnDumpDebugPrintf(pvDumpDebugFile, fmt,         \
  75                                            ## __VA_ARGS__);              \
  76                 else                                                     \
  77                         pr_err(fmt "\n", ## __VA_ARGS__);                \
  78         } while (0)
  79
  80 #define FILE_NAME "pvr_sync_file"
  81
  82 struct sw_sync_create_fence_data {
  83         __u32 value;
  84         char name[32];
  85         __s32 fence;
  86 };
  87 #define SW_SYNC_IOC_MAGIC 'W'
  88 #define SW_SYNC_IOC_CREATE_FENCE \
  89         (_IOWR(SW_SYNC_IOC_MAGIC, 0, struct sw_sync_create_fence_data))
  90 #define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32)
  91
  92 /* Global data for the sync driver */
  93 static struct {
  94         struct pvr_fence_context *foreign_fence_context;
  95         PFN_SYNC_CHECKPOINT_STRUCT sync_checkpoint_ops;
  96 } pvr_sync_data;
  97
  98 #if defined(NO_HARDWARE)
  99 static DEFINE_MUTEX(pvr_timeline_active_list_lock);
 100 static struct list_head pvr_timeline_active_list;
 101 #endif
 102
 103 /* This is the actual timeline metadata. We might keep this around after the
 104  * base sync driver has destroyed the pvr_sync_timeline_wrapper object.
 105  */
 106 struct pvr_sync_timeline {
 107         char name[32];
 108         void *file_handle;
 109         bool is_sw;
 110         /* Fence context used for hw fences */
 111         struct pvr_fence_context *hw_fence_context;
 112         /* Timeline and context for sw fences */
 113         struct pvr_counting_fence_timeline *sw_fence_timeline;
 114 #if defined(NO_HARDWARE)
 115         /* List of all timelines (used to advance all timelines in nohw builds) */
 116         struct list_head list;
 117 #endif
 118 };
 119
 120 static
 121 void pvr_sync_free_checkpoint_list_mem(void *mem_ptr)
 122 {
 123         kfree(mem_ptr);
 124 }
 125
 126 #if defined(NO_HARDWARE)
 127 /* function used to signal pvr fence in nohw builds */
 128 static
 129 void pvr_sync_nohw_signal_fence(void *fence_data_to_signal)
 130 {
 131         struct pvr_sync_timeline *this_timeline;
 132
 133         mutex_lock(&pvr_timeline_active_list_lock);
 134         list_for_each_entry(this_timeline, &pvr_timeline_active_list, list) {
 135                 pvr_fence_context_signal_fences_nohw(this_timeline->hw_fence_context);
 136         }
 137         mutex_unlock(&pvr_timeline_active_list_lock);
 138 }
 139 #endif
 140
 141 static struct pvr_sync_timeline *pvr_sync_timeline_fget(int fd)
 142 {
 143         struct file *file = fget(fd);
 144         struct pvr_sync_timeline *timeline;
 145
 146         if (!file)
 147                 return NULL;
 148
 149         timeline = pvr_sync_get_api_priv(file);
 150         if (!timeline)
 151                 fput(file);
 152
 153         return timeline;
 154 }
 155
 156 static void pvr_sync_timeline_fput(struct pvr_sync_timeline *timeline)
 157 {
 158         struct file *file = pvr_sync_get_file_struct(timeline->file_handle);
 159
 160         if (file)
 161                 fput(file);
 162         else
 163                 pr_err(FILE_NAME ": %s: Timeline incomplete\n", __func__);
 164 }
 165
 166 /* ioctl and fops handling */
 167
 168 int pvr_sync_api_init(void *file_handle, void **api_priv)
 169 {
 170         struct pvr_sync_timeline *timeline;
 171         char task_comm[TASK_COMM_LEN];
 172
 173         get_task_comm(task_comm, current);
 174
 175         timeline = kzalloc(sizeof(*timeline), GFP_KERNEL);
 176         if (!timeline)
 177                 return -ENOMEM;
 178
 179         strlcpy(timeline->name, task_comm, sizeof(timeline->name));
 180         timeline->file_handle = file_handle;
 181         timeline->is_sw = false;
 182
 183         *api_priv = (void *)timeline;
 184
 185         return 0;
 186 }
 187
 188 int pvr_sync_api_deinit(void *api_priv, bool is_sw)
 189 {
 190         struct pvr_sync_timeline *timeline = api_priv;
 191
 192         if (!timeline)
 193                 return 0;
 194
 195         if (timeline->sw_fence_timeline) {
 196                 /* This makes sure any outstanding SW syncs are marked as
 197                  * complete at timeline close time. Otherwise it'll leak the
 198                  * timeline (as outstanding fences hold a ref) and possibly
 199                  * wedge the system if something is waiting on one of those
 200                  * fences
 201                  */
 202                 pvr_counting_fence_timeline_force_complete(
 203                         timeline->sw_fence_timeline);
 204                 pvr_counting_fence_timeline_put(timeline->sw_fence_timeline);
 205         }
 206
 207         if (timeline->hw_fence_context) {
 208 #if defined(NO_HARDWARE)
 209                 mutex_lock(&pvr_timeline_active_list_lock);
 210                 list_del(&timeline->list);
 211                 mutex_unlock(&pvr_timeline_active_list_lock);
 212 #endif
 213                 pvr_fence_context_destroy(timeline->hw_fence_context);
 214         }
 215
 216         kfree(timeline);
 217
 218         return 0;
 219 }
 220
 221 /*
 222  * This is the function that kick code will call in order to 'finalise' a
 223  * created output fence just prior to returning from the kick function.
 224  * The OS native sync code needs to implement a function meeting this
 225  * specification - the implementation may be a nop if the OS does not need
 226  * to perform any actions at this point.
 227  *
 228  * Input: fence_fd            The PVRSRV_FENCE to be 'finalised'. This value
 229  *                            will have been returned by an earlier call to
 230  *                            pvr_sync_create_fence().
 231  * Input: finalise_data       The finalise data returned by an earlier call
 232  *                            to pvr_sync_create_fence().
 233  */
 234 static enum PVRSRV_ERROR_TAG
 235 pvr_sync_finalise_fence(PVRSRV_FENCE fence_fd, void *finalise_data)
 236 {
 237         struct sync_file *sync_file = finalise_data;
 238         struct pvr_fence *pvr_fence;
 239
 240         if (!sync_file || (fence_fd < 0)) {
 241                 pr_err(FILE_NAME ": %s: Invalid input fence\n", __func__);
 242                 return PVRSRV_ERROR_INVALID_PARAMS;
 243         }
 244
 245         pvr_fence = to_pvr_fence(sync_file->fence);
 246
 247         if (!pvr_fence) {
 248                 pr_err(FILE_NAME ": %s: Fence not a pvr fence\n", __func__);
 249                 return PVRSRV_ERROR_INVALID_PARAMS;
 250         }
 251
 252         /* pvr fences can be signalled any time after creation */
 253         dma_fence_enable_sw_signaling(&pvr_fence->base);
 254
 255         fd_install(fence_fd, sync_file->file);
 256
 257         return PVRSRV_OK;
 258 }
 259
 260 /*
 261  * This is the function that kick code will call in order to obtain a new
 262  * PVRSRV_FENCE from the OS native sync code and the PSYNC_CHECKPOINT used
 263  * in that fence. The OS native sync code needs to implement a function
 264  * meeting this specification.
 265  *
 266  * Input: device                   Device node to use in creating a hw_fence_ctx
 267  * Input: fence_name               A string to annotate the fence with (for
 268  *                                 debug).
 269  * Input: timeline                 The timeline on which the new fence is to be
 270  *                                 created.
 271  * Output: new_fence               The new PVRSRV_FENCE to be returned by the
 272  *                                 kick call.
 273  * Output: fence_uid               Unique ID of the update fence.
 274  * Output: fence_finalise_data     Pointer to data needed to finalise the fence.
 275  * Output: new_checkpoint_handle   The PSYNC_CHECKPOINT used by the new fence.
 276  */
 277 static enum PVRSRV_ERROR_TAG
 278 pvr_sync_create_fence(
 279                       struct _PVRSRV_DEVICE_NODE_ *device,
 280                       const char *fence_name,
 281                       PVRSRV_TIMELINE new_fence_timeline,
 282                       PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
 283                       PVRSRV_FENCE *new_fence, u64 *fence_uid,
 284                       void **fence_finalise_data,
 285                       PSYNC_CHECKPOINT *new_checkpoint_handle,
 286                       void **timeline_update_sync,
 287                       __u32 *timeline_update_value)
 288 {
 289         PVRSRV_ERROR err = PVRSRV_OK;
 290         PVRSRV_FENCE new_fence_fd = -1;
 291         struct pvr_sync_timeline *timeline;
 292         struct pvr_fence *pvr_fence;
 293         PSYNC_CHECKPOINT checkpoint;
 294         struct sync_file *sync_file;
 295
 296         if (new_fence_timeline < 0 || !new_fence || !new_checkpoint_handle
 297                 || !fence_finalise_data) {
 298                 pr_err(FILE_NAME ": %s: Invalid input params\n", __func__);
 299                 err =  PVRSRV_ERROR_INVALID_PARAMS;
 300                 goto err_out;
 301         }
 302
 303         /* We reserve the new fence FD before taking any operations
 304          * as we do not want to fail (e.g. run out of FDs)
 305          */
 306         new_fence_fd = get_unused_fd_flags(O_CLOEXEC);
 307         if (new_fence_fd < 0) {
 308                 pr_err(FILE_NAME ": %s: Failed to get fd\n", __func__);
 309                 err = PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
 310                 goto err_out;
 311         }
 312
 313         timeline = pvr_sync_timeline_fget(new_fence_timeline);
 314         if (!timeline) {
 315                 pr_err(FILE_NAME ": %s: Failed to open supplied timeline fd (%d)\n",
 316                         __func__, new_fence_timeline);
 317                 err = PVRSRV_ERROR_INVALID_PARAMS;
 318                 goto err_put_fd;
 319         }
 320
 321         if (timeline->is_sw) {
 322                 /* This should never happen! */
 323                 pr_err(FILE_NAME ": %s: Request to create a pvr fence on sw timeline (%d)\n",
 324                         __func__, new_fence_timeline);
 325                 err = PVRSRV_ERROR_INVALID_PARAMS;
 326                 goto err_put_timeline;
 327         }
 328
 329         if (!timeline->hw_fence_context) {
 330                 /* First time we use this timeline, so create a context. */
 331                 timeline->hw_fence_context =
 332                         pvr_fence_context_create(
 333                                 device,
 334                                 NativeSyncGetFenceStatusWq(),
 335                                 timeline->name);
 336                 if (!timeline->hw_fence_context) {
 337                         pr_err(FILE_NAME ": %s: Failed to create fence context (%d)\n",
 338                                __func__, new_fence_timeline);
 339                         err = PVRSRV_ERROR_OUT_OF_MEMORY;
 340                         goto err_put_timeline;
 341                 }
 342 #if defined(NO_HARDWARE)
 343                 /* Add timeline to active list */
 344                 INIT_LIST_HEAD(&timeline->list);
 345                 mutex_lock(&pvr_timeline_active_list_lock);
 346                 list_add_tail(&timeline->list, &pvr_timeline_active_list);
 347                 mutex_unlock(&pvr_timeline_active_list_lock);
 348 #endif
 349         }
 350
 351         pvr_fence = pvr_fence_create(timeline->hw_fence_context,
 352                                                                  psSyncCheckpointContext,
 353                                                                  new_fence_timeline,
 354                                                                  fence_name);
 355         if (!pvr_fence) {
 356                 pr_err(FILE_NAME ": %s: Failed to create new pvr_fence\n",
 357                         __func__);
 358                 err = PVRSRV_ERROR_OUT_OF_MEMORY;
 359                 goto err_put_timeline;
 360         }
 361
 362         checkpoint = pvr_fence_get_checkpoint(pvr_fence);
 363         if (!checkpoint) {
 364                 pr_err(FILE_NAME ": %s: Failed to get fence checkpoint\n",
 365                         __func__);
 366                 err = PVRSRV_ERROR_OUT_OF_MEMORY;
 367                 goto err_destroy_fence;
 368         }
 369
 370         sync_file = sync_file_create(&pvr_fence->base);
 371         if (!sync_file) {
 372                 pr_err(FILE_NAME ": %s: Failed to create sync_file\n",
 373                         __func__);
 374                 err = PVRSRV_ERROR_OUT_OF_MEMORY;
 375                 goto err_destroy_fence;
 376         }
 377         strlcpy(sync_file_user_name(sync_file),
 378                 pvr_fence->name,
 379                 sizeof(sync_file_user_name(sync_file)));
 380         dma_fence_put(&pvr_fence->base);
 381
 382         *new_fence = new_fence_fd;
 383         *fence_finalise_data = sync_file;
 384         *new_checkpoint_handle = checkpoint;
 385         *fence_uid = OSGetCurrentClientProcessIDKM();
 386         *fence_uid = (*fence_uid << 32) | (new_fence_fd & U32_MAX);
 387         /* not used but don't want to return dangling pointers */
 388         *timeline_update_sync = NULL;
 389         *timeline_update_value = 0;
 390
 391         pvr_sync_timeline_fput(timeline);
 392 err_out:
 393         return err;
 394
 395 err_destroy_fence:
 396         pvr_fence_destroy(pvr_fence);
 397 err_put_timeline:
 398         pvr_sync_timeline_fput(timeline);
 399 err_put_fd:
 400         put_unused_fd(new_fence_fd);
 401         *fence_uid = PVRSRV_NO_FENCE;
 402         goto err_out;
 403 }
 404
 405 /*
 406  * This is the function that kick code will call in order to 'rollback' a
 407  * created output fence should an error occur when submitting the kick.
 408  * The OS native sync code needs to implement a function meeting this
 409  * specification.
 410  *
 411  * Input: fence_to_rollback The PVRSRV_FENCE to be 'rolled back'. The fence
 412  *                          should be destroyed and any actions taken due to
 413  *                          its creation that need to be undone should be
 414  *                          reverted.
 415  * Input: finalise_data     The finalise data for the fence to be 'rolled back'.
 416  */
 417 static enum PVRSRV_ERROR_TAG
 418 pvr_sync_rollback_fence_data(PVRSRV_FENCE fence_to_rollback,
 419                              void *fence_data_to_rollback)
 420 {
 421         struct sync_file *sync_file = fence_data_to_rollback;
 422         struct pvr_fence *pvr_fence;
 423
 424         if (!sync_file || fence_to_rollback < 0) {
 425                 pr_err(FILE_NAME ": %s: Invalid fence (%d)\n", __func__,
 426                         fence_to_rollback);
 427                 return PVRSRV_ERROR_INVALID_PARAMS;
 428         }
 429
 430         pvr_fence = to_pvr_fence(sync_file->fence);
 431         if (!pvr_fence) {
 432                 pr_err(FILE_NAME
 433                         ": %s: Non-PVR fence (%p)\n",
 434                         __func__, sync_file->fence);
 435                 return PVRSRV_ERROR_INVALID_PARAMS;
 436         }
 437
 438         fput(sync_file->file);
 439
 440         put_unused_fd(fence_to_rollback);
 441
 442         return PVRSRV_OK;
 443 }
 444
 445 /*
 446  * This is the function that kick code will call in order to obtain a list of
 447  * the PSYNC_CHECKPOINTs for a given PVRSRV_FENCE passed to a kick function.
 448  * The OS native sync code will allocate the memory to hold the returned list
 449  * of PSYNC_CHECKPOINT ptrs. The caller will free this memory once it has
 450  * finished referencing it.
 451  *
 452  * Input: fence                     The input (check) fence
 453  * Output: nr_checkpoints           The number of PVRSRV_SYNC_CHECKPOINT ptrs
 454  *                                  returned in the checkpoint_handles
 455  *                                  parameter.
 456  * Output: fence_uid                Unique ID of the check fence
 457  * Input/Output: checkpoint_handles The returned list of PVRSRV_SYNC_CHECKPOINTs.
 458  */
 459 static enum PVRSRV_ERROR_TAG
 460 pvr_sync_resolve_fence(PSYNC_CHECKPOINT_CONTEXT psSyncCheckpointContext,
 461                        PVRSRV_FENCE fence_to_resolve, u32 *nr_checkpoints,
 462                        PSYNC_CHECKPOINT **checkpoint_handles, u64 *fence_uid)
 463 {
 464         PSYNC_CHECKPOINT *checkpoints = NULL;
 465         unsigned int i, num_fences = 0, num_used_fences = 0;
 466         struct dma_fence **fences = NULL;
 467         struct dma_fence *fence;
 468         PVRSRV_ERROR err = PVRSRV_OK;
 469
 470         if (!nr_checkpoints || !checkpoint_handles || !fence_uid) {
 471                 pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
 472                         __func__);
 473                 err =  PVRSRV_ERROR_INVALID_PARAMS;
 474                 goto err_out;
 475         }
 476
 477         *nr_checkpoints = 0;
 478         *checkpoint_handles = NULL;
 479         *fence_uid = 0;
 480
 481         if (fence_to_resolve < 0)
 482                 goto err_out;
 483
 484         fence = sync_file_get_fence(fence_to_resolve);
 485         if (!fence) {
 486                 pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
 487                         __func__, fence_to_resolve);
 488                 err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
 489                 goto err_out;
 490         }
 491
 492         if (dma_fence_is_array(fence)) {
 493                 struct dma_fence_array *array = to_dma_fence_array(fence);
 494
 495                 if (array) {
 496                         fences = array->fences;
 497                         num_fences = array->num_fences;
 498                 }
 499         } else {
 500                 fences = &fence;
 501                 num_fences = 1;
 502         }
 503
 504         checkpoints = kmalloc_array(num_fences, sizeof(PSYNC_CHECKPOINT),
 505                               GFP_KERNEL);
 506         if (!checkpoints) {
 507                 err = PVRSRV_ERROR_OUT_OF_MEMORY;
 508                 goto err_put_fence;
 509         }
 510         for (i = 0; i < num_fences; i++) {
 511                 /*
 512                  * Only return the checkpoint if the fence is still active.
 513                  * Don't checked for signalled on PDUMP drivers as we need
 514                  * to make sure that all fences make it to the pdump.
 515                  */
 516 #if !defined(PDUMP)
 517                 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
 518                               &fences[i]->flags))
 519 #endif
 520                 {
 521                         struct pvr_fence *pvr_fence =
 522                                 pvr_fence_create_from_fence(
 523                                         pvr_sync_data.foreign_fence_context,
 524                                         psSyncCheckpointContext,
 525                                         fences[i],
 526                                         fence_to_resolve,
 527                                         "foreign");
 528                         if (!pvr_fence) {
 529                                 pr_err(FILE_NAME ": %s: Failed to create fence\n",
 530                                        __func__);
 531                                 err = PVRSRV_ERROR_OUT_OF_MEMORY;
 532                                 goto err_free_checkpoints;
 533                         }
 534                         checkpoints[num_used_fences] =
 535                                 pvr_fence_get_checkpoint(pvr_fence);
 536                         SyncCheckpointTakeRef(checkpoints[num_used_fences]);
 537                         ++num_used_fences;
 538                         dma_fence_put(&pvr_fence->base);
 539                 }
 540         }
 541         /* If we don't return any checkpoints, delete the array because
 542          * the caller will not.
 543          */
 544         if (num_used_fences == 0) {
 545                 kfree(checkpoints);
 546                 checkpoints = NULL;
 547         }
 548
 549         *checkpoint_handles = checkpoints;
 550         *nr_checkpoints = num_used_fences;
 551         *fence_uid = OSGetCurrentClientProcessIDKM();
 552         *fence_uid = (*fence_uid << 32) | (fence_to_resolve & U32_MAX);
 553
 554 err_put_fence:
 555         dma_fence_put(fence);
 556 err_out:
 557         return err;
 558
 559 err_free_checkpoints:
 560         for (i = 0; i < num_used_fences; i++) {
 561                 if (checkpoints[i])
 562                         SyncCheckpointDropRef(checkpoints[i]);
 563         }
 564         kfree(checkpoints);
 565         goto err_put_fence;
 566 }
 567
 568 /*
 569  * This is the function that driver code will call in order to request the
 570  * sync implementation to output debug information relating to any sync
 571  * checkpoints it may have created which appear in the provided array of
 572  * FW addresses of Unified Fence Objects (UFOs).
 573  *
 574  * Input: nr_ufos             The number of FW addresses provided in the
 575  *                            vaddrs parameter.
 576  * Input: vaddrs              The array of FW addresses of UFOs. The sync
 577  *                            implementation should check each of these to
 578  *                            see if any relate to sync checkpoints it has
 579  *                            created and where they do output debug information
 580  *                            pertaining to the native/fallback sync with
 581  *                            which it is associated.
 582  */
 583 static u32
 584 pvr_sync_dump_info_on_stalled_ufos(u32 nr_ufos, u32 *vaddrs)
 585 {
 586         return pvr_fence_dump_info_on_stalled_ufos(pvr_sync_data.foreign_fence_context,
 587                                                    nr_ufos,
 588                                                    vaddrs);
 589 }
 590
 591 #if defined(PDUMP)
 592 static enum PVRSRV_ERROR_TAG
 593 pvr_sync_fence_get_checkpoints(PVRSRV_FENCE fence_to_pdump, u32 *nr_checkpoints,
 594                                 struct SYNC_CHECKPOINT_TAG ***checkpoint_handles)
 595 {
 596         struct dma_fence **fences = NULL;
 597         struct dma_fence *fence;
 598         struct pvr_fence *pvr_fence;
 599         struct SYNC_CHECKPOINT_TAG **checkpoints = NULL;
 600         unsigned int i, num_fences, num_used_fences = 0;
 601         enum PVRSRV_ERROR_TAG err;
 602
 603         if (fence_to_pdump < 0) {
 604                 err =  PVRSRV_ERROR_INVALID_PARAMS;
 605                 goto err_out;
 606         }
 607
 608         if (!nr_checkpoints || !checkpoint_handles) {
 609                 pr_err(FILE_NAME ": %s: Invalid input checkpoint pointer\n",
 610                         __func__);
 611                 err =  PVRSRV_ERROR_INVALID_PARAMS;
 612                 goto err_out;
 613         }
 614
 615         fence = sync_file_get_fence(fence_to_pdump);
 616         if (!fence) {
 617                 pr_err(FILE_NAME ": %s: Failed to read sync private data for fd %d\n",
 618                         __func__, fence_to_pdump);
 619                 err = PVRSRV_ERROR_HANDLE_NOT_FOUND;
 620                 goto err_out;
 621         }
 622
 623         if (dma_fence_is_array(fence)) {
 624                 struct dma_fence_array *array = to_dma_fence_array(fence);
 625
 626                 fences = array->fences;
 627                 num_fences = array->num_fences;
 628         } else {
 629                 fences = &fence;
 630                 num_fences = 1;
 631         }
 632
 633         checkpoints = kmalloc_array(num_fences, sizeof(*checkpoints),
 634                               GFP_KERNEL);
 635         if (!checkpoints) {
 636                 pr_err("pvr_sync_file: %s: Failed to alloc memory for returned list of sync checkpoints\n",
 637                         __func__);
 638                 err = PVRSRV_ERROR_OUT_OF_MEMORY;
 639                 goto err_put_fence;
 640         }
 641
 642         for (i = 0; i < num_fences; i++) {
 643                 pvr_fence = to_pvr_fence(fences[i]);
 644                 if (!pvr_fence)
 645                         continue;
 646                 checkpoints[num_used_fences] = pvr_fence_get_checkpoint(pvr_fence);
 647                 ++num_used_fences;
 648         }
 649
 650         *checkpoint_handles = checkpoints;
 651         *nr_checkpoints = num_used_fences;
 652         err =  PVRSRV_OK;
 653
 654 err_put_fence:
 655         dma_fence_put(fence);
 656 err_out:
 657         return err;
 658 }
 659 #endif
 660
 661 int pvr_sync_api_rename(void *api_priv, void *user_data)
 662 {
 663         struct pvr_sync_timeline *timeline = api_priv;
 664         struct pvr_sync_rename_ioctl_data *data = user_data;
 665
 666         data->szName[sizeof(data->szName) - 1] = '\0';
 667         strlcpy(timeline->name, data->szName, sizeof(timeline->name));
 668         if (timeline->hw_fence_context)
 669                 strlcpy(timeline->hw_fence_context->name, data->szName,
 670                         sizeof(timeline->hw_fence_context->name));
 671
 672         return 0;
 673 }
 674
 675 int pvr_sync_api_force_sw_only(void *api_priv, void **api_priv_new)
 676 {
 677         struct pvr_sync_timeline *timeline = api_priv;
 678
 679         /* Already in SW mode? */
 680         if (timeline->sw_fence_timeline)
 681                 return 0;
 682
 683         /* Create a sw_sync timeline with the old GPU timeline's name */
 684         timeline->sw_fence_timeline = pvr_counting_fence_timeline_create(
 685                 timeline->name);
 686         if (!timeline->sw_fence_timeline)
 687                 return -ENOMEM;
 688
 689         timeline->is_sw = true;
 690
 691         return 0;
 692 }
 693
 694 int pvr_sync_api_sw_create_fence(void *api_priv, void *user_data)
 695 {
 696         struct pvr_sync_timeline *timeline = api_priv;
 697         struct pvr_sw_sync_create_fence_data *data = user_data;
 698         struct sync_file *sync_file;
 699         int fd = get_unused_fd_flags(O_CLOEXEC);
 700         struct dma_fence *fence;
 701         int err;
 702
 703         if (fd < 0) {
 704                 pr_err(FILE_NAME ": %s: Failed to find unused fd (%d)\n",
 705                        __func__, fd);
 706                 err = -EMFILE;
 707                 goto err_out;
 708         }
 709
 710         fence = pvr_counting_fence_create(timeline->sw_fence_timeline, &data->sync_pt_idx);
 711         if (!fence) {
 712                 pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
 713                        __func__, fd);
 714                 err = -ENOMEM;
 715                 goto err_put_fd;
 716         }
 717
 718         sync_file = sync_file_create(fence);
 719         dma_fence_put(fence);
 720         if (!sync_file) {
 721                 pr_err(FILE_NAME ": %s: Failed to create a sync point (%d)\n",
 722                         __func__, fd);
 723                 err = -ENOMEM;
 724                 goto err_put_fd;
 725         }
 726
 727         data->fence = fd;
 728
 729         fd_install(fd, sync_file->file);
 730
 731         return 0;
 732
 733 err_put_fd:
 734         put_unused_fd(fd);
 735 err_out:
 736         return err;
 737 }
 738
 739 int pvr_sync_api_sw_inc(void *api_priv, void *user_data)
 740 {
 741         struct pvr_sync_timeline *timeline = api_priv;
 742         struct pvr_sw_timeline_advance_data *data = user_data;
 743         bool res;
 744
 745         res = pvr_counting_fence_timeline_inc(timeline->sw_fence_timeline, &data->sync_pt_idx);
 746
 747         /* pvr_counting_fence_timeline_inc won't allow sw timeline to be
 748          * advanced beyond the last defined point
 749          */
 750         if (!res) {
 751                 pr_err("pvr_sync_file: attempt to advance SW timeline beyond last defined point\n");
 752                 return -EPERM;
 753         }
 754
 755         return 0;
 756 }
 757
 758 static void
 759 pvr_sync_debug_request_heading(void *data, u32 verbosity,
 760                                 DUMPDEBUG_PRINTF_FUNC *pfnDumpDebugPrintf,
 761                                 void *pvDumpDebugFile)
 762 {
 763         if (DD_VERB_LVL_ENABLED(verbosity, DEBUG_REQUEST_VERBOSITY_MEDIUM))
 764                 PVR_DUMPDEBUG_LOG(pfnDumpDebugPrintf, pvDumpDebugFile,
 765                                   "------[ Native Fence Sync: timelines ]------");
 766 }
 767
 768 enum PVRSRV_ERROR_TAG pvr_sync_register_functions(void)
 769 {
 770         /* Register the resolve fence and create fence functions with
 771          * sync_checkpoint.c
 772          * The pvr_fence context registers its own EventObject callback to
 773          * update sync status
 774          */
 775         /* Initialise struct and register with sync_checkpoint.c */
 776         pvr_sync_data.sync_checkpoint_ops.pfnFenceResolve = pvr_sync_resolve_fence;
 777         pvr_sync_data.sync_checkpoint_ops.pfnFenceCreate = pvr_sync_create_fence;
 778         pvr_sync_data.sync_checkpoint_ops.pfnFenceDataRollback = pvr_sync_rollback_fence_data;
 779         pvr_sync_data.sync_checkpoint_ops.pfnFenceFinalise = pvr_sync_finalise_fence;
 780 #if defined(NO_HARDWARE)
 781         pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = pvr_sync_nohw_signal_fence;
 782 #else
 783         pvr_sync_data.sync_checkpoint_ops.pfnNoHWUpdateTimelines = NULL;
 784 #endif
 785         pvr_sync_data.sync_checkpoint_ops.pfnFreeCheckpointListMem =
 786                 pvr_sync_free_checkpoint_list_mem;
 787         pvr_sync_data.sync_checkpoint_ops.pfnDumpInfoOnStalledUFOs =
 788                 pvr_sync_dump_info_on_stalled_ufos;
 789         strlcpy(pvr_sync_data.sync_checkpoint_ops.pszImplName, "pvr_sync_file",
 790                 SYNC_CHECKPOINT_IMPL_MAX_STRLEN);
 791 #if defined(PDUMP)
 792         pvr_sync_data.sync_checkpoint_ops.pfnSyncFenceGetCheckpoints =
 793                 pvr_sync_fence_get_checkpoints;
 794 #endif
 795
 796         return SyncCheckpointRegisterFunctions(&pvr_sync_data.sync_checkpoint_ops);
 797 }
 798
 799 int pvr_sync_init(void)
 800 {
 801         int err;
 802
 803         pvr_sync_data.foreign_fence_context =
 804                         pvr_fence_foreign_context_create(
 805                                         NativeSyncGetFenceStatusWq(),
 806                                         "foreign_sync");
 807         if (!pvr_sync_data.foreign_fence_context) {
 808                 pr_err(FILE_NAME ": %s: Failed to create foreign sync context\n",
 809                         __func__);
 810                 err = -ENOMEM;
 811                 goto err_out;
 812         }
 813
 814 #if defined(NO_HARDWARE)
 815         INIT_LIST_HEAD(&pvr_timeline_active_list);
 816 #endif
 817
 818         err = pvr_sync_ioctl_init();
 819         if (err) {
 820                 pr_err(FILE_NAME ": %s: Failed to register pvr_sync device (%d)\n",
 821                        __func__, err);
 822                 goto err_ioctl_init;
 823         }
 824
 825         return 0;
 826
 827 err_ioctl_init:
 828         pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
 829         pvr_fence_cleanup();
 830 err_out:
 831         return err;
 832 }
 833
 834 void pvr_sync_deinit(void)
 835 {
 836         pvr_sync_ioctl_deinit();
 837         pvr_fence_context_destroy(pvr_sync_data.foreign_fence_context);
 838         pvr_fence_cleanup();
 839 }
 840
 841 enum PVRSRV_ERROR_TAG pvr_sync_device_init(struct device *dev)
 842 {
 843         struct drm_device *ddev = dev_get_drvdata(dev);
 844         struct pvr_drm_private *priv = ddev->dev_private;
 845         enum PVRSRV_ERROR_TAG error;
 846
 847         error = PVRSRVRegisterDeviceDbgRequestNotify(
 848                                 &priv->sync_debug_notify_handle,
 849                                 priv->dev_node,
 850                                 pvr_sync_debug_request_heading,
 851                                 DEBUG_REQUEST_LINUXFENCE,
 852                                 NULL);
 853         if (error != PVRSRV_OK) {
 854                 pr_err("%s: failed to register debug request callback (%s)\n",
 855                        __func__, PVRSRVGetErrorString(error));
 856                 goto err_out;
 857         }
 858
 859         /* Register the foreign sync context debug notifier on each device */
 860         error = pvr_fence_context_register_dbg(
 861                                 &priv->sync_foreign_debug_notify_handle,
 862                                 priv->dev_node,
 863                                 pvr_sync_data.foreign_fence_context);
 864         if (error != PVRSRV_OK) {
 865                 pr_err("%s: failed to register fence debug request callback (%s)\n",
 866                        __func__, PVRSRVGetErrorString(error));
 867                 goto err_context_regdbg;
 868         }
 869
 870 #if defined(NO_HARDWARE)
 871         INIT_LIST_HEAD(&pvr_timeline_active_list);
 872 #endif
 873
 874         return PVRSRV_OK;
 875
 876 err_context_regdbg:
 877         PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
 878 err_out:
 879         return error;
 880 }
 881
 882 void pvr_sync_device_deinit(struct device *dev)
 883 {
 884         struct drm_device *ddev = dev_get_drvdata(dev);
 885         struct pvr_drm_private *priv = ddev->dev_private;
 886
 887         PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_foreign_debug_notify_handle);
 888         PVRSRVUnregisterDeviceDbgRequestNotify(priv->sync_debug_notify_handle);
 889 }
 890
 891 enum PVRSRV_ERROR_TAG pvr_sync_fence_wait(void *fence, u32 timeout_in_ms)
 892 {
 893         long timeout = msecs_to_jiffies(timeout_in_ms);
 894         int err;
 895
 896         err = dma_fence_wait_timeout(fence, true, timeout);
 897         /*
 898          * dma_fence_wait_timeout returns:
 899          * - the remaining timeout on success
 900          * - 0 on timeout
 901          * - -ERESTARTSYS if interrupted
 902          */
 903         if (err > 0)
 904                 return PVRSRV_OK;
 905         else if (err == 0)
 906                 return PVRSRV_ERROR_TIMEOUT;
 907
 908         return PVRSRV_ERROR_FAILED_DEPENDENCIES;
 909 }
 910
 911 enum PVRSRV_ERROR_TAG pvr_sync_fence_release(void *fence)
 912 {
 913         dma_fence_put(fence);
 914
 915         return PVRSRV_OK;
 916 }
 917
 918 enum PVRSRV_ERROR_TAG pvr_sync_fence_get(int fence_fd, void **fence_out)
 919 {
 920         struct dma_fence *fence;
 921
 922         fence = sync_file_get_fence(fence_fd);
 923         if (fence == NULL)
 924                 return PVRSRV_ERROR_INVALID_PARAMS;
 925
 926         *fence_out = fence;
 927
 928         return PVRSRV_OK;
 929 }
 930
 931 enum PVRSRV_ERROR_TAG
 932 pvr_sync_sw_timeline_fence_create(struct _PVRSRV_DEVICE_NODE_ *pvrsrv_dev_node,
 933                                   int timeline_fd,
 934                                   const char *fence_name,
 935                                   int *fence_fd_out,
 936                                   u64 *sync_pt_idx)
 937 {
 938         enum PVRSRV_ERROR_TAG srv_err;
 939         struct pvr_sync_timeline *timeline;
 940         struct dma_fence *fence = NULL;
 941         struct sync_file *sync_file = NULL;
 942         int fd;
 943
 944         (void)(pvrsrv_dev_node);
 945
 946         fd = get_unused_fd_flags(O_CLOEXEC);
 947         if (fd < 0)
 948                 return PVRSRV_ERROR_UNABLE_TO_ADD_HANDLE;
 949
 950         timeline = pvr_sync_timeline_fget(timeline_fd);
 951         if (!timeline) {
 952                 /* unrecognised timeline */
 953                 srv_err = PVRSRV_ERROR_RESOURCE_UNAVAILABLE;
 954                 goto err_put_fd;
 955         }
 956         if (!timeline->is_sw) {
 957                 pvr_sync_timeline_fput(timeline);
 958                 srv_err = PVRSRV_ERROR_INVALID_PARAMS;
 959                 goto err_put_fd;
 960         }
 961
 962         fence = pvr_counting_fence_create(timeline->sw_fence_timeline, sync_pt_idx);
 963         pvr_sync_timeline_fput(timeline);
 964         if (!fence) {
 965                 srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
 966                 goto err_put_fd;
 967         }
 968
 969         sync_file = sync_file_create(fence);
 970         dma_fence_put(fence);
 971         if (!sync_file) {
 972                 srv_err = PVRSRV_ERROR_OUT_OF_MEMORY;
 973                 goto err_put_fd;
 974         }
 975
 976         fd_install(fd, sync_file->file);
 977
 978         *fence_fd_out = fd;
 979
 980         return PVRSRV_OK;
 981
 982 err_put_fd:
 983         put_unused_fd(fd);
 984         return srv_err;
 985 }
 986
 987 enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_advance(void *timeline, u64 *sync_pt_idx)
 988 {
 989         if (timeline == NULL)
 990                 return PVRSRV_ERROR_INVALID_PARAMS;
 991
 992         pvr_counting_fence_timeline_inc(timeline, sync_pt_idx);
 993
 994         return PVRSRV_OK;
 995 }
 996
 997 enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_release(void *timeline)
 998 {
 999         if (timeline == NULL)
1000                 return PVRSRV_ERROR_INVALID_PARAMS;
1001
1002         pvr_counting_fence_timeline_put(timeline);
1003
1004         return PVRSRV_OK;
1005 }
1006
1007 enum PVRSRV_ERROR_TAG pvr_sync_sw_timeline_get(int timeline_fd,
1008                                            void **timeline_out)
1009 {
1010         struct pvr_counting_fence_timeline *sw_timeline;
1011         struct pvr_sync_timeline *timeline;
1012
1013         timeline = pvr_sync_timeline_fget(timeline_fd);
1014         if (!timeline)
1015                 return PVRSRV_ERROR_INVALID_PARAMS;
1016
1017         sw_timeline =
1018                 pvr_counting_fence_timeline_get(timeline->sw_fence_timeline);
1019         pvr_sync_timeline_fput(timeline);
1020         if (!sw_timeline)
1021                 return PVRSRV_ERROR_INVALID_PARAMS;
1022
1023         *timeline_out = sw_timeline;
1024
1025         return PVRSRV_OK;
1026 }
1027 static void _dump_sync_point(struct dma_fence *fence,
1028                                                           DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
1029                                                           void *dump_debug_file)
1030 {
1031         const struct dma_fence_ops *fence_ops = fence->ops;
1032         bool signaled = dma_fence_is_signaled(fence);
1033         char time[16] = { '\0' };
1034
1035         fence_ops->timeline_value_str(fence, time, sizeof(time));
1036
1037         PVR_DUMPDEBUG_LOG(dump_debug_printf,
1038                                           dump_debug_file,
1039                                           "<%p> Seq#=%llu TS=%s State=%s TLN=%s",
1040                                           fence,
1041                                           (u64) fence->seqno,
1042                                           time,
1043                                           (signaled) ? "Signalled" : "Active",
1044                                           fence_ops->get_timeline_name(fence));
1045 }
1046
1047 static void _dump_fence(struct dma_fence *fence,
1048                         DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
1049                         void *dump_debug_file)
1050 {
1051         if (dma_fence_is_array(fence)) {
1052                 struct dma_fence_array *fence_array = to_dma_fence_array(fence);
1053                 int i;
1054
1055                 if (fence_array) {
1056                         PVR_DUMPDEBUG_LOG(dump_debug_printf,
1057                                           dump_debug_file,
1058                                           "Fence: [%p] Sync Points:\n",
1059                                           fence_array);
1060
1061                         for (i = 0; i < fence_array->num_fences; i++)
1062                                 _dump_sync_point(fence_array->fences[i],
1063                                                  dump_debug_printf,
1064                                                  dump_debug_file);
1065                 }
1066
1067         } else {
1068                 _dump_sync_point(fence, dump_debug_printf, dump_debug_file);
1069         }
1070 }
1071
1072 enum PVRSRV_ERROR_TAG
1073 sync_dump_fence(void *sw_fence_obj,
1074                 DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
1075                 void *dump_debug_file)
1076 {
1077         struct dma_fence *fence = (struct dma_fence *) sw_fence_obj;
1078
1079         _dump_fence(fence, dump_debug_printf, dump_debug_file);
1080
1081         return PVRSRV_OK;
1082 }
1083
1084 enum PVRSRV_ERROR_TAG
1085 sync_sw_dump_timeline(void *sw_timeline_obj,
1086                       DUMPDEBUG_PRINTF_FUNC *dump_debug_printf,
1087                       void *dump_debug_file)
1088 {
1089         pvr_counting_fence_timeline_dump_timeline(sw_timeline_obj,
1090                                                   dump_debug_printf,
1091                                                   dump_debug_file);
1092
1093         return PVRSRV_OK;
1094 }