drivers/gpu/arm/mali400/mali/common/mali_pp_scheduler.c

   1 /*
   2  * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_pp_scheduler.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_kernel_core.h"
  14 #include "mali_osk.h"
  15 #include "mali_osk_list.h"
  16 #include "mali_scheduler.h"
  17 #include "mali_pp.h"
  18 #include "mali_pp_job.h"
  19 #include "mali_group.h"
  20 #include "mali_pm.h"
  21
  22 #if defined(CONFIG_SYNC)
  23 #define MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE 1
  24 #endif
  25
  26 /* Maximum of 8 PP cores (a group can only have maximum of 1 PP core) */
  27 #define MALI_MAX_NUMBER_OF_PP_GROUPS 9
  28
  29 static mali_bool mali_pp_scheduler_is_suspended(void);
  30 static void mali_pp_scheduler_do_schedule(void *arg);
  31 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
  32 static void mali_pp_scheduler_do_job_delete(void *arg);
  33 #endif
  34
  35 static u32 pp_version = 0;
  36
  37 /* Physical job queue */
  38 static _MALI_OSK_LIST_HEAD_STATIC_INIT(job_queue);              /* List of physical jobs with some unscheduled work */
  39 static u32 job_queue_depth = 0;
  40
  41 /* Physical groups */
  42 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_working);     /* List of physical groups with working jobs on the pp core */
  43 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_idle);        /* List of physical groups with idle jobs on the pp core */
  44
  45 /* Virtual job queue (Mali-450 only) */
  46 static _MALI_OSK_LIST_HEAD_STATIC_INIT(virtual_job_queue);      /* List of unstarted jobs for the virtual group */
  47 static u32 virtual_job_queue_depth = 0;
  48
  49 /* Virtual group (Mali-450 only) */
  50 static struct mali_group *virtual_group = NULL;                 /* Virtual group (if any) */
  51 static mali_bool virtual_group_working = MALI_FALSE;            /* Flag which indicates whether the virtual group is working or idle */
  52
  53 /* Number of physical cores */
  54 static u32 num_cores = 0;
  55
  56 /* Variables to allow safe pausing of the scheduler */
  57 static _mali_osk_wait_queue_t *pp_scheduler_working_wait_queue = NULL;
  58 static u32 pause_count = 0;
  59
  60 static _mali_osk_lock_t *pp_scheduler_lock = NULL;
  61 /* Contains tid of thread that locked the scheduler or 0, if not locked */
  62 MALI_DEBUG_CODE(static u32 pp_scheduler_lock_owner = 0);
  63
  64 static _mali_osk_wq_work_t *pp_scheduler_wq_schedule = NULL;
  65
  66 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
  67 static _mali_osk_wq_work_t *pp_scheduler_wq_job_delete = NULL;
  68 static _mali_osk_lock_t *pp_scheduler_job_delete_lock = NULL;
  69 static _MALI_OSK_LIST_HEAD_STATIC_INIT(pp_scheduler_job_deletion_queue);
  70 #endif
  71
  72 _mali_osk_errcode_t mali_pp_scheduler_initialize(void)
  73 {
  74         struct mali_group *group;
  75         struct mali_pp_core *pp_core;
  76         _mali_osk_lock_flags_t lock_flags;
  77         u32 num_groups;
  78         u32 i;
  79
  80 #if defined(MALI_UPPER_HALF_SCHEDULING)
  81         lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK_IRQ | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
  82 #else
  83         lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
  84 #endif
  85
  86         _MALI_OSK_INIT_LIST_HEAD(&job_queue);
  87         _MALI_OSK_INIT_LIST_HEAD(&group_list_working);
  88         _MALI_OSK_INIT_LIST_HEAD(&group_list_idle);
  89
  90         _MALI_OSK_INIT_LIST_HEAD(&virtual_job_queue);
  91
  92         pp_scheduler_lock = _mali_osk_lock_init(lock_flags, 0, _MALI_OSK_LOCK_ORDER_SCHEDULER);
  93         if (NULL == pp_scheduler_lock)
  94         {
  95                 return _MALI_OSK_ERR_NOMEM;
  96         }
  97
  98         pp_scheduler_working_wait_queue = _mali_osk_wait_queue_init();
  99         if (NULL == pp_scheduler_working_wait_queue)
 100         {
 101                 _mali_osk_lock_term(pp_scheduler_lock);
 102                 return _MALI_OSK_ERR_NOMEM;
 103         }
 104
 105         pp_scheduler_wq_schedule = _mali_osk_wq_create_work(mali_pp_scheduler_do_schedule, NULL);
 106         if (NULL == pp_scheduler_wq_schedule)
 107         {
 108                 _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
 109                 _mali_osk_lock_term(pp_scheduler_lock);
 110                 return _MALI_OSK_ERR_NOMEM;
 111         }
 112
 113 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
 114         pp_scheduler_wq_job_delete = _mali_osk_wq_create_work(mali_pp_scheduler_do_job_delete, NULL);
 115         if (NULL == pp_scheduler_wq_job_delete)
 116         {
 117                 _mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
 118                 _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
 119                 _mali_osk_lock_term(pp_scheduler_lock);
 120                 return _MALI_OSK_ERR_NOMEM;
 121         }
 122
 123         pp_scheduler_job_delete_lock = _mali_osk_lock_init(_MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK_IRQ |_MALI_OSK_LOCKFLAG_NONINTERRUPTABLE, 0, _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
 124         if (NULL == pp_scheduler_job_delete_lock)
 125         {
 126                 _mali_osk_wq_delete_work(pp_scheduler_wq_job_delete);
 127                 _mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
 128                 _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
 129                 _mali_osk_lock_term(pp_scheduler_lock);
 130                 return _MALI_OSK_ERR_NOMEM;
 131         }
 132 #endif
 133
 134         num_groups = mali_group_get_glob_num_groups();
 135
 136         /* Do we have a virtual group? */
 137         for (i = 0; i < num_groups; i++)
 138         {
 139                 group = mali_group_get_glob_group(i);
 140
 141                 if (mali_group_is_virtual(group))
 142                 {
 143                         MALI_DEBUG_PRINT(3, ("Found virtual group %p\n", group));
 144
 145                         virtual_group = group;
 146                         break;
 147                 }
 148         }
 149
 150         /* Find all the available PP cores */
 151         for (i = 0; i < num_groups; i++)
 152         {
 153                 group = mali_group_get_glob_group(i);
 154                 pp_core = mali_group_get_pp_core(group);
 155
 156                 if (NULL != pp_core && !mali_group_is_virtual(group))
 157                 {
 158                         if (0 == pp_version)
 159                         {
 160                                 /* Retrieve PP version from the first available PP core */
 161                                 pp_version = mali_pp_core_get_version(pp_core);
 162                         }
 163
 164                         if (NULL != virtual_group)
 165                         {
 166                                 /* Add all physical PP cores to the virtual group */
 167                                 mali_group_lock(virtual_group);
 168                                 group->state = MALI_GROUP_STATE_JOINING_VIRTUAL;
 169                                 mali_group_add_group(virtual_group, group);
 170                                 mali_group_unlock(virtual_group);
 171                         }
 172                         else
 173                         {
 174                                 _mali_osk_list_add(&group->pp_scheduler_list, &group_list_idle);
 175                         }
 176
 177                         num_cores++;
 178                 }
 179         }
 180
 181         return _MALI_OSK_ERR_OK;
 182 }
 183
 184 void mali_pp_scheduler_terminate(void)
 185 {
 186         struct mali_group *group, *temp;
 187
 188         /* Delete all groups owned by scheduler */
 189         if (NULL != virtual_group)
 190         {
 191                 mali_group_delete(virtual_group);
 192         }
 193
 194         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working));
 195         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, pp_scheduler_list)
 196         {
 197                 mali_group_delete(group);
 198         }
 199
 200 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
 201         _mali_osk_lock_term(pp_scheduler_job_delete_lock);
 202         _mali_osk_wq_delete_work(pp_scheduler_wq_job_delete);
 203 #endif
 204
 205         _mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
 206         _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
 207         _mali_osk_lock_term(pp_scheduler_lock);
 208 }
 209
 210 MALI_STATIC_INLINE void mali_pp_scheduler_lock(void)
 211 {
 212         if(_MALI_OSK_ERR_OK != _mali_osk_lock_wait(pp_scheduler_lock, _MALI_OSK_LOCKMODE_RW))
 213         {
 214                 /* Non-interruptable lock failed: this should never happen. */
 215                 MALI_DEBUG_ASSERT(0);
 216         }
 217         MALI_DEBUG_PRINT(5, ("Mali PP scheduler: PP scheduler lock taken\n"));
 218         MALI_DEBUG_ASSERT(0 == pp_scheduler_lock_owner);
 219         MALI_DEBUG_CODE(pp_scheduler_lock_owner = _mali_osk_get_tid());
 220 }
 221
 222 MALI_STATIC_INLINE void mali_pp_scheduler_unlock(void)
 223 {
 224         MALI_DEBUG_PRINT(5, ("Mali PP scheduler: Releasing PP scheduler lock\n"));
 225         MALI_DEBUG_ASSERT(_mali_osk_get_tid() == pp_scheduler_lock_owner);
 226         MALI_DEBUG_CODE(pp_scheduler_lock_owner = 0);
 227         _mali_osk_lock_signal(pp_scheduler_lock, _MALI_OSK_LOCKMODE_RW);
 228 }
 229
 230 #ifdef DEBUG
 231 MALI_STATIC_INLINE void mali_pp_scheduler_assert_locked(void)
 232 {
 233         MALI_DEBUG_ASSERT(_mali_osk_get_tid() == pp_scheduler_lock_owner);
 234 }
 235 #define MALI_ASSERT_PP_SCHEDULER_LOCKED() mali_pp_scheduler_assert_locked()
 236 #else
 237 #define MALI_ASSERT_PP_SCHEDULER_LOCKED()
 238 #endif
 239
 240 /**
 241  * Returns a physical job if a physical job is ready to run (no barrier present)
 242  */
 243 MALI_STATIC_INLINE struct mali_pp_job *mali_pp_scheduler_get_physical_job(void)
 244 {
 245         MALI_ASSERT_PP_SCHEDULER_LOCKED();
 246
 247         if (!_mali_osk_list_empty(&job_queue))
 248         {
 249                 struct mali_pp_job *job;
 250
 251                 MALI_DEBUG_ASSERT(job_queue_depth > 0);
 252                 job = _MALI_OSK_LIST_ENTRY(job_queue.next, struct mali_pp_job, list);
 253
 254                 if (!mali_pp_job_has_active_barrier(job))
 255                 {
 256                         return job;
 257                 }
 258         }
 259
 260         return NULL;
 261 }
 262
 263 MALI_STATIC_INLINE void mali_pp_scheduler_dequeue_physical_job(struct mali_pp_job *job)
 264 {
 265         MALI_ASSERT_PP_SCHEDULER_LOCKED();
 266         MALI_DEBUG_ASSERT(job_queue_depth > 0);
 267
 268         /* Remove job from queue */
 269         if (!mali_pp_job_has_unstarted_sub_jobs(job))
 270         {
 271                 /* All sub jobs have been started: remove job from queue */
 272                 _mali_osk_list_delinit(&job->list);
 273         }
 274
 275         --job_queue_depth;
 276 }
 277
 278 /**
 279  * Returns a virtual job if a virtual job is ready to run (no barrier present)
 280  */
 281 MALI_STATIC_INLINE struct mali_pp_job *mali_pp_scheduler_get_virtual_job(void)
 282 {
 283         MALI_ASSERT_PP_SCHEDULER_LOCKED();
 284         MALI_DEBUG_ASSERT_POINTER(virtual_group);
 285
 286         if (!_mali_osk_list_empty(&virtual_job_queue))
 287         {
 288                 struct mali_pp_job *job;
 289
 290                 MALI_DEBUG_ASSERT(virtual_job_queue_depth > 0);
 291                 job = _MALI_OSK_LIST_ENTRY(virtual_job_queue.next, struct mali_pp_job, list);
 292
 293                 if (!mali_pp_job_has_active_barrier(job))
 294                 {
 295                         return job;
 296                 }
 297         }
 298
 299         return NULL;
 300 }
 301
 302 MALI_STATIC_INLINE void mali_pp_scheduler_dequeue_virtual_job(struct mali_pp_job *job)
 303 {
 304         MALI_ASSERT_PP_SCHEDULER_LOCKED();
 305         MALI_DEBUG_ASSERT(virtual_job_queue_depth > 0);
 306
 307         /* Remove job from queue */
 308         _mali_osk_list_delinit(&job->list);
 309         --virtual_job_queue_depth;
 310 }
 311
 312 /**
 313  * Checks if the criteria is met for removing a physical core from virtual group
 314  */
 315 MALI_STATIC_INLINE mali_bool mali_pp_scheduler_can_move_virtual_to_physical(void)
 316 {
 317         MALI_ASSERT_PP_SCHEDULER_LOCKED();
 318         MALI_DEBUG_ASSERT(NULL != virtual_group);
 319         MALI_ASSERT_GROUP_LOCKED(virtual_group);
 320         /*
 321          * The criteria for taking out a physical group from a virtual group are the following:
 322          * - There virtual group is idle
 323          * - There are currently no physical groups (idle and working)
 324          * - There are physical jobs to be scheduled (without a barrier)
 325          */
 326         return (!virtual_group_working) &&
 327                _mali_osk_list_empty(&group_list_idle) &&
 328                _mali_osk_list_empty(&group_list_working) &&
 329                (NULL != mali_pp_scheduler_get_physical_job());
 330 }
 331
 332 MALI_STATIC_INLINE struct mali_group *mali_pp_scheduler_acquire_physical_group(void)
 333 {
 334         MALI_ASSERT_PP_SCHEDULER_LOCKED();
 335
 336         if (!_mali_osk_list_empty(&group_list_idle))
 337         {
 338                 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Acquiring physical group from idle list\n"));
 339                 return _MALI_OSK_LIST_ENTRY(group_list_idle.next, struct mali_group, pp_scheduler_list);
 340         }
 341         else if (NULL != virtual_group)
 342         {
 343                 MALI_ASSERT_GROUP_LOCKED(virtual_group);
 344                 if (mali_pp_scheduler_can_move_virtual_to_physical())
 345                 {
 346                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Acquiring physical group from virtual group\n"));
 347                         return mali_group_acquire_group(virtual_group);
 348                 }
 349         }
 350
 351         return NULL;
 352 }
 353
 354 static void mali_pp_scheduler_schedule(void)
 355 {
 356         struct mali_group* physical_groups_to_start[MALI_MAX_NUMBER_OF_PP_GROUPS-1];
 357         struct mali_pp_job* physical_jobs_to_start[MALI_MAX_NUMBER_OF_PP_GROUPS-1];
 358         u32 physical_subjobs_to_start[MALI_MAX_NUMBER_OF_PP_GROUPS-1];
 359         int num_physical_jobs_to_start = 0;
 360         int i;
 361
 362         if (NULL != virtual_group)
 363         {
 364                 /* Need to lock the virtual group because we might need to grab a physical group from it */
 365                 mali_group_lock(virtual_group);
 366         }
 367
 368         mali_pp_scheduler_lock();
 369         if (pause_count > 0)
 370         {
 371                 /* Scheduler is suspended, don't schedule any jobs */
 372                 mali_pp_scheduler_unlock();
 373                 if (NULL != virtual_group)
 374                 {
 375                         mali_group_unlock(virtual_group);
 376                 }
 377                 return;
 378         }
 379
 380         /* Find physical job(s) to schedule first */
 381         while (1)
 382         {
 383                 struct mali_group *group;
 384                 struct mali_pp_job *job;
 385                 u32 subjob;
 386
 387                 job = mali_pp_scheduler_get_physical_job();
 388                 if (NULL == job)
 389                 {
 390                         break; /* No job, early out */
 391                 }
 392
 393                 MALI_DEBUG_ASSERT(!mali_pp_job_is_virtual(job));
 394                 MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
 395                 MALI_DEBUG_ASSERT(1 <= mali_pp_job_get_sub_job_count(job));
 396
 397                 /* Acquire a physical group, either from the idle list or from the virtual group.
 398                  * In case the group was acquired from the virtual group, it's state will be
 399                  * LEAVING_VIRTUAL and must be set to IDLE before it can be used. */
 400                 group = mali_pp_scheduler_acquire_physical_group();
 401                 if (NULL == group)
 402                 {
 403                         /* Could not get a group to run the job on, early out */
 404                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: No more physical groups available.\n"));
 405                         break;
 406                 }
 407
 408                 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Acquired physical group %p\n", group));
 409
 410                 /* Mark subjob as started */
 411                 subjob = mali_pp_job_get_first_unstarted_sub_job(job);
 412                 mali_pp_job_mark_sub_job_started(job, subjob);
 413
 414                 /* Remove job from queue (if we now got the last subjob) */
 415                 mali_pp_scheduler_dequeue_physical_job(job);
 416
 417                 /* Move group to working list */
 418                 _mali_osk_list_move(&(group->pp_scheduler_list), &group_list_working);
 419
 420                 /* Keep track of this group, so that we actually can start the job once we are done with the scheduler lock we are now holding */
 421                 physical_groups_to_start[num_physical_jobs_to_start] = group;
 422                 physical_jobs_to_start[num_physical_jobs_to_start] = job;
 423                 physical_subjobs_to_start[num_physical_jobs_to_start] = subjob;
 424                 ++num_physical_jobs_to_start;
 425
 426                 MALI_DEBUG_ASSERT(num_physical_jobs_to_start < MALI_MAX_NUMBER_OF_PP_GROUPS);
 427         }
 428
 429         /* See if we have a virtual job to schedule */
 430         if (NULL != virtual_group)
 431         {
 432                 if (!virtual_group_working)
 433                 {
 434                         struct mali_pp_job *job = mali_pp_scheduler_get_virtual_job();
 435                         if (NULL != job)
 436                         {
 437                                 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
 438                                 MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
 439                                 MALI_DEBUG_ASSERT(1 == mali_pp_job_get_sub_job_count(job));
 440
 441                                 /* Mark the one and only subjob as started */
 442                                 mali_pp_job_mark_sub_job_started(job, 0);
 443
 444                                 /* Remove job from queue */
 445                                 mali_pp_scheduler_dequeue_virtual_job(job);
 446
 447                                 /* Virtual group is now working */
 448                                 virtual_group_working = MALI_TRUE;
 449
 450                                 /*
 451                                  * We no longer need the scheduler lock,
 452                                  * but we still need the virtual lock in order to start the virtual job
 453                                  */
 454                                 mali_pp_scheduler_unlock();
 455
 456                                 /* Start job */
 457                                 if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(virtual_group, job, 0))
 458                                 {
 459                                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Virtual job %u (0x%08X) part %u/%u started (from schedule)\n",
 460                                                              mali_pp_job_get_id(job), job, 1,
 461                                                              mali_pp_job_get_sub_job_count(job)));
 462                                 }
 463                                 else
 464                                 {
 465                                         MALI_DEBUG_ASSERT(0);
 466                                 }
 467
 468                                 /* And now we are all done with the virtual_group lock as well */
 469                                 mali_group_unlock(virtual_group);
 470                         }
 471                         else
 472                         {
 473                                 /* No virtual job, release the two locks we are holding */
 474                                 mali_pp_scheduler_unlock();
 475                                 mali_group_unlock(virtual_group);
 476                         }
 477                 }
 478                 else
 479                 {
 480                         /* Virtual core busy, release the two locks we are holding */
 481                         mali_pp_scheduler_unlock();
 482                         mali_group_unlock(virtual_group);
 483                 }
 484
 485         }
 486         else
 487         {
 488                 /* There is no virtual group, release the only lock we are holding */
 489                 mali_pp_scheduler_unlock();
 490         }
 491
 492         /*
 493          * Now we have released the scheduler lock, and we are ready to kick of the actual starting of the
 494          * physical jobs.
 495          * The reason we want to wait until we have released the scheduler lock is that job start actually
 496          * may take quite a bit of time (quite many registers needs to be written). This will allow new jobs
 497          * from user space to come in, and post processing of other PP jobs to happen at the same time as we
 498          * start jobs.
 499          */
 500         for (i = 0; i < num_physical_jobs_to_start; i++)
 501         {
 502                 struct mali_group *group = physical_groups_to_start[i];
 503                 struct mali_pp_job *job  = physical_jobs_to_start[i];
 504                 u32 sub_job              = physical_subjobs_to_start[i];
 505
 506                 MALI_DEBUG_ASSERT_POINTER(group);
 507                 MALI_DEBUG_ASSERT_POINTER(job);
 508
 509                 mali_group_lock(group);
 510
 511                 /* In case this group was acquired from a virtual core, update it's state to IDLE */
 512                 group->state = MALI_GROUP_STATE_IDLE;
 513
 514                 if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(group, job, sub_job))
 515                 {
 516                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Physical job %u (0x%08X) part %u/%u started (from schedule)\n",
 517                                              mali_pp_job_get_id(job), job, sub_job + 1,
 518                                              mali_pp_job_get_sub_job_count(job)));
 519                 }
 520                 else
 521                 {
 522                         MALI_DEBUG_ASSERT(0);
 523                 }
 524
 525                 mali_group_unlock(group);
 526
 527                 /* @@@@ todo: remove the return value from mali_group_start_xx_job, since we can't fail on Mali-300++ */
 528         }
 529 }
 530
 531 static void mali_pp_scheduler_return_job_to_user(struct mali_pp_job *job, mali_bool deferred)
 532 {
 533         if (MALI_FALSE == mali_pp_job_use_no_notification(job))
 534         {
 535                 u32 i;
 536                 u32 sub_jobs = mali_pp_job_get_sub_job_count(job);
 537                 mali_bool success = mali_pp_job_was_success(job);
 538
 539                 _mali_uk_pp_job_finished_s *jobres = job->finished_notification->result_buffer;
 540                 _mali_osk_memset(jobres, 0, sizeof(_mali_uk_pp_job_finished_s)); /* @@@@ can be removed once we initialize all members in this struct */
 541                 jobres->user_job_ptr = mali_pp_job_get_user_id(job);
 542                 if (MALI_TRUE == success)
 543                 {
 544                         jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
 545                 }
 546                 else
 547                 {
 548                         jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
 549                 }
 550
 551                 for (i = 0; i < sub_jobs; i++)
 552                 {
 553                         jobres->perf_counter0[i] = mali_pp_job_get_perf_counter_value0(job, i);
 554                         jobres->perf_counter1[i] = mali_pp_job_get_perf_counter_value1(job, i);
 555                 }
 556
 557                 mali_session_send_notification(mali_pp_job_get_session(job), job->finished_notification);
 558                 job->finished_notification = NULL;
 559         }
 560
 561 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
 562         if (MALI_TRUE == deferred)
 563         {
 564                 /* The deletion of the job object (releasing sync refs etc) must be done in a different context */
 565                 _mali_osk_lock_wait(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
 566
 567                 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); /* This job object should not be on any list */
 568                 _mali_osk_list_addtail(&job->list, &pp_scheduler_job_deletion_queue);
 569
 570                 _mali_osk_lock_signal(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
 571
 572                 _mali_osk_wq_schedule_work(pp_scheduler_wq_job_delete);
 573         }
 574         else
 575         {
 576                 mali_pp_job_delete(job);
 577         }
 578 #else
 579         MALI_DEBUG_ASSERT(MALI_FALSE == deferred); /* no use cases need this in this configuration */
 580         mali_pp_job_delete(job);
 581 #endif
 582 }
 583
 584 static void mali_pp_scheduler_do_schedule(void *arg)
 585 {
 586         MALI_IGNORE(arg);
 587
 588         mali_pp_scheduler_schedule();
 589 }
 590
 591 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
 592 static void mali_pp_scheduler_do_job_delete(void *arg)
 593 {
 594         _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
 595         struct mali_pp_job *job;
 596         struct mali_pp_job *tmp;
 597
 598         MALI_IGNORE(arg);
 599
 600         _mali_osk_lock_wait(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
 601
 602         /*
 603          * Quickly "unhook" the jobs pending to be deleted, so we can release the lock before
 604          * we start deleting the job objects (without any locks held
 605          */
 606         _mali_osk_list_move_list(&pp_scheduler_job_deletion_queue, &list);
 607
 608         _mali_osk_lock_signal(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
 609
 610         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, struct mali_pp_job, list)
 611         {
 612                 mali_pp_job_delete(job); /* delete the job object itself */
 613         }
 614 }
 615 #endif
 616
 617 void mali_pp_scheduler_job_done(struct mali_group *group, struct mali_pp_job *job, u32 sub_job, mali_bool success)
 618 {
 619         mali_bool job_is_done;
 620         mali_bool barrier_enforced = MALI_FALSE;
 621
 622         MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) part %u/%u completed (%s)\n",
 623                              mali_pp_job_is_virtual(job) ? "Virtual" : "Physical",
 624                              mali_pp_job_get_id(job),
 625                              job, sub_job + 1,
 626                              mali_pp_job_get_sub_job_count(job),
 627                              success ? "success" : "failure"));
 628         MALI_ASSERT_GROUP_LOCKED(group);
 629         mali_pp_scheduler_lock();
 630
 631         mali_pp_job_mark_sub_job_completed(job, success);
 632
 633         MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job) == mali_group_is_virtual(group));
 634
 635         job_is_done = mali_pp_job_is_complete(job);
 636
 637         if (job_is_done)
 638         {
 639                 struct mali_session_data *session = mali_pp_job_get_session(job);
 640                 struct mali_pp_job *job_head;
 641
 642                 /* Remove job from session list */
 643                 _mali_osk_list_del(&job->session_list);
 644
 645                 /* Send notification back to user space */
 646                 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: All parts completed for %s job %u (0x%08X)\n",
 647                                      mali_pp_job_is_virtual(job) ? "virtual" : "physical",
 648                                      mali_pp_job_get_id(job), job));
 649 #if defined(CONFIG_SYNC)
 650 /* MALI_SEC */
 651 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
 652                 if (job->sync_point)
 653                 {
 654                         int error;
 655                         if (success) error = 0;
 656                         else error = -EFAULT;
 657                         MALI_DEBUG_PRINT(4, ("Sync: Signal %spoint for job %d\n",
 658                                              success ? "" : "failed ",
 659                                              mali_pp_job_get_id(job)));
 660                         mali_sync_signal_pt(job->sync_point, error);
 661                 }
 662 #endif
 663 #endif
 664
 665 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
 666                 mali_pp_scheduler_return_job_to_user(job, MALI_TRUE);
 667 #else
 668                 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE);
 669 #endif
 670
 671                 mali_pm_core_event(MALI_CORE_EVENT_PP_STOP);
 672
 673                 /* Resolve any barriers */
 674                 if (!_mali_osk_list_empty(&session->job_list))
 675                 {
 676                         job_head = _MALI_OSK_LIST_ENTRY(session->job_list.next, struct mali_pp_job, session_list);
 677                         if (mali_pp_job_has_active_barrier(job_head))
 678                         {
 679                                 barrier_enforced = MALI_TRUE;
 680                                 mali_pp_job_barrier_enforced(job_head);
 681                         }
 682                 }
 683         }
 684
 685         /* If paused, then this was the last job, so wake up sleeping workers */
 686         if (pause_count > 0)
 687         {
 688                 /* Wake up sleeping workers. Their wake-up condition is that
 689                  * num_slots == num_slots_idle, so unless we are done working, no
 690                  * threads will actually be woken up.
 691                  */
 692                 _mali_osk_wait_queue_wake_up(pp_scheduler_working_wait_queue);
 693                 mali_pp_scheduler_unlock();
 694                 return;
 695         }
 696
 697         if (barrier_enforced)
 698         {
 699                 /* A barrier was resolved, so schedule previously blocked jobs */
 700                 _mali_osk_wq_schedule_work(pp_scheduler_wq_schedule);
 701
 702                 /* TODO: Subjob optimisation */
 703         }
 704
 705         /* Recycle variables */
 706         job = NULL;
 707         sub_job = 0;
 708
 709         if (mali_group_is_virtual(group))
 710         {
 711                 /* Virtual group */
 712
 713                 /* Now that the virtual group is idle, check if we should reconfigure */
 714                 struct mali_pp_job *physical_job = NULL;
 715                 struct mali_group *physical_group = NULL;
 716
 717                 /* Obey the policy */
 718                 virtual_group_working = MALI_FALSE;
 719
 720                 if (mali_pp_scheduler_can_move_virtual_to_physical())
 721                 {
 722                         /* There is a runnable physical job and we can acquire a physical group */
 723                         physical_job = mali_pp_scheduler_get_physical_job();
 724                         MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(physical_job));
 725
 726                         /* Mark subjob as started */
 727                         sub_job = mali_pp_job_get_first_unstarted_sub_job(physical_job);
 728                         mali_pp_job_mark_sub_job_started(physical_job, sub_job);
 729
 730                         /* Remove job from queue (if we now got the last subjob) */
 731                         mali_pp_scheduler_dequeue_physical_job(physical_job);
 732
 733                         /* Acquire a physical group from the virtual group
 734                          * It's state will be LEAVING_VIRTUAL and must be set to IDLE before it can be used */
 735                         physical_group = mali_group_acquire_group(virtual_group);
 736
 737                         /* Move physical group to the working list, as we will soon start a job on it */
 738                         _mali_osk_list_move(&(physical_group->pp_scheduler_list), &group_list_working);
 739                 }
 740
 741                 /* Start the next virtual job */
 742                 job = mali_pp_scheduler_get_virtual_job();
 743                 if (NULL != job)
 744                 {
 745                         /* There is a runnable virtual job */
 746                         MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
 747                         MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
 748                         MALI_DEBUG_ASSERT(1 == mali_pp_job_get_sub_job_count(job));
 749
 750                         mali_pp_job_mark_sub_job_started(job, 0);
 751
 752                         /* Remove job from queue */
 753                         mali_pp_scheduler_dequeue_virtual_job(job);
 754
 755                         /* Virtual group is now working */
 756                         virtual_group_working = MALI_TRUE;
 757
 758                         mali_pp_scheduler_unlock();
 759
 760                         /* Start job */
 761                         if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(group, job, 0))
 762                         {
 763                                 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Virtual job %u (0x%08X) part %u/%u started (from job_done)\n",
 764                                                      mali_pp_job_get_id(job), job, 1,
 765                                                      mali_pp_job_get_sub_job_count(job)));
 766                         }
 767                         else
 768                         {
 769                                 MALI_DEBUG_ASSERT(0);
 770                         }
 771                 }
 772                 else
 773                 {
 774                         mali_pp_scheduler_unlock();
 775                 }
 776
 777                 /* Start a physical job (if we acquired a physical group earlier) */
 778                 if (NULL != physical_job && NULL != physical_group)
 779                 {
 780                         mali_group_lock(physical_group);
 781
 782                         /* Set the group state from LEAVING_VIRTUAL to IDLE to complete the transition */
 783                         physical_group->state = MALI_GROUP_STATE_IDLE;
 784
 785                         /* Start job on core */
 786                         if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(physical_group, physical_job, sub_job))
 787                         {
 788                                 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Physical job %u (0x%08X) part %u/%u started (from job_done)\n",
 789                                                      mali_pp_job_get_id(physical_job), physical_job, sub_job + 1,
 790                                                      mali_pp_job_get_sub_job_count(physical_job)));
 791                         }
 792                         else
 793                         {
 794                                 /* @@@@ todo: this cant fail on Mali-300+, no need to implement put back of job */
 795                                 MALI_DEBUG_ASSERT(0);
 796                         }
 797
 798                         mali_group_unlock(physical_group);
 799                 }
 800         }
 801         else
 802         {
 803                 /* Physical group */
 804                 job = mali_pp_scheduler_get_physical_job();
 805                 if (NULL != job)
 806                 {
 807                         /* There is a runnable physical job */
 808                         MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
 809
 810                         /* Mark subjob as started */
 811                         sub_job = mali_pp_job_get_first_unstarted_sub_job(job);
 812                         mali_pp_job_mark_sub_job_started(job, sub_job);
 813
 814                         /* Remove job from queue (if we now got the last subjob) */
 815                         mali_pp_scheduler_dequeue_physical_job(job);
 816
 817                         mali_pp_scheduler_unlock();
 818
 819                         /* Group is already on the working list, so start the job */
 820                         if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(group, job, sub_job))
 821                         {
 822                                 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Physical job %u (0x%08X) part %u/%u started (from job_done)\n",
 823                                                      mali_pp_job_get_id(job), job, sub_job + 1,
 824                                                      mali_pp_job_get_sub_job_count(job)));
 825                         }
 826                         else
 827                         {
 828                                 MALI_DEBUG_ASSERT(0);
 829                         }
 830                 }
 831                 else if (NULL != virtual_group)
 832                 {
 833                         /* Rejoin virtual group */
 834                         /* TODO: In the future, a policy check might be useful here */
 835
 836                         /* We're no longer needed on the scheduler list */
 837                         _mali_osk_list_delinit(&(group->pp_scheduler_list));
 838
 839                         /* Make sure no interrupts are handled for this group during
 840                          * the transition from physical to virtual */
 841                         group->state = MALI_GROUP_STATE_JOINING_VIRTUAL;
 842
 843                         mali_pp_scheduler_unlock();
 844                         mali_group_unlock(group);
 845
 846                         mali_group_lock(virtual_group);
 847                         /* TODO: Take group lock also? */
 848                         mali_group_add_group(virtual_group, group);
 849                         mali_group_unlock(virtual_group);
 850
 851                         /* We need to return from this function with the group lock held */
 852                         /* TODO: optimise! */
 853                         mali_group_lock(group);
 854                 }
 855                 else
 856                 {
 857                         _mali_osk_list_move(&(group->pp_scheduler_list), &group_list_idle);
 858                         mali_pp_scheduler_unlock();
 859                 }
 860         }
 861 }
 862
 863 void mali_pp_scheduler_suspend(void)
 864 {
 865         mali_pp_scheduler_lock();
 866         pause_count++; /* Increment the pause_count so that no more jobs will be scheduled */
 867         mali_pp_scheduler_unlock();
 868
 869         /* Go to sleep. When woken up again (in mali_pp_scheduler_job_done), the
 870          * mali_pp_scheduler_suspended() function will be called. This will return true
 871          * iff state is idle and pause_count > 0, so if the core is active this
 872          * will not do anything.
 873          */
 874         _mali_osk_wait_queue_wait_event(pp_scheduler_working_wait_queue, mali_pp_scheduler_is_suspended);
 875 }
 876
 877 void mali_pp_scheduler_resume(void)
 878 {
 879         mali_pp_scheduler_lock();
 880         pause_count--; /* Decrement pause_count to allow scheduling again (if it reaches 0) */
 881         mali_pp_scheduler_unlock();
 882         if (0 == pause_count)
 883         {
 884                 mali_pp_scheduler_schedule();
 885         }
 886 }
 887
 888 MALI_STATIC_INLINE void mali_pp_scheduler_queue_job(struct mali_pp_job *job, struct mali_session_data *session)
 889 {
 890         MALI_DEBUG_ASSERT_POINTER(job);
 891
 892         mali_pm_core_event(MALI_CORE_EVENT_PP_START);
 893
 894         mali_pp_scheduler_lock();
 895
 896         if (mali_pp_job_is_virtual(job))
 897         {
 898                 /* Virtual job */
 899                 virtual_job_queue_depth += 1;
 900                 _mali_osk_list_addtail(&job->list, &virtual_job_queue);
 901         }
 902         else
 903         {
 904                 job_queue_depth += mali_pp_job_get_sub_job_count(job);
 905                 _mali_osk_list_addtail(&job->list, &job_queue);
 906         }
 907
 908         if (mali_pp_job_has_active_barrier(job) && _mali_osk_list_empty(&session->job_list))
 909         {
 910                 /* No running jobs on this session, so barrier condition already met */
 911                 mali_pp_job_barrier_enforced(job);
 912         }
 913
 914         /* Add job to session list */
 915         _mali_osk_list_addtail(&job->session_list, &session->job_list);
 916
 917         MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued\n",
 918                              mali_pp_job_is_virtual(job) ? "Virtual" : "Physical",
 919                              mali_pp_job_get_id(job), job, mali_pp_job_get_sub_job_count(job)));
 920
 921         mali_pp_scheduler_unlock();
 922 }
 923
 924 #if defined(CONFIG_SYNC)
 925 /* MALI_SEC */
 926 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
 927 static void sync_callback(struct sync_fence *fence, struct sync_fence_waiter *waiter)
 928 {
 929         struct mali_pp_job *job = _MALI_OSK_CONTAINER_OF(waiter, struct mali_pp_job, sync_waiter);
 930
 931         /* Schedule sync_callback_work */
 932         _mali_osk_wq_schedule_work(job->sync_work);
 933 }
 934
 935 static void sync_callback_work(void *arg)
 936 {
 937         struct mali_pp_job *job = (struct mali_pp_job *)arg;
 938         struct mali_session_data *session;
 939         int err;
 940
 941         MALI_DEBUG_ASSERT_POINTER(job);
 942
 943         session = job->session;
 944
 945         /* Remove job from session pending job list */
 946         _mali_osk_lock_wait(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
 947         _mali_osk_list_delinit(&job->list);
 948         _mali_osk_lock_signal(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
 949
 950         err = sync_fence_wait(job->pre_fence, 0);
 951         if (likely(0 == err))
 952         {
 953                 MALI_DEBUG_PRINT(3, ("Mali sync: Job %d ready to run\n", mali_pp_job_get_id(job)));
 954
 955                 mali_pp_scheduler_queue_job(job, session);
 956
 957                 mali_pp_scheduler_schedule();
 958         }
 959         else
 960         {
 961                 /* Fence signaled error */
 962                 MALI_DEBUG_PRINT(3, ("Mali sync: Job %d abort due to sync error\n", mali_pp_job_get_id(job)));
 963
 964                 if (job->sync_point) mali_sync_signal_pt(job->sync_point, err);
 965
 966                 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
 967                 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
 968         }
 969 }
 970 #endif
 971 #endif
 972
 973 _mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, _mali_uk_pp_start_job_s *uargs, int *fence)
 974 {
 975         struct mali_session_data *session;
 976         struct mali_pp_job *job;
 977 #if defined(CONFIG_SYNC)
 978 /* MALI_SEC */
 979 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
 980         int post_fence = -1;
 981 #endif
 982 #endif
 983
 984         MALI_DEBUG_ASSERT_POINTER(uargs);
 985         MALI_DEBUG_ASSERT_POINTER(ctx);
 986
 987         session = (struct mali_session_data*)ctx;
 988
 989         job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id());
 990         if (NULL == job)
 991         {
 992                 MALI_PRINT_ERROR(("Failed to create job!\n"));
 993                 return _MALI_OSK_ERR_NOMEM;
 994         }
 995
 996         if (_MALI_OSK_ERR_OK != mali_pp_job_check(job))
 997         {
 998                 /* Not a valid job, return to user immediately */
 999                 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1000                 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1001                 return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1002         }
1003
1004 #if PROFILING_SKIP_PP_JOBS || PROFILING_SKIP_PP_AND_GP_JOBS
1005 #warning PP jobs will not be executed
1006         mali_pp_scheduler_return_job_to_user(job, MALI_FALSE);
1007         return _MALI_OSK_ERR_OK;
1008 #endif
1009
1010 #if defined(CONFIG_SYNC)
1011 /* MALI_SEC */
1012 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
1013         if (_MALI_PP_JOB_FLAG_FENCE & job->uargs.flags)
1014         {
1015                 job->sync_point = mali_stream_create_point(job->uargs.stream);
1016
1017                 if (unlikely(NULL == job->sync_point))
1018                 {
1019                         /* Fence creation failed. */
1020                         MALI_DEBUG_PRINT(2, ("Failed to create sync point for job %d\n", mali_pp_job_get_id(job)));
1021                         mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1022                         mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1023                         return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1024                 }
1025
1026                 post_fence = mali_stream_create_fence(job->sync_point);
1027
1028                 if (unlikely(0 > post_fence))
1029                 {
1030                         /* Fence creation failed. */
1031                         /* mali_stream_create_fence already freed the sync_point */
1032                         MALI_DEBUG_PRINT(2, ("Failed to create fence for job %d\n", mali_pp_job_get_id(job)));
1033                         mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1034                         mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1035                         return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1036                 }
1037
1038                 /* Grab a reference to the fence. It must be around when the
1039                  * job is completed, so the point can be signalled. */
1040                 sync_fence_fdget(post_fence);
1041
1042                 *fence = post_fence;
1043
1044                 MALI_DEBUG_PRINT(3, ("Sync: Created fence %d for job %d\n", post_fence, mali_pp_job_get_id(job)));
1045         }
1046
1047         if (0 < job->uargs.fence)
1048         {
1049                 int pre_fence_fd = job->uargs.fence;
1050                 int err;
1051
1052                 MALI_DEBUG_PRINT(2, ("Sync: Job %d waiting for fence %d\n", mali_pp_job_get_id(job), pre_fence_fd));
1053
1054                 job->pre_fence = sync_fence_fdget(pre_fence_fd); /* Reference will be released when job is deleted. */
1055                 if (NULL == job->pre_fence)
1056                 {
1057                         MALI_DEBUG_PRINT(2, ("Failed to import fence %d\n", pre_fence_fd));
1058                         if (job->sync_point) mali_sync_signal_pt(job->sync_point, -EINVAL);
1059                         mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1060                         mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1061                         return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1062                 }
1063
1064                 job->sync_work = _mali_osk_wq_create_work(sync_callback_work, (void*)job);
1065                 if (NULL == job->sync_work)
1066                 {
1067                         if (job->sync_point) mali_sync_signal_pt(job->sync_point, -ENOMEM);
1068                         mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1069                         mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1070                         return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1071                 }
1072
1073                 /* Add pending job to session pending job list */
1074                 _mali_osk_lock_wait(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1075                 _mali_osk_list_addtail(&job->list, &session->pending_jobs);
1076                 _mali_osk_lock_signal(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1077
1078                 sync_fence_waiter_init(&job->sync_waiter, sync_callback);
1079                 err = sync_fence_wait_async(job->pre_fence, &job->sync_waiter);
1080
1081                 if (0 != err)
1082                 {
1083                         /* No async wait started, remove job from session pending job list */
1084                         _mali_osk_lock_wait(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1085                         _mali_osk_list_delinit(&job->list);
1086                         _mali_osk_lock_signal(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1087                 }
1088
1089                 if (1 == err)
1090                 {
1091                         /* Fence has already signalled */
1092                         mali_pp_scheduler_queue_job(job, session);
1093                         if (0 == _mali_osk_list_empty(&group_list_idle)) mali_pp_scheduler_schedule();
1094                         return _MALI_OSK_ERR_OK;
1095                 }
1096                 else if (0 > err)
1097                 {
1098                         /* Sync fail */
1099                         if (job->sync_point) mali_sync_signal_pt(job->sync_point, err);
1100                         mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1101                         mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1102                         return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1103                 }
1104
1105         }
1106         else
1107 #endif
1108 #endif /* CONFIG_SYNC */
1109         {
1110                 mali_pp_scheduler_queue_job(job, session);
1111
1112                 if (!_mali_osk_list_empty(&group_list_idle) || !virtual_group_working)
1113                 {
1114                         mali_pp_scheduler_schedule();
1115                 }
1116         }
1117
1118         return _MALI_OSK_ERR_OK;
1119 }
1120
1121 _mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args)
1122 {
1123         MALI_DEBUG_ASSERT_POINTER(args);
1124         MALI_DEBUG_ASSERT_POINTER(args->ctx);
1125         args->number_of_cores = num_cores;
1126         return _MALI_OSK_ERR_OK;
1127 }
1128
1129 _mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args)
1130 {
1131         MALI_DEBUG_ASSERT_POINTER(args);
1132         MALI_DEBUG_ASSERT_POINTER(args->ctx);
1133         args->version = pp_version;
1134         return _MALI_OSK_ERR_OK;
1135 }
1136
1137 void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args)
1138 {
1139         struct mali_session_data *session;
1140         struct mali_pp_job *job;
1141         struct mali_pp_job *tmp;
1142
1143         MALI_DEBUG_ASSERT_POINTER(args);
1144         MALI_DEBUG_ASSERT_POINTER(args->ctx);
1145
1146         session = (struct mali_session_data*)args->ctx;
1147
1148         /* Check queue for jobs that match */
1149         mali_pp_scheduler_lock();
1150         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &job_queue, struct mali_pp_job, list)
1151         {
1152                 if (mali_pp_job_get_session(job) == session &&
1153                     mali_pp_job_get_frame_builder_id(job) == (u32)args->fb_id &&
1154                     mali_pp_job_get_flush_id(job) == (u32)args->flush_id)
1155                 {
1156                         if (args->wbx & _MALI_UK_PP_JOB_WB0)
1157                         {
1158                                 mali_pp_job_disable_wb0(job);
1159                         }
1160                         if (args->wbx & _MALI_UK_PP_JOB_WB1)
1161                         {
1162                                 mali_pp_job_disable_wb1(job);
1163                         }
1164                         if (args->wbx & _MALI_UK_PP_JOB_WB2)
1165                         {
1166                                 mali_pp_job_disable_wb2(job);
1167                         }
1168                         break;
1169                 }
1170         }
1171         mali_pp_scheduler_unlock();
1172 }
1173
1174 void mali_pp_scheduler_abort_session(struct mali_session_data *session)
1175 {
1176         struct mali_pp_job *job, *tmp_job;
1177         struct mali_group *group, *tmp_group;
1178         struct mali_group *groups[MALI_MAX_NUMBER_OF_GROUPS];
1179         s32 i = 0;
1180
1181         mali_pp_scheduler_lock();
1182         MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Aborting all jobs from session 0x%08x\n", session));
1183
1184         _MALI_OSK_LIST_FOREACHENTRY(job, tmp_job, &session->job_list, struct mali_pp_job, session_list)
1185         {
1186                 /* Remove job from queue (if it's not queued, list_del has no effect) */
1187                 _mali_osk_list_delinit(&job->list);
1188
1189                 if (mali_pp_job_is_virtual(job))
1190                 {
1191                         MALI_DEBUG_ASSERT(1 == mali_pp_job_get_sub_job_count(job));
1192                         if (0 == mali_pp_job_get_first_unstarted_sub_job(job))
1193                         {
1194                                 --virtual_job_queue_depth;
1195                         }
1196                 }
1197                 else
1198                 {
1199                         job_queue_depth -= mali_pp_job_get_sub_job_count(job) - mali_pp_job_get_first_unstarted_sub_job(job);
1200                 }
1201
1202                 /* Mark all unstarted jobs as failed */
1203                 mali_pp_job_mark_unstarted_failed(job);
1204
1205                 if (mali_pp_job_is_complete(job))
1206                 {
1207                         _mali_osk_list_del(&job->session_list);
1208
1209                         /* It is safe to delete the job, since it won't land in job_done() */
1210                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Aborted PP job 0x%08x\n", job));
1211                         mali_pp_job_delete(job);
1212
1213                         mali_pm_core_event(MALI_CORE_EVENT_PP_STOP);
1214                 }
1215                 else
1216                 {
1217                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Keeping partially started PP job 0x%08x in session\n", job));
1218                 }
1219         }
1220
1221         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working, struct mali_group, pp_scheduler_list)
1222         {
1223                 groups[i++] = group;
1224         }
1225
1226         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_idle, struct mali_group, pp_scheduler_list)
1227         {
1228                 groups[i++] = group;
1229         }
1230
1231         mali_pp_scheduler_unlock();
1232
1233         /* Abort running jobs from this session */
1234         while (i > 0)
1235         {
1236                 mali_group_abort_session(groups[--i], session);
1237         }
1238
1239         if (NULL != virtual_group)
1240         {
1241                 mali_group_abort_session(virtual_group, session);
1242         }
1243 }
1244
1245 static mali_bool mali_pp_scheduler_is_suspended(void)
1246 {
1247         mali_bool ret;
1248
1249         mali_pp_scheduler_lock();
1250         ret = pause_count > 0 && _mali_osk_list_empty(&group_list_working) && !virtual_group_working;
1251         mali_pp_scheduler_unlock();
1252
1253         return ret;
1254 }
1255
1256 int mali_pp_scheduler_get_queue_depth(void)
1257 {
1258         return job_queue_depth;
1259 }
1260
1261 #if MALI_STATE_TRACKING
1262 u32 mali_pp_scheduler_dump_state(char *buf, u32 size)
1263 {
1264         int n = 0;
1265         struct mali_group *group;
1266         struct mali_group *temp;
1267
1268         n += _mali_osk_snprintf(buf + n, size - n, "PP:\n");
1269         n += _mali_osk_snprintf(buf + n, size - n, "\tQueue is %s\n", _mali_osk_list_empty(&job_queue) ? "empty" : "not empty");
1270         n += _mali_osk_snprintf(buf + n, size - n, "\n");
1271
1272         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, pp_scheduler_list)
1273         {
1274                 n += mali_group_dump_state(group, buf + n, size - n);
1275         }
1276
1277         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, pp_scheduler_list)
1278         {
1279                 n += mali_group_dump_state(group, buf + n, size - n);
1280         }
1281
1282         if (NULL != virtual_group)
1283         {
1284                 n += mali_group_dump_state(virtual_group, buf + n, size -n);
1285         }
1286
1287         n += _mali_osk_snprintf(buf + n, size - n, "\n");
1288         return n;
1289 }
1290 #endif
1291
1292 /* This function is intended for power on reset of all cores.
1293  * No locking is done for the list iteration, which can only be safe if the
1294  * scheduler is paused and all cores idle. That is always the case on init and
1295  * power on. */
1296 void mali_pp_scheduler_reset_all_groups(void)
1297 {
1298         struct mali_group *group, *temp;
1299
1300         if (NULL != virtual_group)
1301         {
1302                 mali_group_reset(virtual_group);
1303         }
1304
1305         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working));
1306
1307         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, pp_scheduler_list)
1308         {
1309                 mali_group_reset(group);
1310         }
1311 }
1312
1313 void mali_pp_scheduler_zap_all_active(struct mali_session_data *session)
1314 {
1315         struct mali_group *group, *temp;
1316         struct mali_group *groups[MALI_MAX_NUMBER_OF_GROUPS];
1317         s32 i = 0;
1318
1319         if (NULL != virtual_group)
1320         {
1321                 mali_group_zap_session(virtual_group, session);
1322         }
1323
1324         mali_pp_scheduler_lock();
1325         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, pp_scheduler_list)
1326         {
1327                 groups[i++] = group;
1328         }
1329         mali_pp_scheduler_unlock();
1330
1331         while (i > 0)
1332         {
1333                 mali_group_zap_session(groups[--i], session);
1334         }
1335 }