drivers/gpu/arm/mali400/r4p0_rel0/common/mali_gp_scheduler.c

   1 /*
   2  * Copyright (C) 2012 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_gp_scheduler.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_osk.h"
  14 #include "mali_osk_list.h"
  15 #include "mali_scheduler.h"
  16 #include "mali_gp.h"
  17 #include "mali_gp_job.h"
  18 #include "mali_group.h"
  19 #include "mali_timeline.h"
  20 #include "mali_osk_profiling.h"
  21 #include "mali_kernel_utilization.h"
  22 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
  23 #include <linux/sched.h>
  24 #include <trace/events/gpu.h>
  25 #endif
  26
  27 enum mali_gp_slot_state {
  28         MALI_GP_SLOT_STATE_IDLE,
  29         MALI_GP_SLOT_STATE_WORKING,
  30         MALI_GP_SLOT_STATE_DISABLED,
  31 };
  32
  33 /* A render slot is an entity which jobs can be scheduled onto */
  34 struct mali_gp_slot {
  35         struct mali_group *group;
  36         /*
  37          * We keep track of the state here as well as in the group object
  38          * so we don't need to take the group lock so often (and also avoid clutter with the working lock)
  39          */
  40         enum mali_gp_slot_state state;
  41         u32 returned_cookie;
  42 };
  43
  44 static u32 gp_version = 0;
  45 static _MALI_OSK_LIST_HEAD_STATIC_INIT(job_queue);      /* List of unscheduled jobs. */
  46 static _MALI_OSK_LIST_HEAD_STATIC_INIT(job_queue_high); /* List of unscheduled high priority jobs. */
  47 static struct mali_gp_slot slot;
  48
  49 /* Variables to allow safe pausing of the scheduler */
  50 static _mali_osk_wait_queue_t *gp_scheduler_working_wait_queue = NULL;
  51 static u32 pause_count = 0;
  52
  53 static mali_bool mali_gp_scheduler_is_suspended(void *data);
  54 static void mali_gp_scheduler_job_queued(void);
  55 static void mali_gp_scheduler_job_completed(void);
  56
  57 #if defined(MALI_UPPER_HALF_SCHEDULING)
  58 static _mali_osk_spinlock_irq_t *gp_scheduler_lock = NULL;
  59 #else
  60 static _mali_osk_spinlock_t *gp_scheduler_lock = NULL;
  61 #endif /* defined(MALI_UPPER_HALF_SCHEDULING) */
  62
  63 _mali_osk_errcode_t mali_gp_scheduler_initialize(void)
  64 {
  65         u32 num_groups;
  66         u32 i;
  67         _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK;
  68
  69 #if defined(MALI_UPPER_HALF_SCHEDULING)
  70         gp_scheduler_lock = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER);
  71 #else
  72         gp_scheduler_lock = _mali_osk_spinlock_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_SCHEDULER);
  73 #endif /* defined(MALI_UPPER_HALF_SCHEDULING) */
  74         if (NULL == gp_scheduler_lock) {
  75                 ret = _MALI_OSK_ERR_NOMEM;
  76                 goto cleanup;
  77         }
  78
  79         gp_scheduler_working_wait_queue = _mali_osk_wait_queue_init();
  80         if (NULL == gp_scheduler_working_wait_queue) {
  81                 ret = _MALI_OSK_ERR_NOMEM;
  82                 goto cleanup;
  83         }
  84
  85         /* Find all the available GP cores */
  86         num_groups = mali_group_get_glob_num_groups();
  87         for (i = 0; i < num_groups; i++) {
  88                 struct mali_group *group = mali_group_get_glob_group(i);
  89                 MALI_DEBUG_ASSERT(NULL != group);
  90                 if (NULL != group) {
  91                         struct mali_gp_core *gp_core = mali_group_get_gp_core(group);
  92                         if (NULL != gp_core) {
  93                                 if (0 == gp_version) {
  94                                         /* Retrieve GP version */
  95                                         gp_version = mali_gp_core_get_version(gp_core);
  96                                 }
  97                                 slot.group = group;
  98                                 slot.state = MALI_GP_SLOT_STATE_IDLE;
  99                                 break; /* There is only one GP, no point in looking for more */
 100                         }
 101                 } else {
 102                         ret = _MALI_OSK_ERR_ITEM_NOT_FOUND;
 103                         goto cleanup;
 104                 }
 105         }
 106
 107         return _MALI_OSK_ERR_OK;
 108
 109 cleanup:
 110         if (NULL != gp_scheduler_working_wait_queue) {
 111                 _mali_osk_wait_queue_term(gp_scheduler_working_wait_queue);
 112                 gp_scheduler_working_wait_queue = NULL;
 113         }
 114
 115         if (NULL != gp_scheduler_lock) {
 116 #if defined(MALI_UPPER_HALF_SCHEDULING)
 117                 _mali_osk_spinlock_irq_term(gp_scheduler_lock);
 118 #else
 119                 _mali_osk_spinlock_term(gp_scheduler_lock);
 120 #endif /* defined(MALI_UPPER_HALF_SCHEDULING) */
 121                 gp_scheduler_lock = NULL;
 122         }
 123
 124         return ret;
 125 }
 126
 127 void mali_gp_scheduler_terminate(void)
 128 {
 129         MALI_DEBUG_ASSERT(   MALI_GP_SLOT_STATE_IDLE     == slot.state
 130                              || MALI_GP_SLOT_STATE_DISABLED == slot.state);
 131         MALI_DEBUG_ASSERT_POINTER(slot.group);
 132         mali_group_delete(slot.group);
 133
 134         _mali_osk_wait_queue_term(gp_scheduler_working_wait_queue);
 135
 136 #if defined(MALI_UPPER_HALF_SCHEDULING)
 137         _mali_osk_spinlock_irq_term(gp_scheduler_lock);
 138 #else
 139         _mali_osk_spinlock_term(gp_scheduler_lock);
 140 #endif /* defined(MALI_UPPER_HALF_SCHEDULING) */
 141 }
 142
 143 MALI_STATIC_INLINE void mali_gp_scheduler_lock(void)
 144 {
 145 #if defined(MALI_UPPER_HALF_SCHEDULING)
 146         _mali_osk_spinlock_irq_lock(gp_scheduler_lock);
 147 #else
 148         _mali_osk_spinlock_lock(gp_scheduler_lock);
 149 #endif /* defined(MALI_UPPER_HALF_SCHEDULING) */
 150         MALI_DEBUG_PRINT(5, ("Mali GP scheduler: GP scheduler lock taken\n"));
 151 }
 152
 153 MALI_STATIC_INLINE void mali_gp_scheduler_unlock(void)
 154 {
 155         MALI_DEBUG_PRINT(5, ("Mali GP scheduler: Releasing GP scheduler lock\n"));
 156 #if defined(MALI_UPPER_HALF_SCHEDULING)
 157         _mali_osk_spinlock_irq_unlock(gp_scheduler_lock);
 158 #else
 159         _mali_osk_spinlock_unlock(gp_scheduler_lock);
 160 #endif /* defined(MALI_UPPER_HALF_SCHEDULING) */
 161 }
 162
 163 #if defined(DEBUG)
 164 #define MALI_ASSERT_GP_SCHEDULER_LOCKED() MALI_DEBUG_ASSERT_LOCK_HELD(gp_scheduler_lock)
 165 #else
 166 #define MALI_ASSERT_GP_SCHEDULER_LOCKED() do {} while (0)
 167 #endif /* defined(DEBUG) */
 168
 169 /* Group and scheduler must be locked when entering this function.  Both will be unlocked before
 170  * exiting. */
 171 static void mali_gp_scheduler_schedule_internal_and_unlock(void)
 172 {
 173         struct mali_gp_job *job = NULL;
 174
 175         MALI_DEBUG_ASSERT_LOCK_HELD(slot.group->lock);
 176         MALI_DEBUG_ASSERT_LOCK_HELD(gp_scheduler_lock);
 177
 178         if (0 < pause_count || MALI_GP_SLOT_STATE_IDLE != slot.state ||
 179             (_mali_osk_list_empty(&job_queue) && _mali_osk_list_empty(&job_queue_high))) {
 180                 mali_gp_scheduler_unlock();
 181                 mali_group_unlock(slot.group);
 182                 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Nothing to schedule (paused=%u, idle slots=%u)\n",
 183                                      pause_count, MALI_GP_SLOT_STATE_IDLE == slot.state ? 1 : 0));
 184 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
 185                 trace_gpu_sched_switch(mali_gp_get_hw_core_desc(group->gp_core), sched_clock(), 0, 0, 0);
 186 #endif
 187                 return; /* Nothing to do, so early out */
 188         }
 189
 190         /* Get next job in queue */
 191         if (!_mali_osk_list_empty(&job_queue_high)) {
 192                 job = _MALI_OSK_LIST_ENTRY(job_queue_high.next, struct mali_gp_job, list);
 193         } else {
 194                 MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job_queue));
 195                 job = _MALI_OSK_LIST_ENTRY(job_queue.next, struct mali_gp_job, list);
 196         }
 197
 198         MALI_DEBUG_ASSERT_POINTER(job);
 199
 200         /* Remove the job from queue */
 201         _mali_osk_list_del(&job->list);
 202
 203         /* Mark slot as busy */
 204         slot.state = MALI_GP_SLOT_STATE_WORKING;
 205
 206         mali_gp_scheduler_unlock();
 207
 208         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Starting job %u (0x%08X)\n", mali_gp_job_get_id(job), job));
 209
 210         mali_group_start_gp_job(slot.group, job);
 211         mali_group_unlock(slot.group);
 212 }
 213
 214 void mali_gp_scheduler_schedule(void)
 215 {
 216         mali_group_lock(slot.group);
 217         mali_gp_scheduler_lock();
 218
 219         mali_gp_scheduler_schedule_internal_and_unlock();
 220 }
 221
 222 static void mali_gp_scheduler_return_job_to_user(struct mali_gp_job *job, mali_bool success)
 223 {
 224         _mali_uk_gp_job_finished_s *jobres = job->finished_notification->result_buffer;
 225         _mali_osk_memset(jobres, 0, sizeof(_mali_uk_gp_job_finished_s)); /* @@@@ can be removed once we initialize all members in this struct */
 226         jobres->user_job_ptr = mali_gp_job_get_user_id(job);
 227         if (MALI_TRUE == success) {
 228                 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
 229         } else {
 230                 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
 231         }
 232
 233         jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job);
 234         jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job);
 235         jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job);
 236
 237         mali_session_send_notification(mali_gp_job_get_session(job), job->finished_notification);
 238         job->finished_notification = NULL;
 239
 240         mali_gp_job_delete(job);
 241         mali_gp_scheduler_job_completed();
 242 }
 243
 244 /* Group must be locked when entering this function.  Will be unlocked before exiting. */
 245 void mali_gp_scheduler_job_done(struct mali_group *group, struct mali_gp_job *job, mali_bool success)
 246 {
 247         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 248
 249         MALI_DEBUG_ASSERT_POINTER(group);
 250         MALI_DEBUG_ASSERT_POINTER(job);
 251
 252         MALI_DEBUG_ASSERT_LOCK_HELD(group->lock);
 253         MALI_DEBUG_ASSERT(slot.group == group);
 254
 255         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) completed (%s)\n", mali_gp_job_get_id(job), job, success ? "success" : "failure"));
 256
 257         /* Release tracker. */
 258         schedule_mask |= mali_timeline_tracker_release(&job->tracker);
 259
 260         /* Signal PP job. */
 261         schedule_mask |= mali_gp_job_signal_pp_tracker(job, success);
 262
 263         mali_gp_scheduler_lock();
 264
 265         /* Mark slot as idle again */
 266         slot.state = MALI_GP_SLOT_STATE_IDLE;
 267
 268         /* If paused, then this was the last job, so wake up sleeping workers */
 269         if (pause_count > 0) {
 270                 _mali_osk_wait_queue_wake_up(gp_scheduler_working_wait_queue);
 271         }
 272
 273         /* Schedule any queued GP jobs on this group. */
 274         mali_gp_scheduler_schedule_internal_and_unlock();
 275
 276         /* GP is now scheduled, removing it from the mask. */
 277         schedule_mask &= ~MALI_SCHEDULER_MASK_GP;
 278
 279         if (MALI_SCHEDULER_MASK_EMPTY != schedule_mask) {
 280                 /* Releasing the tracker activated other jobs that need scheduling. */
 281                 mali_scheduler_schedule_from_mask(schedule_mask, MALI_FALSE);
 282         }
 283
 284         /* Sends the job end message to user space and free the job object */
 285         mali_gp_scheduler_return_job_to_user(job, success);
 286 }
 287
 288 void mali_gp_scheduler_oom(struct mali_group *group, struct mali_gp_job *job)
 289 {
 290         _mali_uk_gp_job_suspended_s * jobres;
 291         _mali_osk_notification_t * notification;
 292
 293         mali_gp_scheduler_lock();
 294
 295         notification = job->oom_notification;
 296         job->oom_notification = NULL;
 297         slot.returned_cookie = mali_gp_job_get_id(job);
 298
 299         jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer;
 300         jobres->user_job_ptr = mali_gp_job_get_user_id(job);
 301         jobres->cookie = mali_gp_job_get_id(job);
 302
 303         mali_gp_scheduler_unlock();
 304
 305         mali_session_send_notification(mali_gp_job_get_session(job), notification);
 306
 307         /*
 308         * If this function failed, then we could return the job to user space right away,
 309         * but there is a job timer anyway that will do that eventually.
 310         * This is not exactly a common case anyway.
 311         */
 312 }
 313
 314 void mali_gp_scheduler_suspend(void)
 315 {
 316         mali_gp_scheduler_lock();
 317         pause_count++; /* Increment the pause_count so that no more jobs will be scheduled */
 318         mali_gp_scheduler_unlock();
 319
 320         _mali_osk_wait_queue_wait_event(gp_scheduler_working_wait_queue, mali_gp_scheduler_is_suspended, NULL);
 321 }
 322
 323 void mali_gp_scheduler_resume(void)
 324 {
 325         mali_gp_scheduler_lock();
 326         pause_count--; /* Decrement pause_count to allow scheduling again (if it reaches 0) */
 327         mali_gp_scheduler_unlock();
 328         if (0 == pause_count) {
 329                 mali_gp_scheduler_schedule();
 330         }
 331 }
 332
 333 mali_timeline_point mali_gp_scheduler_submit_job(struct mali_session_data *session, struct mali_gp_job *job)
 334 {
 335         mali_timeline_point point;
 336
 337         MALI_DEBUG_ASSERT_POINTER(session);
 338         MALI_DEBUG_ASSERT_POINTER(job);
 339
 340         /* We hold a PM reference for every job we hold queued (and running) */
 341         _mali_osk_pm_dev_ref_add();
 342
 343         /* Add job to Timeline system. */
 344         point = mali_timeline_system_add_tracker(session->timeline_system, &job->tracker, MALI_TIMELINE_GP);
 345
 346         return point;
 347 }
 348
 349 _mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx, _mali_uk_gp_start_job_s *uargs)
 350 {
 351         struct mali_session_data *session;
 352         struct mali_gp_job *job;
 353         mali_timeline_point point;
 354         u32 __user *timeline_point_ptr = NULL;
 355
 356         MALI_DEBUG_ASSERT_POINTER(uargs);
 357         MALI_DEBUG_ASSERT_POINTER(ctx);
 358
 359         session = (struct mali_session_data*)ctx;
 360
 361         job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(), NULL);
 362         if (NULL == job) {
 363                 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
 364                 return _MALI_OSK_ERR_NOMEM;
 365         }
 366
 367         timeline_point_ptr = (u32 __user *) job->uargs.timeline_point_ptr;
 368
 369         point = mali_gp_scheduler_submit_job(session, job);
 370
 371         if (0 != _mali_osk_put_user(((u32) point), timeline_point_ptr)) {
 372                 /* Let user space know that something failed after the job was started. */
 373                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 374         }
 375
 376         return _MALI_OSK_ERR_OK;
 377 }
 378
 379 _mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args)
 380 {
 381         MALI_DEBUG_ASSERT_POINTER(args);
 382         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
 383         args->number_of_cores = 1;
 384         return _MALI_OSK_ERR_OK;
 385 }
 386
 387 _mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args)
 388 {
 389         MALI_DEBUG_ASSERT_POINTER(args);
 390         MALI_CHECK_NON_NULL(args->ctx, _MALI_OSK_ERR_INVALID_ARGS);
 391         args->version = gp_version;
 392         return _MALI_OSK_ERR_OK;
 393 }
 394
 395 _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args)
 396 {
 397         struct mali_session_data *session;
 398         struct mali_gp_job *resumed_job;
 399         _mali_osk_notification_t *new_notification = 0;
 400
 401         MALI_DEBUG_ASSERT_POINTER(args);
 402
 403         if (NULL == args->ctx) {
 404                 return _MALI_OSK_ERR_INVALID_ARGS;
 405         }
 406
 407         session = (struct mali_session_data*)args->ctx;
 408         if (NULL == session) {
 409                 return _MALI_OSK_ERR_FAULT;
 410         }
 411
 412         if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code) {
 413                 new_notification = _mali_osk_notification_create(_MALI_NOTIFICATION_GP_STALLED, sizeof(_mali_uk_gp_job_suspended_s));
 414
 415                 if (NULL == new_notification) {
 416                         MALI_PRINT_ERROR(("Mali GP scheduler: Failed to allocate notification object. Will abort GP job.\n"));
 417                         mali_group_lock(slot.group);
 418                         mali_group_abort_gp_job(slot.group, args->cookie);
 419                         mali_group_unlock(slot.group);
 420                         return _MALI_OSK_ERR_FAULT;
 421                 }
 422         }
 423
 424         mali_group_lock(slot.group);
 425
 426         if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code) {
 427                 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Resuming job %u with new heap; 0x%08X - 0x%08X\n", args->cookie, args->arguments[0], args->arguments[1]));
 428
 429                 resumed_job = mali_group_resume_gp_with_new_heap(slot.group, args->cookie, args->arguments[0], args->arguments[1]);
 430                 if (NULL != resumed_job) {
 431                         resumed_job->oom_notification = new_notification;
 432                         mali_group_unlock(slot.group);
 433                         return _MALI_OSK_ERR_OK;
 434                 } else {
 435                         mali_group_unlock(slot.group);
 436                         _mali_osk_notification_delete(new_notification);
 437                         return _MALI_OSK_ERR_FAULT;
 438                 }
 439         }
 440
 441         MALI_DEBUG_PRINT(2, ("Mali GP scheduler: Aborting job %u, no new heap provided\n", args->cookie));
 442         mali_group_abort_gp_job(slot.group, args->cookie);
 443         mali_group_unlock(slot.group);
 444         return _MALI_OSK_ERR_OK;
 445 }
 446
 447 void mali_gp_scheduler_abort_session(struct mali_session_data *session)
 448 {
 449         struct mali_gp_job *job, *tmp;
 450         _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs);
 451
 452         MALI_DEBUG_ASSERT_POINTER(session);
 453         MALI_DEBUG_ASSERT(session->is_aborting);
 454
 455         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Aborting all jobs from session 0x%08X.\n", session));
 456
 457         mali_gp_scheduler_lock();
 458
 459         /* Find all jobs from the aborting session. */
 460         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &job_queue, struct mali_gp_job, list) {
 461                 if (job->session == session) {
 462                         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Removing job %u (0x%08X) from queue.\n", mali_gp_job_get_id(job), job));
 463                         _mali_osk_list_move(&job->list, &removed_jobs);
 464                 }
 465         }
 466
 467         /* Find all high priority jobs from the aborting session. */
 468         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &job_queue_high, struct mali_gp_job, list) {
 469                 if (job->session == session) {
 470                         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Removing job %u (0x%08X) from queue.\n", mali_gp_job_get_id(job), job));
 471                         _mali_osk_list_move(&job->list, &removed_jobs);
 472                 }
 473         }
 474
 475         mali_gp_scheduler_unlock();
 476
 477         /* Release and delete all found jobs from the aborting session. */
 478         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &removed_jobs, struct mali_gp_job, list) {
 479                 mali_timeline_tracker_release(&job->tracker);
 480                 mali_gp_job_signal_pp_tracker(job, MALI_FALSE);
 481                 mali_gp_job_delete(job);
 482                 mali_gp_scheduler_job_completed();
 483         }
 484
 485         /* Abort any running jobs from the session. */
 486         mali_group_abort_session(slot.group, session);
 487 }
 488
 489 static mali_bool mali_gp_scheduler_is_suspended(void *data)
 490 {
 491         mali_bool ret;
 492
 493         /* This callback does not use the data pointer. */
 494         MALI_IGNORE(data);
 495
 496         mali_gp_scheduler_lock();
 497         ret = pause_count > 0 && (slot.state == MALI_GP_SLOT_STATE_IDLE || slot.state == MALI_GP_SLOT_STATE_DISABLED);
 498         mali_gp_scheduler_unlock();
 499
 500         return ret;
 501 }
 502
 503
 504 #if MALI_STATE_TRACKING
 505 u32 mali_gp_scheduler_dump_state(char *buf, u32 size)
 506 {
 507         int n = 0;
 508
 509         n += _mali_osk_snprintf(buf + n, size - n, "GP\n");
 510         n += _mali_osk_snprintf(buf + n, size - n, "\tQueue is %s\n", _mali_osk_list_empty(&job_queue) ? "empty" : "not empty");
 511         n += _mali_osk_snprintf(buf + n, size - n, "\tHigh priority queue is %s\n", _mali_osk_list_empty(&job_queue_high) ? "empty" : "not empty");
 512
 513         n += mali_group_dump_state(slot.group, buf + n, size - n);
 514         n += _mali_osk_snprintf(buf + n, size - n, "\n");
 515
 516         return n;
 517 }
 518 #endif
 519
 520 void mali_gp_scheduler_reset_all_groups(void)
 521 {
 522         if (NULL != slot.group) {
 523                 mali_group_lock(slot.group);
 524                 mali_group_reset(slot.group);
 525                 mali_group_unlock(slot.group);
 526         }
 527 }
 528
 529 void mali_gp_scheduler_zap_all_active(struct mali_session_data *session)
 530 {
 531         if (NULL != slot.group) {
 532                 mali_group_zap_session(slot.group, session);
 533         }
 534 }
 535
 536 void mali_gp_scheduler_enable_group(struct mali_group *group)
 537 {
 538         MALI_DEBUG_ASSERT_POINTER(group);
 539         MALI_DEBUG_ASSERT(slot.group == group);
 540         MALI_DEBUG_PRINT(2, ("Mali GP scheduler: enabling gp group %p\n", group));
 541
 542         mali_group_lock(group);
 543
 544         if (MALI_GROUP_STATE_DISABLED != group->state) {
 545                 mali_group_unlock(group);
 546                 MALI_DEBUG_PRINT(2, ("Mali GP scheduler: gp group %p already enabled\n", group));
 547                 return;
 548         }
 549
 550         mali_gp_scheduler_lock();
 551
 552         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_DISABLED == group->state);
 553         MALI_DEBUG_ASSERT(MALI_GP_SLOT_STATE_DISABLED == slot.state);
 554         slot.state = MALI_GP_SLOT_STATE_IDLE;
 555         group->state = MALI_GROUP_STATE_IDLE;
 556
 557         mali_group_power_on_group(group);
 558         mali_group_reset(group);
 559
 560         /* Pick up any jobs that might have been queued while the GP group was disabled. */
 561         mali_gp_scheduler_schedule_internal_and_unlock();
 562 }
 563
 564 void mali_gp_scheduler_disable_group(struct mali_group *group)
 565 {
 566         MALI_DEBUG_ASSERT_POINTER(group);
 567         MALI_DEBUG_ASSERT(slot.group == group);
 568         MALI_DEBUG_PRINT(2, ("Mali GP scheduler: disabling gp group %p\n", group));
 569
 570         mali_gp_scheduler_suspend();
 571         mali_group_lock(group);
 572         mali_gp_scheduler_lock();
 573
 574         MALI_DEBUG_ASSERT(   MALI_GROUP_STATE_IDLE     == group->state
 575                              || MALI_GROUP_STATE_DISABLED == group->state);
 576
 577         if (MALI_GROUP_STATE_DISABLED == group->state) {
 578                 MALI_DEBUG_ASSERT(MALI_GP_SLOT_STATE_DISABLED == slot.state);
 579                 MALI_DEBUG_PRINT(2, ("Mali GP scheduler: gp group %p already disabled\n", group));
 580         } else {
 581                 MALI_DEBUG_ASSERT(MALI_GP_SLOT_STATE_IDLE == slot.state);
 582                 slot.state = MALI_GP_SLOT_STATE_DISABLED;
 583                 group->state = MALI_GROUP_STATE_DISABLED;
 584
 585                 mali_group_power_off_group(group, MALI_TRUE);
 586         }
 587
 588         mali_gp_scheduler_unlock();
 589         mali_group_unlock(group);
 590         mali_gp_scheduler_resume();
 591 }
 592
 593 static mali_scheduler_mask mali_gp_scheduler_queue_job(struct mali_gp_job *job)
 594 {
 595         _mali_osk_list_t *queue = NULL;
 596         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 597         struct mali_gp_job *iter, *tmp;
 598
 599         MALI_DEBUG_ASSERT_POINTER(job);
 600         MALI_DEBUG_ASSERT_POINTER(job->session);
 601
 602         MALI_DEBUG_ASSERT_LOCK_HELD(gp_scheduler_lock);
 603
 604         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE | MALI_PROFILING_EVENT_CHANNEL_SOFTWARE | MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE, job->pid, job->tid, job->uargs.frame_builder_id, job->uargs.flush_id, 0);
 605
 606         job->cache_order = mali_scheduler_get_new_cache_order();
 607
 608         /* Determine which queue the job should be added to. */
 609         if (job->session->use_high_priority_job_queue) {
 610                 queue = &job_queue_high;
 611         } else {
 612                 queue = &job_queue;
 613         }
 614
 615         /* Find position in queue where job should be added. */
 616         _MALI_OSK_LIST_FOREACHENTRY_REVERSE(iter, tmp, queue, struct mali_gp_job, list) {
 617                 if (mali_gp_job_is_after(job, iter)) {
 618                         break;
 619                 }
 620         }
 621
 622         /* Add job to queue. */
 623         _mali_osk_list_add(&job->list, &iter->list);
 624
 625         /* Set schedule bitmask if the GP core is idle. */
 626         if (MALI_GP_SLOT_STATE_IDLE == slot.state) {
 627                 schedule_mask |= MALI_SCHEDULER_MASK_GP;
 628         }
 629
 630         mali_gp_scheduler_job_queued();
 631
 632 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
 633         trace_gpu_job_enqueue(mali_gp_job_get_tid(job), mali_gp_job_get_id(job), "GP");
 634 #endif
 635
 636         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n", mali_gp_job_get_id(job), job));
 637
 638         return schedule_mask;
 639 }
 640
 641 mali_scheduler_mask mali_gp_scheduler_activate_job(struct mali_gp_job *job)
 642 {
 643         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
 644
 645         MALI_DEBUG_ASSERT_POINTER(job);
 646         MALI_DEBUG_ASSERT_POINTER(job->session);
 647
 648         MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n", mali_gp_job_get_id(job), job));
 649
 650         mali_gp_scheduler_lock();
 651
 652         if (unlikely(job->session->is_aborting)) {
 653                 /* Before checking if the session is aborting, the scheduler must be locked. */
 654                 MALI_DEBUG_ASSERT_LOCK_HELD(gp_scheduler_lock);
 655
 656                 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) activated while session is aborting.\n", mali_gp_job_get_id(job), job));
 657
 658                 /* This job should not be on any list. */
 659                 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list));
 660
 661                 mali_gp_scheduler_unlock();
 662
 663                 /* Release tracker and delete job. */
 664                 mali_timeline_tracker_release(&job->tracker);
 665                 mali_gp_job_signal_pp_tracker(job, MALI_FALSE);
 666                 mali_gp_job_delete(job);
 667                 mali_gp_scheduler_job_completed();
 668
 669                 /* Since we are aborting we ignore the scheduler mask. */
 670                 return MALI_SCHEDULER_MASK_EMPTY;
 671         }
 672
 673         /* GP job is ready to run, queue it. */
 674         schedule_mask = mali_gp_scheduler_queue_job(job);
 675
 676         mali_gp_scheduler_unlock();
 677
 678         return schedule_mask;
 679 }
 680
 681 static void mali_gp_scheduler_job_queued(void)
 682 {
 683         if (mali_utilization_enabled()) {
 684                 /*
 685                  * We cheat a little bit by counting the PP as busy from the time a GP job is queued.
 686                  * This will be fine because we only loose the tiny idle gap between jobs, but
 687                  * we will instead get less utilization work to do (less locks taken)
 688                  */
 689                 mali_utilization_gp_start();
 690         }
 691 }
 692
 693 static void mali_gp_scheduler_job_completed(void)
 694 {
 695         /* Release the PM reference we got in the mali_gp_scheduler_job_queued() function */
 696         _mali_osk_pm_dev_ref_dec();
 697
 698         if (mali_utilization_enabled()) {
 699                 mali_utilization_gp_end();
 700         }
 701 }