2 * Copyright (C) 2011-2012 ARM Limited. All rights reserved.
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
11 #include "mali_pp_scheduler.h"
12 #include "mali_kernel_common.h"
13 #include "mali_kernel_core.h"
15 #include "mali_osk_list.h"
16 #include "mali_scheduler.h"
18 #include "mali_pp_job.h"
19 #include "mali_group.h"
22 #if defined(CONFIG_SYNC)
23 #define MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE 1
26 /* Maximum of 8 PP cores (a group can only have maximum of 1 PP core) */
27 #define MALI_MAX_NUMBER_OF_PP_GROUPS 9
29 static mali_bool mali_pp_scheduler_is_suspended(void);
30 static void mali_pp_scheduler_do_schedule(void *arg);
31 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
32 static void mali_pp_scheduler_do_job_delete(void *arg);
35 static u32 pp_version = 0;
37 /* Physical job queue */
38 static _MALI_OSK_LIST_HEAD_STATIC_INIT(job_queue); /* List of physical jobs with some unscheduled work */
39 static u32 job_queue_depth = 0;
42 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_working); /* List of physical groups with working jobs on the pp core */
43 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_idle); /* List of physical groups with idle jobs on the pp core */
45 /* Virtual job queue (Mali-450 only) */
46 static _MALI_OSK_LIST_HEAD_STATIC_INIT(virtual_job_queue); /* List of unstarted jobs for the virtual group */
47 static u32 virtual_job_queue_depth = 0;
49 /* Virtual group (Mali-450 only) */
50 static struct mali_group *virtual_group = NULL; /* Virtual group (if any) */
51 static mali_bool virtual_group_working = MALI_FALSE; /* Flag which indicates whether the virtual group is working or idle */
53 /* Number of physical cores */
54 static u32 num_cores = 0;
56 /* Variables to allow safe pausing of the scheduler */
57 static _mali_osk_wait_queue_t *pp_scheduler_working_wait_queue = NULL;
58 static u32 pause_count = 0;
60 static _mali_osk_lock_t *pp_scheduler_lock = NULL;
61 /* Contains tid of thread that locked the scheduler or 0, if not locked */
62 MALI_DEBUG_CODE(static u32 pp_scheduler_lock_owner = 0);
64 static _mali_osk_wq_work_t *pp_scheduler_wq_schedule = NULL;
66 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
67 static _mali_osk_wq_work_t *pp_scheduler_wq_job_delete = NULL;
68 static _mali_osk_lock_t *pp_scheduler_job_delete_lock = NULL;
69 static _MALI_OSK_LIST_HEAD_STATIC_INIT(pp_scheduler_job_deletion_queue);
72 _mali_osk_errcode_t mali_pp_scheduler_initialize(void)
74 struct mali_group *group;
75 struct mali_pp_core *pp_core;
76 _mali_osk_lock_flags_t lock_flags;
80 #if defined(MALI_UPPER_HALF_SCHEDULING)
81 lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK_IRQ | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
83 lock_flags = _MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK | _MALI_OSK_LOCKFLAG_NONINTERRUPTABLE;
86 _MALI_OSK_INIT_LIST_HEAD(&job_queue);
87 _MALI_OSK_INIT_LIST_HEAD(&group_list_working);
88 _MALI_OSK_INIT_LIST_HEAD(&group_list_idle);
90 _MALI_OSK_INIT_LIST_HEAD(&virtual_job_queue);
92 pp_scheduler_lock = _mali_osk_lock_init(lock_flags, 0, _MALI_OSK_LOCK_ORDER_SCHEDULER);
93 if (NULL == pp_scheduler_lock)
95 return _MALI_OSK_ERR_NOMEM;
98 pp_scheduler_working_wait_queue = _mali_osk_wait_queue_init();
99 if (NULL == pp_scheduler_working_wait_queue)
101 _mali_osk_lock_term(pp_scheduler_lock);
102 return _MALI_OSK_ERR_NOMEM;
105 pp_scheduler_wq_schedule = _mali_osk_wq_create_work(mali_pp_scheduler_do_schedule, NULL);
106 if (NULL == pp_scheduler_wq_schedule)
108 _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
109 _mali_osk_lock_term(pp_scheduler_lock);
110 return _MALI_OSK_ERR_NOMEM;
113 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
114 pp_scheduler_wq_job_delete = _mali_osk_wq_create_work(mali_pp_scheduler_do_job_delete, NULL);
115 if (NULL == pp_scheduler_wq_job_delete)
117 _mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
118 _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
119 _mali_osk_lock_term(pp_scheduler_lock);
120 return _MALI_OSK_ERR_NOMEM;
123 pp_scheduler_job_delete_lock = _mali_osk_lock_init(_MALI_OSK_LOCKFLAG_ORDERED | _MALI_OSK_LOCKFLAG_SPINLOCK_IRQ |_MALI_OSK_LOCKFLAG_NONINTERRUPTABLE, 0, _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
124 if (NULL == pp_scheduler_job_delete_lock)
126 _mali_osk_wq_delete_work(pp_scheduler_wq_job_delete);
127 _mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
128 _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
129 _mali_osk_lock_term(pp_scheduler_lock);
130 return _MALI_OSK_ERR_NOMEM;
134 num_groups = mali_group_get_glob_num_groups();
136 /* Do we have a virtual group? */
137 for (i = 0; i < num_groups; i++)
139 group = mali_group_get_glob_group(i);
141 if (mali_group_is_virtual(group))
143 MALI_DEBUG_PRINT(3, ("Found virtual group %p\n", group));
145 virtual_group = group;
150 /* Find all the available PP cores */
151 for (i = 0; i < num_groups; i++)
153 group = mali_group_get_glob_group(i);
154 pp_core = mali_group_get_pp_core(group);
156 if (NULL != pp_core && !mali_group_is_virtual(group))
160 /* Retrieve PP version from the first available PP core */
161 pp_version = mali_pp_core_get_version(pp_core);
164 if (NULL != virtual_group)
166 /* Add all physical PP cores to the virtual group */
167 mali_group_lock(virtual_group);
168 group->state = MALI_GROUP_STATE_JOINING_VIRTUAL;
169 mali_group_add_group(virtual_group, group);
170 mali_group_unlock(virtual_group);
174 _mali_osk_list_add(&group->pp_scheduler_list, &group_list_idle);
181 return _MALI_OSK_ERR_OK;
184 void mali_pp_scheduler_terminate(void)
186 struct mali_group *group, *temp;
188 /* Delete all groups owned by scheduler */
189 if (NULL != virtual_group)
191 mali_group_delete(virtual_group);
194 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working));
195 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, pp_scheduler_list)
197 mali_group_delete(group);
200 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
201 _mali_osk_lock_term(pp_scheduler_job_delete_lock);
202 _mali_osk_wq_delete_work(pp_scheduler_wq_job_delete);
205 _mali_osk_wq_delete_work(pp_scheduler_wq_schedule);
206 _mali_osk_wait_queue_term(pp_scheduler_working_wait_queue);
207 _mali_osk_lock_term(pp_scheduler_lock);
210 MALI_STATIC_INLINE void mali_pp_scheduler_lock(void)
212 if(_MALI_OSK_ERR_OK != _mali_osk_lock_wait(pp_scheduler_lock, _MALI_OSK_LOCKMODE_RW))
214 /* Non-interruptable lock failed: this should never happen. */
215 MALI_DEBUG_ASSERT(0);
217 MALI_DEBUG_PRINT(5, ("Mali PP scheduler: PP scheduler lock taken\n"));
218 MALI_DEBUG_ASSERT(0 == pp_scheduler_lock_owner);
219 MALI_DEBUG_CODE(pp_scheduler_lock_owner = _mali_osk_get_tid());
222 MALI_STATIC_INLINE void mali_pp_scheduler_unlock(void)
224 MALI_DEBUG_PRINT(5, ("Mali PP scheduler: Releasing PP scheduler lock\n"));
225 MALI_DEBUG_ASSERT(_mali_osk_get_tid() == pp_scheduler_lock_owner);
226 MALI_DEBUG_CODE(pp_scheduler_lock_owner = 0);
227 _mali_osk_lock_signal(pp_scheduler_lock, _MALI_OSK_LOCKMODE_RW);
231 MALI_STATIC_INLINE void mali_pp_scheduler_assert_locked(void)
233 MALI_DEBUG_ASSERT(_mali_osk_get_tid() == pp_scheduler_lock_owner);
235 #define MALI_ASSERT_PP_SCHEDULER_LOCKED() mali_pp_scheduler_assert_locked()
237 #define MALI_ASSERT_PP_SCHEDULER_LOCKED()
241 * Returns a physical job if a physical job is ready to run (no barrier present)
243 MALI_STATIC_INLINE struct mali_pp_job *mali_pp_scheduler_get_physical_job(void)
245 MALI_ASSERT_PP_SCHEDULER_LOCKED();
247 if (!_mali_osk_list_empty(&job_queue))
249 struct mali_pp_job *job;
251 MALI_DEBUG_ASSERT(job_queue_depth > 0);
252 job = _MALI_OSK_LIST_ENTRY(job_queue.next, struct mali_pp_job, list);
254 if (!mali_pp_job_has_active_barrier(job))
263 MALI_STATIC_INLINE void mali_pp_scheduler_dequeue_physical_job(struct mali_pp_job *job)
265 MALI_ASSERT_PP_SCHEDULER_LOCKED();
266 MALI_DEBUG_ASSERT(job_queue_depth > 0);
268 /* Remove job from queue */
269 if (!mali_pp_job_has_unstarted_sub_jobs(job))
271 /* All sub jobs have been started: remove job from queue */
272 _mali_osk_list_delinit(&job->list);
279 * Returns a virtual job if a virtual job is ready to run (no barrier present)
281 MALI_STATIC_INLINE struct mali_pp_job *mali_pp_scheduler_get_virtual_job(void)
283 MALI_ASSERT_PP_SCHEDULER_LOCKED();
284 MALI_DEBUG_ASSERT_POINTER(virtual_group);
286 if (!_mali_osk_list_empty(&virtual_job_queue))
288 struct mali_pp_job *job;
290 MALI_DEBUG_ASSERT(virtual_job_queue_depth > 0);
291 job = _MALI_OSK_LIST_ENTRY(virtual_job_queue.next, struct mali_pp_job, list);
293 if (!mali_pp_job_has_active_barrier(job))
302 MALI_STATIC_INLINE void mali_pp_scheduler_dequeue_virtual_job(struct mali_pp_job *job)
304 MALI_ASSERT_PP_SCHEDULER_LOCKED();
305 MALI_DEBUG_ASSERT(virtual_job_queue_depth > 0);
307 /* Remove job from queue */
308 _mali_osk_list_delinit(&job->list);
309 --virtual_job_queue_depth;
313 * Checks if the criteria is met for removing a physical core from virtual group
315 MALI_STATIC_INLINE mali_bool mali_pp_scheduler_can_move_virtual_to_physical(void)
317 MALI_ASSERT_PP_SCHEDULER_LOCKED();
318 MALI_DEBUG_ASSERT(NULL != virtual_group);
319 MALI_ASSERT_GROUP_LOCKED(virtual_group);
321 * The criteria for taking out a physical group from a virtual group are the following:
322 * - There virtual group is idle
323 * - There are currently no physical groups (idle and working)
324 * - There are physical jobs to be scheduled (without a barrier)
326 return (!virtual_group_working) &&
327 _mali_osk_list_empty(&group_list_idle) &&
328 _mali_osk_list_empty(&group_list_working) &&
329 (NULL != mali_pp_scheduler_get_physical_job());
332 MALI_STATIC_INLINE struct mali_group *mali_pp_scheduler_acquire_physical_group(void)
334 MALI_ASSERT_PP_SCHEDULER_LOCKED();
336 if (!_mali_osk_list_empty(&group_list_idle))
338 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Acquiring physical group from idle list\n"));
339 return _MALI_OSK_LIST_ENTRY(group_list_idle.next, struct mali_group, pp_scheduler_list);
341 else if (NULL != virtual_group)
343 MALI_ASSERT_GROUP_LOCKED(virtual_group);
344 if (mali_pp_scheduler_can_move_virtual_to_physical())
346 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Acquiring physical group from virtual group\n"));
347 return mali_group_acquire_group(virtual_group);
354 static void mali_pp_scheduler_schedule(void)
356 struct mali_group* physical_groups_to_start[MALI_MAX_NUMBER_OF_PP_GROUPS-1];
357 struct mali_pp_job* physical_jobs_to_start[MALI_MAX_NUMBER_OF_PP_GROUPS-1];
358 u32 physical_subjobs_to_start[MALI_MAX_NUMBER_OF_PP_GROUPS-1];
359 int num_physical_jobs_to_start = 0;
362 if (NULL != virtual_group)
364 /* Need to lock the virtual group because we might need to grab a physical group from it */
365 mali_group_lock(virtual_group);
368 mali_pp_scheduler_lock();
371 /* Scheduler is suspended, don't schedule any jobs */
372 mali_pp_scheduler_unlock();
373 if (NULL != virtual_group)
375 mali_group_unlock(virtual_group);
380 /* Find physical job(s) to schedule first */
383 struct mali_group *group;
384 struct mali_pp_job *job;
387 job = mali_pp_scheduler_get_physical_job();
390 break; /* No job, early out */
393 MALI_DEBUG_ASSERT(!mali_pp_job_is_virtual(job));
394 MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
395 MALI_DEBUG_ASSERT(1 <= mali_pp_job_get_sub_job_count(job));
397 /* Acquire a physical group, either from the idle list or from the virtual group.
398 * In case the group was acquired from the virtual group, it's state will be
399 * LEAVING_VIRTUAL and must be set to IDLE before it can be used. */
400 group = mali_pp_scheduler_acquire_physical_group();
403 /* Could not get a group to run the job on, early out */
404 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: No more physical groups available.\n"));
408 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Acquired physical group %p\n", group));
410 /* Mark subjob as started */
411 subjob = mali_pp_job_get_first_unstarted_sub_job(job);
412 mali_pp_job_mark_sub_job_started(job, subjob);
414 /* Remove job from queue (if we now got the last subjob) */
415 mali_pp_scheduler_dequeue_physical_job(job);
417 /* Move group to working list */
418 _mali_osk_list_move(&(group->pp_scheduler_list), &group_list_working);
420 /* Keep track of this group, so that we actually can start the job once we are done with the scheduler lock we are now holding */
421 physical_groups_to_start[num_physical_jobs_to_start] = group;
422 physical_jobs_to_start[num_physical_jobs_to_start] = job;
423 physical_subjobs_to_start[num_physical_jobs_to_start] = subjob;
424 ++num_physical_jobs_to_start;
426 MALI_DEBUG_ASSERT(num_physical_jobs_to_start < MALI_MAX_NUMBER_OF_PP_GROUPS);
429 /* See if we have a virtual job to schedule */
430 if (NULL != virtual_group)
432 if (!virtual_group_working)
434 struct mali_pp_job *job = mali_pp_scheduler_get_virtual_job();
437 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
438 MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
439 MALI_DEBUG_ASSERT(1 == mali_pp_job_get_sub_job_count(job));
441 /* Mark the one and only subjob as started */
442 mali_pp_job_mark_sub_job_started(job, 0);
444 /* Remove job from queue */
445 mali_pp_scheduler_dequeue_virtual_job(job);
447 /* Virtual group is now working */
448 virtual_group_working = MALI_TRUE;
451 * We no longer need the scheduler lock,
452 * but we still need the virtual lock in order to start the virtual job
454 mali_pp_scheduler_unlock();
457 if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(virtual_group, job, 0))
459 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Virtual job %u (0x%08X) part %u/%u started (from schedule)\n",
460 mali_pp_job_get_id(job), job, 1,
461 mali_pp_job_get_sub_job_count(job)));
465 MALI_DEBUG_ASSERT(0);
468 /* And now we are all done with the virtual_group lock as well */
469 mali_group_unlock(virtual_group);
473 /* No virtual job, release the two locks we are holding */
474 mali_pp_scheduler_unlock();
475 mali_group_unlock(virtual_group);
480 /* Virtual core busy, release the two locks we are holding */
481 mali_pp_scheduler_unlock();
482 mali_group_unlock(virtual_group);
488 /* There is no virtual group, release the only lock we are holding */
489 mali_pp_scheduler_unlock();
493 * Now we have released the scheduler lock, and we are ready to kick of the actual starting of the
495 * The reason we want to wait until we have released the scheduler lock is that job start actually
496 * may take quite a bit of time (quite many registers needs to be written). This will allow new jobs
497 * from user space to come in, and post processing of other PP jobs to happen at the same time as we
500 for (i = 0; i < num_physical_jobs_to_start; i++)
502 struct mali_group *group = physical_groups_to_start[i];
503 struct mali_pp_job *job = physical_jobs_to_start[i];
504 u32 sub_job = physical_subjobs_to_start[i];
506 MALI_DEBUG_ASSERT_POINTER(group);
507 MALI_DEBUG_ASSERT_POINTER(job);
509 mali_group_lock(group);
511 /* In case this group was acquired from a virtual core, update it's state to IDLE */
512 group->state = MALI_GROUP_STATE_IDLE;
514 if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(group, job, sub_job))
516 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Physical job %u (0x%08X) part %u/%u started (from schedule)\n",
517 mali_pp_job_get_id(job), job, sub_job + 1,
518 mali_pp_job_get_sub_job_count(job)));
522 MALI_DEBUG_ASSERT(0);
525 mali_group_unlock(group);
527 /* @@@@ todo: remove the return value from mali_group_start_xx_job, since we can't fail on Mali-300++ */
531 static void mali_pp_scheduler_return_job_to_user(struct mali_pp_job *job, mali_bool deferred)
533 if (MALI_FALSE == mali_pp_job_use_no_notification(job))
536 u32 sub_jobs = mali_pp_job_get_sub_job_count(job);
537 mali_bool success = mali_pp_job_was_success(job);
539 _mali_uk_pp_job_finished_s *jobres = job->finished_notification->result_buffer;
540 _mali_osk_memset(jobres, 0, sizeof(_mali_uk_pp_job_finished_s)); /* @@@@ can be removed once we initialize all members in this struct */
541 jobres->user_job_ptr = mali_pp_job_get_user_id(job);
542 if (MALI_TRUE == success)
544 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
548 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
551 for (i = 0; i < sub_jobs; i++)
553 jobres->perf_counter0[i] = mali_pp_job_get_perf_counter_value0(job, i);
554 jobres->perf_counter1[i] = mali_pp_job_get_perf_counter_value1(job, i);
557 mali_session_send_notification(mali_pp_job_get_session(job), job->finished_notification);
558 job->finished_notification = NULL;
561 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
562 if (MALI_TRUE == deferred)
564 /* The deletion of the job object (releasing sync refs etc) must be done in a different context */
565 _mali_osk_lock_wait(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
567 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&job->list)); /* This job object should not be on any list */
568 _mali_osk_list_addtail(&job->list, &pp_scheduler_job_deletion_queue);
570 _mali_osk_lock_signal(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
572 _mali_osk_wq_schedule_work(pp_scheduler_wq_job_delete);
576 mali_pp_job_delete(job);
579 MALI_DEBUG_ASSERT(MALI_FALSE == deferred); /* no use cases need this in this configuration */
580 mali_pp_job_delete(job);
584 static void mali_pp_scheduler_do_schedule(void *arg)
588 mali_pp_scheduler_schedule();
591 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
592 static void mali_pp_scheduler_do_job_delete(void *arg)
594 _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
595 struct mali_pp_job *job;
596 struct mali_pp_job *tmp;
600 _mali_osk_lock_wait(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
603 * Quickly "unhook" the jobs pending to be deleted, so we can release the lock before
604 * we start deleting the job objects (without any locks held
606 _mali_osk_list_move_list(&pp_scheduler_job_deletion_queue, &list);
608 _mali_osk_lock_signal(pp_scheduler_job_delete_lock, _MALI_OSK_LOCKMODE_RW);
610 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list, struct mali_pp_job, list)
612 mali_pp_job_delete(job); /* delete the job object itself */
617 void mali_pp_scheduler_job_done(struct mali_group *group, struct mali_pp_job *job, u32 sub_job, mali_bool success)
619 mali_bool job_is_done;
620 mali_bool barrier_enforced = MALI_FALSE;
622 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) part %u/%u completed (%s)\n",
623 mali_pp_job_is_virtual(job) ? "Virtual" : "Physical",
624 mali_pp_job_get_id(job),
626 mali_pp_job_get_sub_job_count(job),
627 success ? "success" : "failure"));
628 MALI_ASSERT_GROUP_LOCKED(group);
629 mali_pp_scheduler_lock();
631 mali_pp_job_mark_sub_job_completed(job, success);
633 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job) == mali_group_is_virtual(group));
635 job_is_done = mali_pp_job_is_complete(job);
639 struct mali_session_data *session = mali_pp_job_get_session(job);
640 struct mali_pp_job *job_head;
642 /* Remove job from session list */
643 _mali_osk_list_del(&job->session_list);
645 /* Send notification back to user space */
646 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: All parts completed for %s job %u (0x%08X)\n",
647 mali_pp_job_is_virtual(job) ? "virtual" : "physical",
648 mali_pp_job_get_id(job), job));
649 #if defined(CONFIG_SYNC)
651 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
655 if (success) error = 0;
656 else error = -EFAULT;
657 MALI_DEBUG_PRINT(4, ("Sync: Signal %spoint for job %d\n",
658 success ? "" : "failed ",
659 mali_pp_job_get_id(job)));
660 mali_sync_signal_pt(job->sync_point, error);
665 #if defined(MALI_PP_SCHEDULER_USE_DEFERRED_JOB_DELETE)
666 mali_pp_scheduler_return_job_to_user(job, MALI_TRUE);
668 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE);
671 mali_pm_core_event(MALI_CORE_EVENT_PP_STOP);
673 /* Resolve any barriers */
674 if (!_mali_osk_list_empty(&session->job_list))
676 job_head = _MALI_OSK_LIST_ENTRY(session->job_list.next, struct mali_pp_job, session_list);
677 if (mali_pp_job_has_active_barrier(job_head))
679 barrier_enforced = MALI_TRUE;
680 mali_pp_job_barrier_enforced(job_head);
685 /* If paused, then this was the last job, so wake up sleeping workers */
688 /* Wake up sleeping workers. Their wake-up condition is that
689 * num_slots == num_slots_idle, so unless we are done working, no
690 * threads will actually be woken up.
692 _mali_osk_wait_queue_wake_up(pp_scheduler_working_wait_queue);
693 mali_pp_scheduler_unlock();
697 if (barrier_enforced)
699 /* A barrier was resolved, so schedule previously blocked jobs */
700 _mali_osk_wq_schedule_work(pp_scheduler_wq_schedule);
702 /* TODO: Subjob optimisation */
705 /* Recycle variables */
709 if (mali_group_is_virtual(group))
713 /* Now that the virtual group is idle, check if we should reconfigure */
714 struct mali_pp_job *physical_job = NULL;
715 struct mali_group *physical_group = NULL;
717 /* Obey the policy */
718 virtual_group_working = MALI_FALSE;
720 if (mali_pp_scheduler_can_move_virtual_to_physical())
722 /* There is a runnable physical job and we can acquire a physical group */
723 physical_job = mali_pp_scheduler_get_physical_job();
724 MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(physical_job));
726 /* Mark subjob as started */
727 sub_job = mali_pp_job_get_first_unstarted_sub_job(physical_job);
728 mali_pp_job_mark_sub_job_started(physical_job, sub_job);
730 /* Remove job from queue (if we now got the last subjob) */
731 mali_pp_scheduler_dequeue_physical_job(physical_job);
733 /* Acquire a physical group from the virtual group
734 * It's state will be LEAVING_VIRTUAL and must be set to IDLE before it can be used */
735 physical_group = mali_group_acquire_group(virtual_group);
737 /* Move physical group to the working list, as we will soon start a job on it */
738 _mali_osk_list_move(&(physical_group->pp_scheduler_list), &group_list_working);
741 /* Start the next virtual job */
742 job = mali_pp_scheduler_get_virtual_job();
745 /* There is a runnable virtual job */
746 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
747 MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
748 MALI_DEBUG_ASSERT(1 == mali_pp_job_get_sub_job_count(job));
750 mali_pp_job_mark_sub_job_started(job, 0);
752 /* Remove job from queue */
753 mali_pp_scheduler_dequeue_virtual_job(job);
755 /* Virtual group is now working */
756 virtual_group_working = MALI_TRUE;
758 mali_pp_scheduler_unlock();
761 if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(group, job, 0))
763 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Virtual job %u (0x%08X) part %u/%u started (from job_done)\n",
764 mali_pp_job_get_id(job), job, 1,
765 mali_pp_job_get_sub_job_count(job)));
769 MALI_DEBUG_ASSERT(0);
774 mali_pp_scheduler_unlock();
777 /* Start a physical job (if we acquired a physical group earlier) */
778 if (NULL != physical_job && NULL != physical_group)
780 mali_group_lock(physical_group);
782 /* Set the group state from LEAVING_VIRTUAL to IDLE to complete the transition */
783 physical_group->state = MALI_GROUP_STATE_IDLE;
785 /* Start job on core */
786 if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(physical_group, physical_job, sub_job))
788 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Physical job %u (0x%08X) part %u/%u started (from job_done)\n",
789 mali_pp_job_get_id(physical_job), physical_job, sub_job + 1,
790 mali_pp_job_get_sub_job_count(physical_job)));
794 /* @@@@ todo: this cant fail on Mali-300+, no need to implement put back of job */
795 MALI_DEBUG_ASSERT(0);
798 mali_group_unlock(physical_group);
804 job = mali_pp_scheduler_get_physical_job();
807 /* There is a runnable physical job */
808 MALI_DEBUG_ASSERT(mali_pp_job_has_unstarted_sub_jobs(job));
810 /* Mark subjob as started */
811 sub_job = mali_pp_job_get_first_unstarted_sub_job(job);
812 mali_pp_job_mark_sub_job_started(job, sub_job);
814 /* Remove job from queue (if we now got the last subjob) */
815 mali_pp_scheduler_dequeue_physical_job(job);
817 mali_pp_scheduler_unlock();
819 /* Group is already on the working list, so start the job */
820 if (_MALI_OSK_ERR_OK == mali_group_start_pp_job(group, job, sub_job))
822 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Physical job %u (0x%08X) part %u/%u started (from job_done)\n",
823 mali_pp_job_get_id(job), job, sub_job + 1,
824 mali_pp_job_get_sub_job_count(job)));
828 MALI_DEBUG_ASSERT(0);
831 else if (NULL != virtual_group)
833 /* Rejoin virtual group */
834 /* TODO: In the future, a policy check might be useful here */
836 /* We're no longer needed on the scheduler list */
837 _mali_osk_list_delinit(&(group->pp_scheduler_list));
839 /* Make sure no interrupts are handled for this group during
840 * the transition from physical to virtual */
841 group->state = MALI_GROUP_STATE_JOINING_VIRTUAL;
843 mali_pp_scheduler_unlock();
844 mali_group_unlock(group);
846 mali_group_lock(virtual_group);
847 /* TODO: Take group lock also? */
848 mali_group_add_group(virtual_group, group);
849 mali_group_unlock(virtual_group);
851 /* We need to return from this function with the group lock held */
852 /* TODO: optimise! */
853 mali_group_lock(group);
857 _mali_osk_list_move(&(group->pp_scheduler_list), &group_list_idle);
858 mali_pp_scheduler_unlock();
863 void mali_pp_scheduler_suspend(void)
865 mali_pp_scheduler_lock();
866 pause_count++; /* Increment the pause_count so that no more jobs will be scheduled */
867 mali_pp_scheduler_unlock();
869 /* Go to sleep. When woken up again (in mali_pp_scheduler_job_done), the
870 * mali_pp_scheduler_suspended() function will be called. This will return true
871 * iff state is idle and pause_count > 0, so if the core is active this
872 * will not do anything.
874 _mali_osk_wait_queue_wait_event(pp_scheduler_working_wait_queue, mali_pp_scheduler_is_suspended);
877 void mali_pp_scheduler_resume(void)
879 mali_pp_scheduler_lock();
880 pause_count--; /* Decrement pause_count to allow scheduling again (if it reaches 0) */
881 mali_pp_scheduler_unlock();
882 if (0 == pause_count)
884 mali_pp_scheduler_schedule();
888 MALI_STATIC_INLINE void mali_pp_scheduler_queue_job(struct mali_pp_job *job, struct mali_session_data *session)
890 MALI_DEBUG_ASSERT_POINTER(job);
892 mali_pm_core_event(MALI_CORE_EVENT_PP_START);
894 mali_pp_scheduler_lock();
896 if (mali_pp_job_is_virtual(job))
899 virtual_job_queue_depth += 1;
900 _mali_osk_list_addtail(&job->list, &virtual_job_queue);
904 job_queue_depth += mali_pp_job_get_sub_job_count(job);
905 _mali_osk_list_addtail(&job->list, &job_queue);
908 if (mali_pp_job_has_active_barrier(job) && _mali_osk_list_empty(&session->job_list))
910 /* No running jobs on this session, so barrier condition already met */
911 mali_pp_job_barrier_enforced(job);
914 /* Add job to session list */
915 _mali_osk_list_addtail(&job->session_list, &session->job_list);
917 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued\n",
918 mali_pp_job_is_virtual(job) ? "Virtual" : "Physical",
919 mali_pp_job_get_id(job), job, mali_pp_job_get_sub_job_count(job)));
921 mali_pp_scheduler_unlock();
924 #if defined(CONFIG_SYNC)
926 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
927 static void sync_callback(struct sync_fence *fence, struct sync_fence_waiter *waiter)
929 struct mali_pp_job *job = _MALI_OSK_CONTAINER_OF(waiter, struct mali_pp_job, sync_waiter);
931 /* Schedule sync_callback_work */
932 _mali_osk_wq_schedule_work(job->sync_work);
935 static void sync_callback_work(void *arg)
937 struct mali_pp_job *job = (struct mali_pp_job *)arg;
938 struct mali_session_data *session;
941 MALI_DEBUG_ASSERT_POINTER(job);
943 session = job->session;
945 /* Remove job from session pending job list */
946 _mali_osk_lock_wait(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
947 _mali_osk_list_delinit(&job->list);
948 _mali_osk_lock_signal(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
950 err = sync_fence_wait(job->pre_fence, 0);
951 if (likely(0 == err))
953 MALI_DEBUG_PRINT(3, ("Mali sync: Job %d ready to run\n", mali_pp_job_get_id(job)));
955 mali_pp_scheduler_queue_job(job, session);
957 mali_pp_scheduler_schedule();
961 /* Fence signaled error */
962 MALI_DEBUG_PRINT(3, ("Mali sync: Job %d abort due to sync error\n", mali_pp_job_get_id(job)));
964 if (job->sync_point) mali_sync_signal_pt(job->sync_point, err);
966 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
967 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
973 _mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx, _mali_uk_pp_start_job_s *uargs, int *fence)
975 struct mali_session_data *session;
976 struct mali_pp_job *job;
977 #if defined(CONFIG_SYNC)
979 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
984 MALI_DEBUG_ASSERT_POINTER(uargs);
985 MALI_DEBUG_ASSERT_POINTER(ctx);
987 session = (struct mali_session_data*)ctx;
989 job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id());
992 MALI_PRINT_ERROR(("Failed to create job!\n"));
993 return _MALI_OSK_ERR_NOMEM;
996 if (_MALI_OSK_ERR_OK != mali_pp_job_check(job))
998 /* Not a valid job, return to user immediately */
999 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1000 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1001 return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1004 #if PROFILING_SKIP_PP_JOBS || PROFILING_SKIP_PP_AND_GP_JOBS
1005 #warning PP jobs will not be executed
1006 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE);
1007 return _MALI_OSK_ERR_OK;
1010 #if defined(CONFIG_SYNC)
1012 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3,4,0)
1013 if (_MALI_PP_JOB_FLAG_FENCE & job->uargs.flags)
1015 job->sync_point = mali_stream_create_point(job->uargs.stream);
1017 if (unlikely(NULL == job->sync_point))
1019 /* Fence creation failed. */
1020 MALI_DEBUG_PRINT(2, ("Failed to create sync point for job %d\n", mali_pp_job_get_id(job)));
1021 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1022 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1023 return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1026 post_fence = mali_stream_create_fence(job->sync_point);
1028 if (unlikely(0 > post_fence))
1030 /* Fence creation failed. */
1031 /* mali_stream_create_fence already freed the sync_point */
1032 MALI_DEBUG_PRINT(2, ("Failed to create fence for job %d\n", mali_pp_job_get_id(job)));
1033 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1034 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1035 return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1038 /* Grab a reference to the fence. It must be around when the
1039 * job is completed, so the point can be signalled. */
1040 sync_fence_fdget(post_fence);
1042 *fence = post_fence;
1044 MALI_DEBUG_PRINT(3, ("Sync: Created fence %d for job %d\n", post_fence, mali_pp_job_get_id(job)));
1047 if (0 < job->uargs.fence)
1049 int pre_fence_fd = job->uargs.fence;
1052 MALI_DEBUG_PRINT(2, ("Sync: Job %d waiting for fence %d\n", mali_pp_job_get_id(job), pre_fence_fd));
1054 job->pre_fence = sync_fence_fdget(pre_fence_fd); /* Reference will be released when job is deleted. */
1055 if (NULL == job->pre_fence)
1057 MALI_DEBUG_PRINT(2, ("Failed to import fence %d\n", pre_fence_fd));
1058 if (job->sync_point) mali_sync_signal_pt(job->sync_point, -EINVAL);
1059 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1060 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1061 return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1064 job->sync_work = _mali_osk_wq_create_work(sync_callback_work, (void*)job);
1065 if (NULL == job->sync_work)
1067 if (job->sync_point) mali_sync_signal_pt(job->sync_point, -ENOMEM);
1068 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1069 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1070 return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1073 /* Add pending job to session pending job list */
1074 _mali_osk_lock_wait(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1075 _mali_osk_list_addtail(&job->list, &session->pending_jobs);
1076 _mali_osk_lock_signal(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1078 sync_fence_waiter_init(&job->sync_waiter, sync_callback);
1079 err = sync_fence_wait_async(job->pre_fence, &job->sync_waiter);
1083 /* No async wait started, remove job from session pending job list */
1084 _mali_osk_lock_wait(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1085 _mali_osk_list_delinit(&job->list);
1086 _mali_osk_lock_signal(session->pending_jobs_lock, _MALI_OSK_LOCKMODE_RW);
1091 /* Fence has already signalled */
1092 mali_pp_scheduler_queue_job(job, session);
1093 if (0 == _mali_osk_list_empty(&group_list_idle)) mali_pp_scheduler_schedule();
1094 return _MALI_OSK_ERR_OK;
1099 if (job->sync_point) mali_sync_signal_pt(job->sync_point, err);
1100 mali_pp_job_mark_sub_job_completed(job, MALI_FALSE); /* Flagging the job as failed. */
1101 mali_pp_scheduler_return_job_to_user(job, MALI_FALSE); /* This will also delete the job object */
1102 return _MALI_OSK_ERR_OK; /* User is notified via a notification, so this call is ok */
1108 #endif /* CONFIG_SYNC */
1110 mali_pp_scheduler_queue_job(job, session);
1112 if (!_mali_osk_list_empty(&group_list_idle) || !virtual_group_working)
1114 mali_pp_scheduler_schedule();
1118 return _MALI_OSK_ERR_OK;
1121 _mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args)
1123 MALI_DEBUG_ASSERT_POINTER(args);
1124 MALI_DEBUG_ASSERT_POINTER(args->ctx);
1125 args->number_of_cores = num_cores;
1126 return _MALI_OSK_ERR_OK;
1129 _mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args)
1131 MALI_DEBUG_ASSERT_POINTER(args);
1132 MALI_DEBUG_ASSERT_POINTER(args->ctx);
1133 args->version = pp_version;
1134 return _MALI_OSK_ERR_OK;
1137 void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args)
1139 struct mali_session_data *session;
1140 struct mali_pp_job *job;
1141 struct mali_pp_job *tmp;
1143 MALI_DEBUG_ASSERT_POINTER(args);
1144 MALI_DEBUG_ASSERT_POINTER(args->ctx);
1146 session = (struct mali_session_data*)args->ctx;
1148 /* Check queue for jobs that match */
1149 mali_pp_scheduler_lock();
1150 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &job_queue, struct mali_pp_job, list)
1152 if (mali_pp_job_get_session(job) == session &&
1153 mali_pp_job_get_frame_builder_id(job) == (u32)args->fb_id &&
1154 mali_pp_job_get_flush_id(job) == (u32)args->flush_id)
1156 if (args->wbx & _MALI_UK_PP_JOB_WB0)
1158 mali_pp_job_disable_wb0(job);
1160 if (args->wbx & _MALI_UK_PP_JOB_WB1)
1162 mali_pp_job_disable_wb1(job);
1164 if (args->wbx & _MALI_UK_PP_JOB_WB2)
1166 mali_pp_job_disable_wb2(job);
1171 mali_pp_scheduler_unlock();
1174 void mali_pp_scheduler_abort_session(struct mali_session_data *session)
1176 struct mali_pp_job *job, *tmp_job;
1177 struct mali_group *group, *tmp_group;
1178 struct mali_group *groups[MALI_MAX_NUMBER_OF_GROUPS];
1181 mali_pp_scheduler_lock();
1182 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Aborting all jobs from session 0x%08x\n", session));
1184 _MALI_OSK_LIST_FOREACHENTRY(job, tmp_job, &session->job_list, struct mali_pp_job, session_list)
1186 /* Remove job from queue (if it's not queued, list_del has no effect) */
1187 _mali_osk_list_delinit(&job->list);
1189 if (mali_pp_job_is_virtual(job))
1191 MALI_DEBUG_ASSERT(1 == mali_pp_job_get_sub_job_count(job));
1192 if (0 == mali_pp_job_get_first_unstarted_sub_job(job))
1194 --virtual_job_queue_depth;
1199 job_queue_depth -= mali_pp_job_get_sub_job_count(job) - mali_pp_job_get_first_unstarted_sub_job(job);
1202 /* Mark all unstarted jobs as failed */
1203 mali_pp_job_mark_unstarted_failed(job);
1205 if (mali_pp_job_is_complete(job))
1207 _mali_osk_list_del(&job->session_list);
1209 /* It is safe to delete the job, since it won't land in job_done() */
1210 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Aborted PP job 0x%08x\n", job));
1211 mali_pp_job_delete(job);
1213 mali_pm_core_event(MALI_CORE_EVENT_PP_STOP);
1217 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Keeping partially started PP job 0x%08x in session\n", job));
1221 _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working, struct mali_group, pp_scheduler_list)
1223 groups[i++] = group;
1226 _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_idle, struct mali_group, pp_scheduler_list)
1228 groups[i++] = group;
1231 mali_pp_scheduler_unlock();
1233 /* Abort running jobs from this session */
1236 mali_group_abort_session(groups[--i], session);
1239 if (NULL != virtual_group)
1241 mali_group_abort_session(virtual_group, session);
1245 static mali_bool mali_pp_scheduler_is_suspended(void)
1249 mali_pp_scheduler_lock();
1250 ret = pause_count > 0 && _mali_osk_list_empty(&group_list_working) && !virtual_group_working;
1251 mali_pp_scheduler_unlock();
1256 int mali_pp_scheduler_get_queue_depth(void)
1258 return job_queue_depth;
1261 #if MALI_STATE_TRACKING
1262 u32 mali_pp_scheduler_dump_state(char *buf, u32 size)
1265 struct mali_group *group;
1266 struct mali_group *temp;
1268 n += _mali_osk_snprintf(buf + n, size - n, "PP:\n");
1269 n += _mali_osk_snprintf(buf + n, size - n, "\tQueue is %s\n", _mali_osk_list_empty(&job_queue) ? "empty" : "not empty");
1270 n += _mali_osk_snprintf(buf + n, size - n, "\n");
1272 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, pp_scheduler_list)
1274 n += mali_group_dump_state(group, buf + n, size - n);
1277 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, pp_scheduler_list)
1279 n += mali_group_dump_state(group, buf + n, size - n);
1282 if (NULL != virtual_group)
1284 n += mali_group_dump_state(virtual_group, buf + n, size -n);
1287 n += _mali_osk_snprintf(buf + n, size - n, "\n");
1292 /* This function is intended for power on reset of all cores.
1293 * No locking is done for the list iteration, which can only be safe if the
1294 * scheduler is paused and all cores idle. That is always the case on init and
1296 void mali_pp_scheduler_reset_all_groups(void)
1298 struct mali_group *group, *temp;
1300 if (NULL != virtual_group)
1302 mali_group_reset(virtual_group);
1305 MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working));
1307 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, pp_scheduler_list)
1309 mali_group_reset(group);
1313 void mali_pp_scheduler_zap_all_active(struct mali_session_data *session)
1315 struct mali_group *group, *temp;
1316 struct mali_group *groups[MALI_MAX_NUMBER_OF_GROUPS];
1319 if (NULL != virtual_group)
1321 mali_group_zap_session(virtual_group, session);
1324 mali_pp_scheduler_lock();
1325 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, pp_scheduler_list)
1327 groups[i++] = group;
1329 mali_pp_scheduler_unlock();
1333 mali_group_zap_session(groups[--i], session);